diff --git a/.gitignore b/.gitignore index b67dc8e6499..011a4b005dc 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ tools/cli/cloudmonkey/precache.py tools/marvin/marvin/cloudstackAPI/ tools/marvin/build/ tools/cli/build/ +tools/appliance/systemvmtemplate/packer_cache/ *.jar *.war *.mar diff --git a/agent/src/main/java/com/cloud/agent/AgentShell.java b/agent/src/main/java/com/cloud/agent/AgentShell.java index 01654ac9caa..0ab5b7863a8 100644 --- a/agent/src/main/java/com/cloud/agent/AgentShell.java +++ b/agent/src/main/java/com/cloud/agent/AgentShell.java @@ -16,6 +16,26 @@ // under the License. package com.cloud.agent; +import com.cloud.agent.Agent.ExitStatus; +import com.cloud.agent.dao.StorageComponent; +import com.cloud.agent.dao.impl.PropertiesStorage; +import com.cloud.resource.ServerResource; +import com.cloud.utils.LogUtils; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.ProcessUtil; +import com.cloud.utils.PropertiesUtil; +import com.cloud.utils.backoff.BackoffAlgorithm; +import com.cloud.utils.backoff.impl.ConstantTimeBackoff; +import com.cloud.utils.exception.CloudRuntimeException; +import com.google.common.base.Strings; +import org.apache.commons.daemon.Daemon; +import org.apache.commons.daemon.DaemonContext; +import org.apache.commons.daemon.DaemonInitException; +import org.apache.commons.lang.math.NumberUtils; +import org.apache.log4j.Logger; +import org.apache.log4j.xml.DOMConfigurator; + +import javax.naming.ConfigurationException; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -30,28 +50,6 @@ import java.util.Map; import java.util.Properties; import java.util.UUID; -import javax.naming.ConfigurationException; - -import org.apache.commons.daemon.Daemon; -import org.apache.commons.daemon.DaemonContext; -import org.apache.commons.daemon.DaemonInitException; -import org.apache.commons.lang.math.NumberUtils; -import org.apache.log4j.Logger; -import org.apache.log4j.xml.DOMConfigurator; - -import com.cloud.agent.Agent.ExitStatus; -import com.cloud.agent.dao.StorageComponent; -import com.cloud.agent.dao.impl.PropertiesStorage; -import com.cloud.resource.ServerResource; -import com.cloud.utils.LogUtils; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.ProcessUtil; -import com.cloud.utils.PropertiesUtil; -import com.cloud.utils.backoff.BackoffAlgorithm; -import com.cloud.utils.backoff.impl.ConstantTimeBackoff; -import com.cloud.utils.exception.CloudRuntimeException; -import com.google.common.base.Strings; - public class AgentShell implements IAgentShell, Daemon { private static final Logger s_logger = Logger.getLogger(AgentShell.class.getName()); @@ -423,13 +421,13 @@ public class AgentShell implements IAgentShell, Daemon { } catch (final ClassNotFoundException e) { throw new ConfigurationException("Resource class not found: " + name + " due to: " + e.toString()); } catch (final SecurityException e) { - throw new ConfigurationException("Security excetion when loading resource: " + name + " due to: " + e.toString()); + throw new ConfigurationException("Security exception when loading resource: " + name + " due to: " + e.toString()); } catch (final NoSuchMethodException e) { - throw new ConfigurationException("Method not found excetion when loading resource: " + name + " due to: " + e.toString()); + throw new ConfigurationException("Method not found exception when loading resource: " + name + " due to: " + e.toString()); } catch (final IllegalArgumentException e) { - throw new ConfigurationException("Illegal argument excetion when loading resource: " + name + " due to: " + e.toString()); + throw new ConfigurationException("Illegal argument exception when loading resource: " + name + " due to: " + e.toString()); } catch (final InstantiationException e) { - throw new ConfigurationException("Instantiation excetion when loading resource: " + name + " due to: " + e.toString()); + throw new ConfigurationException("Instantiation exception when loading resource: " + name + " due to: " + e.toString()); } catch (final IllegalAccessException e) { throw new ConfigurationException("Illegal access exception when loading resource: " + name + " due to: " + e.toString()); } catch (final InvocationTargetException e) { diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index bf43940c0a0..5fbbc872d34 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -73,4 +73,5 @@ public interface VmDetailConstants { String DISK_OFFERING = "diskOffering"; String DEPLOY_AS_IS_CONFIGURATION = "configurationId"; + String CKS_CONTROL_NODE_LOGIN_USER = "controlNodeLoginUser"; } diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 86753525154..588ed92a0d8 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -73,10 +73,10 @@ - def projectVersion = project.version + def projectVersion = project.properties['project.systemvm.template.version'] String[] versionParts = projectVersion.tokenize('.') - pom.properties['cs.version'] = "4.16" - pom.properties['patch.version'] = "0" + pom.properties['cs.version'] = versionParts[0] + "." + versionParts[1] + pom.properties['patch.version'] = versionParts[2] @@ -146,7 +146,7 @@ bash templateConfig.sh - ${project.version} + ${project.systemvm.template.version} diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index cf3f7282f9f..3c6abbc6e86 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -367,10 +367,11 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker { return; } - SystemVmTemplateRegistration.parseMetadataFile(); - final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue); - SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.valueOf(currentVersion.getMajorRelease()) + "." + String.valueOf(currentVersion.getMinorRelease()); - SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(currentVersion.getPatchRelease()); + String csVersion = SystemVmTemplateRegistration.parseMetadataFile(); + final CloudStackVersion sysVmVersion = CloudStackVersion.parse(csVersion); + final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue); + SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.valueOf(sysVmVersion.getMajorRelease()) + "." + String.valueOf(sysVmVersion.getMinorRelease()); + SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease()); s_logger.info("DB version = " + dbVersion + " Code Version = " + currentVersion); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index d788adac824..1eb3fdd20bb 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -54,7 +54,7 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreDaoImpl; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.commons.codec.digest.DigestUtils; +import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.ini4j.Ini; @@ -64,7 +64,6 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -351,16 +350,6 @@ public class SystemVmTemplateRegistration { } } - private String calculateChecksum(File file) { - try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { - return DigestUtils.md5Hex(is); - } catch (IOException e) { - String errMsg = "Failed to calculate template checksum"; - LOGGER.error(errMsg, e); - throw new CloudRuntimeException(errMsg, e); - } - } - public Long getRegisteredTemplateId(Pair hypervisorAndTemplateName) { VMTemplateVO vmTemplate = vmTemplateDao.findLatestTemplateByName(hypervisorAndTemplateName.second()); Long templateId = null; @@ -371,24 +360,24 @@ public class SystemVmTemplateRegistration { } private static String fetchTemplatesPath() { - String filePath = RELATIVE_TEMPLATE_PATH + METADATA_FILE_NAME; + String filePath = RELATIVE_TEMPLATE_PATH + METADATA_FILE_NAME; + LOGGER.debug(String.format("Looking for file [ %s ] in the classpath.", filePath)); + File metaFile = new File(filePath); + String templatePath = null; + if (metaFile.exists()) { + templatePath = RELATIVE_TEMPLATE_PATH; + } + if (templatePath == null) { + filePath = ABSOLUTE_TEMPLATE_PATH + METADATA_FILE_NAME; + metaFile = new File(filePath); + templatePath = ABSOLUTE_TEMPLATE_PATH; LOGGER.debug(String.format("Looking for file [ %s ] in the classpath.", filePath)); - File metaFile = new File(filePath); - String templatePath = null; - if (metaFile.exists()) { - templatePath = RELATIVE_TEMPLATE_PATH; - } - if (templatePath == null) { - filePath = ABSOLUTE_TEMPLATE_PATH + METADATA_FILE_NAME; - metaFile = new File(filePath); - templatePath = ABSOLUTE_TEMPLATE_PATH; - LOGGER.debug(String.format("Looking for file [ %s ] in the classpath.", filePath)); - if (!metaFile.exists()) { - String errMsg = String.format("Unable to locate metadata file in your setup at %s", filePath.toString()); - LOGGER.error(errMsg); - throw new CloudRuntimeException(errMsg); - } + if (!metaFile.exists()) { + String errMsg = String.format("Unable to locate metadata file in your setup at %s", filePath.toString()); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); } + } return templatePath; } @@ -481,7 +470,7 @@ public class SystemVmTemplateRegistration { template.setCrossZones(true); template.setHypervisorType(details.getHypervisorType()); template.setState(VirtualMachineTemplate.State.Inactive); - template.setDeployAsIs(Hypervisor.HypervisorType.VMware.equals(details.getHypervisorType())); + template.setDeployAsIs(false); template = vmTemplateDao.persist(template); return template; } @@ -577,12 +566,12 @@ public class SystemVmTemplateRegistration { } private void updateTemplateTablesOnFailure(long templateId) { - VMTemplateVO template = vmTemplateDao.createForUpdate(templateId); - template.setState(VirtualMachineTemplate.State.Inactive); - vmTemplateDao.update(template.getId(), template); - vmTemplateDao.remove(templateId); - TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByTemplate(template.getId(), DataStoreRole.Image); - templateDataStoreDao.remove(templateDataStoreVO.getId()); + VMTemplateVO template = vmTemplateDao.createForUpdate(templateId); + template.setState(VirtualMachineTemplate.State.Inactive); + vmTemplateDao.update(template.getId(), template); + vmTemplateDao.remove(templateId); + TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByTemplate(template.getId(), DataStoreRole.Image); + templateDataStoreDao.remove(templateDataStoreVO.getId()); } public static void unmountStore(String filePath) { @@ -603,7 +592,7 @@ public class SystemVmTemplateRegistration { } private void setupTemplate(String templateName, Pair hypervisorAndTemplateName, - String destTempFolder) throws CloudRuntimeException { + String destTempFolder) throws CloudRuntimeException { String setupTmpltScript = Script.findScript(storageScriptsDir, "setup-sysvm-tmplt"); if (setupTmpltScript == null) { throw new CloudRuntimeException("Unable to find the createtmplt.sh"); @@ -651,7 +640,7 @@ public class SystemVmTemplateRegistration { } public void registerTemplate(Pair hypervisorAndTemplateName, - Pair storeUrlAndId, VMTemplateVO templateVO, String filePath) { + Pair storeUrlAndId, VMTemplateVO templateVO, String filePath) { Long templateId = null; try { templateId = templateVO.getId(); @@ -690,7 +679,13 @@ public class SystemVmTemplateRegistration { } } - public static void parseMetadataFile() { + /** + * This method parses the metadata file consisting of the systemVM templates information + * @return the version of the systemvm template that is to be used. This is done to in order + * to fallback on the latest available version of the systemVM template when there does not + * exist a template corresponding to the current code version. + */ + public static String parseMetadataFile() { try { Ini ini = new Ini(); ini.load(new FileReader(METADATA_FILE)); @@ -702,6 +697,8 @@ public class SystemVmTemplateRegistration { NewTemplateChecksum.put(hypervisorType, section.get("checksum")); NewTemplateUrl.put(hypervisorType, section.get("downloadurl")); } + Ini.Section section = ini.get("default"); + return section.get("version"); } catch (Exception e) { String errMsg = String.format("Failed to parse systemVM template metadata file: %s", METADATA_FILE); LOGGER.error(errMsg, e); @@ -735,7 +732,7 @@ public class SystemVmTemplateRegistration { } File tempFile = new File(TEMPLATES_PATH + matchedTemplate); - String templateChecksum = calculateChecksum(tempFile); + String templateChecksum = DigestHelper.calculateChecksum(tempFile); if (!templateChecksum.equals(NewTemplateChecksum.get(getHypervisorType(hypervisor)))) { LOGGER.error(String.format("Checksum mismatch: %s != %s ", templateChecksum, NewTemplateChecksum.get(getHypervisorType(hypervisor)))); templatesFound = false; @@ -812,9 +809,6 @@ public class SystemVmTemplateRegistration { private void updateRegisteredTemplateDetails(Long templateId, Map.Entry hypervisorAndTemplateName) { VMTemplateVO templateVO = vmTemplateDao.findById(templateId); templateVO.setTemplateType(Storage.TemplateType.SYSTEM); - if (Hypervisor.HypervisorType.VMware == templateVO.getHypervisorType()) { - templateVO.setDeployAsIs(true); - } boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); if (!updated) { String errMsg = String.format("updateSystemVmTemplates:Exception while updating template with id %s to be marked as 'system'", templateId); @@ -834,9 +828,6 @@ public class SystemVmTemplateRegistration { private void updateTemplateUrlAndChecksum(VMTemplateVO templateVO, Map.Entry hypervisorAndTemplateName) { templateVO.setUrl(NewTemplateUrl.get(hypervisorAndTemplateName.getKey())); templateVO.setChecksum(NewTemplateChecksum.get(hypervisorAndTemplateName.getKey())); - if (Hypervisor.HypervisorType.VMware == templateVO.getHypervisorType()) { - templateVO.setDeployAsIs(true); - } boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); if (!updated) { String errMsg = String.format("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type %s", hypervisorAndTemplateName.getKey().name()); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql b/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql index 650aa0313b8..d53e2181cef 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql @@ -128,6 +128,29 @@ CALL `cloud_usage`.`IDEMPOTENT_ADD_UNIQUE_INDEX`('cloud_usage.usage_volume', 'id -- Add PK to cloud_usage.usage_vpn_user CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_vpn_user', 'id', 'BIGINT(20) NOT NULL AUTO_INCREMENT FIRST, ADD PRIMARY KEY (`id`)'); +UPDATE `cloud`.`vm_template` SET deploy_as_is = 0 WHERE id = 8; + +CREATE PROCEDURE `cloud`.`UPDATE_KUBERNETES_NODE_DETAILS`() +BEGIN + DECLARE vmid BIGINT +; DECLARE done TINYINT DEFAULT FALSE +; DECLARE vmidcursor CURSOR FOR SELECT DISTINCT(vm_id) FROM `cloud`.`kubernetes_cluster_vm_map` +; DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE +; OPEN vmidcursor +; vmid_loop:LOOP + FETCH NEXT FROM vmidcursor INTO vmid +; IF done THEN + LEAVE vmid_loop +; ELSE + INSERT `cloud`.`user_vm_details` (vm_id, name, value, display) VALUES (vmid, 'controlNodeLoginUser', 'core', 1) +; END IF +; END LOOP +; CLOSE vmidcursor +; END; + +CALL `cloud`.`UPDATE_KUBERNETES_NODE_DETAILS`(); +DROP PROCEDURE IF EXISTS `cloud`.`UPDATE_KUBERNETES_NODE_DETAILS`; + -- Add support for VMware 7.0.2.0 INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '7.0.2.0', 1024, 0, 59, 64, 1, 1); -- Copy VMware 7.0.1.0 hypervisor guest OS mappings to VMware 7.0.2.0 diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index c3093530e7b..891c73db1bb 100644 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -23,8 +23,10 @@ function getTemplateVersion() { subversion1="$(cut -d'.' -f1 <<<"$version")" subversion2="$(cut -d'.' -f2 <<<"$version")" minorversion="$(cut -d'.' -f3 <<<"$version")" + securityversion="$(cut -d'.' -f4 <<<"$version")" export CS_VERSION="${subversion1}"."${subversion2}" export CS_MINOR_VERSION="${minorversion}" + export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}" } function getGenericName() { @@ -52,12 +54,14 @@ function getChecksum() { function createMetadataFile() { local fileData=$(cat $SOURCEFILE) + echo -e "["default"]\nversion = $VERSION.${securityversion}\n" >> $METADATAFILE for i in "${!templates[@]}" do section="$i" hvName=$(getGenericName $i) - templatename="systemvm-${i}-${CS_VERSION}.${CS_MINOR_VERSION}" - checksum=$(getChecksum "$fileData" $hvName) + + templatename="systemvm-${i}-${VERSION}" + checksum=$(getChecksum "$fileData" "$VERSION-$hvName") downloadurl="${templates[$i]}" filename=$(echo ${downloadurl##*'/'}) echo -e "["$section"]\ntemplatename = $templatename\nchecksum = $checksum\ndownloadurl = $downloadurl\nfilename = $filename\n" >> $METADATAFILE @@ -66,12 +70,12 @@ function createMetadataFile() { declare -A templates getTemplateVersion $1 -templates=( ["kvm"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-${CS_VERSION}.${CS_MINOR_VERSION}-kvm.qcow2.bz2" - ["vmware"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-${CS_VERSION}.${CS_MINOR_VERSION}-vmware.ova" - ["xenserver"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-xen.vhd.bz2" - ["hyperv"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-hyperv.vhd.zip" - ["lxc"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-kvm.qcow2.bz2" - ["ovm3"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-ovm.raw.bz2" ) +templates=( ["kvm"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-kvm.qcow2.bz2" + ["vmware"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-vmware.ova" + ["xenserver"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-xen.vhd.bz2" + ["hyperv"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-hyperv.vhd.zip" + ["lxc"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-kvm.qcow2.bz2" + ["ovm3"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-ovm.raw.bz2" ) PARENTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/dist/systemvm-templates/" diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index aec96fe6778..550ac74df4b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -16,57 +16,6 @@ // under the License. package com.cloud.kubernetes.cluster; -import java.math.BigInteger; -import java.net.MalformedURLException; -import java.net.URL; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; - -import org.apache.cloudstack.acl.ControlledEntity; -import org.apache.cloudstack.acl.SecurityChecker; -import org.apache.cloudstack.annotation.AnnotationService; -import org.apache.cloudstack.annotation.dao.AnnotationDao; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ApiConstants.VMDetails; -import org.apache.cloudstack.api.ResponseObject.ResponseView; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; -import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; -import org.apache.cloudstack.api.response.KubernetesClusterResponse; -import org.apache.cloudstack.api.response.ListResponse; -import org.apache.cloudstack.api.response.UserVmResponse; -import org.apache.cloudstack.config.ApiServiceConfiguration; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.NetworkOfferingJoinDao; import com.cloud.api.query.dao.TemplateJoinDao; @@ -165,6 +114,53 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.annotation.AnnotationService; +import org.apache.cloudstack.annotation.dao.AnnotationDao; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiConstants.VMDetails; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; +import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.config.ApiServiceConfiguration; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.math.BigInteger; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -428,21 +424,18 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne private void validateDockerRegistryParams(final String dockerRegistryUserName, final String dockerRegistryPassword, - final String dockerRegistryUrl, - final String dockerRegistryEmail) { + final String dockerRegistryUrl) { // if no params related to docker registry specified then nothing to validate so return true if ((dockerRegistryUserName == null || dockerRegistryUserName.isEmpty()) && (dockerRegistryPassword == null || dockerRegistryPassword.isEmpty()) && - (dockerRegistryUrl == null || dockerRegistryUrl.isEmpty()) && - (dockerRegistryEmail == null || dockerRegistryEmail.isEmpty())) { + (dockerRegistryUrl == null || dockerRegistryUrl.isEmpty())) { return; } // all params related to docker registry must be specified or nothing if (!((dockerRegistryUserName != null && !dockerRegistryUserName.isEmpty()) && (dockerRegistryPassword != null && !dockerRegistryPassword.isEmpty()) && - (dockerRegistryUrl != null && !dockerRegistryUrl.isEmpty()) && - (dockerRegistryEmail != null && !dockerRegistryEmail.isEmpty()))) { + (dockerRegistryUrl != null && !dockerRegistryUrl.isEmpty()))) { throw new InvalidParameterValueException("All the docker private registry parameters (username, password, url, email) required are specified"); } @@ -451,12 +444,6 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } catch (MalformedURLException e) { throw new InvalidParameterValueException("Invalid docker registry url specified"); } - - Pattern VALID_EMAIL_ADDRESS_REGEX = Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,6}$", Pattern.CASE_INSENSITIVE); - Matcher matcher = VALID_EMAIL_ADDRESS_REGEX.matcher(dockerRegistryEmail); - if (!matcher.find()) { - throw new InvalidParameterValueException("Invalid docker registry email specified"); - } } private DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { @@ -619,7 +606,6 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); - final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); final Long nodeRootDiskSize = cmd.getNodeRootDiskSize(); final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress(); @@ -706,7 +692,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne throw new InvalidParameterValueException("Given service offering ID: %s is not suitable for Kubernetes cluster"); } - validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl, dockerRegistryEmail); + validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl); Network network = null; if (networkId != null) { @@ -729,7 +715,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } if (!KubernetesClusterExperimentalFeaturesEnabled.value() && (!Strings.isNullOrEmpty(dockerRegistryUrl) || - !Strings.isNullOrEmpty(dockerRegistryUserName) || !Strings.isNullOrEmpty(dockerRegistryEmail) || !Strings.isNullOrEmpty(dockerRegistryPassword))) { + !Strings.isNullOrEmpty(dockerRegistryUserName) || !Strings.isNullOrEmpty(dockerRegistryPassword))) { throw new CloudRuntimeException(String.format("Private registry for the Kubernetes cluster is an experimental feature. Use %s configuration for enabling experimental features", KubernetesClusterExperimentalFeaturesEnabled.key())); } } @@ -779,7 +765,6 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); - final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); final boolean networkCleanup = cmd.getNetworkId() == null; Transaction.execute(new TransactionCallbackNoReturn() { @Override @@ -797,9 +782,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne if (!Strings.isNullOrEmpty(dockerRegistryUrl)) { details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_URL, dockerRegistryUrl, true)); } - if (!Strings.isNullOrEmpty(dockerRegistryEmail)) { - details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_EMAIL, dockerRegistryEmail, true)); - } + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.USERNAME, "admin", true)); SecureRandom random = new SecureRandom(); String randomPassword = new BigInteger(130, random).toString(32); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 62d45a3e028..60111d2a397 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -17,27 +17,6 @@ package com.cloud.kubernetes.cluster.actionworkers; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; - -import javax.inject.Inject; - -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.ca.CAManager; -import org.apache.cloudstack.config.ApiServiceConfiguration; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.io.IOUtils; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.VlanDao; @@ -78,14 +57,40 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.fsm.StateMachine2; import com.cloud.utils.ssh.SshHelper; +import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.UserVmService; +import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.UserVmDetailsDao; import com.google.common.base.Strings; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.ca.CAManager; +import org.apache.cloudstack.config.ApiServiceConfiguration; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; public class KubernetesClusterActionWorker { - public static final String CLUSTER_NODE_VM_USER = "core"; + public static final String CLUSTER_NODE_VM_USER = "cloud"; public static final int CLUSTER_API_PORT = 6443; public static final int CLUSTER_NODES_DEFAULT_START_SSH_PORT = 2222; @@ -120,6 +125,8 @@ public class KubernetesClusterActionWorker { @Inject protected UserVmDao userVmDao; @Inject + protected UserVmDetailsDao userVmDetailsDao; + @Inject protected UserVmService userVmService; @Inject protected VlanDao vlanDao; @@ -175,6 +182,27 @@ public class KubernetesClusterActionWorker { return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), StringUtils.getPreferredCharset()); } + protected String getControlNodeLoginUser() { + List vmMapVOList = getKubernetesClusterVMMaps(); + if (vmMapVOList.size() > 0) { + long vmId = vmMapVOList.get(0).getVmId(); + UserVmVO userVM = userVmDao.findById(vmId); + if (userVM == null) { + throw new CloudRuntimeException("Failed to find login user, Unable to log in to node to fetch details"); + } + Set vm = new HashSet<>(); + vm.add(userVM.getName()); + UserVmDetailVO vmDetail = userVmDetailsDao.findDetail(vmId, VmDetailConstants.CKS_CONTROL_NODE_LOGIN_USER); + if (vmDetail != null && !org.apache.commons.lang3.StringUtils.isEmpty(vmDetail.getValue())) { + return vmDetail.getValue(); + } else { + return CLUSTER_NODE_VM_USER; + } + } else { + return CLUSTER_NODE_VM_USER; + } + } + protected void logMessage(final Level logLevel, final String message, final Exception e) { if (logLevel == Level.INFO) { if (LOGGER.isInfoEnabled()) { @@ -410,6 +438,20 @@ public class KubernetesClusterActionWorker { return vmList; } + protected void updateLoginUserDetails(List clusterVMs) { + if (clusterVMs == null) { + clusterVMs = getKubernetesClusterVMMaps().stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + } + if (!CollectionUtils.isEmpty(clusterVMs)) { + for (Long vmId : clusterVMs) { + UserVm controlNode = userVmDao.findById(vmId); + if (controlNode != null) { + userVmDetailsDao.addDetail(vmId, VmDetailConstants.CKS_CONTROL_NODE_LOGIN_USER, CLUSTER_NODE_VM_USER, true); + } + } + } + } + protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); try { @@ -424,13 +466,14 @@ public class KubernetesClusterActionWorker { protected boolean createCloudStackSecret(String[] keys) { File pkFile = getManagementServerSshPublicKeyFile(); Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + List vmMapVOList = getKubernetesClusterVMMaps(); publicIpAddress = publicIpSshPort.first(); sshPort = publicIpSshPort.second(); try { final String command = String.format("sudo %s/%s -u '%s' -k '%s' -s '%s'", scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]); - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); return result.first(); } catch (Exception e) { @@ -469,10 +512,10 @@ public class KubernetesClusterActionWorker { protected void copyScriptFile(String nodeAddress, final int sshPort, File file, String desitnation) { try { - SshHelper.scpTo(nodeAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + SshHelper.scpTo(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, "~/", file.getAbsolutePath(), "0755"); String cmdStr = String.format("sudo mv ~/%s %s/%s", file.getName(), scriptPath, desitnation); - SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, cmdStr, 10000, 10000, 10 * 60 * 1000); } catch (Exception e) { throw new CloudRuntimeException(e); @@ -496,7 +539,7 @@ public class KubernetesClusterActionWorker { publicIpAddress = publicIpSshPort.first(); sshPort = publicIpSshPort.second(); - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, commands.toString(), 10000, 10000, 60000); return result.first(); } catch (Exception e) { @@ -521,7 +564,7 @@ public class KubernetesClusterActionWorker { try { String command = String.format("sudo %s/%s", scriptPath, deployProviderScriptFilename); - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); // Maybe the file isn't present. Try and copy it @@ -536,7 +579,7 @@ public class KubernetesClusterActionWorker { } // If at first you don't succeed ... - result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); if (!result.first()) { throw new CloudRuntimeException(result.second()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 595aab93999..67fb093534b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -17,26 +17,6 @@ package com.cloud.kubernetes.cluster.actionworkers; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import javax.inject.Inject; - -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; -import org.apache.cloudstack.api.command.user.vm.StartVMCmd; -import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Level; - import com.cloud.capacity.CapacityManager; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; @@ -74,9 +54,9 @@ import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceManager; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.LaunchPermissionDao; -import com.cloud.storage.VolumeApiService; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.SSHKeyPairVO; @@ -95,8 +75,27 @@ import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.Nic; import com.cloud.vm.UserVmManager; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; +import org.apache.cloudstack.api.command.user.vm.StartVMCmd; +import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; + +import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -164,56 +163,53 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); - /* genarate /.docker/config.json file on the nodes only if Kubernetes cluster is created to - * use docker private registry */ - String dockerUserName = null; - String dockerPassword = null; - String dockerRegistryUrl = null; - String dockerRegistryEmail = null; - List details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId()); - for (KubernetesClusterDetailsVO detail : details) { - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) { - dockerUserName = detail.getValue(); - } - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) { - dockerPassword = detail.getValue(); - } - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) { - dockerRegistryUrl = detail.getValue(); - } - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_EMAIL)) { - dockerRegistryEmail = detail.getValue(); - } - } - if (!Strings.isNullOrEmpty(dockerUserName) && !Strings.isNullOrEmpty(dockerPassword)) { - // do write file for /.docker/config.json through the code instead of k8s-node.yml as we can no make a section - // optional or conditionally applied - String dockerConfigString = "write_files:\n" + - " - path: /.docker/config.json\n" + - " owner: core:core\n" + - " permissions: '0644'\n" + - " content: |\n" + - " {\n" + - " \"auths\": {\n" + - " {{docker.url}}: {\n" + - " \"auth\": {{docker.secret}},\n" + - " \"email\": {{docker.email}}\n" + - " }\n" + - " }\n" + - " }"; - k8sNodeConfig = k8sNodeConfig.replace("write_files:", dockerConfigString); - final String dockerUrlKey = "{{docker.url}}"; - final String dockerAuthKey = "{{docker.secret}}"; - final String dockerEmailKey = "{{docker.email}}"; - final String usernamePasswordKey = dockerUserName + ":" + dockerPassword; - String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(StringUtils.getPreferredCharset())); - k8sNodeConfig = k8sNodeConfig.replace(dockerUrlKey, "\"" + dockerRegistryUrl + "\""); - k8sNodeConfig = k8sNodeConfig.replace(dockerAuthKey, "\"" + base64Auth + "\""); - k8sNodeConfig = k8sNodeConfig.replace(dockerEmailKey, "\"" + dockerRegistryEmail + "\""); - } + + k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig); + return k8sNodeConfig; } + protected String updateKubeConfigWithRegistryDetails(String k8sConfig) { + /* genarate /etc/containerd/config.toml file on the nodes only if Kubernetes cluster is created to + * use docker private registry */ + String registryUsername = null; + String registryPassword = null; + String registryUrl = null; + + List details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId()); + for (KubernetesClusterDetailsVO detail : details) { + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) { + registryUsername = detail.getValue(); + } + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) { + registryPassword = detail.getValue(); + } + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) { + registryUrl = detail.getValue(); + } + } + if (!Strings.isNullOrEmpty(registryUsername) && !Strings.isNullOrEmpty(registryPassword) && !Strings.isNullOrEmpty(registryUrl)) { + // Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details + String runCmd = "- bash -x /opt/bin/setup-containerd"; + + String registryEp = registryUrl.split("://")[1]; + k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd); + final String registryUrlKey = "{{registry.url}}"; + final String registryUrlEpKey = "{{registry.url.endpoint}}"; + final String registryAuthKey = "{{registry.token}}"; + final String registryUname = "{{registry.username}}"; + final String registryPsswd = "{{registry.password}}"; + + final String usernamePasswordKey = registryUsername + ":" + registryPassword; + String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(StringUtils.getPreferredCharset())); + k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl); + k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp); + k8sConfig = k8sConfig.replace(registryUname, registryUsername); + k8sConfig = k8sConfig.replace(registryPsswd, registryPassword); + k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth); + } + return k8sConfig; + } protected DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { final int cpu_requested = offering.getCpu() * offering.getSpeed(); final long ram_requested = offering.getRamSize() * 1024L * 1024L; @@ -369,6 +365,9 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } + if (Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())) { + customParameterMap.put(VmDetailConstants.ROOT_DISK_CONTROLLER, "scsi"); + } String suffix = Long.toHexString(System.currentTimeMillis()); String hostName = String.format("%s-node-%s", kubernetesClusterNodeNamePrefix, suffix); String k8sNodeConfig = null; @@ -616,7 +615,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu if (enable) { String command = String.format("sudo /opt/bin/autoscale-kube-cluster -i %s -e -M %d -m %d", kubernetesCluster.getUuid(), maxSize, minSize); - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); // Maybe the file isn't present. Try and copy it @@ -631,7 +630,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu } // If at first you don't succeed ... - result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); if (!result.first()) { throw new CloudRuntimeException(result.second()); @@ -639,7 +638,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu } updateKubernetesClusterEntry(true, minSize, maxSize); } else { - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, String.format("sudo /opt/bin/autoscale-kube-cluster -d"), 10000, 10000, 60000); if (!result.first()) { @@ -654,6 +653,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu return false; } finally { // Deploying the autoscaler might fail but it can be deployed manually too, so no need to go to an alert state + updateLoginUserDetails(null); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 0c2c17d3aa4..aa9606d473f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -26,6 +26,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.context.CallContext; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Level; @@ -192,13 +193,13 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif while (retryCounter < retries) { retryCounter++; try { - Pair result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, + Pair result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), pkFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), 10000, 10000, 60000); if (!result.first()) { LOGGER.warn(String.format("Draining node: %s on VM : %s in Kubernetes cluster : %s unsuccessful", hostName, userVm.getDisplayName(), kubernetesCluster.getName())); } else { - result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, + result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), pkFile, null, String.format("sudo /opt/bin/kubectl delete node %s", hostName), 10000, 10000, 30000); if (result.first()) { @@ -359,6 +360,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif launchPermissionDao.persist(launchPermission); try { clusterVMs = provisionKubernetesClusterNodeVms((int)(newVmCount + kubernetesCluster.getNodeCount()), (int)kubernetesCluster.getNodeCount(), publicIpAddress); + updateLoginUserDetails(clusterVMs.stream().map(InternalIdentity::getId).collect(Collectors.toList())); } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, unable to provision node VM in the cluster", kubernetesCluster.getName()), e); } @@ -372,7 +374,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); kubernetesClusterVO.setNodeCount(clusterSize); boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesClusterVO, publicIpAddress, sshPort, - CLUSTER_NODE_VM_USER, sshKeyFile, scaleTimeoutTime, 15000); + getControlNodeLoginUser(), sshKeyFile, scaleTimeoutTime, 15000); detachIsoKubernetesVMs(clusterVMs); deleteTemplateLaunchPermission(); if (!readyNodesCountValid) { // Scaling failed diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index b6da75dba81..f9d42f61756 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -17,24 +17,6 @@ package com.cloud.kubernetes.cluster.actionworkers; -import java.io.IOException; -import java.net.InetAddress; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.framework.ca.Certificate; -import org.apache.cloudstack.utils.security.CertUtils; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Level; - import com.cloud.dc.DataCenter; import com.cloud.dc.Vlan; import com.cloud.dc.VlanVO; @@ -74,7 +56,27 @@ import com.cloud.vm.ReservationContext; import com.cloud.vm.ReservationContextImpl; import com.cloud.vm.UserVmManager; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDetailConstants; import com.google.common.base.Strings; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.framework.ca.Certificate; +import org.apache.cloudstack.utils.security.CertUtils; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker { @@ -175,6 +177,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion()); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); + return k8sControlNodeConfig; } @@ -197,6 +201,9 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } + if (Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())) { + customParameterMap.put(VmDetailConstants.ROOT_DISK_CONTROLLER, "scsi"); + } String suffix = Long.toHexString(System.currentTimeMillis()); String hostName = String.format("%s-control-%s", kubernetesClusterNodeNamePrefix, suffix); boolean haSupported = isKubernetesVersionSupportsHA(); @@ -237,6 +244,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); + return k8sControlNodeConfig; } @@ -253,6 +262,9 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } + if (Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())) { + customParameterMap.put(VmDetailConstants.ROOT_DISK_CONTROLLER, "scsi"); + } String suffix = Long.toHexString(System.currentTimeMillis()); String hostName = String.format("%s-control-%s", kubernetesClusterNodeNamePrefix, suffix); String k8sControlNodeConfig = null; @@ -455,7 +467,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return true; } } - String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, timeoutTime); + String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, timeoutTime); if (!Strings.isNullOrEmpty(kubeConfig)) { final String controlVMPrivateIpAddress = getControlVmPrivateIp(); if (!Strings.isNullOrEmpty(controlVMPrivateIpAddress)) { @@ -475,7 +487,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return true; } } - if (KubernetesClusterUtil.isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, timeoutTime, 15000)) { + if (KubernetesClusterUtil.isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, timeoutTime, 15000)) { kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(true), false); return true; } @@ -571,7 +583,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif sshPort = publicIpSshPort.second(); updateKubernetesClusterEntryEndpoint(); boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, - CLUSTER_NODE_VM_USER, sshKeyFile, startTimeoutTime, 15000); + getControlNodeLoginUser(), sshKeyFile, startTimeoutTime, 15000); detachIsoKubernetesVMs(clusterVMs); if (!readyNodesCountValid) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s as it does not have desired number of nodes in ready state", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); @@ -584,6 +596,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } taintControlNodes(); deployProvider(); + updateLoginUserDetails(clusterVMs.stream().map(InternalIdentity::getId).collect(Collectors.toList())); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } @@ -638,7 +651,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } long actualNodeCount = 0; try { - actualNodeCount = KubernetesClusterUtil.getKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile); + actualNodeCount = KubernetesClusterUtil.getKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile); } catch (Exception e) { return false; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index ad289bf81a9..6e22c9f336a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -63,7 +63,7 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke private Pair runInstallScriptOnVM(final UserVm vm, final int index) throws Exception { int nodeSshPort = sshPort == 22 ? sshPort : sshPort + index; String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; - SshHelper.scpTo(nodeAddress, nodeSshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + SshHelper.scpTo(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null, "~/", upgradeScriptFile.getAbsolutePath(), "0755"); String cmdStr = String.format("sudo ./%s %s %s %s %s", upgradeScriptFile.getName(), @@ -71,7 +71,7 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke index == 0 ? "true" : "false", KubernetesVersionManagerImpl.compareSemanticVersions(upgradeVersion.getSemanticVersion(), "1.15.0") < 0 ? "true" : "false", Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType())); - return SshHelper.sshExecute(nodeAddress, nodeSshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + return SshHelper.sshExecute(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null, cmdStr, 10000, 10000, 10 * 60 * 1000); } @@ -90,7 +90,7 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke vm.getDisplayName(), kubernetesCluster.getName(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); } try { - result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), 10000, 10000, 60000); } catch (Exception e) { @@ -114,11 +114,11 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke if (System.currentTimeMillis() > upgradeTimeoutTime) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out", kubernetesCluster.getName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } - if (!KubernetesClusterUtil.uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), vm, upgradeTimeoutTime, 15000)) { + if (!KubernetesClusterUtil.uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), vm, upgradeTimeoutTime, 15000)) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to uncordon Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } if (i == 0) { // Wait for control node to get in Ready state - if (!KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), hostName, upgradeTimeoutTime, 15000)) { + if (!KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName, upgradeTimeoutTime, 15000)) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get control Kubernetes node on VM : %s in ready state", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 376c148abb7..d767c2b52bf 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -16,10 +16,10 @@ // under the License. package org.apache.cloudstack.api.command.user.kubernetes.cluster; -import java.security.InvalidParameterException; - -import javax.inject.Inject; - +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ACL; @@ -41,10 +41,8 @@ import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; -import com.cloud.kubernetes.cluster.KubernetesCluster; -import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; -import com.cloud.kubernetes.cluster.KubernetesClusterService; -import com.cloud.utils.exception.CloudRuntimeException; +import javax.inject.Inject; +import java.security.InvalidParameterException; @APICommand(name = CreateKubernetesClusterCmd.APINAME, description = "Creates a Kubernetes cluster", @@ -140,10 +138,6 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { description = "URL for the docker image private registry") private String dockerRegistryUrl; - @Parameter(name = ApiConstants.DOCKER_REGISTRY_EMAIL, type = CommandType.STRING, - description = "email of the docker image private registry user") - private String dockerRegistryEmail; - @Parameter(name = ApiConstants.NODE_ROOT_DISK_SIZE, type = CommandType.LONG, description = "root disk size in GB for each node") private Long nodeRootDiskSize; @@ -226,10 +220,6 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { return dockerRegistryUrl; } - public String getDockerRegistryEmail() { - return dockerRegistryEmail; - } - public Long getNodeRootDiskSize() { if (nodeRootDiskSize != null) { if (nodeRootDiskSize < DEFAULT_NODE_ROOT_DISK_SIZE) { diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 0891d2015b3..3156f378674 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -18,7 +18,7 @@ --- users: - - name: core + - name: cloud sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash ssh_authorized_keys: @@ -31,7 +31,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -107,6 +107,8 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + output=`ls ${BINARIES_DIR}/docker/` if [ "$output" != "" ]; then while read -r line; do @@ -118,7 +120,7 @@ write_files: fi retval=0 set +e - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -165,7 +167,7 @@ write_files: fi systemctl enable kubelet && systemctl start kubelet - modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then crucial_cmd_attempts=1 @@ -176,7 +178,7 @@ write_files: fi retval=0 set +e - kubeadm config images pull + kubeadm config images pull --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -192,7 +194,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -210,16 +212,33 @@ write_files: fi kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification - sudo touch /home/core/success - echo "true" > /home/core/success + sudo touch /home/cloud/success + echo "true" > /home/cloud/success + + - path: /opt/bin/setup-containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] Type=simple @@ -241,6 +260,10 @@ write_files: ExecStart=/opt/bin/deploy-kube-system runcmd: + - chown -R cloud:cloud /home/cloud/.ssh + - containerd config default > /etc/containerd/config.toml + - sed -i '/\[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options\]/a '"\\ SystemdCgroup=true"'' /etc/containerd/config.toml + - systemctl restart containerd - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 322f08099cf..4dcaf496072 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -18,7 +18,7 @@ --- users: - - name: core + - name: cloud sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash ssh_authorized_keys: @@ -51,7 +51,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -127,6 +127,8 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + output=`ls ${BINARIES_DIR}/docker/` if [ "$output" != "" ]; then while read -r line; do @@ -138,7 +140,7 @@ write_files: fi retval=0 set +e - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -187,7 +189,7 @@ write_files: fi systemctl enable kubelet && systemctl start kubelet - modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then crucial_cmd_attempts=1 @@ -198,7 +200,7 @@ write_files: fi retval=0 set +e - kubeadm config images pull + kubeadm config images pull --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -216,7 +218,7 @@ write_files: fi retval=0 set +e - kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} + kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -231,7 +233,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -267,16 +269,34 @@ write_files: /opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true /opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true - sudo touch /home/core/success - echo "true" > /home/core/success + sudo touch /home/cloud/success + echo "true" > /home/cloud/success + + - path: /opt/bin/setup-containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd + - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] Type=simple @@ -298,6 +318,10 @@ write_files: ExecStart=/opt/bin/deploy-kube-system runcmd: + - chown -R cloud:cloud /home/cloud/.ssh + - containerd config default > /etc/containerd/config.toml + - sed -i '/\[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options\]/a '"\\ SystemdCgroup=true"'' /etc/containerd/config.toml + - systemctl restart containerd - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 28ba43e7584..eedc1d0c8c8 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -18,7 +18,7 @@ --- users: - - name: core + - name: cloud sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash ssh_authorized_keys: @@ -31,7 +31,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -107,6 +107,8 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + output=`ls ${BINARIES_DIR}/docker/` if [ "$output" != "" ]; then while read -r line; do @@ -118,7 +120,7 @@ write_files: fi retval=0 set +e - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -165,7 +167,7 @@ write_files: fi systemctl enable kubelet && systemctl start kubelet - modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then crucial_cmd_attempts=1 @@ -176,7 +178,7 @@ write_files: fi retval=0 set +e - kubeadm config images pull + kubeadm config images pull --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -192,7 +194,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -210,16 +212,33 @@ write_files: fi kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --discovery-token-unsafe-skip-ca-verification - sudo touch /home/core/success - echo "true" > /home/core/success + sudo touch /home/cloud/success + echo "true" > /home/cloud/success + + - path: /opt/bin/setup-containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] Type=simple @@ -241,5 +260,9 @@ write_files: ExecStart=/opt/bin/deploy-kube-system runcmd: + - chown -R cloud:cloud /home/cloud/.ssh + - containerd config default > /etc/containerd/config.toml + - sed -i '/\[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options\]/a '"\\ SystemdCgroup=true"'' /etc/containerd/config.toml + - systemctl restart containerd - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index 7e0c3c00a8a..b85ea00e9ed 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -93,7 +93,7 @@ if [ -d "$BINARIES_DIR" ]; then output=`ls ${BINARIES_DIR}/docker/` if [ "$output" != "" ]; then while read -r line; do - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" done <<< "$output" fi if [ -e "${BINARIES_DIR}/provider.yaml" ]; then diff --git a/pom.xml b/pom.xml index 5721b3e3b60..f1ba427171f 100644 --- a/pom.xml +++ b/pom.xml @@ -49,6 +49,7 @@ UTF-8 UTF-8 + 4.16.1.0 11 diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index ba3dca77a8d..ce7626c98c2 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -98,19 +98,18 @@ provider_conf_file="${working_dir}/provider.yaml" curl -sSL ${PROVIDER_URL} -o ${provider_conf_file} echo "Fetching k8s docker images..." -docker -v +ctr -v if [ $? -ne 0 ]; then - echo "Installing docker..." + echo "Installing containerd..." if [ -f /etc/redhat-release ]; then sudo yum -y remove docker-common docker container-selinux docker-selinux docker-engine sudo yum -y install lvm2 device-mapper device-mapper-persistent-data device-mapper-event device-mapper-libs device-mapper-event-libs sudo yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.107-3.el7.noarch.rpm - sudo wget https://download.docker.com/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo && sudo yum -y install docker-ce - sudo systemctl enable docker && sudo systemctl start docker + sudo yum install -y containerd.io elif [ -f /etc/lsb-release ]; then - sudo apt update && sudo apt install docker.io -y - sudo systemctl enable docker && sudo systemctl start docker + sudo apt update && sudo apt install containerd.io -y fi + sudo systemctl enable containerd && sudo systemctl start containerd fi mkdir -p "${working_dir}/docker" output=`${k8s_dir}/kubeadm config images list --kubernetes-version=${RELEASE}` @@ -130,11 +129,14 @@ provider_image=`grep "image:" ${provider_conf_file} | cut -d ':' -f2- | tr -d ' output=`printf "%s\n" ${output} ${provider_image}` while read -r line; do - echo "Downloading docker image $line ---" - sudo docker pull "$line" + echo "Downloading image $line ---" + if [[ $line == kubernetesui* ]] || [[ $line == apache* ]]; then + line="docker.io/${line}" + fi + sudo ctr image pull "$line" image_name=`echo "$line" | grep -oE "[^/]+$"` - sudo docker save "$line" > "${working_dir}/docker/$image_name.tar" - sudo docker image rm "$line" + sudo ctr image export "${working_dir}/docker/$image_name.tar" "$line" + sudo ctr image rm "$line" done <<< "$output" echo "Restore kubeadm permissions..." diff --git a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh index a864d188d00..ac672a26539 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh +++ b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh @@ -61,9 +61,7 @@ setup_k8s_node() { log_it "Starting cloud-init services" systemctl enable --now --no-block containerd - systemctl enable --now --no-block docker.socket - systemctl enable --now --no-block docker.service - if [ -f /home/core/success ]; then + if [ -f /home/cloud/success ]; then systemctl stop cloud-init cloud-config cloud-final systemctl disable cloud-init cloud-config cloud-final else diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 0387ddf9cc1..46f1f5d02bd 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -101,12 +101,12 @@ class TestKubernetesCluster(cloudstackTestCase): (cls.services["cks_kubernetes_versions"]["1.20.9"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.20.9"]["url"], e)) if cls.setup_failed == False: try: - cls.kubernetes_version_1_21_3 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.21.3"]) - cls.kubernetes_version_ids.append(cls.kubernetes_version_1_21_3.id) + cls.kubernetes_version_1_21_5 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.21.5"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_1_21_5.id) except Exception as e: cls.setup_failed = True cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % - (cls.services["cks_kubernetes_versions"]["1.21.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.21.3"]["url"], e)) + (cls.services["cks_kubernetes_versions"]["1.21.5"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.21.5"]["url"], e)) if cls.setup_failed == False: cks_offering_data = cls.services["cks_service_offering"] @@ -349,7 +349,7 @@ class TestKubernetesCluster(cloudstackTestCase): if self.setup_failed == True: self.fail("Setup incomplete") global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(version=self.kubernetes_version_1_21_3) + k8s_cluster = self.getValidKubernetesCluster(version=self.kubernetes_version_1_21_5) self.debug("Downgrading Kubernetes cluster with ID: %s to a lower version. This should fail!" % k8s_cluster.id) @@ -362,7 +362,7 @@ class TestKubernetesCluster(cloudstackTestCase): self.debug("Upgrading Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_3.id) + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_5.id) return @attr(tags=["advanced", "smoke"], required_hardware="true") @@ -381,12 +381,12 @@ class TestKubernetesCluster(cloudstackTestCase): time.sleep(self.services["sleep"]) self.debug("Upgrading Kubernetes cluster with ID: %s" % k8s_cluster.id) try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1_21_3.id) + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1_21_5.id) except Exception as e: self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) self.fail("Failed to upgrade Kubernetes cluster due to: %s" % e) - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_3.id) + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_5.id) return @attr(tags=["advanced", "smoke"], required_hardware="true") @@ -434,7 +434,7 @@ class TestKubernetesCluster(cloudstackTestCase): if self.setup_failed == True: self.fail("Setup incomplete") global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(version=self.kubernetes_version_1_21_3) + k8s_cluster = self.getValidKubernetesCluster(version=self.kubernetes_version_1_21_5) self.debug("Autoscaling Kubernetes cluster with ID: %s" % k8s_cluster.id) try: @@ -535,12 +535,12 @@ class TestKubernetesCluster(cloudstackTestCase): self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1_21_3.id) + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1_21_5.id) except Exception as e: self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_3.id) + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_5.id) self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) return diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index db3eec5cb40..67bb03491e4 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -19,7 +19,7 @@ set -e set -x -CLOUDSTACK_RELEASE=4.16.0 +CLOUDSTACK_RELEASE=4.16.1 function configure_apache2() { # Enable ssl, rewrite and auth @@ -50,9 +50,12 @@ function configure_cacerts() { CDIR=$(pwd) cd /tmp # Add LetsEncrypt ca-cert - wget https://letsencrypt.org/certs/lets-encrypt-x3-cross-signed.der - keytool -trustcacerts -keystore /etc/ssl/certs/java/cacerts -storepass changeit -noprompt -importcert -alias letsencryptauthorityx3cross -file lets-encrypt-x3-cross-signed.der - rm -f lets-encrypt-x3-cross-signed.der + wget https://letsencrypt.org/certs/lets-encrypt-r3.der + wget https://letsencrypt.org/certs/isrgrootx1.der + + keytool -trustcacerts -keystore /etc/ssl/certs/java/cacerts -storepass changeit -noprompt -importcert -alias letsencryptauthorityr3 -file lets-encrypt-r3.der + keytool -trustcacerts -keystore /etc/ssl/certs/java/cacerts -storepass changeit -noprompt -importcert -alias letsencryptauthorityx1 -file isrgrootx1.der + rm -f lets-encrypt-r3.der isrgrootx1.der cd $CDIR } @@ -126,10 +129,6 @@ function configure_services() { # Disable container services systemctl disable containerd - systemctl disable docker.service - systemctl stop docker.service - systemctl disable docker.socket - systemctl stop docker.socket # Disable cloud init by default cat < /etc/cloud/cloud.cfg.d/cloudstack.cfg diff --git a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh index 555a00c28ed..3b4ef653728 100644 --- a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh +++ b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh @@ -98,7 +98,7 @@ function install_packages() { apt-key fingerprint 0EBFCD88 add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" apt-get update - ${apt_get} install docker-ce docker-ce-cli containerd.io + ${apt_get} install containerd.io apt_clean diff --git a/tools/appliance/systemvmtemplate/template.json b/tools/appliance/systemvmtemplate/template.json index bd932bf6cde..31eb2ea5abf 100644 --- a/tools/appliance/systemvmtemplate/template.json +++ b/tools/appliance/systemvmtemplate/template.json @@ -27,8 +27,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:5f6aed67b159d7ccc1a90df33cc8a314aa278728a6f50707ebf10c02e46664e383ca5fa19163b0a1c6a4cb77a39587881584b00b45f512b4a470f1138eaa1801", - "iso_url": "https://cdimage.debian.org/debian-cd/11.0.0/amd64/iso-cd/debian-11.0.0-amd64-netinst.iso", + "iso_checksum": "sha512:c685b85cf9f248633ba3cd2b9f9e781fa03225587e0c332aef2063f6877a1f0622f56d44cf0690087b0ca36883147ecb5593e3da6f965968402cdbdf12f6dd74", + "iso_url": "https://cdimage.debian.org/debian-cd/11.2.0/amd64/iso-cd/debian-11.2.0-amd64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemuargs": [ diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 08f0ccf5fc2..582a71d682e 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -47,6 +47,7 @@ "label.accept.project.invitation": "Accept project invitation", "label.access": "Access", "label.accesskey": "Access Key", +"label.acess.kubernetes.nodes": "Access Kubernetes Nodes", "label.account": "Account", "label.account.and.security.group": "Account - Security group", "label.account.details": "Account details", @@ -1233,6 +1234,7 @@ "label.keypair": "SSH Key Pair", "label.kubeconfig.cluster": "Kubernetes Cluster Config", "label.kubernetes": "Kubernetes", +"label.kubernetes.access.details": "The kubernetes nodes can be accessed via ssh using:
ssh -i [ssh_key] -p [port_number] cloud@[public_ip_address]

where,
ssh_key: points to the ssh private key file corresponding to the key that was associated while creating the Kubernetes cluster. If no ssh key was provided during Kubernetes cluster creation, use the ssh private key of the management server.
port_number: can be obtained from the Port Forwarding Tab (Public Port column)", "label.kubernetes.cluster": "Kubernetes cluster", "label.kubernetes.cluster.create": "Create Kubernetes Cluster", "label.kubernetes.cluster.delete": "Delete Kubernetes Cluster", diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index 360525397f6..3bd6d578cd7 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -100,7 +100,6 @@ export default { return (['Admin'].includes(store.userInfo.roletype) || // If admin or owner or belongs to current project (record.domainid === store.userInfo.domainid && record.account === store.userInfo.account) || (record.domainid === store.userInfo.domainid && record.projectid && store.project && store.project.id && record.projectid === store.project.id)) && - record.templatetype !== 'SYSTEM' && record.isready }, popup: true, diff --git a/ui/src/views/compute/CreateKubernetesCluster.vue b/ui/src/views/compute/CreateKubernetesCluster.vue index 6b6b81f6791..52aa831d75c 100644 --- a/ui/src/views/compute/CreateKubernetesCluster.vue +++ b/ui/src/views/compute/CreateKubernetesCluster.vue @@ -226,14 +226,6 @@ }]" :placeholder="apiParams.dockerregistryurl.description"/> - - - -
diff --git a/ui/src/views/compute/KubernetesServiceTab.vue b/ui/src/views/compute/KubernetesServiceTab.vue index 97f1c5456ec..039d09eab87 100644 --- a/ui/src/views/compute/KubernetesServiceTab.vue +++ b/ui/src/views/compute/KubernetesServiceTab.vue @@ -88,6 +88,9 @@

{{ $t('label.more.access.dashboard.ui') }}, https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/#accessing-the-dashboard-ui

+ +

+ { private static final Logger s_logger = Logger.getLogger(Script.class); @@ -320,7 +319,7 @@ public class Script implements Callable { try { _logger.trace("Checking exit value of process"); _process.exitValue(); - _logger.trace("Script ran within the alloted time"); + _logger.trace("Script ran within the allotted time"); } catch (IllegalThreadStateException e) { _logger.warn("Interrupting script."); _isTimeOut = true; diff --git a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java index 291f8c1b43f..dc4333be6a1 100644 --- a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java +++ b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java @@ -19,16 +19,15 @@ package com.cloud.utils.ssh; -import java.io.IOException; -import java.io.InputStream; - +import com.google.common.base.Strings; +import com.trilead.ssh2.ChannelCondition; +import com.trilead.ssh2.Session; import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; -import com.google.common.base.Strings; -import com.trilead.ssh2.ChannelCondition; -import com.trilead.ssh2.Session; +import java.io.IOException; +import java.io.InputStream; public class SSHCmdHelper { private static final Logger s_logger = Logger.getLogger(SSHCmdHelper.class); @@ -242,8 +241,8 @@ public class SSHCmdHelper { } return result; } catch (Exception e) { - s_logger.debug("Ssh executed failed", e); - throw new SshException("Ssh executed failed " + e.getMessage()); + s_logger.debug("SSH execution failed", e); + throw new SshException("SSH execution failed " + e.getMessage()); } finally { if (sshSession != null) sshSession.close(); diff --git a/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java b/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java index f856b1f2892..a8307cc751f 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java @@ -16,18 +16,23 @@ // under the License. package org.apache.cloudstack.utils.security; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Paths; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.HashMap; import java.util.Map; public class DigestHelper { - + public static final Logger s_logger = Logger.getLogger(DigestHelper.class.getName()); public static ChecksumValue digest(String algorithm, InputStream is) throws NoSuchAlgorithmException, IOException { MessageDigest digest = MessageDigest.getInstance(algorithm); ChecksumValue checksum = null; @@ -131,4 +136,14 @@ public class DigestHelper { public static String getHashValueFromChecksumValue(String checksum) { return isAlgorithmPresent(checksum) ? new ChecksumValue(checksum).getChecksum() : checksum; } + + public static String calculateChecksum(File file) { + try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { + return org.apache.commons.codec.digest.DigestUtils.md5Hex(is); + } catch (IOException e) { + String errMsg = "Failed to calculate template checksum"; + s_logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } }