Merge remote-tracking branch 'origin/4.18'

agent/bindir/libvirtqemuhook.in

@@ -61,21 +61,23 @@ def handleMigrateBegin():
     try:
         domain = parse(sys.stdin)
         for interface in domain.getElementsByTagName("interface"):
-            source = interface.getElementsByTagName("source")[0]
+            sources = interface.getElementsByTagName("source")
-            bridge = source.getAttribute("bridge")
+            if sources.length > 0:
-            if isOldStyleBridge(bridge):
+                source = interface.getElementsByTagName("source")[0]
-                vlanId = bridge.replace("cloudVirBr", "")
+                bridge = source.getAttribute("bridge")
-                phyDev = getGuestNetworkDevice()
+                if isOldStyleBridge(bridge):
-            elif isNewStyleBridge(bridge):
+                    vlanId = bridge.replace("cloudVirBr", "")
-                vlanId = re.sub(r"br(\w+)-", "", bridge)
-                phyDev = re.sub(r"-(\d+)$", "" , re.sub(r"^br", "" ,bridge))
-                netlib = networkConfig()
-                if not netlib.isNetworkDev(phyDev):
                     phyDev = getGuestNetworkDevice()
-            else:
+                elif isNewStyleBridge(bridge):
-                continue
+                    vlanId = re.sub(r"br(\w+)-", "", bridge)
-            newBrName = "br" + phyDev + "-" + vlanId
+                    phyDev = re.sub(r"-(\d+)$", "" , re.sub(r"^br", "" ,bridge))
-            source.setAttribute("bridge", newBrName)
+                    netlib = networkConfig()
+                    if not netlib.isNetworkDev(phyDev):
+                        phyDev = getGuestNetworkDevice()
+                else:
+                    continue
+                newBrName = "br" + phyDev + "-" + vlanId
+                source.setAttribute("bridge", newBrName)
         print(domain.toxml())
     except:
         pass

api/src/main/java/org/apache/cloudstack/api/command/admin/annotation/AddAnnotationCmd.java

@@ -41,7 +41,7 @@ public class AddAnnotationCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ANNOTATION, type = CommandType.STRING, description = "the annotation text")
     private String annotation;
 
-    @Parameter(name = ApiConstants.ENTITY_TYPE, type = CommandType.STRING, description = "the entity type (only HOST is allowed atm)")
+    @Parameter(name = ApiConstants.ENTITY_TYPE, type = CommandType.STRING, description = "The following entity types are allowed VM, VOLUME, SNAPSHOT, VM_SNAPSHOT, INSTANCE_GROUP, SSH_KEYPAIR, USER_DATA, NETWORK, VPC, PUBLIC_IP_ADDRESS, VPN_CUSTOMER_GATEWAY, TEMPLATE, ISO, KUBERNETES_CLUSTER, SERVICE_OFFERING, DISK_OFFERING, NETWORK_OFFERING, ZONE, POD, CLUSTER, HOST, DOMAIN, PRIMARY_STORAGE, SECONDARY_STORAGE, VR, SYSTEM_VM, AUTOSCALE_VM_GROUP, MANAGEMENT_SERVER")
     private String entityType;
 
     @Parameter(name = ApiConstants.ENTITY_ID, type = CommandType.STRING, description = "the id of the entity to annotate")

core/src/main/java/com/cloud/agent/api/MigrateCommand.java

@@ -171,6 +171,7 @@ public class MigrateCommand extends Command {
         private final DriverType driverType;
         private final Source source;
         private final String sourceText;
+        private final String backingStoreText;
         private boolean isSourceDiskOnStorageFileSystem;
 
         public MigrateDiskInfo(final String serialNumber, final DiskType diskType, final DriverType driverType, final Source source, final String sourceText) {
@@ -179,6 +180,16 @@ public class MigrateCommand extends Command {
             this.driverType = driverType;
             this.source = source;
             this.sourceText = sourceText;
+            this.backingStoreText = null;
+        }
+
+        public MigrateDiskInfo(final String serialNumber, final DiskType diskType, final DriverType driverType, final Source source, final String sourceText, final String backingStoreText) {
+            this.serialNumber = serialNumber;
+            this.diskType = diskType;
+            this.driverType = driverType;
+            this.source = source;
+            this.sourceText = sourceText;
+            this.backingStoreText = backingStoreText;
         }
 
         public String getSerialNumber() {
@@ -201,6 +212,8 @@ public class MigrateCommand extends Command {
             return sourceText;
         }
 
+        public String getBackingStoreText() { return backingStoreText; }
+
         public boolean isSourceDiskOnStorageFileSystem() {
             return isSourceDiskOnStorageFileSystem;
         }

core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java

@@ -86,6 +86,7 @@ public class TemplateObjectTO implements DataTO {
         this.hypervisorType = template.getHypervisorType();
         this.deployAsIs = template.isDeployAsIs();
         this.deployAsIsConfiguration = template.getDeployAsIsConfiguration();
+        this.directDownload = template.isDirectDownload();
     }
 
     @Override

engine/schema/pom.xml

@@ -57,12 +57,6 @@
             <artifactId>ini4j</artifactId>
             <version>${cs.ini.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter</artifactId>
-            <version>${cs.junit.jupiter.version}</version>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
     <build>
         <plugins>

engine/schema/src/test/java/com/cloud/upgrade/DatabaseVersionHierarchyTest.java

@@ -26,15 +26,15 @@ import com.cloud.upgrade.dao.Upgrade41520to41600;
 import com.cloud.upgrade.dao.Upgrade41720to41800;
 import com.cloud.upgrade.dao.Upgrade481to490;
 import org.apache.cloudstack.utils.CloudStackVersion;
-import org.junit.jupiter.api.BeforeAll;
+import org.junit.BeforeClass;
-import org.junit.jupiter.api.Test;
+import org.junit.Test;
 
 import java.io.InputStream;
 import java.sql.Connection;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.Assert.assertEquals;
 
-class DatabaseVersionHierarchyTest {
+public class DatabaseVersionHierarchyTest {
 
     private static DatabaseVersionHierarchy hierarchy;
 
@@ -71,8 +71,8 @@ class DatabaseVersionHierarchyTest {
 
     }
 
-    @BeforeAll
+    @BeforeClass
-    static void init() {
+    public static void init() {
         DatabaseVersionHierarchy.DatabaseVersionHierarchyBuilder builder = DatabaseVersionHierarchy.builder()
                 .next("0.0.5", new DummyUpgrade())
                 .next("1.0.0.0", new DummyUpgrade())
@@ -95,23 +95,23 @@ class DatabaseVersionHierarchyTest {
     }
 
     @Test
-    void getRecentVersionMiddle() {
+    public void getRecentVersionMiddle() {
         assertEquals("2.0.0", hierarchy.getRecentVersion(CloudStackVersion.parse("2.2.2")).toString());
     }
     @Test
-    void getRecentVersionEarly() {
+    public void getRecentVersionEarly() {
         assertEquals(null, hierarchy.getRecentVersion(CloudStackVersion.parse("0.0.2")));
     }
     @Test
-    void getRecentVersionStart() {
+    public void getRecentVersionStart() {
         assertEquals(null, hierarchy.getRecentVersion(CloudStackVersion.parse("0.0.5")));
     }
     @Test
-    void getRecentVersionJust() {
+    public void getRecentVersionJust() {
         assertEquals("0.0.5", hierarchy.getRecentVersion(CloudStackVersion.parse("0.0.9")).toString());
     }
     @Test
-    void getRecentVersionExact() {
+    public void getRecentVersionExact() {
         assertEquals("0.0.5", hierarchy.getRecentVersion(CloudStackVersion.parse("1.0.0.0")).toString());
     }
 }

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java

@@ -149,9 +149,9 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot
      * Configures a {@link MigrateDiskInfo} object configured for migrating a File System volume and calls rootImageProvisioning.
      */
     @Override
-    protected MigrateCommand.MigrateDiskInfo configureMigrateDiskInfo(VolumeInfo srcVolumeInfo, String destPath) {
+    protected MigrateCommand.MigrateDiskInfo configureMigrateDiskInfo(VolumeInfo srcVolumeInfo, String destPath, String backingPath) {
-        return new MigrateCommand.MigrateDiskInfo(srcVolumeInfo.getPath(), MigrateCommand.MigrateDiskInfo.DiskType.FILE, MigrateCommand.MigrateDiskInfo.DriverType.QCOW2,
+            return new MigrateCommand.MigrateDiskInfo(srcVolumeInfo.getPath(), MigrateCommand.MigrateDiskInfo.DiskType.FILE, MigrateCommand.MigrateDiskInfo.DriverType.QCOW2,
-                MigrateCommand.MigrateDiskInfo.Source.FILE, destPath);
+                    MigrateCommand.MigrateDiskInfo.Source.FILE, destPath, backingPath);
     }
 
     /**
@@ -163,6 +163,15 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot
         return new File(destStoragePool.getPath(), destVolumeInfo.getUuid()).getAbsolutePath();
     }
 
+    @Override
+    protected String generateBackingPath(StoragePoolVO destStoragePool, VolumeInfo destVolumeInfo) {
+        String templateInstallPath = getVolumeBackingFile(destVolumeInfo);
+        if (templateInstallPath == null) {
+            return null;
+        }
+        return new File(destStoragePool.getPath(), templateInstallPath).getAbsolutePath();
+    }
+
     /**
      * Returns the template UUID with the given id. If the template ID is null, it returns null.
      */
@@ -201,6 +210,12 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot
             return;
         }
 
+        TemplateInfo directDownloadTemplateInfo = templateDataFactory.getReadyBypassedTemplateOnPrimaryStore(srcVolumeInfo.getTemplateId(), destDataStore.getId(), destHost.getId());
+        if (directDownloadTemplateInfo != null) {
+            LOGGER.debug(String.format("Template %s was of direct download type and successfully staged to primary store %s", directDownloadTemplateInfo.getId(), directDownloadTemplateInfo.getDataStore().getId()));
+            return;
+        }
+
         VMTemplateStoragePoolVO sourceVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(destStoragePool.getId(), srcVolumeInfo.getTemplateId(), null);
         if (sourceVolumeTemplateStoragePoolVO == null && (isStoragePoolTypeInList(destStoragePool.getPoolType(), StoragePoolType.Filesystem, StoragePoolType.SharedMountPoint))) {
             DataStore sourceTemplateDataStore = dataStoreManagerImpl.getRandomImageStore(srcVolumeInfo.getDataCenterId());

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java

@@ -1871,7 +1871,8 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy {
                             MigrateCommand.MigrateDiskInfo.Source.FILE,
                             connectHostToVolume(destHost, destVolumeInfo.getPoolId(), volumeIdentifier));
                 } else {
-                    migrateDiskInfo = configureMigrateDiskInfo(srcVolumeInfo, destPath);
+                    String backingPath = generateBackingPath(destStoragePool, destVolumeInfo);
+                    migrateDiskInfo = configureMigrateDiskInfo(srcVolumeInfo, destPath, backingPath);
                     migrateDiskInfo.setSourceDiskOnStorageFileSystem(isStoragePoolTypeOfFile(sourceStoragePool));
                     migrateDiskInfoList.add(migrateDiskInfo);
                     prepareDiskWithSecretConsumerDetail(vmTO, srcVolumeInfo, destVolumeInfo.getPath());
@@ -1994,14 +1995,18 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy {
         return connectHostToVolume(destHost, destVolumeInfo.getPoolId(), destVolumeInfo.get_iScsiName());
     }
 
+    protected String generateBackingPath(StoragePoolVO destStoragePool, VolumeInfo destVolumeInfo) {
+        return null;
+    }
+
     /**
      * Configures a {@link MigrateDiskInfo} object with disk type of BLOCK, Driver type RAW and Source DEV
      */
-    protected MigrateCommand.MigrateDiskInfo configureMigrateDiskInfo(VolumeInfo srcVolumeInfo, String destPath) {
+    protected MigrateCommand.MigrateDiskInfo configureMigrateDiskInfo(VolumeInfo srcVolumeInfo, String destPath, String backingPath) {
         return new MigrateCommand.MigrateDiskInfo(srcVolumeInfo.getPath(),
                 MigrateCommand.MigrateDiskInfo.DiskType.BLOCK,
                 MigrateCommand.MigrateDiskInfo.DriverType.RAW,
-                MigrateCommand.MigrateDiskInfo.Source.DEV, destPath);
+                MigrateCommand.MigrateDiskInfo.Source.DEV, destPath, backingPath);
     }
 
     /**
@@ -2023,7 +2028,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy {
     /*
      * Return backing file for volume (if any), only for KVM volumes
      */
-    private String getVolumeBackingFile(VolumeInfo srcVolumeInfo) {
+    String getVolumeBackingFile(VolumeInfo srcVolumeInfo) {
         if (srcVolumeInfo.getHypervisorType() == HypervisorType.KVM &&
                 srcVolumeInfo.getTemplateId() != null && srcVolumeInfo.getPoolId() != null) {
             VMTemplateVO template = _vmTemplateDao.findById(srcVolumeInfo.getTemplateId());

engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java

@@ -240,7 +240,7 @@ public class KvmNonManagedStorageSystemDataMotionTest {
     public void configureMigrateDiskInfoTest() {
         VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
         Mockito.doReturn("volume path").when(srcVolumeInfo).getPath();
-        MigrateCommand.MigrateDiskInfo migrateDiskInfo = kvmNonManagedStorageDataMotionStrategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath");
+        MigrateCommand.MigrateDiskInfo migrateDiskInfo = kvmNonManagedStorageDataMotionStrategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath", null);
         Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DiskType.FILE, migrateDiskInfo.getDiskType());
         Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DriverType.QCOW2, migrateDiskInfo.getDriverType());
         Assert.assertEquals(MigrateCommand.MigrateDiskInfo.Source.FILE, migrateDiskInfo.getSource());
@@ -248,6 +248,19 @@ public class KvmNonManagedStorageSystemDataMotionTest {
         Assert.assertEquals("volume path", migrateDiskInfo.getSerialNumber());
     }
 
+    @Test
+    public void configureMigrateDiskInfoWithBackingTest() {
+        VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn("volume path").when(srcVolumeInfo).getPath();
+        MigrateCommand.MigrateDiskInfo migrateDiskInfo = kvmNonManagedStorageDataMotionStrategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath", "backingPath");
+        Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DiskType.FILE, migrateDiskInfo.getDiskType());
+        Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DriverType.QCOW2, migrateDiskInfo.getDriverType());
+        Assert.assertEquals(MigrateCommand.MigrateDiskInfo.Source.FILE, migrateDiskInfo.getSource());
+        Assert.assertEquals("destPath", migrateDiskInfo.getSourceText());
+        Assert.assertEquals("volume path", migrateDiskInfo.getSerialNumber());
+        Assert.assertEquals("backingPath", migrateDiskInfo.getBackingStoreText());
+    }
+
     @Test
     public void shouldMigrateVolumeTest() {
         StoragePoolVO sourceStoragePool = Mockito.spy(new StoragePoolVO());

engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategyTest.java

@@ -192,7 +192,7 @@ public class StorageSystemDataMotionStrategyTest {
     public void configureMigrateDiskInfoTest() {
         VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
         Mockito.doReturn("volume path").when(srcVolumeInfo).getPath();
-        MigrateCommand.MigrateDiskInfo migrateDiskInfo = strategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath");
+        MigrateCommand.MigrateDiskInfo migrateDiskInfo = strategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath", null);
         Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DiskType.BLOCK, migrateDiskInfo.getDiskType());
         Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DriverType.RAW, migrateDiskInfo.getDriverType());
         Assert.assertEquals(MigrateCommand.MigrateDiskInfo.Source.DEV, migrateDiskInfo.getSource());
@@ -200,6 +200,19 @@ public class StorageSystemDataMotionStrategyTest {
         Assert.assertEquals("volume path", migrateDiskInfo.getSerialNumber());
     }
 
+    @Test
+    public void configureMigrateDiskInfoWithBackingTest() {
+        VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
+        Mockito.doReturn("volume path").when(srcVolumeInfo).getPath();
+        MigrateCommand.MigrateDiskInfo migrateDiskInfo = strategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath", "backingPath");
+        Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DiskType.BLOCK, migrateDiskInfo.getDiskType());
+        Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DriverType.RAW, migrateDiskInfo.getDriverType());
+        Assert.assertEquals(MigrateCommand.MigrateDiskInfo.Source.DEV, migrateDiskInfo.getSource());
+        Assert.assertEquals("destPath", migrateDiskInfo.getSourceText());
+        Assert.assertEquals("volume path", migrateDiskInfo.getSerialNumber());
+        Assert.assertEquals("backingPath", migrateDiskInfo.getBackingStoreText());
+    }
+
     @Test
     public void setVolumePathTest() {
         VolumeVO volume = new VolumeVO("name", 0l, 0l, 0l, 0l, 0l, "folder", "path", Storage.ProvisioningType.THIN, 0l, Volume.Type.ROOT);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java

@@ -575,6 +575,16 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
                                     diskNode.appendChild(newChildSourceNode);
                                 } else if (migrateStorageManaged && "auth".equals(diskChildNode.getNodeName())) {
                                     diskNode.removeChild(diskChildNode);
+                                } else if ("backingStore".equals(diskChildNode.getNodeName()) && migrateDiskInfo.getBackingStoreText() != null) {
+                                    for (int b = 0; b < diskChildNode.getChildNodes().getLength(); b++) {
+                                        Node backingChild = diskChildNode.getChildNodes().item(b);
+                                        if ("source".equals(backingChild.getNodeName())) {
+                                            diskChildNode.removeChild(backingChild);
+                                            Element newChildBackingElement = doc.createElement("source");
+                                            newChildBackingElement.setAttribute(migrateDiskInfo.getSource().toString(), migrateDiskInfo.getBackingStoreText());
+                                            diskChildNode.appendChild(newChildBackingElement);
+                                        }
+                                    }
                                 } else if ("encryption".equals(diskChildNode.getNodeName())) {
                                     for (int s = 0; s < diskChildNode.getChildNodes().getLength(); s++) {
                                         Node encryptionChild = diskChildNode.getChildNodes().item(s);

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java

@@ -794,6 +794,51 @@ public class LibvirtMigrateCommandWrapperTest {
         assertXpath(doc, "/domain/devices/disk/encryption/secret/@uuid", expectedSecretUuid);
     }
 
+    @Test
+    public void testReplaceBackingStore() throws Exception {
+        Map<String, MigrateDiskInfo> mapMigrateStorage = new HashMap<String, MigrateDiskInfo>();
+        final String xmlDesc =
+                "<domain type='kvm' id='3'>" +
+                        "  <devices>" +
+                        "    <disk type='file' device='disk'>\n" +
+                        "      <driver name='qemu' type='qcow2' cache='none'/>\n" +
+                        "      <source file='/mnt/07eb495b-5590-3877-9fb7-23c6e9a40d40/bf8621b3-027c-497d-963b-06319650f048'/>\n" +
+                        "      <target dev='vdb' bus='virtio'/>\n" +
+                        "      <serial>bf8621b3027c497d963b</serial>\n" +
+                        "      <alias name='virtio-disk1'/>\n" +
+                        "      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>\n" +
+                        "      <backingStore type='file' index='1'>\n" +
+                        "        <format type='raw'/>\n" +
+                        "        <source file='/mnt/07eb495b-5590-3877-9fb7-23c6e9a40d40/bb4d4df4-c004-11e5-94ed-5254001daa61'/>\n" +
+                        "        <backingStore/>\n" +
+                        "      </backingStore>\n" +
+                        "      <encryption format='luks'>\n" +
+                        "        <secret type='passphrase' uuid='5644d664-a238-3a9b-811c-961f609d29f4'/>\n" +
+                        "      </encryption>\n" +
+                        "    </disk>\n" +
+                        "  </devices>" +
+                        "</domain>";
+
+        final String volumeFile = "3530f749-82fd-458e-9485-a357e6e541db";
+        final String backingFile = "0bc745b6-f3d7-44a9-ad8e-68904b77e2ab";
+        String newDiskPath = "/mnt/2d0435e1-99e0-4f1d-94c0-bee1f6f8b99e/" + volumeFile;
+        String newBackingStorePath = "/mnt/2d0435e1-99e0-4f1d-94c0-bee1f6f8b99e/" + backingFile;
+        MigrateDiskInfo diskInfo = new MigrateDiskInfo("123456", DiskType.BLOCK, DriverType.RAW, Source.FILE, newDiskPath, newBackingStorePath);
+        mapMigrateStorage.put("/mnt/07eb495b-5590-3877-9fb7-23c6e9a40d40/bf8621b3-027c-497d-963b-06319650f048", diskInfo);
+
+        final String result = libvirtMigrateCmdWrapper.replaceStorage(xmlDesc, mapMigrateStorage, false);
+        InputStream in = IOUtils.toInputStream(result);
+        DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
+        DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
+        Document doc = docBuilder.parse(in);
+        assertXpath(doc, "/domain/devices/disk/backingStore/source/@file", newBackingStorePath);
+        assertXpath(doc, "/domain/devices/disk/source/@file", newDiskPath);
+        assertXpath(doc, "/domain/devices/disk/serial", "bf8621b3027c497d963b");
+
+        final String expectedSecretUuid = LibvirtComputingResource.generateSecretUUIDFromString(volumeFile);
+        assertXpath(doc, "/domain/devices/disk/encryption/secret/@uuid", expectedSecretUuid);
+    }
+
     public void testReplaceStorageXmlDiskNotManagedStorage() throws ParserConfigurationException, TransformerException, SAXException, IOException {
         final LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper();
         String destDisk1FileName = "XXXXXXXXXXXXXX";

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -602,6 +602,32 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         });
     }
 
+    protected void validateIpAddressRelatedConfigValues(final String configName, final String value) {
+        if (!configName.endsWith(".ip") && !configName.endsWith(".ipaddress") && !configName.endsWith(".iprange")) {
+            return;
+        }
+        if (StringUtils.isEmpty(value)) {
+            return;
+        }
+        final ConfigKey<?> configKey = _configDepot.get(configName);
+        if (configKey == null || !String.class.equals(configKey.type())) {
+            return;
+        }
+        boolean err = (configName.endsWith(".ip") || configName.endsWith(".ipaddress")) && !NetUtils.isValidIp4(value);
+        if (configName.endsWith(".iprange")) {
+            err = true;
+            if (value.contains("-")) {
+                String[] ips = value.split("-");
+                if (ips.length == 2 && NetUtils.isValidIp4(ips[0]) && NetUtils.isValidIp4(ips[1])) {
+                    err = false;
+                }
+            }
+        }
+        if (err) {
+            throw new InvalidParameterValueException("Invalid IP address value(s) specified for the config value");
+        }
+    }
+
     @Override
     public boolean start() {
 
@@ -874,6 +900,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
             catergory = config.getCategory();
         }
 
+        validateIpAddressRelatedConfigValues(name, value);
+
         if (value == null) {
             return _configDao.findByName(name);
         }

server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java

@@ -4260,7 +4260,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
                         _volsDao.update(volumeToAttach.getId(), volumeToAttach);
                     }
 
-                    if (host != null && volumeToAttachStoragePool.getPoolType() == Storage.StoragePoolType.PowerFlex) {
+                    if (host != null && volumeToAttachStoragePool != null && volumeToAttachStoragePool.getPoolType() == Storage.StoragePoolType.PowerFlex) {
                         // Unmap the volume on PowerFlex/ScaleIO pool for stopped VM
                         volService.revokeAccess(volFactory.getVolume(volumeToAttach.getId()), host, dataStore);
                     }

server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java

@@ -742,7 +742,7 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
             executorService.scheduleWithFixedDelay(
                     new DirectDownloadCertificateUploadBackgroundTask(this, hostDao, dataCenterDao,
                             directDownloadCertificateDao, directDownloadCertificateHostMapDao),
-                    60L, DirectDownloadCertificateUploadInterval.value(), TimeUnit.HOURS);
+                    1L, DirectDownloadCertificateUploadInterval.value(), TimeUnit.HOURS);
         }
         return true;
     }

server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java

@@ -16,22 +16,37 @@
 // under the License.
 package com.cloud.configuration;
 
-import com.cloud.utils.net.NetUtils;
+import java.util.List;
+
+import org.apache.cloudstack.framework.config.ConfigDepot;
+import org.apache.cloudstack.framework.config.ConfigKey;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.powermock.api.mockito.PowerMockito;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
-import java.util.List;
+import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.storage.StorageManager;
+import com.cloud.utils.net.NetUtils;
 
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(NetUtils.class)
 public class ConfigurationManagerImplTest {
+    @Mock
+    ConfigDepot configDepot;
     ConfigurationManagerImpl configurationManagerImplSpy = Mockito.spy(new ConfigurationManagerImpl());
+
+    @Before
+    public void setUp() throws Exception {
+        configurationManagerImplSpy._configDepot = configDepot;
+    }
+
     @Test
     public void validateIfIntValueIsInRangeTestValidValueReturnNull() {
         String testVariable = configurationManagerImplSpy.validateIfIntValueIsInRange("String name", "3", "1-5");
@@ -191,4 +206,50 @@ public class ConfigurationManagerImplTest {
         String testVariable = configurationManagerImplSpy.validateRangeOther("NameTest1", "ThisShouldNotWork", "ThisShouldWork,ThisShouldAlsoWork,SoShouldThis");
         Assert.assertNotNull(testVariable);
     }
+
+    @Test
+    public void testValidateIpAddressRelatedConfigValuesUnrelated() {
+        configurationManagerImplSpy.validateIpAddressRelatedConfigValues(StorageManager.PreferredStoragePool.key(), "something");
+        configurationManagerImplSpy.validateIpAddressRelatedConfigValues("config.ip", "");
+        Mockito.when(configurationManagerImplSpy._configDepot.get("config.ip")).thenReturn(null);
+        configurationManagerImplSpy.validateIpAddressRelatedConfigValues("config.ip", "something");
+        ConfigKey<?> key = StorageManager.MountDisabledStoragePool;
+        Mockito.doReturn(key).when(configurationManagerImplSpy._configDepot).get(StorageManager.MountDisabledStoragePool.key());
+        configurationManagerImplSpy.validateIpAddressRelatedConfigValues(StorageManager.MountDisabledStoragePool.key(), "false");
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateIpAddressRelatedConfigValuesInvalidIp() {
+        ConfigKey<String> key = StorageManager.PreferredStoragePool; // Any ConfigKey of String type
+        Mockito.doReturn(key).when(configurationManagerImplSpy._configDepot).get("config.ip");
+        configurationManagerImplSpy.validateIpAddressRelatedConfigValues("config.ip", "abcdefg");
+    }
+
+    @Test
+    public void testValidateIpAddressRelatedConfigValuesValidIp() {
+        ConfigKey<String> key = StorageManager.PreferredStoragePool; // Any ConfigKey of String type
+        Mockito.doReturn(key).when(configurationManagerImplSpy._configDepot).get("config.ip");
+        configurationManagerImplSpy.validateIpAddressRelatedConfigValues("config.ip", "192.168.1.1");
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateIpAddressRelatedConfigValuesInvalidIpRange() {
+        ConfigKey<String> key = StorageManager.PreferredStoragePool; // Any ConfigKey of String type. RemoteAccessVpnManagerImpl.RemoteAccessVpnClientIpRange not accessible here
+        Mockito.doReturn(key).when(configurationManagerImplSpy._configDepot).get("config.iprange");
+        configurationManagerImplSpy.validateIpAddressRelatedConfigValues("config.iprange", "xyz-192.168.1.20");
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateIpAddressRelatedConfigValuesInvalidIpRange1() {
+        ConfigKey<String> key = StorageManager.PreferredStoragePool; // Any ConfigKey of String type. RemoteAccessVpnManagerImpl.RemoteAccessVpnClientIpRange not accessible here
+        Mockito.doReturn(key).when(configurationManagerImplSpy._configDepot).get("config.iprange");
+        configurationManagerImplSpy.validateIpAddressRelatedConfigValues("config.iprange", "192.168.1.20");
+    }
+
+    @Test
+    public void testValidateIpAddressRelatedConfigValuesValidIpRange() {
+        ConfigKey<String> key = StorageManager.PreferredStoragePool; // Any ConfigKey of String type. RemoteAccessVpnManagerImpl.RemoteAccessVpnClientIpRange not accessible here
+        Mockito.doReturn(key).when(configurationManagerImplSpy._configDepot).get("config.iprange");
+        configurationManagerImplSpy.validateIpAddressRelatedConfigValues("config.iprange", "192.168.1.1-192.168.1.100");
+    }
 }

systemvm/agent/noVNC/core/rfb.js

@@ -433,21 +433,30 @@ export default class RFB extends EventTargetMixin {
         this._resumeAuthentication();
     }
 
-    sendText(text) {
+     sendText(text) {
-        for (var i = 0; i < text.length; i++) {
+        const sleep = (time) => {
-            const character = text.charAt(i);
+            return new Promise(resolve => setTimeout(resolve, time))
-            var charCode = USKeyTable[character] || false;
+        }
-            if (charCode) {
+
-                this.sendKey(charCode, character, true);
+        const keyboardTypeText = async () => {
-                this.sendKey(charCode, character, false);
+            for (var i = 0; i < text.length; i++) {
-            } else {
+                const character = text.charAt(i);
-                charCode = text.charCodeAt(i)
+                var charCode = USKeyTable[character] || false;
-                this.sendKey(KeyTable.XK_Shift_L, "ShiftLeft", true);
+                if (charCode) {
-                this.sendKey(charCode, character, true);
+                    this.sendKey(charCode, character, true);
-                this.sendKey(charCode, character, false);
+                    this.sendKey(charCode, character, false);
-                this.sendKey(KeyTable.XK_Shift_L, "ShiftLeft", false);
+                } else {
+                    charCode = text.charCodeAt(i)
+                    this.sendKey(KeyTable.XK_Shift_L, "ShiftLeft", true);
+                    this.sendKey(charCode, character, true);
+                    this.sendKey(charCode, character, false);
+                    this.sendKey(KeyTable.XK_Shift_L, "ShiftLeft", false);
+                }
+                await sleep(25)
             }
         }
+
+        keyboardTypeText()
     }
 
     sendCtrlAltDel() {

test/integration/component/test_configdrive.py

@@ -1025,7 +1025,7 @@ class ConfigDriveUtils:
         :rtype: str
         """
         self.debug("Updating userdata for VM - %s" % vm.name)
-        updated_user_data = base64.encodestring(new_user_data.encode()).decode()
+        updated_user_data = base64.encodebytes(new_user_data.encode()).decode()
         with self.stopped_vm(vm):
             vm.update(self.api_client, userdata=updated_user_data)
 

test/integration/component/test_deploy_vm_userdata_multi_nic.py

@@ -126,7 +126,7 @@ class TestDeployVmWithUserDataMultiNic(cloudstackTestCase):
         """Test userdata update when non default nic is without userdata for deploy and update
         """
 
-        self.userdata = base64.encodestring(self.userdata.encode()).decode()
+        self.userdata = base64.encodebytes(self.userdata.encode()).decode()
 
         network1 = Network.create(
             self.apiclient,

test/integration/component/test_deploy_vm_userdata_reg.py

@@ -99,7 +99,7 @@ class TestDeployVmWithUserData(cloudstackTestCase):
         # py2 didn't insert any new-lines
         # so we now do the encoding in the stored userdata string and remove the '\n's
         # to get a good easy string compare in the assert later on.
-        cls.userdata = base64.encodestring(cls.userdata.encode()).decode().replace('\n', '')
+        cls.userdata = base64.encodebytes(cls.userdata.encode()).decode().replace('\n', '')
         cls.user_data_2k= ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(2000))
         cls.user_data_2kl = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(1900))
 

tools/marvin/marvin/cloudstackConnection.py

@@ -147,7 +147,7 @@ class CSConnection(object):
                 ).replace("+", "%20")]
             ) for r in params]
         )
-        signature = base64.encodestring(
+        signature = base64.encodebytes(
             hmac.new(self.securityKey.encode('utf-8'),
                      hash_str.encode('utf-8'),
                      hashlib.sha1).digest()).strip()

tools/ngui/requester.py

@@ -68,7 +68,7 @@ def make_request(command, args, logger, host, port,
                        str.lower(urllib.quote_plus(str(r[1]))).replace("+",
                        "%20")]) for r in request])
 
-    sig = urllib.quote_plus(base64.encodestring(hmac.new(secretkey, hashStr,
+    sig = urllib.quote_plus(base64.encodebytes(hmac.new(secretkey, hashStr,
                             hashlib.sha1).digest()).strip())
     request_url += "&signature=%s" % sig
     request_url = "%s://%s:%s%s?%s" % (protocol, host, port, path, request_url)

ui/src/views/tools/ImportUnmanagedInstance.vue

@@ -177,6 +177,7 @@
                 :maxCpu="getMaxCpu()"
                 :minMemory="getMinMemory()"
                 :maxMemory="getMaxMemory()"
+                :cpuSpeed="getCPUSpeed()"
                 @update-iops-value="updateFieldValue"
                 @update-compute-cpunumber="updateFieldValue"
                 @update-compute-cpuspeed="updateFieldValue"
@@ -522,6 +523,15 @@ export default {
       }
       return 'serviceofferingdetails' in this.computeOffering ? this.computeOffering.serviceofferingdetails.maxmemory * 1 : Number.MAX_SAFE_INTEGER
     },
+    getCPUSpeed () {
+      if (!this.computeOffering) {
+        return 0
+      }
+      if (this.computeOffering.cpuspeed) {
+        return this.computeOffering.cpuspeed * 1
+      }
+      return this.resource.cpuspeed * 1 || 0
+    },
     fetchOptions (param, name, exclude) {
       if (exclude && exclude.length > 0) {
         if (exclude.includes(name)) {