Merge release branch 4.18 to main

* 4.18:
  UI: allow new keys for VM details (#7793)
  Refactoring StorPool's smoke tests (#7392)
  UI: decode userdata in EditVM dialog (#7796)
  packaging: unalias cp before package upgrade (#7722)
  make NoopDbUpgrade do a systemvm template check (#7564)
  UI unit test: fix expected values (#7792)

engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java

@@ -446,10 +446,11 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
     }
 
     @VisibleForTesting
-    protected static final class NoopDbUpgrade implements DbUpgrade {
+    protected static final class NoopDbUpgrade implements DbUpgrade, DbUpgradeSystemVmTemplate {
 
         private final String upgradedVersion;
         private final String[] upgradeRange;
+        private SystemVmTemplateRegistration systemVmTemplateRegistration;
 
         private NoopDbUpgrade(final CloudStackVersion fromVersion, final CloudStackVersion toVersion) {
 
@@ -490,5 +491,19 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
             return new InputStream[0];
         }
 
+        private void initSystemVmTemplateRegistration() {
+            systemVmTemplateRegistration = new SystemVmTemplateRegistration("");
+        }
+
+        @Override
+        public void updateSystemVmTemplates(Connection conn) {
+            s_logger.debug("Updating System Vm template IDs");
+            initSystemVmTemplateRegistration();
+            try {
+                systemVmTemplateRegistration.updateSystemVmTemplates(conn);
+            } catch (Exception e) {
+                throw new CloudRuntimeException("Failed to find / register SystemVM template(s)");
+            }
+        }
     }
 }

packaging/centos8/cloud.spec

@@ -392,6 +392,7 @@ install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-inte
 
 %posttrans common
 
+unalias cp
 python_dir=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")
 if [ ! -z $python_dir ];then
   cp -f -r /usr/share/cloudstack-common/python-site/* $python_dir/

plugins/storage/volume/storpool/pom.xml

@@ -61,6 +61,11 @@
             <artifactId>mockito-inline</artifactId>
             <version>4.7.0</version>
         </dependency>
+        <dependency>
+            <groupId>pl.project13.maven</groupId>
+            <artifactId>git-commit-id-plugin</artifactId>
+            <version>4.9.10</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>
@@ -75,6 +80,35 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>pl.project13.maven</groupId>
+                <artifactId>git-commit-id-plugin</artifactId>
+                <version>4.9.10</version>
+                <executions>
+                    <execution>
+                        <id>get-the-git-infos</id>
+                        <goals>
+                            <goal>revision</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <dotGitDirectory>${project.basedir}/.git</dotGitDirectory>
+                    <prefix>git</prefix>
+                    <verbose>false</verbose>
+                    <generateGitPropertiesFile>true</generateGitPropertiesFile>
+                    <generateGitPropertiesFilename>${project.build.outputDirectory}/git.properties</generateGitPropertiesFilename>
+                    <format>json</format>
+                    <excludeProperties>
+                        <excludeProperty>git.*.email</excludeProperty>
+                    </excludeProperties>
+                    <gitDescribe>
+                        <skip>false</skip>
+                        <always>false</always>
+                        <dirty>-dirty</dirty>
+                    </gitDescribe>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 </project>

test/integration/plugins/storpool/MigrateVolumeToStorPool.py

@@ -78,10 +78,19 @@ class TestMigrateVolumeToAnotherPool(cloudstackTestCase):
 
     @classmethod
     def setUpCloudStack(cls):
-        cls.spapi = spapi.Api(host="10.2.23.248", port="81", auth="6549874687", multiCluster=True)
+        config = cls.getClsConfig()
+        StorPoolHelper.logger = cls
+
+        zone = config.zones[0]
+        assert zone is not None
+
+        cls.spapi = spapi.Api(host=zone.spEndpoint, port=zone.spEndpointPort, auth=zone.spAuthToken, multiCluster=True)
         testClient = super(TestMigrateVolumeToAnotherPool, cls).getClsTestClient()
         cls.apiclient = testClient.getApiClient()
 
+        cls.zone = list_zones(cls.apiclient, name=zone.name)[0]
+        assert cls.zone is not None
+
         cls._cleanup = []
 
         cls.unsupportedHypervisor = False
@@ -93,14 +102,6 @@ class TestMigrateVolumeToAnotherPool(cloudstackTestCase):
         cls.services = testClient.getParsedTestDataConfig()
         # Get Zone, Domain and templates
         cls.domain = get_domain(cls.apiclient)
-        cls.zone = None
-        zones = list_zones(cls.apiclient)
-
-        for z in zones:
-            if z.name == cls.getClsConfig().mgtSvr[0].zone:
-                cls.zone = z
-
-        assert cls.zone is not None
 
         td = TestData()
         cls.testdata = td.testdata

test/integration/plugins/storpool/TestStorPoolVolumes.py

@@ -77,6 +77,13 @@ class TestStoragePool(cloudstackTestCase):
 
     @classmethod
     def setUpCloudStack(cls):
+        config = cls.getClsConfig()
+        StorPoolHelper.logger = cls
+
+        zone = config.zones[0]
+        assert zone is not None
+
+        cls.spapi = spapi.Api(host=zone.spEndpoint, port=zone.spEndpointPort, auth=zone.spAuthToken, multiCluster=True)
         testClient = super(TestStoragePool, cls).getClsTestClient()
 
         cls._cleanup = []
@@ -94,20 +101,16 @@ class TestStoragePool(cloudstackTestCase):
 
         # Get Zone, Domain and templates
         cls.domain = get_domain(cls.apiclient)
-        cls.zone = None
+        cls.zone = list_zones(cls.apiclient, name=zone.name)[0]
-        zones = list_zones(cls.apiclient)
+        cls.debug(cls.zone)
-
+        cls.debug(list_zones(cls.apiclient, name=zone.name))
-        for z in zones:
-            if z.name == cls.getClsConfig().mgtSvr[0].zone:
-                cls.zone = z
-
         assert cls.zone is not None
 
         cls.sp_template_1 = "ssd"
         storpool_primary_storage = {
             "name" : cls.sp_template_1,
             "zoneid": cls.zone.id,
-            "url": "SP_API_HTTP=10.2.23.248:81;SP_AUTH_TOKEN=6549874687;SP_TEMPLATE=%s" % cls.sp_template_1,
+            "url": "SP_API_HTTP=%s:%s;SP_AUTH_TOKEN=%s;SP_TEMPLATE=%s" % (zone.spEndpoint, zone.spEndpointPort, zone.spAuthToken, cls.sp_template_1),
             "scope": "zone",
             "capacitybytes": 564325555333,
             "capacityiops": 155466,
@@ -117,8 +120,6 @@ class TestStoragePool(cloudstackTestCase):
             }
 
         cls.storpool_primary_storage = storpool_primary_storage
-        host, port, auth = cls.getCfgFromUrl(url = storpool_primary_storage["url"])
-        cls.spapi = spapi.Api(host=host, port=port, auth=auth, multiCluster=True)
 
         storage_pool = list_storage_pools(
             cls.apiclient,
@@ -166,7 +167,7 @@ class TestStoragePool(cloudstackTestCase):
         storpool_primary_storage2 = {
             "name" : cls.sp_template_2,
             "zoneid": cls.zone.id,
-            "url": "SP_API_HTTP=10.2.23.248:81;SP_AUTH_TOKEN=6549874687;SP_TEMPLATE=%s" % cls.sp_template_2,
+            "url": "SP_API_HTTP=%s:%s;SP_AUTH_TOKEN=%s;SP_TEMPLATE=%s" % (zone.spEndpoint, zone.spEndpointPort, zone.spAuthToken, cls.sp_template_2),
             "scope": "zone",
             "capacitybytes": 564325555333,
             "capacityiops": 1554,

test/integration/plugins/storpool/TestTagsOnStorPool.py

@@ -77,7 +77,13 @@ class TestStoragePool(cloudstackTestCase):
 
     @classmethod
     def setUpCloudStack(cls):
-        cls.spapi = spapi.Api(host="10.2.23.248", port="81", auth="6549874687", multiCluster=True)
+        config = cls.getClsConfig()
+        StorPoolHelper.logger = cls
+
+        zone = config.zones[0]
+        assert zone is not None
+
+        cls.spapi = spapi.Api(host=zone.spEndpoint, port=zone.spEndpointPort, auth=zone.spAuthToken, multiCluster=True)
         testClient = super(TestStoragePool, cls).getClsTestClient()
         cls.apiclient = testClient.getApiClient()
         cls.unsupportedHypervisor = False
@@ -91,12 +97,10 @@ class TestStoragePool(cloudstackTestCase):
         cls.services = testClient.getParsedTestDataConfig()
         # Get Zone, Domain and templates
         cls.domain = get_domain(cls.apiclient)
-        cls.zone = None
+        cls.zone = list_zones(cls.apiclient, name=zone.name)[0]
-        zones = list_zones(cls.apiclient)
+        cls.debug(cls.zone)
-
+        cls.debug(list_zones(cls.apiclient, name=zone.name))
-        for z in zones:
+        assert cls.zone is not None
-            if z.name == cls.getClsConfig().mgtSvr[0].zone:
-                cls.zone = z
 
         assert cls.zone is not None
 

test/integration/plugins/storpool/TestVmSnapshots.py

@@ -60,6 +60,12 @@ class TestVmSnapshot(cloudstackTestCase):
 
     @classmethod
     def setUpCloudStack(cls):
+        config = cls.getClsConfig()
+        StorPoolHelper.logger = cls
+
+        zone = config.zones[0]
+        assert zone is not None
+
         testClient = super(TestVmSnapshot, cls).getClsTestClient()
         cls.apiclient = testClient.getApiClient()
         cls._cleanup = []
@@ -74,13 +80,9 @@ class TestVmSnapshot(cloudstackTestCase):
         cls.services = testClient.getParsedTestDataConfig()
         # Get Zone, Domain and templates
         cls.domain = get_domain(cls.apiclient)
-        cls.zone = None
+        cls.zone = list_zones(cls.apiclient, name=zone.name)[0]
-        zones = list_zones(cls.apiclient)
+        cls.debug(cls.zone)
-
+        cls.debug(list_zones(cls.apiclient, name=zone.name))
-        for z in zones:
-            if z.name == cls.getClsConfig().mgtSvr[0].zone:
-                cls.zone = z
-
         assert cls.zone is not None
 
         cls.cluster = list_clusters(cls.apiclient)[0]

test/integration/plugins/storpool/sp_util.py

@@ -301,6 +301,12 @@ class TestData():
             },
         }
 class StorPoolHelper():
+    def setUpClass(cls):
+        cls.logger = None
+
+    @classmethod
+    def logging(cls):
+        return cls.logger
 
     @classmethod
     def create_template_from_snapshot(self, apiclient, services, snapshotid=None, volumeid=None):

ui/public/locales/en.json

@@ -780,7 +780,6 @@
 "label.egressdefaultpolicy": "Default egress policy",
 "label.elastic": "Elastic",
 "label.email": "Email",
-"label.enabled": "Enabled",
 "label.enable.autoscale.vmgroup": "Enable AutoScale VM Group",
 "label.enable.host": "Enable Host",
 "label.enable.network.offering": "Enable network offering",
@@ -2058,6 +2057,7 @@
 "label.uk.keyboard": "UK keyboard",
 "label.unauthorized": "Unauthorized",
 "label.unavailable": "Unavailable",
+"label.undefined": "Undefined",
 "label.unit": "Usage unit",
 "label.unknown": "Unknown",
 "label.unlimited": "Unlimited",

ui/src/components/view/DetailSettings.vue

@@ -170,7 +170,11 @@ export default {
         return []
       }
       if (!Array.isArray(this.detailOptions[this.newKey])) {
+        if (this.detailOptions[this.newKey]) {
           return { value: this.detailOptions[this.newKey] }
+        } else {
+          return ''
+        }
       }
       return this.detailOptions[this.newKey].map(value => {
         return { value: value }

ui/src/views/compute/EditVM.vue

@@ -283,6 +283,10 @@ export default {
         this.$notifyError(error)
       }).finally(() => { this.groups.loading = false })
     },
+    decodeUserData (userdata) {
+      const decodedData = Buffer.from(userdata, 'base64')
+      return decodedData.toString('utf-8')
+    },
     fetchUserData () {
       const params = {
         id: this.resource.id,
@@ -290,7 +294,7 @@ export default {
       }
 
       api('listVirtualMachines', params).then(json => {
-        this.form.userdata = atob(json.listvirtualmachinesresponse.virtualmachine[0].userdata || '')
+        this.form.userdata = this.decodeUserData(json.listvirtualmachinesresponse.virtualmachine[0].userdata || '')
       })
     },
     handleSubmit () {

ui/tests/unit/views/AutogenView.spec.js

@@ -1146,10 +1146,9 @@ describe('Views > AutogenView.vue', () => {
         ])
         expect(wrapper.vm.showAction).toBeTruthy()
         expect(listUuidOpts).toHaveBeenCalledTimes(4)
-        expect(listUuidOpts).toHaveBeenCalledWith({ name: 'id', type: 'uuid' })
+        expect(listUuidOpts).toHaveBeenCalledWith({ name: 'id', type: 'uuid' }, undefined)
-        expect(listUuidOpts).toHaveBeenCalledWith({ name: 'column1', type: 'list' })
+        expect(listUuidOpts).toHaveBeenCalledWith({ name: 'column1', type: 'list' }, undefined)
-        expect(listUuidOpts).toHaveBeenCalledWith({ name: 'column2', type: 'string' })
+        expect(listUuidOpts).toHaveBeenCalledWith({ name: 'column2', type: 'string' }, undefined)
-        expect(listUuidOpts).toHaveBeenCalledWith({ name: 'account', type: 'string' })
         done()
       })