CLOUDSTACK-10227: Stabilization fixes for 4.11.0.0 (#2403)

This fixes regression failures seen in Trillian, fixes NPEs that cause Travis related failures.
This also removes the aria2 dependency from rpms that require users to enable/install epel-release.
This finally updates the checksums for 4.11 systemvmtemplates in db upgrade path.

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

api/src/org/apache/cloudstack/api/response/HostHAResponse.java

@@ -84,6 +84,9 @@ public final class HostHAResponse extends BaseResponse {
 
     public void setHaState(HAConfig.HAState haState) {
         this.haState = haState;
+        if (haState == null) {
+            this.haState = HAConfig.HAState.Disabled;
+        }
     }
 
     public String getProvider() {

engine/schema/src/com/cloud/upgrade/dao/Upgrade41000to41100.java

@@ -167,7 +167,6 @@ public class Upgrade41000to41100 implements DbUpgrade {
 
         final Map<Hypervisor.HypervisorType, String> newTemplateUrl = new HashMap<Hypervisor.HypervisorType, String>() {
             {
-                // FIXME: upload templates
                 put(Hypervisor.HypervisorType.KVM, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.0-kvm.qcow2.bz2");
                 put(Hypervisor.HypervisorType.VMware, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.0-vmware.ova");
                 put(Hypervisor.HypervisorType.XenServer, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.0-xen.vhd.bz2");
@@ -179,13 +178,12 @@ public class Upgrade41000to41100 implements DbUpgrade {
 
         final Map<Hypervisor.HypervisorType, String> newTemplateChecksum = new HashMap<Hypervisor.HypervisorType, String>() {
             {
-                // FIXME: update checksums?
-                put(Hypervisor.HypervisorType.KVM, "bc2eac46f16a2ece6c19d4b89db41de3");
-                put(Hypervisor.HypervisorType.XenServer, "908c28a8d4c232f960e0f84af7f86c80");
-                put(Hypervisor.HypervisorType.VMware, "970bfb070a80bd74820881d8149643c1");
-                put(Hypervisor.HypervisorType.Hyperv, "0adb35bd9f92e80d3fc63fcdd9bb55e5");
-                put(Hypervisor.HypervisorType.LXC, "bc2eac46f16a2ece6c19d4b89db41de3");
-                put(Hypervisor.HypervisorType.Ovm3, "94a41f0a5361933813bb34a51df56f56");
+                put(Hypervisor.HypervisorType.KVM, "2d8d1e4eacc976814b97f02849481433");
+                put(Hypervisor.HypervisorType.XenServer, "a5ecf7ed485e2da5ec1993069aa60553");
+                put(Hypervisor.HypervisorType.VMware, "84dab5d1e8267b5dc85eb4eaa21a1efe");
+                put(Hypervisor.HypervisorType.Hyperv, "d592ab6a2271303fe75b6a27f8e6bd53");
+                put(Hypervisor.HypervisorType.LXC, "2d8d1e4eacc976814b97f02849481433");
+                put(Hypervisor.HypervisorType.Ovm3, "60fe2227b89a8980ee09f89dc7b19582");
             }
         };
 

packaging/centos63/cloud.spec

@@ -130,8 +130,6 @@ Requires: perl
 Requires: libvirt-python
 Requires: qemu-img
 Requires: qemu-kvm
-Requires: epel-release
-Requires: aria2
 Provides: cloud-agent
 Obsoletes: cloud-agent < 4.1.0
 Obsoletes: cloud-agent-libs < 4.1.0

packaging/centos7/cloud.spec

@@ -111,8 +111,6 @@ Requires: perl
 Requires: libvirt-python
 Requires: qemu-img
 Requires: qemu-kvm
-Requires: epel-release
-Requires: aria2
 Provides: cloud-agent
 Group: System Environment/Libraries
 %description agent

server/src/com/cloud/template/HypervisorTemplateAdapter.java

@@ -451,6 +451,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
 
         if (imageStores == null || imageStores.size() == 0) {
             // already destroyed on image stores
+            success = true;
             s_logger.info("Unable to find image store still having template: " + template.getName() + ", so just mark the template removed");
         } else {
             // Make sure the template is downloaded to all found image stores
@@ -536,10 +537,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
                                 templateZoneDao.remove(templateZone.getId());
                             }
                         }
-                    } catch (InterruptedException e) {
-                        s_logger.debug("Delete template Failed", e);
-                        throw new CloudRuntimeException("Delete template Failed", e);
-                    } catch (ExecutionException e) {
+                    } catch (InterruptedException|ExecutionException e) {
                         s_logger.debug("Delete template Failed", e);
                         throw new CloudRuntimeException("Delete template Failed", e);
                     }
@@ -551,7 +549,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
 
         }
         if (success) {
-            if ((imageStores.size() > 1) && (profile.getZoneIdList() != null)) {
+            if ((imageStores != null && imageStores.size() > 1) && (profile.getZoneIdList() != null)) {
                 //if template is stored in more than one image stores, and the zone id is not null, then don't delete other templates.
                 return success;
             }

server/src/org/apache/cloudstack/ha/HAManagerImpl.java

@@ -199,13 +199,16 @@ public final class HAManagerImpl extends ManagerBase implements HAManager, Clust
 
     private HAResource validateAndFindHAResource(final HAConfig haConfig) {
         HAResource resource = null;
+        if (haConfig == null) {
+            return null;
+        }
         if (haConfig.getResourceType() == HAResource.ResourceType.Host) {
             final Host host = hostDao.findById(haConfig.getResourceId());
             if (host != null && host.getRemoved() != null) {
                 return null;
             }
             resource = host;
-            if (resource == null && haConfig.getState() != HAConfig.HAState.Disabled) {
+            if (haConfig.getState() == null || (resource == null && haConfig.getState() != HAConfig.HAState.Disabled)) {
                 disableHA(haConfig.getResourceId(), haConfig.getResourceType());
                 return null;
             }
@@ -224,6 +227,9 @@ public final class HAManagerImpl extends ManagerBase implements HAManager, Clust
     }
 
     private HAProvider<HAResource> validateAndFindHAProvider(final HAConfig haConfig, final HAResource resource) {
+        if (haConfig == null) {
+            return null;
+        }
         final HAProvider<HAResource> haProvider = haProviderMap.get(haConfig.getHaProvider());
         if (haProvider != null && !haProvider.isEligible(resource)) {
             if (haConfig.getState() != HAConfig.HAState.Ineligible) {
@@ -639,6 +645,10 @@ public final class HAManagerImpl extends ManagerBase implements HAManager, Clust
                 }
                 final List<HAConfig> haConfigList = new ArrayList<HAConfig>(haConfigDao.listAll());
                 for (final HAConfig haConfig : haConfigList) {
+                    if (haConfig == null) {
+                        continue;
+                    }
+
                     if (!checkHAOwnership(haConfig)) {
                         continue;
                     }

test/integration/smoke/test_hostha_simulator.py

@@ -352,6 +352,7 @@ class TestHostHA(cloudstackTestCase):
             Zone > Cluster > Host
         """
         host = self.getHost()
+        self.configureAndDisableHostHa(host.id)
         self.configureAndEnableHostHa()
 
         self.checkSyncToState('Available')

test/integration/smoke/test_public_ip_range.py

@@ -218,7 +218,7 @@ class TestDedicatePublicIPRange(cloudstackTestCase):
                     return True, response[0].id
             return False, None
 
-        res, systemvmId = wait_until(3, 100, checkSystemVMUp)
+        res, systemvmId = wait_until(3, 200, checkSystemVMUp)
         if not res:
             raise Exception("Failed to wait for systemvm to be running")
         return systemvmId
@@ -336,13 +336,13 @@ class TestDedicatePublicIPRange(cloudstackTestCase):
         return False
 
     @attr(tags = ["advanced", "publiciprange", "dedicate", "release"], required_hardware="false")
-    def test_dedicate_public_ip_range_for_system_vms_cpvm(self):
-        """Test CPVM Public IP
+    def test_dedicate_public_ip_range_for_system_vms_01_ssvm(self):
+        """Test SSVM Public IP
         """
         self.debug("Precondition: No public IP range dedicated for system vms in the environment")
         if self.exists_public_ip_range_for_system_vms(self.services["zoneid"]):
             self.skipTest("An existing IP range defined for system vms, aborting test")
-        
+
         services = {
             "gateway":"192.168.100.1",
             "netmask":"255.255.255.0",
@@ -355,13 +355,13 @@ class TestDedicatePublicIPRange(cloudstackTestCase):
 
         self.base_system_vm(
             services,
-            'consoleproxy'
+            'secondarystoragevm'
         )
         return
 
     @attr(tags = ["advanced", "publiciprange", "dedicate", "release"], required_hardware="false")
-    def test_dedicate_public_ip_range_for_system_vms_ssvm(self):
-        """Test SSVM Public IP
+    def test_dedicate_public_ip_range_for_system_vms_02_cpvm(self):
+        """Test CPVM Public IP
         """
         self.debug("Precondition: No public IP range dedicated for system vms in the environment")
         if self.exists_public_ip_range_for_system_vms(self.services["zoneid"]):
@@ -379,6 +379,6 @@ class TestDedicatePublicIPRange(cloudstackTestCase):
 
         self.base_system_vm(
             services,
-            'secondarystoragevm'
+            'consoleproxy'
         )
         return

test/integration/smoke/test_ssvm.py

@@ -198,7 +198,6 @@ class TestSSVMs(cloudstackTestCase):
                 True,
                 "Check list response returns a valid list"
             )
-            iprange = ipranges_response[0]
 
             # Fetch corresponding Physical Network of SSVM's Zone
             listphyntwk = PhysicalNetwork.list(
@@ -213,9 +212,14 @@ class TestSSVMs(cloudstackTestCase):
                         self.apiclient,
                         physicalnetworkid=listphyntwk[0].id),
                     list) is True):
-                self.assertEqual(
-                    ssvm.gateway,
-                    iprange.gateway,
+                gatewayFound = False
+                for iprange in ipranges_response:
+                    if ssvm.gateway == iprange.gateway:
+                        gatewayFound = True
+                        break
+
+                self.assertTrue(
+                    gatewayFound,
                     "Check gateway with that of corresponding ip range"
                 )
 
@@ -333,7 +337,6 @@ class TestSSVMs(cloudstackTestCase):
                 True,
                 "Check list response returns a valid list"
             )
-            iprange = ipranges_response[0]
 
             # Fetch corresponding Physical Network of SSVM's Zone
             listphyntwk = PhysicalNetwork.list(
@@ -348,13 +351,13 @@ class TestSSVMs(cloudstackTestCase):
                         self.apiclient,
                         physicalnetworkid=listphyntwk[0].id),
                     list) is True):
-                cpvmValidGateway = False
+                gatewayFound = False
                 for iprange in ipranges_response:
                     if iprange.gateway == cpvm.gateway:
-                        cpvmValidGateway = True
+                        gatewayFound = True
                         break
                 self.assertTrue(
-                    cpvmValidGateway,
+                    gatewayFound,
                     "Check gateway with that of corresponding ip range"
                 )
 

test/integration/smoke/test_templates.py

@@ -540,7 +540,6 @@ class TestTemplates(cloudstackTestCase):
         cls.services["template"]["ostypeid"] = template.ostypeid
         cls.services["template_2"]["ostypeid"] = template.ostypeid
         cls.services["ostypeid"] = template.ostypeid
-        cls.services["isdynamicallyscalable"] = template.isdynamicallyscalable
         cls.account = Account.create(
                             cls.apiclient,
                             cls.services["account"],
@@ -590,6 +589,7 @@ class TestTemplates(cloudstackTestCase):
                                          account=cls.account.name,
                                          domainid=cls.account.domainid
                                          )
+        cls.services["isdynamicallyscalable"] = cls.template_1.isdynamicallyscalable
         cls.template_2 = Template.create(
                                          cls.apiclient,
                                          cls.services["template_2"],

test/integration/smoke/test_volumes.py

@@ -246,8 +246,6 @@ class TestCreateVolume(cloudstackTestCase):
                 ret = checkVolumeSize(ssh_handle=ssh,volume_name=volume_name,size_to_verify=vol_sz)
             elif list_volume_response[0].hypervisor.lower() == "hyperv":
                 ret = checkVolumeSize(ssh_handle=ssh,volume_name="/dev/sdb",size_to_verify=vol_sz)
-            elif list_volume_response[0].hypervisor.lower() == "vmware":
-                ret = checkVolumeSize(ssh_handle=ssh,volume_name="/dev/sdb",size_to_verify=vol_sz)
             else:
                 ret = checkVolumeSize(ssh_handle=ssh,size_to_verify=vol_sz)
             self.debug(" Volume Size Expected %s  Actual :%s" %(vol_sz,ret[1]))

ui/scripts/templates.js

@@ -258,8 +258,9 @@
                                                     $form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide();
                                                     $form.find('.form-item[rel=directdownload]').hide();
 
-                                                    if (isAdmin())
+                                                    if (isAdmin()) {
                                                         $form.find('.form-item[rel=xenserverToolsVersion61plus]').css('display', 'inline-block');
+                                                    }
                                                 } else if ($(this).val() == "KVM") {
                                                     $form.find('.form-item[rel=rootDiskControllerType]').hide();
                                                     $form.find('.form-item[rel=nicAdapterType]').hide();
@@ -267,7 +268,9 @@
                                                     $form.find('.form-item[rel=xenserverToolsVersion61plus]').hide();
                                                     $form.find('.form-item[rel=rootDiskControllerTypeKVM]').css('display', 'inline-block');
                                                     $form.find('.form-item[rel=xenserverToolsVersion61plus]').css('display', 'inline-block');
-                                                    $form.find('.form-item[rel=directdownload]').css('display', 'inline-block');
+                                                    if (isAdmin()) {
+                                                      $form.find('.form-item[rel=directdownload]').css('display', 'inline-block');
+                                                    }
                                                 } else {
                                                     $form.find('.form-item[rel=rootDiskControllerType]').hide();
                                                     $form.find('.form-item[rel=nicAdapterType]').hide();

vmware-base/src/com/cloud/hypervisor/vmware/mo/HostMO.java

@@ -1039,7 +1039,7 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost {
     @Override
     public boolean isHyperHostConnected() throws Exception {
         HostRuntimeInfo runtimeInfo = (HostRuntimeInfo)_context.getVimClient().getDynamicProperty(_mor, "runtime");
-        return runtimeInfo.getConnectionState() == HostSystemConnectionState.CONNECTED;
+        return runtimeInfo != null && runtimeInfo.getConnectionState() == HostSystemConnectionState.CONNECTED;
     }
 
     public boolean revertToSnapshot(ManagedObjectReference morSnapshot) throws Exception {