Merge branch '4.11'

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41000to41100.java

@@ -22,15 +22,11 @@ import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.log4j.Logger;
 
-import com.cloud.hypervisor.Hypervisor;
 import com.cloud.utils.PropertiesUtil;
 import com.cloud.utils.exception.CloudRuntimeException;
 
@@ -68,7 +64,6 @@ public class Upgrade41000to41100 implements DbUpgrade {
     public void performDataMigration(Connection conn) {
         checkAndEnableDynamicRoles(conn);
         validateUserDataInBase64(conn);
-        updateSystemVmTemplates(conn);
     }
 
     private void checkAndEnableDynamicRoles(final Connection conn) {
@@ -126,176 +121,6 @@ public class Upgrade41000to41100 implements DbUpgrade {
         }
     }
 
-    @SuppressWarnings("serial")
-    private void updateSystemVmTemplates(final Connection conn) {
-        LOG.debug("Updating System Vm template IDs");
-        // Get all hypervisors in use
-        final Set<Hypervisor.HypervisorType> hypervisorsListInUse = new HashSet<Hypervisor.HypervisorType>();
-        try (PreparedStatement pstmt = conn.prepareStatement("select distinct(hypervisor_type) from `cloud`.`cluster` where removed is null"); ResultSet rs = pstmt.executeQuery()) {
-            while (rs.next()) {
-                switch (Hypervisor.HypervisorType.getType(rs.getString(1))) {
-                    case XenServer:
-                        hypervisorsListInUse.add(Hypervisor.HypervisorType.XenServer);
-                        break;
-                    case KVM:
-                        hypervisorsListInUse.add(Hypervisor.HypervisorType.KVM);
-                        break;
-                    case VMware:
-                        hypervisorsListInUse.add(Hypervisor.HypervisorType.VMware);
-                        break;
-                    case Hyperv:
-                        hypervisorsListInUse.add(Hypervisor.HypervisorType.Hyperv);
-                        break;
-                    case LXC:
-                        hypervisorsListInUse.add(Hypervisor.HypervisorType.LXC);
-                        break;
-                    case Ovm3:
-                        hypervisorsListInUse.add(Hypervisor.HypervisorType.Ovm3);
-                        break;
-                    default:
-                        break;
-                }
-            }
-        } catch (final SQLException e) {
-            LOG.error("updateSystemVmTemplates: Exception caught while getting hypervisor types from clusters: " + e.getMessage());
-            throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting hypervisor types from clusters", e);
-        }
-
-        final Map<Hypervisor.HypervisorType, String> NewTemplateNameList = new HashMap<Hypervisor.HypervisorType, String>() {
-            {
-                put(Hypervisor.HypervisorType.KVM, "systemvm-kvm-4.11");
-                put(Hypervisor.HypervisorType.VMware, "systemvm-vmware-4.11");
-                put(Hypervisor.HypervisorType.XenServer, "systemvm-xenserver-4.11");
-                put(Hypervisor.HypervisorType.Hyperv, "systemvm-hyperv-4.11");
-                put(Hypervisor.HypervisorType.LXC, "systemvm-lxc-4.11");
-                put(Hypervisor.HypervisorType.Ovm3, "systemvm-ovm3-4.11");
-            }
-        };
-
-        final Map<Hypervisor.HypervisorType, String> routerTemplateConfigurationNames = new HashMap<Hypervisor.HypervisorType, String>() {
-            {
-                put(Hypervisor.HypervisorType.KVM, "router.template.kvm");
-                put(Hypervisor.HypervisorType.VMware, "router.template.vmware");
-                put(Hypervisor.HypervisorType.XenServer, "router.template.xenserver");
-                put(Hypervisor.HypervisorType.Hyperv, "router.template.hyperv");
-                put(Hypervisor.HypervisorType.LXC, "router.template.lxc");
-                put(Hypervisor.HypervisorType.Ovm3, "router.template.ovm3");
-            }
-        };
-
-        final Map<Hypervisor.HypervisorType, String> newTemplateUrl = new HashMap<Hypervisor.HypervisorType, String>() {
-            {
-                put(Hypervisor.HypervisorType.KVM, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.0-kvm.qcow2.bz2");
-                put(Hypervisor.HypervisorType.VMware, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.0-vmware.ova");
-                put(Hypervisor.HypervisorType.XenServer, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.0-xen.vhd.bz2");
-                put(Hypervisor.HypervisorType.Hyperv, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.0-hyperv.vhd.zip");
-                put(Hypervisor.HypervisorType.LXC, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.0-kvm.qcow2.bz2");
-                put(Hypervisor.HypervisorType.Ovm3, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.0-ovm.raw.bz2");
-            }
-        };
-
-        final Map<Hypervisor.HypervisorType, String> newTemplateChecksum = new HashMap<Hypervisor.HypervisorType, String>() {
-            {
-                put(Hypervisor.HypervisorType.KVM, "2d8d1e4eacc976814b97f02849481433");
-                put(Hypervisor.HypervisorType.XenServer, "a5ecf7ed485e2da5ec1993069aa60553");
-                put(Hypervisor.HypervisorType.VMware, "84dab5d1e8267b5dc85eb4eaa21a1efe");
-                put(Hypervisor.HypervisorType.Hyperv, "d592ab6a2271303fe75b6a27f8e6bd53");
-                put(Hypervisor.HypervisorType.LXC, "2d8d1e4eacc976814b97f02849481433");
-                put(Hypervisor.HypervisorType.Ovm3, "60fe2227b89a8980ee09f89dc7b19582");
-            }
-        };
-
-        for (final Map.Entry<Hypervisor.HypervisorType, String> hypervisorAndTemplateName : NewTemplateNameList.entrySet()) {
-            LOG.debug("Updating " + hypervisorAndTemplateName.getKey() + " System Vms");
-            try (PreparedStatement pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name = ? and removed is null order by id desc limit 1")) {
-                // Get 4.11.0 systemvm template id for corresponding hypervisor
-                long templateId = -1;
-                pstmt.setString(1, hypervisorAndTemplateName.getValue());
-                try (ResultSet rs = pstmt.executeQuery()) {
-                    if (rs.next()) {
-                        templateId = rs.getLong(1);
-                    }
-                } catch (final SQLException e) {
-                    LOG.error("updateSystemVmTemplates: Exception caught while getting ids of templates: " + e.getMessage());
-                    throw new CloudRuntimeException("updateSystemVmTemplates: Exception caught while getting ids of templates", e);
-                }
-
-                // change template type to SYSTEM
-                if (templateId != -1) {
-                    try (PreparedStatement templ_type_pstmt = conn.prepareStatement("update `cloud`.`vm_template` set type='SYSTEM' where id = ?");) {
-                        templ_type_pstmt.setLong(1, templateId);
-                        templ_type_pstmt.executeUpdate();
-                    } catch (final SQLException e) {
-                        LOG.error("updateSystemVmTemplates:Exception while updating template with id " + templateId + " to be marked as 'system': " + e.getMessage());
-                        throw new CloudRuntimeException("updateSystemVmTemplates:Exception while updating template with id " + templateId + " to be marked as 'system'", e);
-                    }
-                    // update template ID of system Vms
-                    try (PreparedStatement update_templ_id_pstmt = conn
-                            .prepareStatement("update `cloud`.`vm_instance` set vm_template_id = ? where type <> 'User' and hypervisor_type = ?");) {
-                        update_templ_id_pstmt.setLong(1, templateId);
-                        update_templ_id_pstmt.setString(2, hypervisorAndTemplateName.getKey().toString());
-                        update_templ_id_pstmt.executeUpdate();
-                    } catch (final Exception e) {
-                        LOG.error("updateSystemVmTemplates:Exception while setting template for " + hypervisorAndTemplateName.getKey().toString() + " to " + templateId
-                                + ": " + e.getMessage());
-                        throw new CloudRuntimeException("updateSystemVmTemplates:Exception while setting template for " + hypervisorAndTemplateName.getKey().toString() + " to "
-                                + templateId, e);
-                    }
-
-                    // Change value of global configuration parameter
-                    // router.template.* for the corresponding hypervisor
-                    try (PreparedStatement update_pstmt = conn.prepareStatement("UPDATE `cloud`.`configuration` SET value = ? WHERE name = ?");) {
-                        update_pstmt.setString(1, hypervisorAndTemplateName.getValue());
-                        update_pstmt.setString(2, routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey()));
-                        update_pstmt.executeUpdate();
-                    } catch (final SQLException e) {
-                        LOG.error("updateSystemVmTemplates:Exception while setting " + routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey()) + " to "
-                                + hypervisorAndTemplateName.getValue() + ": " + e.getMessage());
-                        throw new CloudRuntimeException("updateSystemVmTemplates:Exception while setting "
-                                + routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey()) + " to " + hypervisorAndTemplateName.getValue(), e);
-                    }
-
-                    // Change value of global configuration parameter
-                    // minreq.sysvmtemplate.version for the ACS version
-                    try (PreparedStatement update_pstmt = conn.prepareStatement("UPDATE `cloud`.`configuration` SET value = ? WHERE name = ?");) {
-                        update_pstmt.setString(1, "4.11.0");
-                        update_pstmt.setString(2, "minreq.sysvmtemplate.version");
-                        update_pstmt.executeUpdate();
-                    } catch (final SQLException e) {
-                        LOG.error("updateSystemVmTemplates:Exception while setting 'minreq.sysvmtemplate.version' to 4.11.0: " + e.getMessage());
-                        throw new CloudRuntimeException("updateSystemVmTemplates:Exception while setting 'minreq.sysvmtemplate.version' to 4.11.0", e);
-                    }
-                } else {
-                    if (hypervisorsListInUse.contains(hypervisorAndTemplateName.getKey())) {
-                        throw new CloudRuntimeException(getUpgradedVersion() + hypervisorAndTemplateName.getKey() + " SystemVm template not found. Cannot upgrade system Vms");
-                    } else {
-                        LOG.warn(getUpgradedVersion() + hypervisorAndTemplateName.getKey() + " SystemVm template not found. " + hypervisorAndTemplateName.getKey()
-                                + " hypervisor is not used, so not failing upgrade");
-                        // Update the latest template URLs for corresponding
-                        // hypervisor
-                        try (PreparedStatement update_templ_url_pstmt = conn
-                                .prepareStatement("UPDATE `cloud`.`vm_template` SET url = ? , checksum = ? WHERE hypervisor_type = ? AND type = 'SYSTEM' AND removed is null order by id desc limit 1");) {
-                            update_templ_url_pstmt.setString(1, newTemplateUrl.get(hypervisorAndTemplateName.getKey()));
-                            update_templ_url_pstmt.setString(2, newTemplateChecksum.get(hypervisorAndTemplateName.getKey()));
-                            update_templ_url_pstmt.setString(3, hypervisorAndTemplateName.getKey().toString());
-                            update_templ_url_pstmt.executeUpdate();
-                        } catch (final SQLException e) {
-                            LOG.error("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type "
-                                    + hypervisorAndTemplateName.getKey().toString() + ": " + e.getMessage());
-                            throw new CloudRuntimeException("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type "
-                                    + hypervisorAndTemplateName.getKey().toString(), e);
-                        }
-                    }
-                }
-            } catch (final SQLException e) {
-                LOG.error("updateSystemVmTemplates:Exception while getting ids of templates: " + e.getMessage());
-                throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting ids of templates", e);
-            }
-        }
-        LOG.debug("Updating System Vm Template IDs Complete");
-    }
-
-
     @Override
     public InputStream[] getCleanupScripts() {
         final String scriptFile = "META-INF/db/schema-41000to41100-cleanup.sql";

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41100to41110.java

@@ -19,12 +19,24 @@
 
 package com.cloud.upgrade.dao;
 
-import com.cloud.utils.exception.CloudRuntimeException;
-
 import java.io.InputStream;
 import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 public class Upgrade41100to41110 implements DbUpgrade {
+    final static Logger LOG = Logger.getLogger(Upgrade41000to41100.class);
+
     @Override
     public String[] getUpgradableVersionRange() {
         return new String[]{"4.11.0.0", "4.11.1.0"};
@@ -53,7 +65,175 @@ public class Upgrade41100to41110 implements DbUpgrade {
 
     @Override
     public void performDataMigration(Connection conn) {
+        updateSystemVmTemplates(conn);
+    }
 
+    @SuppressWarnings("serial")
+    private void updateSystemVmTemplates(final Connection conn) {
+        LOG.debug("Updating System Vm template IDs");
+        final Set<Hypervisor.HypervisorType> hypervisorsListInUse = new HashSet<Hypervisor.HypervisorType>();
+        try (PreparedStatement pstmt = conn.prepareStatement("select distinct(hypervisor_type) from `cloud`.`cluster` where removed is null"); ResultSet rs = pstmt.executeQuery()) {
+            while (rs.next()) {
+                switch (Hypervisor.HypervisorType.getType(rs.getString(1))) {
+                    case XenServer:
+                        hypervisorsListInUse.add(Hypervisor.HypervisorType.XenServer);
+                        break;
+                    case KVM:
+                        hypervisorsListInUse.add(Hypervisor.HypervisorType.KVM);
+                        break;
+                    case VMware:
+                        hypervisorsListInUse.add(Hypervisor.HypervisorType.VMware);
+                        break;
+                    case Hyperv:
+                        hypervisorsListInUse.add(Hypervisor.HypervisorType.Hyperv);
+                        break;
+                    case LXC:
+                        hypervisorsListInUse.add(Hypervisor.HypervisorType.LXC);
+                        break;
+                    case Ovm3:
+                        hypervisorsListInUse.add(Hypervisor.HypervisorType.Ovm3);
+                        break;
+                    default:
+                        break;
+                }
+            }
+        } catch (final SQLException e) {
+            LOG.error("updateSystemVmTemplates: Exception caught while getting hypervisor types from clusters: " + e.getMessage());
+            throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting hypervisor types from clusters", e);
+        }
+
+        final Map<Hypervisor.HypervisorType, String> NewTemplateNameList = new HashMap<Hypervisor.HypervisorType, String>() {
+            {
+                put(Hypervisor.HypervisorType.KVM, "systemvm-kvm-4.11.1");
+                put(Hypervisor.HypervisorType.VMware, "systemvm-vmware-4.11.1");
+                put(Hypervisor.HypervisorType.XenServer, "systemvm-xenserver-4.11.1");
+                put(Hypervisor.HypervisorType.Hyperv, "systemvm-hyperv-4.11.1");
+                put(Hypervisor.HypervisorType.LXC, "systemvm-lxc-4.11.1");
+                put(Hypervisor.HypervisorType.Ovm3, "systemvm-ovm3-4.11.1");
+            }
+        };
+
+        final Map<Hypervisor.HypervisorType, String> routerTemplateConfigurationNames = new HashMap<Hypervisor.HypervisorType, String>() {
+            {
+                put(Hypervisor.HypervisorType.KVM, "router.template.kvm");
+                put(Hypervisor.HypervisorType.VMware, "router.template.vmware");
+                put(Hypervisor.HypervisorType.XenServer, "router.template.xenserver");
+                put(Hypervisor.HypervisorType.Hyperv, "router.template.hyperv");
+                put(Hypervisor.HypervisorType.LXC, "router.template.lxc");
+                put(Hypervisor.HypervisorType.Ovm3, "router.template.ovm3");
+            }
+        };
+
+        final Map<Hypervisor.HypervisorType, String> newTemplateUrl = new HashMap<Hypervisor.HypervisorType, String>() {
+            {
+                put(Hypervisor.HypervisorType.KVM, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.1-kvm.qcow2.bz2");
+                put(Hypervisor.HypervisorType.VMware, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.1-vmware.ova");
+                put(Hypervisor.HypervisorType.XenServer, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.1-xen.vhd.bz2");
+                put(Hypervisor.HypervisorType.Hyperv, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.1-hyperv.vhd.zip");
+                put(Hypervisor.HypervisorType.LXC, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.1-kvm.qcow2.bz2");
+                put(Hypervisor.HypervisorType.Ovm3, "https://download.cloudstack.org/systemvm/4.11/systemvmtemplate-4.11.1-ovm.raw.bz2");
+            }
+        };
+
+        final Map<Hypervisor.HypervisorType, String> newTemplateChecksum = new HashMap<Hypervisor.HypervisorType, String>() {
+            {
+                put(Hypervisor.HypervisorType.KVM, "6019c2ed1a13669dcf334fe380c776b0");
+                put(Hypervisor.HypervisorType.XenServer, "f2245e912c856ab610d91f88c362a1f9");
+                put(Hypervisor.HypervisorType.VMware, "1dbcd051fcfcd0fd568ff6eb5294988a");
+                put(Hypervisor.HypervisorType.Hyperv, "e68ec90f0dc06821d94a2ee0e88fa646");
+                put(Hypervisor.HypervisorType.LXC, "6019c2ed1a13669dcf334fe380c776b0");
+                put(Hypervisor.HypervisorType.Ovm3, "cd2ac8dcdaf6c05d75e29cb39ee9a10f");
+            }
+        };
+
+        for (final Map.Entry<Hypervisor.HypervisorType, String> hypervisorAndTemplateName : NewTemplateNameList.entrySet()) {
+            LOG.debug("Updating " + hypervisorAndTemplateName.getKey() + " System Vms");
+            try (PreparedStatement pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name = ? and removed is null order by id desc limit 1")) {
+                // Get 4.11.0 systemvm template id for corresponding hypervisor
+                long templateId = -1;
+                pstmt.setString(1, hypervisorAndTemplateName.getValue());
+                try (ResultSet rs = pstmt.executeQuery()) {
+                    if (rs.next()) {
+                        templateId = rs.getLong(1);
+                    }
+                } catch (final SQLException e) {
+                    LOG.error("updateSystemVmTemplates: Exception caught while getting ids of templates: " + e.getMessage());
+                    throw new CloudRuntimeException("updateSystemVmTemplates: Exception caught while getting ids of templates", e);
+                }
+
+                // change template type to SYSTEM
+                if (templateId != -1) {
+                    try (PreparedStatement templ_type_pstmt = conn.prepareStatement("update `cloud`.`vm_template` set type='SYSTEM' where id = ?");) {
+                        templ_type_pstmt.setLong(1, templateId);
+                        templ_type_pstmt.executeUpdate();
+                    } catch (final SQLException e) {
+                        LOG.error("updateSystemVmTemplates:Exception while updating template with id " + templateId + " to be marked as 'system': " + e.getMessage());
+                        throw new CloudRuntimeException("updateSystemVmTemplates:Exception while updating template with id " + templateId + " to be marked as 'system'", e);
+                    }
+                    // update template ID of system Vms
+                    try (PreparedStatement update_templ_id_pstmt = conn
+                            .prepareStatement("update `cloud`.`vm_instance` set vm_template_id = ? where type <> 'User' and hypervisor_type = ?");) {
+                        update_templ_id_pstmt.setLong(1, templateId);
+                        update_templ_id_pstmt.setString(2, hypervisorAndTemplateName.getKey().toString());
+                        update_templ_id_pstmt.executeUpdate();
+                    } catch (final Exception e) {
+                        LOG.error("updateSystemVmTemplates:Exception while setting template for " + hypervisorAndTemplateName.getKey().toString() + " to " + templateId
+                                + ": " + e.getMessage());
+                        throw new CloudRuntimeException("updateSystemVmTemplates:Exception while setting template for " + hypervisorAndTemplateName.getKey().toString() + " to "
+                                + templateId, e);
+                    }
+
+                    // Change value of global configuration parameter
+                    // router.template.* for the corresponding hypervisor
+                    try (PreparedStatement update_pstmt = conn.prepareStatement("UPDATE `cloud`.`configuration` SET value = ? WHERE name = ?");) {
+                        update_pstmt.setString(1, hypervisorAndTemplateName.getValue());
+                        update_pstmt.setString(2, routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey()));
+                        update_pstmt.executeUpdate();
+                    } catch (final SQLException e) {
+                        LOG.error("updateSystemVmTemplates:Exception while setting " + routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey()) + " to "
+                                + hypervisorAndTemplateName.getValue() + ": " + e.getMessage());
+                        throw new CloudRuntimeException("updateSystemVmTemplates:Exception while setting "
+                                + routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey()) + " to " + hypervisorAndTemplateName.getValue(), e);
+                    }
+
+                    // Change value of global configuration parameter
+                    // minreq.sysvmtemplate.version for the ACS version
+                    try (PreparedStatement update_pstmt = conn.prepareStatement("UPDATE `cloud`.`configuration` SET value = ? WHERE name = ?");) {
+                        update_pstmt.setString(1, "4.11.1");
+                        update_pstmt.setString(2, "minreq.sysvmtemplate.version");
+                        update_pstmt.executeUpdate();
+                    } catch (final SQLException e) {
+                        LOG.error("updateSystemVmTemplates:Exception while setting 'minreq.sysvmtemplate.version' to 4.11.1: " + e.getMessage());
+                        throw new CloudRuntimeException("updateSystemVmTemplates:Exception while setting 'minreq.sysvmtemplate.version' to 4.11.1", e);
+                    }
+                } else {
+                    if (hypervisorsListInUse.contains(hypervisorAndTemplateName.getKey())) {
+                        throw new CloudRuntimeException(getUpgradedVersion() + hypervisorAndTemplateName.getKey() + " SystemVm template not found. Cannot upgrade system Vms");
+                    } else {
+                        LOG.warn(getUpgradedVersion() + hypervisorAndTemplateName.getKey() + " SystemVm template not found. " + hypervisorAndTemplateName.getKey()
+                                + " hypervisor is not used, so not failing upgrade");
+                        // Update the latest template URLs for corresponding
+                        // hypervisor
+                        try (PreparedStatement update_templ_url_pstmt = conn
+                                .prepareStatement("UPDATE `cloud`.`vm_template` SET url = ? , checksum = ? WHERE hypervisor_type = ? AND type = 'SYSTEM' AND removed is null order by id desc limit 1");) {
+                            update_templ_url_pstmt.setString(1, newTemplateUrl.get(hypervisorAndTemplateName.getKey()));
+                            update_templ_url_pstmt.setString(2, newTemplateChecksum.get(hypervisorAndTemplateName.getKey()));
+                            update_templ_url_pstmt.setString(3, hypervisorAndTemplateName.getKey().toString());
+                            update_templ_url_pstmt.executeUpdate();
+                        } catch (final SQLException e) {
+                            LOG.error("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type "
+                                    + hypervisorAndTemplateName.getKey().toString() + ": " + e.getMessage());
+                            throw new CloudRuntimeException("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type "
+                                    + hypervisorAndTemplateName.getKey().toString(), e);
+                        }
+                    }
+                }
+            } catch (final SQLException e) {
+                LOG.error("updateSystemVmTemplates:Exception while getting ids of templates: " + e.getMessage());
+                throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting ids of templates", e);
+            }
+        }
+        LOG.debug("Updating System Vm Template IDs Complete");
     }
 
     @Override

plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java

@@ -20,6 +20,7 @@ package org.apache.cloudstack.ca.provider;
 import java.io.IOException;
 import java.io.StringReader;
 import java.math.BigInteger;
+import java.net.InetAddress;
 import java.security.InvalidKeyException;
 import java.security.KeyManagementException;
 import java.security.KeyPair;
@@ -34,6 +35,7 @@ import java.security.UnrecoverableKeyException;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
 import java.security.spec.InvalidKeySpecException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -45,6 +47,7 @@ import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.TrustManagerFactory;
+import javax.xml.bind.DatatypeConverter;
 
 import org.apache.cloudstack.ca.CAManager;
 import org.apache.cloudstack.framework.ca.CAProvider;
@@ -55,9 +58,15 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.utils.security.CertUtils;
 import org.apache.cloudstack.utils.security.KeyStoreUtils;
 import org.apache.log4j.Logger;
-import org.bouncycastle.jce.PKCS10CertificationRequest;
+import org.bouncycastle.asn1.pkcs.Attribute;
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.asn1.x509.GeneralName;
+import org.bouncycastle.asn1.x509.GeneralNames;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequest;
 import org.bouncycastle.util.io.pem.PemObject;
 import org.bouncycastle.util.io.pem.PemReader;
 
@@ -137,7 +146,17 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con
         return new Certificate(clientCertificate, keyPair.getPrivate(), Collections.singletonList(caCertificate));
     }
 
-    private Certificate generateCertificateUsingCsr(final String csr, final List<String> domainNames, final List<String> ipAddresses, final int validityDays) throws NoSuchAlgorithmException, InvalidKeyException, NoSuchProviderException, CertificateException, SignatureException, IOException, OperatorCreationException {
+    private Certificate generateCertificateUsingCsr(final String csr, final List<String> names, final List<String> ips, final int validityDays) throws NoSuchAlgorithmException, InvalidKeyException, NoSuchProviderException, CertificateException, SignatureException, IOException, OperatorCreationException {
+        final List<String> dnsNames = new ArrayList<>();
+        final List<String> ipAddresses = new ArrayList<>();
+
+        if (names != null) {
+            dnsNames.addAll(names);
+        }
+        if (ips != null) {
+            ipAddresses.addAll(ips);
+        }
+
         PemObject pemObject = null;
 
         try {
@@ -151,13 +170,33 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con
             throw new CloudRuntimeException("Unable to read/process CSR: " + csr);
         }
 
-        final PKCS10CertificationRequest request = new PKCS10CertificationRequest(pemObject.getContent());
+        final JcaPKCS10CertificationRequest request = new JcaPKCS10CertificationRequest(pemObject.getContent());
+        final String subject = request.getSubject().toString();
+        for (final Attribute attribute : request.getAttributes()) {
+            if (attribute == null) {
+                continue;
+            }
+            if (attribute.getAttrType().equals(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest)) {
+                final Extensions extensions = Extensions.getInstance(attribute.getAttrValues().getObjectAt(0));
+                final GeneralNames gns = GeneralNames.fromExtensions(extensions, Extension.subjectAlternativeName);
+                if (gns != null && gns.getNames() != null && gns.getNames().length > 0) {
+                    for (final GeneralName name : gns.getNames()) {
+                        if (name.getTagNo() == GeneralName.dNSName) {
+                            dnsNames.add(name.getName().toString());
+                        }
+                        if (name.getTagNo() == GeneralName.iPAddress) {
+                            final InetAddress address = InetAddress.getByAddress(DatatypeConverter.parseHexBinary(name.getName().toString().substring(1)));
+                            ipAddresses.add(address.toString().replace("/", ""));
+                        }
+                    }
+                }
+            }
+        }
 
-        final String subject = request.getCertificationRequestInfo().getSubject().toString();
         final X509Certificate clientCertificate = CertUtils.generateV3Certificate(
                 caCertificate, caKeyPair, request.getPublicKey(),
                 subject, CAManager.CertSignatureAlgorithm.value(),
-                validityDays, domainNames, ipAddresses);
+                validityDays, dnsNames, ipAddresses);
         return new Certificate(clientCertificate, null, Collections.singletonList(caCertificate));
     }
 

plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java

@@ -152,7 +152,7 @@ public class SolidFireHostListener implements HypervisorHostListener {
     }
 
     private void handleVMware(HostVO host, boolean add, ModifyTargetsCommand.TargetTypeToRemove targetTypeToRemove) {
-        if (HypervisorType.VMware.equals(host.getHypervisorType())) {
+        if (host != null && HypervisorType.VMware.equals(host.getHypervisorType())) {
             List<StoragePoolVO> storagePools = _storagePoolDao.findPoolsByProvider(SolidFireUtil.PROVIDER_NAME);
 
             if (storagePools != null && storagePools.size() > 0) {

plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java

@@ -32,7 +32,6 @@ import java.security.PublicKey;
 import java.security.SignatureException;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
-import java.security.spec.InvalidKeySpecException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -147,11 +146,11 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage
             try {
                 KeyPair keyPair = CertUtils.generateRandomKeyPair(4096);
                 _ksDao.save(SAMLPluginConstants.SAMLSP_KEYPAIR,
-                        CertUtils.privateKeyToPem(keyPair.getPrivate()),
+                        SAMLUtils.encodePrivateKey(keyPair.getPrivate()),
-                        CertUtils.publicKeyToPem(keyPair.getPublic()), "samlsp-keypair");
+                        SAMLUtils.encodePublicKey(keyPair.getPublic()), "samlsp-keypair");
                 keyStoreVO = _ksDao.findByName(SAMLPluginConstants.SAMLSP_KEYPAIR);
                 s_logger.info("No SAML keystore found, created and saved a new Service Provider keypair");
-            } catch (final NoSuchProviderException | NoSuchAlgorithmException | IOException e) {
+            } catch (final NoSuchProviderException | NoSuchAlgorithmException e) {
                 s_logger.error("Unable to create and save SAML keypair, due to: ", e);
             }
         }
@@ -166,19 +165,8 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage
         KeyPair spKeyPair = null;
         X509Certificate spX509Key = null;
         if (keyStoreVO != null) {
-
+            final PrivateKey privateKey = SAMLUtils.decodePrivateKey(keyStoreVO.getCertificate());
-            PrivateKey privateKey = null;
+            final PublicKey publicKey = SAMLUtils.decodePublicKey(keyStoreVO.getKey());
-            try {
-                privateKey = CertUtils.pemToPrivateKey(keyStoreVO.getCertificate());
-            } catch (final InvalidKeySpecException | IOException e) {
-                s_logger.error("Failed to read private key, due to error: ", e);
-            }
-            PublicKey publicKey = null;
-            try {
-                publicKey = CertUtils.pemToPublicKey(keyStoreVO.getKey());
-            } catch (final InvalidKeySpecException | IOException e) {
-                s_logger.error("Failed to read public key, due to error: ", e);
-            }
             if (privateKey != null && publicKey != null) {
                 spKeyPair = new KeyPair(publicKey, privateKey);
                 KeystoreVO x509VO = _ksDao.findByName(SAMLPluginConstants.SAMLSP_X509CERT);

plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAMLUtils.java

@@ -28,15 +28,20 @@ import java.math.BigInteger;
 import java.net.URLEncoder;
 import java.nio.charset.Charset;
 import java.security.InvalidKeyException;
+import java.security.KeyFactory;
 import java.security.KeyPair;
 import java.security.NoSuchAlgorithmException;
 import java.security.NoSuchProviderException;
 import java.security.PrivateKey;
+import java.security.PublicKey;
 import java.security.SecureRandom;
 import java.security.Signature;
 import java.security.SignatureException;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
 import java.util.List;
 import java.util.zip.Deflater;
 import java.util.zip.DeflaterOutputStream;
@@ -264,12 +269,6 @@ public class SAMLUtils {
         return url;
     }
 
-    public static X509Certificate generateRandomX509Certificate(KeyPair keyPair) throws NoSuchAlgorithmException, NoSuchProviderException, CertificateException, SignatureException, InvalidKeyException, OperatorCreationException {
-        return CertUtils.generateV1Certificate(keyPair,
-                "CN=ApacheCloudStack", "CN=ApacheCloudStack",
-                3, "SHA256WithRSA");
-    }
-
     public static void setupSamlUserCookies(final LoginCmdResponse loginResponse, final HttpServletResponse resp) throws IOException {
         resp.addCookie(new Cookie("userid", URLEncoder.encode(loginResponse.getUserId(), HttpUtils.UTF_8)));
         resp.addCookie(new Cookie("domainid", URLEncoder.encode(loginResponse.getDomainId(), HttpUtils.UTF_8)));
@@ -284,4 +283,82 @@ public class SAMLUtils {
         resp.addHeader("SET-COOKIE", String.format("%s=%s;HttpOnly", ApiConstants.SESSIONKEY, loginResponse.getSessionKey()));
     }
 
+    /**
+     * Returns base64 encoded PublicKey
+     * @param key PublicKey
+     * @return public key encoded string
+     */
+    public static String encodePublicKey(PublicKey key) {
+        try {
+            KeyFactory keyFactory = CertUtils.getKeyFactory();
+            if (keyFactory == null) return null;
+            X509EncodedKeySpec spec = keyFactory.getKeySpec(key, X509EncodedKeySpec.class);
+            return new String(org.bouncycastle.util.encoders.Base64.encode(spec.getEncoded()), Charset.forName("UTF-8"));
+        } catch (InvalidKeySpecException e) {
+            s_logger.error("Unable to create KeyFactory:" + e.getMessage());
+        }
+        return null;
+    }
+
+    /**
+     * Returns base64 encoded PrivateKey
+     * @param key PrivateKey
+     * @return privatekey encoded string
+     */
+    public static String encodePrivateKey(PrivateKey key) {
+        try {
+            KeyFactory keyFactory = CertUtils.getKeyFactory();
+            if (keyFactory == null) return null;
+            PKCS8EncodedKeySpec spec = keyFactory.getKeySpec(key,
+                    PKCS8EncodedKeySpec.class);
+            return new String(org.bouncycastle.util.encoders.Base64.encode(spec.getEncoded()), Charset.forName("UTF-8"));
+        } catch (InvalidKeySpecException e) {
+            s_logger.error("Unable to create KeyFactory:" + e.getMessage());
+        }
+        return null;
+    }
+
+    /**
+     * Decodes base64 encoded public key to PublicKey
+     * @param publicKey encoded public key string
+     * @return returns PublicKey
+     */
+    public static PublicKey decodePublicKey(String publicKey) {
+        byte[] sigBytes = org.bouncycastle.util.encoders.Base64.decode(publicKey);
+        X509EncodedKeySpec x509KeySpec = new X509EncodedKeySpec(sigBytes);
+        KeyFactory keyFactory = CertUtils.getKeyFactory();
+        if (keyFactory == null)
+            return null;
+        try {
+            return keyFactory.generatePublic(x509KeySpec);
+        } catch (InvalidKeySpecException e) {
+            s_logger.error("Unable to create PrivateKey from privateKey string:" + e.getMessage());
+        }
+        return null;
+    }
+
+    /**
+     * Decodes base64 encoded private key to PrivateKey
+     * @param privateKey encoded private key string
+     * @return returns PrivateKey
+     */
+    public static PrivateKey decodePrivateKey(String privateKey) {
+        byte[] sigBytes = org.bouncycastle.util.encoders.Base64.decode(privateKey);
+        PKCS8EncodedKeySpec pkscs8KeySpec = new PKCS8EncodedKeySpec(sigBytes);
+        KeyFactory keyFactory = CertUtils.getKeyFactory();
+        if (keyFactory == null)
+            return null;
+        try {
+            return keyFactory.generatePrivate(pkscs8KeySpec);
+        } catch (InvalidKeySpecException e) {
+            s_logger.error("Unable to create PrivateKey from privateKey string:" + e.getMessage());
+        }
+        return null;
+    }
+
+    public static X509Certificate generateRandomX509Certificate(KeyPair keyPair) throws NoSuchAlgorithmException, NoSuchProviderException, CertificateException, SignatureException, InvalidKeyException, OperatorCreationException {
+        return CertUtils.generateV1Certificate(keyPair,
+                "CN=ApacheCloudStack", "CN=ApacheCloudStack",
+                3, "SHA256WithRSA");
+    }
 }

plugins/user-authenticators/saml2/src/test/java/org/apache/cloudstack/SAMLUtilsTest.java

@@ -64,12 +64,14 @@ public class SAMLUtilsTest extends TestCase {
     public void testX509Helpers() throws Exception {
         KeyPair keyPair = CertUtils.generateRandomKeyPair(4096);
 
-        String privateKeyString = CertUtils.privateKeyToPem(keyPair.getPrivate());
+        String privateKeyString = SAMLUtils.encodePrivateKey(keyPair.getPrivate());
-        String publicKeyString = CertUtils.publicKeyToPem(keyPair.getPublic());
+        String publicKeyString = SAMLUtils.encodePublicKey(keyPair.getPublic());
 
-        PrivateKey privateKey = CertUtils.pemToPrivateKey(privateKeyString);
+        PrivateKey privateKey = SAMLUtils.decodePrivateKey(privateKeyString);
-        PublicKey publicKey = CertUtils.pemToPublicKey(publicKeyString);
+        PublicKey publicKey = SAMLUtils.decodePublicKey(publicKeyString);
 
+        assertNotNull(privateKey);
+        assertNotNull(publicKey);
         assertTrue(privateKey.equals(keyPair.getPrivate()));
         assertTrue(publicKey.equals(keyPair.getPublic()));
     }

scripts/util/keystore-setup

@@ -42,7 +42,8 @@ keytool -genkey -storepass "$KS_PASS" -keypass "$KS_PASS" -alias "$ALIAS" -keyal
 
 # Generate CSR
 rm -f "$CSR_FILE"
-keytool -certreq -storepass "$KS_PASS" -alias "$ALIAS" -file $CSR_FILE -keystore "$KS_FILE" > /dev/null 2>&1
+addresses=$(ip address | grep inet | awk '{print $2}' | sed 's/\/.*//g' | grep -v '^169.254.' | grep -v '^127.0.0.1' | grep -v '^::1' | sed 's/^/ip:/g' | tr '\r\n' ',')
+keytool -certreq -storepass "$KS_PASS" -alias "$ALIAS" -file $CSR_FILE -keystore "$KS_FILE" -ext san="$addresses" > /dev/null 2>&1
 cat "$CSR_FILE"
 
 # Fix file permissions

systemvm/debian/opt/cloud/bin/configure.py

@@ -957,6 +957,7 @@ class IpTablesExecutor:
         acls.process()
 
         acls = CsAcl('firewallrules', self.config)
+        acls.flushAllowAllEgressRules()
         acls.process()
 
         fwd = CsForwardingRules("forwardingrules", self.config)

systemvm/debian/opt/cloud/bin/setup/cloud-early-config

@@ -71,16 +71,16 @@ config_guest() {
 
 get_boot_params() {
   case $HYPERVISOR in
-     xen-pv)
+     xen-pv|xen-domU)
           cat /proc/cmdline > $CMDLINE
           sed -i "s/%/ /g" $CMDLINE
           ;;
      xen-hvm)
-          if [ ! -f /usr/sbin/xenstore-read ]; then
+          if [ ! -f /usr/bin/xenstore-read ]; then
             log_it "ERROR: xentools not installed, cannot found xenstore-read" && exit 5
           fi
-          /usr/sbin/xenstore-read vm-data/cloudstack/init > /var/cache/cloud/cmdline
+          /usr/bin/xenstore-read vm-data/cloudstack/init > $CMDLINE
-          sed -i "s/%/ /g" /var/cache/cloud/cmdline
+          sed -i "s/%/ /g" $CMDLINE
           ;;
      kvm)
           VPORT=$(find /dev/virtio-ports -type l -name '*.vport' 2>/dev/null|head -1)

systemvm/debian/opt/cloud/bin/setup/patchsystemvm.sh

@@ -37,6 +37,8 @@ patch_systemvm() {
       echo "Restored keystore file and certs using backup" >> $logfile
    fi
    rm -fr $backupfolder
+   # Import global cacerts into 'cloud' service's keystore
+   keytool -importkeystore -srckeystore /etc/ssl/certs/java/cacerts -destkeystore /usr/local/cloud/systemvm/certs/realhostip.keystore -srcstorepass changeit -deststorepass vmops.com -noprompt || true
    return 0
 }
 

test/integration/smoke/test_service_offerings.py

@@ -448,7 +448,7 @@ class TestCpuCapServiceOfferings(cloudstackTestCase):
         cls.apiclient = testClient.getApiClient()
         cls.services = testClient.getParsedTestDataConfig()
         cls.hypervisor = testClient.getHypervisorInfo()
-
+        cls._cleanup = []
         cls.hypervisorNotSupported = False
         if cls.hypervisor.lower() not in ["kvm"]:
             cls.hypervisorNotSupported = True
@@ -516,7 +516,6 @@ class TestCpuCapServiceOfferings(cloudstackTestCase):
             cls.offering,
             cls.account
         ]
-        return
 
     @classmethod
     def tearDownClass(cls):

test/integration/smoke/test_vm_life_cycle.py

@@ -794,6 +794,10 @@ class TestSecuredVmMigration(cloudstackTestCase):
         cls.apiclient = testClient.getApiClient()
         cls.services = testClient.getParsedTestDataConfig()
         cls.hypervisor = testClient.getHypervisorInfo()
+        cls._cleanup = []
+
+        if cls.hypervisor.lower() not in ["kvm"]:
+            return
 
         # Get Zone, Domain and templates
         domain = get_domain(cls.apiclient)
@@ -846,12 +850,12 @@ class TestSecuredVmMigration(cloudstackTestCase):
         self.apiclient = self.testClient.getApiClient()
         self.dbclient = self.testClient.getDbConnection()
         self.cleanup = []
-        self.updateConfiguration("ca.plugin.root.auth.strictness", "false")
-        self.make_all_hosts_secure()
-
         if self.hypervisor.lower() not in ["kvm"]:
             self.skipTest("Secured migration is not supported on other than KVM")
 
+        self.updateConfiguration("ca.plugin.root.auth.strictness", "false")
+        self.make_all_hosts_secure()
+
     def tearDown(self):
         self.make_all_hosts_secure()
 

tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh

@@ -19,7 +19,7 @@
 set -e
 set -x
 
-CLOUDSTACK_RELEASE=4.11.0
+CLOUDSTACK_RELEASE=4.11.1
 
 function configure_apache2() {
    # Enable ssl, rewrite and auth
@@ -31,6 +31,16 @@ function configure_apache2() {
    sed -i 's/SSLProtocol .*$/SSLProtocol TLSv1.2/g' /etc/apache2/mods-available/ssl.conf
 }
 
+function configure_cacerts() {
+  CDIR=$(pwd)
+  cd /tmp
+  # Add LetsEncrypt ca-cert
+  wget https://letsencrypt.org/certs/lets-encrypt-x3-cross-signed.der
+  keytool -trustcacerts -keystore /etc/ssl/certs/java/cacerts -storepass changeit -noprompt -importcert -alias letsencryptauthorityx3cross -file lets-encrypt-x3-cross-signed.der
+  rm -f lets-encrypt-x3-cross-signed.der
+  cd $CDIR
+}
+
 function install_cloud_scripts() {
   # ./cloud_scripts/ has been put there by ../../cloud_scripts_shar_archive.sh
   rsync -av ./cloud_scripts/ /
@@ -102,6 +112,7 @@ function configure_services() {
   configure_apache2
   configure_strongswan
   configure_issue
+  configure_cacerts
 }
 
 return 2>/dev/null || configure_services

tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh

@@ -86,9 +86,9 @@ function install_packages() {
   fi
 
   # Install xenserver guest utilities as debian repos don't have it
-  wget https://mirrors.kernel.org/ubuntu/pool/universe/x/xe-guest-utilities/xe-guest-utilities_7.4.0-0ubuntu1_amd64.deb
+  wget https://mirrors.kernel.org/ubuntu/pool/main/x/xe-guest-utilities/xe-guest-utilities_7.10.0-0ubuntu1_amd64.deb
-  dpkg -i xe-guest-utilities_7.4.0-0ubuntu1_amd64.deb
+  dpkg -i xe-guest-utilities_7.10.0-0ubuntu1_amd64.deb
-  rm -f xe-guest-utilities_7.4.0-0ubuntu1_amd64.deb
+  rm -f xe-guest-utilities_7.10.0-0ubuntu1_amd64.deb
 }
 
 return 2>/dev/null || install_packages

utils/src/main/java/org/apache/cloudstack/utils/security/CertUtils.java

@@ -40,6 +40,7 @@ import java.security.spec.InvalidKeySpecException;
 import java.security.spec.PKCS8EncodedKeySpec;
 import java.security.spec.X509EncodedKeySpec;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 
 import javax.security.auth.x500.X500Principal;
@@ -219,7 +220,7 @@ public class CertUtils {
 
         final List<ASN1Encodable> subjectAlternativeNames = new ArrayList<ASN1Encodable>();
         if (publicIPAddresses != null) {
-            for (final String publicIPAddress: publicIPAddresses) {
+            for (final String publicIPAddress: new HashSet<>(publicIPAddresses)) {
                 if (Strings.isNullOrEmpty(publicIPAddress)) {
                     continue;
                 }
@@ -227,7 +228,7 @@ public class CertUtils {
             }
         }
         if (dnsNames != null) {
-            for (final String dnsName : dnsNames) {
+            for (final String dnsName : new HashSet<>(dnsNames)) {
                 if (Strings.isNullOrEmpty(dnsName)) {
                     continue;
                 }