Merge branch '4.11'

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41000to41100.java

@@ -31,6 +31,7 @@ import org.apache.commons.codec.binary.Base64;
 import org.apache.log4j.Logger;
 
 import com.cloud.hypervisor.Hypervisor;
+import com.cloud.utils.PropertiesUtil;
 import com.cloud.utils.exception.CloudRuntimeException;
 
 public class Upgrade41000to41100 implements DbUpgrade {
@@ -65,10 +66,27 @@ public class Upgrade41000to41100 implements DbUpgrade {
 
     @Override
     public void performDataMigration(Connection conn) {
+        checkAndEnableDynamicRoles(conn);
         validateUserDataInBase64(conn);
         updateSystemVmTemplates(conn);
     }
 
+    private void checkAndEnableDynamicRoles(final Connection conn) {
+        final Map<String, String> apiMap = PropertiesUtil.processConfigFile(new String[] { "commands.properties" });
+        if (apiMap == null || apiMap.isEmpty()) {
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("No commands.properties file was found, enabling dynamic roles by setting dynamic.apichecker.enabled to true if not already enabled.");
+            }
+            try (final PreparedStatement updateStatement = conn.prepareStatement("INSERT INTO cloud.configuration (category, instance, name, default_value, value) VALUES ('Advanced', 'DEFAULT', 'dynamic.apichecker.enabled', 'false', 'true') ON DUPLICATE KEY UPDATE value='true'")) {
+                updateStatement.executeUpdate();
+            } catch (SQLException e) {
+                LOG.error("Failed to set dynamic.apichecker.enabled to true, please run migrate-dynamicroles.py script to manually migrate to dynamic roles.", e);
+            }
+        } else {
+            LOG.warn("Old commands.properties static checker is deprecated, please use migrate-dynamicroles.py to migrate to dynamic roles. Refer http://docs.cloudstack.apache.org/projects/cloudstack-administration/en/latest/accounts.html#using-dynamic-roles");
+        }
+    }
+
     private void validateUserDataInBase64(Connection conn) {
         try (final PreparedStatement selectStatement = conn.prepareStatement("SELECT `id`, `user_data` FROM `cloud`.`user_vm` WHERE `user_data` IS NOT NULL;");
              final ResultSet selectResultSet = selectStatement.executeQuery()) {

plugins/acl/static-role-based/src/main/java/org/apache/cloudstack/acl/StaticRoleBasedAPIAccessChecker.java

@@ -39,6 +39,7 @@ import com.cloud.utils.component.PluggableService;
 
 // This is the default API access checker that grab's the user's account
 // based on the account type, access is granted
+@Deprecated
 public class StaticRoleBasedAPIAccessChecker extends AdapterBase implements APIChecker {
 
     protected static final Logger LOGGER = Logger.getLogger(StaticRoleBasedAPIAccessChecker.class);

scripts/util/migrate-dynamicroles.py

@@ -55,6 +55,14 @@ def migrateApiRolePermissions(apis, conn):
             if (octetKey[role] & int(apis[api])) > 0:
                 runSql(conn, "INSERT INTO `cloud`.`role_permissions` (`uuid`, `role_id`, `rule`, `permission`, `sort_order`) values (UUID(), %d, '%s', 'ALLOW', %d);" % (role, api, sortOrder))
                 sortOrder += 1
+    print("Static role permissions from commands.properties have been migrated into the db")
+
+
+def enableDynamicApiChecker(conn):
+    runSql(conn, "UPDATE `cloud`.`configuration` SET value='true' where name='dynamic.apichecker.enabled'")
+    conn.commit()
+    conn.close()
+    print("Dynamic role based API checker has been enabled!")
 
 
 def main():
@@ -71,6 +79,8 @@ def main():
                         help="Host or IP of the MySQL server")
     parser.add_option("-f", "--properties-file", action="store", type="string", dest="commandsfile", default="/etc/cloudstack/management/commands.properties",
                         help="The commands.properties file")
+    parser.add_option("-D", "--default", action="store_true", dest="defaultRules", default=False,
+                        help="")
     parser.add_option("-d", "--dryrun", action="store_true", dest="dryrun", default=False,
                         help="Dry run and debug operations this tool will perform")
     (options, args) = parser.parse_args()
@@ -89,8 +99,14 @@ def main():
             port=int(options.port),
             db=options.db)
 
+    if options.defaultRules:
+        print("Applying the default role permissions, ignoring any provided properties files(s).")
+        enableDynamicApiChecker(conn)
+        sys.exit(0)
+
     if not os.path.isfile(options.commandsfile):
-        print("Provided commands.properties cannot be accessed or does not exist, please check check permissions")
+        print("Provided commands.properties cannot be accessed or does not exist.")
+        print("Please check passed options, or run only with --default option to use the default role permissions.")
         sys.exit(1)
 
     while True:
@@ -122,15 +138,8 @@ def main():
 
     # Migrate rules from commands.properties to cloud.role_permissions
     migrateApiRolePermissions(apiMap, conn)
-    print("Static role permissions from commands.properties have been migrated into the db")
-
-    # Enable dynamic role based API checker
-    runSql(conn, "UPDATE `cloud`.`configuration` SET value='true' where name='dynamic.apichecker.enabled'")
-    conn.commit()
-    conn.close()
-
-    print("Dynamic role based API checker has been enabled!")
 
+    enableDynamicApiChecker(conn)
 
 if __name__ == '__main__':
     main()

scripts/vm/network/security_group.py

@@ -951,16 +951,15 @@ def parse_network_rules(rules):
   if rules is None or len(rules) == 0:
     return ret
 
-  lines = rules.split(';')[:-1]
+  lines = rules.split('NEXT;')[:-1]
   for line in lines:
-    tokens = line.split(':', 4)
+    tokens = line.split(';', 3)
-    if len(tokens) != 5:
+    if len(tokens) != 4:
       continue
 
-    ruletype = tokens[0]
+    ruletype, protocol = tokens[0].split(':')
-    protocol = tokens[1]
+    start = int(tokens[1])
-    start = int(tokens[2])
+    end = int(tokens[2])
-    end = int(tokens[3])
     cidrs = tokens.pop();
 
     ipv4 = []

systemvm/debian/opt/cloud/bin/cs/CsAddress.py

@@ -556,14 +556,6 @@ class CsIP:
             if self.config.is_vpc() or self.config.is_router():
                 CsDevice(self.dev, self.config).configure_rp()
 
-                # If redundant then this is dealt with
-                # by the master backup functions
-                if not cmdline.is_redundant():
-                    if method == "add":
-                        CsPasswdSvc(self.address['public_ip']).start()
-                    elif method == "delete":
-                        CsPasswdSvc(self.address['public_ip']).stop()
-
                 logging.error(
                     "Not able to setup source-nat for a regular router yet")
 
@@ -575,6 +567,14 @@ class CsIP:
                 app = CsApache(self)
                 app.setup()
 
+                # If redundant then this is dealt with
+                # by the master backup functions
+                if not cmdline.is_redundant():
+                    if method == "add":
+                        CsPasswdSvc(self.address['public_ip']).start()
+                    elif method == "delete":
+                        CsPasswdSvc(self.address['public_ip']).stop()
+
         if self.get_type() == "public" and self.config.is_vpc() and method == "add":
             if self.address["source_nat"]:
                 vpccidr = cmdline.get_vpccidr()

systemvm/debian/opt/cloud/bin/cs/CsDhcp.py

@@ -49,15 +49,21 @@ class CsDhcp(CsDataBag):
             self.add(self.dbag[item])
         self.write_hosts()
 
-        if self.cloud.is_changed():
-            self.delete_leases()
-
         self.configure_server()
 
-        restart_dnsmasq = self.conf.commit()
+        restart_dnsmasq = False
-        self.cloud.commit()
+
+        if self.conf.commit():
+            restart_dnsmasq = True
+
+        if self.cloud.commit():
+            restart_dnsmasq = True
+
         self.dhcp_opts.commit()
 
+        if restart_dnsmasq:
+            self.delete_leases()
+
         if not self.cl.is_redundant() or self.cl.is_master():
             if restart_dnsmasq:
                 CsHelper.service("dnsmasq", "restart")