Merge branch '4.19'

engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java

@@ -130,6 +130,7 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
     protected static Logger LOGGER = LogManager.getLogger(DatabaseUpgradeChecker.class);
     private final DatabaseVersionHierarchy hierarchy;
     private static final String VIEWS_DIRECTORY = Paths.get("META-INF", "db", "views").toString();
+    private static final String PROCEDURES_DIRECTORY = Paths.get("META-INF", "db", "procedures").toString();
 
     @Inject
     VersionDao _dao;
@@ -298,83 +299,120 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
     }
 
     protected void upgrade(CloudStackVersion dbVersion, CloudStackVersion currentVersion) {
+        executeProcedureScripts();
+        final DbUpgrade[] upgrades = executeUpgrades(dbVersion, currentVersion);
+
+        executeViewScripts();
+        updateSystemVmTemplates(upgrades);
+    }
+
+    protected void executeProcedureScripts() {
+        LOGGER.info(String.format("Executing Stored Procedure scripts that are under resource directory [%s].", PROCEDURES_DIRECTORY));
+        List<String> filesPathUnderViewsDirectory = FileUtil.getFilesPathsUnderResourceDirectory(PROCEDURES_DIRECTORY);
+
+        try (TransactionLegacy txn = TransactionLegacy.open("execute-procedure-scripts")) {
+            Connection conn = txn.getConnection();
+
+            for (String filePath : filesPathUnderViewsDirectory) {
+                LOGGER.debug(String.format("Executing PROCEDURE script [%s].", filePath));
+
+                InputStream viewScript = Thread.currentThread().getContextClassLoader().getResourceAsStream(filePath);
+                runScript(conn, viewScript);
+            }
+
+            LOGGER.info(String.format("Finished execution of PROCEDURE scripts that are under resource directory [%s].", PROCEDURES_DIRECTORY));
+        } catch (SQLException e) {
+            String message = String.format("Unable to execute PROCEDURE scripts due to [%s].", e.getMessage());
+            LOGGER.error(message, e);
+            throw new CloudRuntimeException(message, e);
+        }
+    }
+
+    private DbUpgrade[] executeUpgrades(CloudStackVersion dbVersion, CloudStackVersion currentVersion) {
         LOGGER.info("Database upgrade must be performed from " + dbVersion + " to " + currentVersion);
 
         final DbUpgrade[] upgrades = calculateUpgradePath(dbVersion, currentVersion);
 
         for (DbUpgrade upgrade : upgrades) {
-            VersionVO version;
+            VersionVO version = executeUpgrade(upgrade);
-            LOGGER.debug("Running upgrade " + upgrade.getClass().getSimpleName() + " to upgrade from " + upgrade.getUpgradableVersionRange()[0] + "-" + upgrade
+            executeUpgradeCleanup(upgrade, version);
-                .getUpgradableVersionRange()[1] + " to " + upgrade.getUpgradedVersion());
+        }
-            TransactionLegacy txn = TransactionLegacy.open("Upgrade");
+        return upgrades;
-            txn.start();
+    }
+
+    private VersionVO executeUpgrade(DbUpgrade upgrade) {
+        VersionVO version;
+        LOGGER.debug("Running upgrade " + upgrade.getClass().getSimpleName() + " to upgrade from " + upgrade.getUpgradableVersionRange()[0] + "-" + upgrade
+            .getUpgradableVersionRange()[1] + " to " + upgrade.getUpgradedVersion());
+        TransactionLegacy txn = TransactionLegacy.open("Upgrade");
+        txn.start();
+        try {
+            Connection conn;
             try {
-                Connection conn;
+                conn = txn.getConnection();
-                try {
+            } catch (SQLException e) {
-                    conn = txn.getConnection();
-                } catch (SQLException e) {
-                    String errorMessage = "Unable to upgrade the database";
-                    LOGGER.error(errorMessage, e);
-                    throw new CloudRuntimeException(errorMessage, e);
-                }
-                InputStream[] scripts = upgrade.getPrepareScripts();
-                if (scripts != null) {
-                    for (InputStream script : scripts) {
-                        runScript(conn, script);
-                    }
-                }
-
-                upgrade.performDataMigration(conn);
-
-                version = new VersionVO(upgrade.getUpgradedVersion());
-                version = _dao.persist(version);
-
-                txn.commit();
-            } catch (CloudRuntimeException e) {
                 String errorMessage = "Unable to upgrade the database";
                 LOGGER.error(errorMessage, e);
                 throw new CloudRuntimeException(errorMessage, e);
-            } finally {
+            }
-                txn.close();
+            InputStream[] scripts = upgrade.getPrepareScripts();
+            if (scripts != null) {
+                for (InputStream script : scripts) {
+                    runScript(conn, script);
+                }
             }
 
-            // Run the corresponding '-cleanup.sql' script
+            upgrade.performDataMigration(conn);
-            txn = TransactionLegacy.open("Cleanup");
-            try {
-                LOGGER.info("Cleanup upgrade " + upgrade.getClass().getSimpleName() + " to upgrade from " + upgrade.getUpgradableVersionRange()[0] + "-" + upgrade
-                    .getUpgradableVersionRange()[1] + " to " + upgrade.getUpgradedVersion());
 
-                txn.start();
+            version = new VersionVO(upgrade.getUpgradedVersion());
-                Connection conn;
+            version = _dao.persist(version);
-                try {
-                    conn = txn.getConnection();
-                } catch (SQLException e) {
-                    LOGGER.error("Unable to cleanup the database", e);
-                    throw new CloudRuntimeException("Unable to cleanup the database", e);
-                }
 
-                InputStream[] scripts = upgrade.getCleanupScripts();
+            txn.commit();
-                if (scripts != null) {
+        } catch (CloudRuntimeException e) {
-                    for (InputStream script : scripts) {
+            String errorMessage = "Unable to upgrade the database";
-                        runScript(conn, script);
+            LOGGER.error(errorMessage, e);
-                        LOGGER.debug("Cleanup script " + upgrade.getClass().getSimpleName() + " is executed successfully");
+            throw new CloudRuntimeException(errorMessage, e);
-                    }
+        } finally {
-                }
+            txn.close();
-                txn.commit();
-
-                txn.start();
-                version.setStep(Step.Complete);
-                version.setUpdated(new Date());
-                _dao.update(version.getId(), version);
-                txn.commit();
-                LOGGER.debug("Upgrade completed for version " + version.getVersion());
-            } finally {
-                txn.close();
-            }
         }
+        return version;
+    }
 
-        executeViewScripts();
+    private void executeUpgradeCleanup(DbUpgrade upgrade, VersionVO version) {
-        updateSystemVmTemplates(upgrades);
+        TransactionLegacy txn;
+        // Run the corresponding '-cleanup.sql' script
+        txn = TransactionLegacy.open("Cleanup");
+        try {
+            LOGGER.info("Cleanup upgrade " + upgrade.getClass().getSimpleName() + " to upgrade from " + upgrade.getUpgradableVersionRange()[0] + "-" + upgrade
+                .getUpgradableVersionRange()[1] + " to " + upgrade.getUpgradedVersion());
+
+            txn.start();
+            Connection conn;
+            try {
+                conn = txn.getConnection();
+            } catch (SQLException e) {
+                LOGGER.error("Unable to cleanup the database", e);
+                throw new CloudRuntimeException("Unable to cleanup the database", e);
+            }
+
+            InputStream[] scripts = upgrade.getCleanupScripts();
+            if (scripts != null) {
+                for (InputStream script : scripts) {
+                    runScript(conn, script);
+                    LOGGER.debug("Cleanup script " + upgrade.getClass().getSimpleName() + " is executed successfully");
+                }
+            }
+            txn.commit();
+
+            txn.start();
+            version.setStep(Step.Complete);
+            version.setUpdated(new Date());
+            _dao.update(version.getId(), version);
+            txn.commit();
+            LOGGER.debug("Upgrade completed for version " + version.getVersion());
+        } finally {
+            txn.close();
+        }
     }
 
     protected void executeViewScripts() {

engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDao.java

@@ -38,4 +38,7 @@ public interface AffinityGroupDao extends GenericDao<AffinityGroupVO, Long> {
     AffinityGroupVO findByAccountAndType(Long accountId, String string);
 
     AffinityGroupVO findDomainLevelGroupByType(Long domainId, String string);
+
+    List<AffinityGroupVO> listByIds(List<Long> ids, boolean exclusive);
+
 }

engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDaoImpl.java

@@ -31,6 +31,7 @@ import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 
 public class AffinityGroupDaoImpl extends GenericDaoBase<AffinityGroupVO, Long> implements AffinityGroupDao {
+    private SearchBuilder<AffinityGroupVO> IdsSearch;
     private SearchBuilder<AffinityGroupVO> AccountIdSearch;
     private SearchBuilder<AffinityGroupVO> AccountIdNameSearch;
     private SearchBuilder<AffinityGroupVO> AccountIdNamesSearch;
@@ -47,6 +48,10 @@ public class AffinityGroupDaoImpl extends GenericDaoBase<AffinityGroupVO, Long>
 
     @PostConstruct
     protected void init() {
+        IdsSearch = createSearchBuilder();
+        IdsSearch.and("idIn", IdsSearch.entity().getId(), SearchCriteria.Op.IN);
+        IdsSearch.done();
+
         AccountIdSearch = createSearchBuilder();
         AccountIdSearch.and("accountId", AccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
         AccountIdSearch.done();
@@ -158,4 +163,11 @@ public class AffinityGroupDaoImpl extends GenericDaoBase<AffinityGroupVO, Long>
         sc.setJoinParameters("domainTypeSearch", "domainId", domainId);
         return findOneBy(sc);
     }
+
+    @Override
+    public List<AffinityGroupVO> listByIds(List<Long> ids, boolean exclusive) {
+        SearchCriteria<AffinityGroupVO> sc = IdsSearch.create();
+        sc.setParameters("idIn", ids.toArray());
+        return lockRows(sc, null, exclusive);
+    }
 }

engine/schema/src/main/resources/META-INF/db/procedures/cloud.add_guest_os_and_hypervisor_mapping.sql

@@ -0,0 +1,49 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- PR#4699 Drop the procedure `ADD_GUEST_OS_AND_HYPERVISOR_MAPPING` if it already exist.
+DROP PROCEDURE IF EXISTS `cloud`.`ADD_GUEST_OS_AND_HYPERVISOR_MAPPING`;
+
+-- PR#4699 Create the procedure `ADD_GUEST_OS_AND_HYPERVISOR_MAPPING` to add guest_os and guest_os_hypervisor mapping.
+CREATE PROCEDURE `cloud`.`ADD_GUEST_OS_AND_HYPERVISOR_MAPPING` (
+    IN guest_os_category_id bigint(20) unsigned,
+    IN guest_os_display_name VARCHAR(255),
+    IN guest_os_hypervisor_hypervisor_type VARCHAR(32),
+    IN guest_os_hypervisor_hypervisor_version VARCHAR(32),
+    IN guest_os_hypervisor_guest_os_name VARCHAR(255)
+        )
+BEGIN
+INSERT  INTO cloud.guest_os (uuid, category_id, display_name, created)
+SELECT 	UUID(), guest_os_category_id, guest_os_display_name, now()
+FROM    DUAL
+WHERE 	not exists( SELECT  1
+                     FROM    cloud.guest_os
+                     WHERE   cloud.guest_os.category_id = guest_os_category_id
+                       AND     cloud.guest_os.display_name = guest_os_display_name)
+
+;	INSERT  INTO cloud.guest_os_hypervisor (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created)
+     SELECT 	UUID(), guest_os_hypervisor_hypervisor_type, guest_os_hypervisor_hypervisor_version, guest_os_hypervisor_guest_os_name, guest_os.id, now()
+     FROM 	cloud.guest_os
+     WHERE 	guest_os.category_id = guest_os_category_id
+       AND 	guest_os.display_name = guest_os_display_name
+       AND	NOT EXISTS (SELECT  1
+                          FROM    cloud.guest_os_hypervisor as hypervisor
+                          WHERE   hypervisor_type = guest_os_hypervisor_hypervisor_type
+                            AND     hypervisor_version = guest_os_hypervisor_hypervisor_version
+                            AND     hypervisor.guest_os_id = guest_os.id
+                            AND     hypervisor.guest_os_name = guest_os_hypervisor_guest_os_name)
+;END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_column.sql

@@ -0,0 +1,27 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- in cloud
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_COLUMN`;
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_COLUMN` (
+        IN in_table_name VARCHAR(200),
+		IN in_column_name VARCHAR(200),
+		IN in_column_definition VARCHAR(1000)
+)
+BEGIN
+
+    DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_key.sql

@@ -0,0 +1,27 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_KEY`;
+
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_KEY` (
+		IN in_index_name VARCHAR(200)
+    , IN in_table_name VARCHAR(200)
+    , IN in_key_definition VARCHAR(1000)
+)
+BEGIN
+
+    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', ' ADD KEY ') ; SET @ddl = CONCAT(@ddl, ' ', in_index_name); SET @ddl = CONCAT(@ddl, ' ', in_key_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_unique_index.sql

@@ -0,0 +1,26 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Idempotent ADD UNIQUE INDEX
+DROP PROCEDURE IF EXISTS `cloud_usage`.`IDEMPOTENT_ADD_UNIQUE_INDEX`;
+CREATE PROCEDURE `cloud_usage`.`IDEMPOTENT_ADD_UNIQUE_INDEX` (
+    IN in_table_name VARCHAR(200)
+, IN in_index_name VARCHAR(200)
+, IN in_index_definition VARCHAR(1000)
+)
+BEGIN
+    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD UNIQUE INDEX ', in_index_name); SET @ddl = CONCAT(@ddl, ' ', in_index_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_unique_key.sql

@@ -0,0 +1,26 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Idempotent ADD UNIQUE KEY
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`;
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY` (
+    IN in_table_name VARCHAR(200)
+, IN in_key_name VARCHAR(200)
+, IN in_key_definition VARCHAR(1000)
+)
+BEGIN
+    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD UNIQUE KEY ', in_key_name); SET @ddl = CONCAT(@ddl, ' ', in_key_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_change_column.sql

@@ -0,0 +1,27 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- in usage Idempotent CHANGE COLUMN
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_CHANGE_COLUMN`;
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_CHANGE_COLUMN` (
+    IN in_table_name VARCHAR(200)
+    , IN in_column_name VARCHAR(200)
+    , IN in_column_new_name VARCHAR(200)
+    , IN in_column_new_definition VARCHAR(1000)
+)
+BEGIN
+    DECLARE CONTINUE HANDLER FOR 1054 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'CHANGE COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_new_name); SET @ddl = CONCAT(@ddl, ' ', in_column_new_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_create_unique_index.sql

@@ -0,0 +1,27 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_CREATE_UNIQUE_INDEX`;
+
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_CREATE_UNIQUE_INDEX` (
+		IN in_index_name VARCHAR(200)
+    , IN in_table_name VARCHAR(200)
+    , IN in_index_definition VARCHAR(1000)
+)
+BEGIN
+
+    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT('CREATE UNIQUE INDEX ', in_index_name); SET @ddl = CONCAT(@ddl, ' ', ' ON ') ; SET @ddl = CONCAT(@ddl, ' ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', in_index_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_drop_foreign_key.sql

@@ -0,0 +1,25 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`;
+
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY` (
+		IN in_table_name VARCHAR(200),
+        IN in_foreign_key_name VARCHAR(200)
+)
+BEGIN
+    DECLARE CONTINUE HANDLER FOR 1091, 1025 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', ' DROP FOREIGN KEY '); SET @ddl = CONCAT(@ddl, ' ', in_foreign_key_name); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_drop_index.sql

@@ -0,0 +1,27 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- in cloud
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_DROP_INDEX`;
+
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_DROP_INDEX` (
+		IN in_index_name VARCHAR(200)
+    , IN in_table_name VARCHAR(200)
+)
+BEGIN
+
+    DECLARE CONTINUE HANDLER FOR 1091 BEGIN END; SET @ddl = CONCAT('DROP INDEX ', in_index_name); SET @ddl = CONCAT(@ddl, ' ', ' ON ') ; SET @ddl = CONCAT(@ddl, ' ', in_table_name); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_insert_guestos_hypervisor_mapping.sql

@@ -0,0 +1,48 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_INSERT_GUESTOS_HYPERVISOR_MAPPING`;
+
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_INSERT_GUESTOS_HYPERVISOR_MAPPING`(
+                            IN in_hypervisor_type VARCHAR(32),
+                            IN in_hypervisor_version VARCHAR(32),
+                            IN in_guest_os_name VARCHAR(255),
+                            IN in_guest_os_id BIGINT(20) UNSIGNED,
+                            IN is_user_defined int(1) UNSIGNED)
+BEGIN
+        IF NOT EXISTS ((SELECT * FROM `cloud`.`guest_os_hypervisor` WHERE
+            hypervisor_type=in_hypervisor_type AND
+            hypervisor_version=in_hypervisor_version AND
+            guest_os_id = in_guest_os_id))
+        THEN
+                INSERT INTO `cloud`.`guest_os_hypervisor` (
+                        uuid,
+                        hypervisor_type,
+                        hypervisor_version,
+                        guest_os_name,
+                        guest_os_id,
+                        created,
+                        is_user_defined)
+                        VALUES (
+                            UUID(),
+                            in_hypervisor_type,
+                            in_hypervisor_version,
+                            in_guest_os_name,
+                            in_guest_os_id,
+                            utc_timestamp(),
+                            is_user_defined
+                        ); END IF; END;;

engine/schema/src/main/resources/META-INF/db/procedures/usage.idempotent_add_column.sql

@@ -0,0 +1,26 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- in usage
+DROP PROCEDURE IF EXISTS `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`;
+CREATE PROCEDURE `cloud_usage`.`IDEMPOTENT_ADD_COLUMN` (
+    IN in_table_name VARCHAR(200)
+, IN in_column_name VARCHAR(200)
+, IN in_column_definition VARCHAR(1000)
+)
+BEGIN
+    DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/usage.idempotent_change_column.sql

@@ -0,0 +1,27 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- in usage Idempotent CHANGE COLUMN
+DROP PROCEDURE IF EXISTS `cloud_usage`.`IDEMPOTENT_CHANGE_COLUMN`;
+CREATE PROCEDURE `cloud_usage`.`IDEMPOTENT_CHANGE_COLUMN` (
+    IN in_table_name VARCHAR(200)
+    , IN in_old_column_name VARCHAR(200)
+    , IN in_new_column_name VARCHAR(200)
+    , IN in_column_definition VARCHAR(1000)
+)
+BEGIN
+    DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', ' CHANGE COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_old_column_name); SET @ddl = CONCAT(@ddl, ' ', in_new_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/procedures/usage.idempotent_drop_index.sql

@@ -0,0 +1,25 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- in usage
+DROP PROCEDURE IF EXISTS `cloud_usage`.`IDEMPOTENT_DROP_INDEX`;
+CREATE PROCEDURE `cloud_usage`.`IDEMPOTENT_DROP_INDEX` (
+    IN in_index_name VARCHAR(200)
+, IN in_table_name VARCHAR(200)
+)
+BEGIN
+    DECLARE CONTINUE HANDLER FOR 1091 BEGIN END; SET @ddl = CONCAT('DROP INDEX ', in_index_name); SET @ddl = CONCAT(@ddl, ' ', ' ON ') ; SET @ddl = CONCAT(@ddl, ' ', in_table_name); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/schema-41000to41100.sql

@@ -19,51 +19,6 @@
 -- Schema upgrade from 4.10.0.0 to 4.11.0.0
 --;
 
---;
--- Stored procedure to do idempotent column add;
---;
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_COLUMN`;
-
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_COLUMN` (
-		IN in_table_name VARCHAR(200)
-    , IN in_column_name VARCHAR(200)
-    , IN in_column_definition VARCHAR(1000)
-)
-BEGIN
-
-    DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`;
-
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY` (
-		IN in_table_name VARCHAR(200)
-    , IN in_foreign_key_name VARCHAR(200)
-)
-BEGIN
-
-    DECLARE CONTINUE HANDLER FOR 1091 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', ' DROP FOREIGN KEY '); SET @ddl = CONCAT(@ddl, ' ', in_foreign_key_name); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_DROP_INDEX`;
-
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_DROP_INDEX` (
-		IN in_index_name VARCHAR(200)
-    , IN in_table_name VARCHAR(200)
-)
-BEGIN
-
-    DECLARE CONTINUE HANDLER FOR 1091 BEGIN END; SET @ddl = CONCAT('DROP INDEX ', in_index_name); SET @ddl = CONCAT(@ddl, ' ', ' ON ') ; SET @ddl = CONCAT(@ddl, ' ', in_table_name); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_CREATE_UNIQUE_INDEX`;
-
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_CREATE_UNIQUE_INDEX` (
-		IN in_index_name VARCHAR(200)
-    , IN in_table_name VARCHAR(200)
-    , IN in_index_definition VARCHAR(1000)
-)
-BEGIN
-
-    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT('CREATE UNIQUE INDEX ', in_index_name); SET @ddl = CONCAT(@ddl, ' ', ' ON ') ; SET @ddl = CONCAT(@ddl, ' ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', in_index_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
 -- Add For VPC flag
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.network_offerings','for_vpc', 'INT(1) NOT NULL DEFAULT 0');
 

engine/schema/src/main/resources/META-INF/db/schema-41520to41600.sql

@@ -123,20 +123,6 @@ CREATE VIEW `cloud`.`service_offering_view` AS
     GROUP BY
         `service_offering`.`id`;
 
---;
--- Stored procedure to do idempotent column add;
--- This is copied from schema-41000to41100.sql
---;
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_COLUMN`;
-
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_COLUMN` (
-    IN in_table_name VARCHAR(200),
-    IN in_column_name VARCHAR(200),
-    IN in_column_definition VARCHAR(1000)
-)
-BEGIN
-
-    DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
 
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.account','created', 'datetime DEFAULT NULL COMMENT ''date created'' AFTER `state` ');
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.domain','created', 'datetime DEFAULT NULL COMMENT ''date created'' AFTER `next_child_seq` ');
@@ -730,39 +716,6 @@ ALTER TABLE `cloud`.`annotations` ADD COLUMN `admins_only` tinyint(1) unsigned N
 -- Add uuid for ssh keypairs
 ALTER TABLE `cloud`.`ssh_keypairs` ADD COLUMN `uuid` varchar(40) AFTER `id`;
 
--- PR#4699 Drop the procedure `ADD_GUEST_OS_AND_HYPERVISOR_MAPPING` if it already exist.
-DROP PROCEDURE IF EXISTS `cloud`.`ADD_GUEST_OS_AND_HYPERVISOR_MAPPING`;
-
--- PR#4699 Create the procedure `ADD_GUEST_OS_AND_HYPERVISOR_MAPPING` to add guest_os and guest_os_hypervisor mapping.
-CREATE PROCEDURE `cloud`.`ADD_GUEST_OS_AND_HYPERVISOR_MAPPING` (
-    IN guest_os_category_id bigint(20) unsigned,
-    IN guest_os_display_name VARCHAR(255),
-    IN guest_os_hypervisor_hypervisor_type VARCHAR(32),
-    IN guest_os_hypervisor_hypervisor_version VARCHAR(32),
-    IN guest_os_hypervisor_guest_os_name VARCHAR(255)
-)
-BEGIN
-	INSERT  INTO cloud.guest_os (uuid, category_id, display_name, created)
-	SELECT 	UUID(), guest_os_category_id, guest_os_display_name, now()
-	FROM    DUAL
-	WHERE 	not exists( SELECT  1
-	                    FROM    cloud.guest_os
-	                    WHERE   cloud.guest_os.category_id = guest_os_category_id
-	                    AND     cloud.guest_os.display_name = guest_os_display_name)
-
-;	INSERT  INTO cloud.guest_os_hypervisor (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created)
-	SELECT 	UUID(), guest_os_hypervisor_hypervisor_type, guest_os_hypervisor_hypervisor_version, guest_os_hypervisor_guest_os_name, guest_os.id, now()
-	FROM 	cloud.guest_os
-	WHERE 	guest_os.category_id = guest_os_category_id
-	AND 	guest_os.display_name = guest_os_display_name
-	AND	NOT EXISTS (SELECT  1
-	                    FROM    cloud.guest_os_hypervisor as hypervisor
-	                    WHERE   hypervisor_type = guest_os_hypervisor_hypervisor_type
-	                    AND     hypervisor_version = guest_os_hypervisor_hypervisor_version
-	                    AND     hypervisor.guest_os_id = guest_os.id
-	                    AND     hypervisor.guest_os_name = guest_os_hypervisor_guest_os_name)
-;END;
-
 -- PR#4699 Call procedure `ADD_GUEST_OS_AND_HYPERVISOR_MAPPING` to add new data to guest_os and guest_os_hypervisor.
 CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (10, 'Ubuntu 20.04 LTS', 'KVM', 'default', 'Ubuntu 20.04 LTS');
 CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (10, 'Ubuntu 21.04', 'KVM', 'default', 'Ubuntu 21.04');

engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql

@@ -21,60 +21,6 @@
 
 ALTER TABLE `cloud`.`vm_work_job` ADD COLUMN `secondary_object` char(100) COMMENT 'any additional item that must be checked during queueing' AFTER `vm_instance_id`;
 
--- Stored procedures to handle cloud and cloud_schema changes
-
--- Idempotent ADD COLUMN
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_COLUMN`;
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_COLUMN` (
-    IN in_table_name VARCHAR(200)
-, IN in_column_name VARCHAR(200)
-, IN in_column_definition VARCHAR(1000)
-)
-BEGIN
-    DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
--- Idempotent ADD COLUMN
-DROP PROCEDURE IF EXISTS `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`;
-CREATE PROCEDURE `cloud_usage`.`IDEMPOTENT_ADD_COLUMN` (
-    IN in_table_name VARCHAR(200)
-, IN in_column_name VARCHAR(200)
-, IN in_column_definition VARCHAR(1000)
-)
-BEGIN
-    DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
--- Idempotent DROP INDEX
-DROP PROCEDURE IF EXISTS `cloud_usage`.`IDEMPOTENT_DROP_INDEX`;
-CREATE PROCEDURE `cloud_usage`.`IDEMPOTENT_DROP_INDEX` (
-    IN in_index_name VARCHAR(200)
-, IN in_table_name VARCHAR(200)
-)
-BEGIN
-    DECLARE CONTINUE HANDLER FOR 1091 BEGIN END; SET @ddl = CONCAT('DROP INDEX ', in_index_name); SET @ddl = CONCAT(@ddl, ' ', ' ON ') ; SET @ddl = CONCAT(@ddl, ' ', in_table_name); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
--- Idempotent ADD UNIQUE INDEX
-DROP PROCEDURE IF EXISTS `cloud_usage`.`IDEMPOTENT_ADD_UNIQUE_INDEX`;
-CREATE PROCEDURE `cloud_usage`.`IDEMPOTENT_ADD_UNIQUE_INDEX` (
-    IN in_table_name VARCHAR(200)
-, IN in_index_name VARCHAR(200)
-, IN in_index_definition VARCHAR(1000)
-)
-BEGIN
-    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD UNIQUE INDEX ', in_index_name); SET @ddl = CONCAT(@ddl, ' ', in_index_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
--- Idempotent CHANGE COLUMN
-DROP PROCEDURE IF EXISTS `cloud_usage`.`IDEMPOTENT_CHANGE_COLUMN`;
-CREATE PROCEDURE `cloud_usage`.`IDEMPOTENT_CHANGE_COLUMN` (
-    IN in_table_name VARCHAR(200)
-, IN in_old_column_name VARCHAR(200)
-, IN in_new_column_name VARCHAR(200)
-, IN in_column_definition VARCHAR(1000)
-)
-BEGIN
-    DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', ' CHANGE COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_old_column_name); SET @ddl = CONCAT(@ddl, ' ', in_new_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
--- Invoke stored procedures to add primary keys on missing tables
-
 -- Add PK to cloud.op_user_stats_log
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.op_user_stats_log', 'id', 'BIGINT(20) NOT NULL AUTO_INCREMENT FIRST, ADD PRIMARY KEY (`id`)');
 

engine/schema/src/main/resources/META-INF/db/schema-41610to41700.sql

@@ -219,21 +219,6 @@ CREATE VIEW `cloud`.`service_offering_view` AS
         `service_offering`.`id`;
 
 
---;
--- Stored procedure to do idempotent column add;
--- This is copied from schema-41000to41100.sql
---;
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_COLUMN`;
-
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_COLUMN` (
-    IN in_table_name VARCHAR(200),
-    IN in_column_name VARCHAR(200),
-    IN in_column_definition VARCHAR(1000)
-)
-BEGIN
-
-    DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes','external_uuid', 'VARCHAR(40) DEFAULT null ');
 
 DROP VIEW IF EXISTS `cloud`.`volume_view`;
@@ -937,35 +922,5 @@ INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hype
 -- Copy XenServer 8.2.0 hypervisor guest OS mappings to XenServer 8.2.1
 INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '8.2.1', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='8.2.0';
 
-DROP PROCEDURE IF EXISTS `cloud`.`ADD_GUEST_OS_AND_HYPERVISOR_MAPPING`;
-CREATE PROCEDURE `cloud`.`ADD_GUEST_OS_AND_HYPERVISOR_MAPPING` (
-    IN guest_os_category_id bigint(20) unsigned,
-    IN guest_os_display_name VARCHAR(255),
-    IN guest_os_hypervisor_hypervisor_type VARCHAR(32),
-    IN guest_os_hypervisor_hypervisor_version VARCHAR(32),
-    IN guest_os_hypervisor_guest_os_name VARCHAR(255)
-        )
-BEGIN
-INSERT  INTO cloud.guest_os (uuid, category_id, display_name, created)
-SELECT 	UUID(), guest_os_category_id, guest_os_display_name, now()
-FROM    DUAL
-WHERE 	not exists( SELECT  1
-                     FROM    cloud.guest_os
-                     WHERE   cloud.guest_os.category_id = guest_os_category_id
-                       AND     cloud.guest_os.display_name = guest_os_display_name)
-
-;	INSERT  INTO cloud.guest_os_hypervisor (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created)
-     SELECT 	UUID(), guest_os_hypervisor_hypervisor_type, guest_os_hypervisor_hypervisor_version, guest_os_hypervisor_guest_os_name, guest_os.id, now()
-     FROM 	cloud.guest_os
-     WHERE 	guest_os.category_id = guest_os_category_id
-       AND 	guest_os.display_name = guest_os_display_name
-       AND	NOT EXISTS (SELECT  1
-                          FROM    cloud.guest_os_hypervisor as hypervisor
-                          WHERE   hypervisor_type = guest_os_hypervisor_hypervisor_type
-                            AND     hypervisor_version = guest_os_hypervisor_hypervisor_version
-                            AND     hypervisor.guest_os_id = guest_os.id
-                            AND     hypervisor.guest_os_name = guest_os_hypervisor_guest_os_name)
-;END;
-
 CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (2, 'Debian GNU/Linux 11 (64-bit)', 'XenServer', '8.2.1', 'Debian Bullseye 11');
 CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (2, 'Debian GNU/Linux 11 (32-bit)', 'XenServer', '8.2.1', 'Debian Bullseye 11');

engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql

@@ -214,16 +214,6 @@ CREATE VIEW `cloud`.`domain_router_view` AS
             and async_job.instance_type = 'DomainRouter'
             and async_job.job_status = 0;
 
--- Idempotent ADD COLUMN
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_COLUMN`;
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_COLUMN` (
-    IN in_table_name VARCHAR(200)
-, IN in_column_name VARCHAR(200)
-, IN in_column_definition VARCHAR(1000)
-)
-BEGIN
-    DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
 -- Add passphrase table
 CREATE TABLE IF NOT EXISTS `cloud`.`passphrase` (
     `id` bigint unsigned NOT NULL auto_increment,
@@ -433,45 +423,6 @@ WHERE roles.role_type != 'Admin' AND roles.is_default = 1 AND role_perm.rule = '
 
 -- VM autoscaling
 
--- Idempotent ADD COLUMN
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_COLUMN`;
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_COLUMN` (
-    IN in_table_name VARCHAR(200)
-, IN in_column_name VARCHAR(200)
-, IN in_column_definition VARCHAR(1000)
-)
-BEGIN
-    DECLARE CONTINUE HANDLER FOR 1060 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
--- Idempotent RENAME COLUMN
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_CHANGE_COLUMN`;
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_CHANGE_COLUMN` (
-    IN in_table_name VARCHAR(200)
-, IN in_column_name VARCHAR(200)
-, IN in_column_new_name VARCHAR(200)
-, IN in_column_new_definition VARCHAR(1000)
-)
-BEGIN
-    DECLARE CONTINUE HANDLER FOR 1054 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'CHANGE COLUMN') ; SET @ddl = CONCAT(@ddl, ' ', in_column_name); SET @ddl = CONCAT(@ddl, ' ', in_column_new_name); SET @ddl = CONCAT(@ddl, ' ', in_column_new_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
--- Idempotent ADD UNIQUE KEY
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`;
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY` (
-    IN in_table_name VARCHAR(200)
-, IN in_key_name VARCHAR(200)
-, IN in_key_definition VARCHAR(1000)
-)
-BEGIN
-    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', 'ADD UNIQUE KEY ', in_key_name); SET @ddl = CONCAT(@ddl, ' ', in_key_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
-
--- Idempotent DROP FOREIGN KEY
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`;
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY` (
-    IN in_table_name VARCHAR(200)
-, IN in_foreign_key_name VARCHAR(200)
-)
-BEGIN
-    DECLARE CONTINUE HANDLER FOR 1091, 1025 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', ' DROP FOREIGN KEY '); SET @ddl = CONCAT(@ddl, ' ', in_foreign_key_name); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
 
 -- Add column 'supports_vm_autoscaling' to 'network_offerings' table
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.network_offerings', 'supports_vm_autoscaling', 'boolean default false');
@@ -1186,16 +1137,6 @@ CREATE TABLE IF NOT EXISTS `cloud`.`tungsten_lb_health_monitor` (
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
 --- #6888 add index to speed up querying IPs in the network-tab
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_KEY`;
-
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_KEY` (
-		IN in_index_name VARCHAR(200)
-    , IN in_table_name VARCHAR(200)
-    , IN in_key_definition VARCHAR(1000)
-)
-BEGIN
-
-    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT('ALTER TABLE ', in_table_name); SET @ddl = CONCAT(@ddl, ' ', ' ADD KEY ') ; SET @ddl = CONCAT(@ddl, ' ', in_index_name); SET @ddl = CONCAT(@ddl, ' ', in_key_definition); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
 
 CALL `cloud`.`IDEMPOTENT_ADD_KEY`('i_user_ip_address_state','user_ip_address', '(state)');
 

engine/schema/src/main/resources/META-INF/db/schema-4910to4920.sql

@@ -19,42 +19,6 @@
 -- Schema upgrade from 4.9.1.0 to 4.9.2.0;
 --;
 
---;
--- Stored procedure to do idempotent insert;
---;
-
-DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_INSERT_GUESTOS_HYPERVISOR_MAPPING`;
-
-CREATE PROCEDURE `cloud`.`IDEMPOTENT_INSERT_GUESTOS_HYPERVISOR_MAPPING`(
-                            IN in_hypervisor_type VARCHAR(32),
-                            IN in_hypervisor_version VARCHAR(32),
-                            IN in_guest_os_name VARCHAR(255),
-                            IN in_guest_os_id BIGINT(20) UNSIGNED,
-                            IN is_user_defined int(1) UNSIGNED)
-BEGIN
-        IF NOT EXISTS ((SELECT * FROM `cloud`.`guest_os_hypervisor` WHERE
-            hypervisor_type=in_hypervisor_type AND
-            hypervisor_version=in_hypervisor_version AND
-            guest_os_id = in_guest_os_id))
-        THEN
-                INSERT INTO `cloud`.`guest_os_hypervisor` (
-                        uuid,
-                        hypervisor_type,
-                        hypervisor_version,
-                        guest_os_name,
-                        guest_os_id,
-                        created,
-                        is_user_defined)
-                        VALUES (
-                            UUID(),
-                            in_hypervisor_type,
-                            in_hypervisor_version,
-                            in_guest_os_name,
-                            in_guest_os_id,
-                            utc_timestamp(),
-                            is_user_defined
-                        ); END IF; END;;
-
 CALL `cloud`.`IDEMPOTENT_INSERT_GUESTOS_HYPERVISOR_MAPPING`('Xenserver', '7.0.0', 'CentOS 4.5 (32-bit)', 1,  0);
 CALL `cloud`.`IDEMPOTENT_INSERT_GUESTOS_HYPERVISOR_MAPPING`('Xenserver', '7.0.0', 'CentOS 4.6 (32-bit)', 2,  0);
 CALL `cloud`.`IDEMPOTENT_INSERT_GUESTOS_HYPERVISOR_MAPPING`('Xenserver', '7.0.0', 'CentOS 4.7 (32-bit)', 3,  0);
@@ -234,5 +198,3 @@ CALL `cloud`.`IDEMPOTENT_INSERT_GUESTOS_HYPERVISOR_MAPPING`('Xenserver', '7.0.0'
 
 CALL `cloud`.`IDEMPOTENT_INSERT_GUESTOS_HYPERVISOR_MAPPING`('Xenserver', '7.0.0', 'Ubuntu Trusty Tahr 14.04', 255, 0);
 CALL `cloud`.`IDEMPOTENT_INSERT_GUESTOS_HYPERVISOR_MAPPING`('Xenserver', '7.0.0', 'Ubuntu Trusty Tahr 14.04', 256, 0);
-
-DROP PROCEDURE `cloud`.`IDEMPOTENT_INSERT_GUESTOS_HYPERVISOR_MAPPING`

plugins/affinity-group-processors/host-affinity/src/main/java/org/apache/cloudstack/affinity/HostAffinityProcessor.java

@@ -23,9 +23,14 @@ import java.util.Map;
 import java.util.Set;
 import java.util.HashSet;
 import java.util.ArrayList;
+import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 
+import com.cloud.utils.db.Transaction;
+import com.cloud.utils.db.TransactionCallback;
+import com.cloud.utils.db.TransactionCallbackNoReturn;
+import com.cloud.utils.db.TransactionStatus;
 import org.apache.commons.collections.CollectionUtils;
 
 import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
@@ -54,9 +59,16 @@ public class HostAffinityProcessor extends AffinityProcessorBase implements Affi
         VirtualMachine vm = vmProfile.getVirtualMachine();
         List<AffinityGroupVMMapVO> vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType());
         if (CollectionUtils.isNotEmpty(vmGroupMappings)) {
-            for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
+            List<Long> affinityGroupIdList = vmGroupMappings.stream().map(AffinityGroupVMMapVO::getAffinityGroupId).collect(Collectors.toList());
-                processAffinityGroup(vmGroupMapping, plan, vm, vmList);
+            Transaction.execute(new TransactionCallbackNoReturn() {
-            }
+                @Override
+                public void doInTransactionWithoutResult(TransactionStatus status) {
+                    _affinityGroupDao.listByIds(affinityGroupIdList, true);
+                    for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
+                        processAffinityGroup(vmGroupMapping, plan, vm, vmList);
+                    }
+                }
+            });
         }
     }
 
@@ -130,16 +142,23 @@ public class HostAffinityProcessor extends AffinityProcessorBase implements Affi
         long plannedHostId = plannedDestination.getHost().getId();
         VirtualMachine vm = vmProfile.getVirtualMachine();
         List<AffinityGroupVMMapVO> vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType());
-
+        if (CollectionUtils.isEmpty(vmGroupMappings)) {
-        if (CollectionUtils.isNotEmpty(vmGroupMappings)) {
+            return true;
-            for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
-                if (!checkAffinityGroup(vmGroupMapping, vm, plannedHostId)) {
-                    return false;
-                }
-            }
         }
+        List<Long> affinityGroupIds = vmGroupMappings.stream().map(AffinityGroupVMMapVO::getAffinityGroupId).collect(Collectors.toList());
+        return Transaction.execute(new TransactionCallback<Boolean>() {
+            @Override
+            public Boolean doInTransaction(TransactionStatus status) {
+                _affinityGroupDao.listByIds(affinityGroupIds, true);
+                for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
+                    if (!checkAffinityGroup(vmGroupMapping, vm, plannedHostId)) {
+                        return false;
+                    }
 
-        return true;
+                }
+                return true;
+            }
+        });
     }
 
     /**

plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java

@@ -19,10 +19,12 @@ package org.apache.cloudstack.affinity;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import org.apache.commons.collections.CollectionUtils;
 
 import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
 import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
@@ -37,6 +39,10 @@ import com.cloud.deploy.DeploymentPlanner.ExcludeList;
 import com.cloud.exception.AffinityConflictException;
 import com.cloud.utils.DateUtil;
 import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.db.Transaction;
+import com.cloud.utils.db.TransactionCallback;
+import com.cloud.utils.db.TransactionCallbackNoReturn;
+import com.cloud.utils.db.TransactionStatus;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineProfile;
@@ -65,40 +71,54 @@ public class HostAntiAffinityProcessor extends AffinityProcessorBase implements
         VirtualMachine vm = vmProfile.getVirtualMachine();
         List<AffinityGroupVMMapVO> vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType());
 
-        for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
+        if (CollectionUtils.isEmpty(vmGroupMappings)) {
-            if (vmGroupMapping != null) {
+            return;
-                AffinityGroupVO group = _affinityGroupDao.findById(vmGroupMapping.getAffinityGroupId());
+        }
-
+        List<Long> affinityGroupIds = vmGroupMappings.stream().map(AffinityGroupVMMapVO::getAffinityGroupId).collect(Collectors.toList());
-                if (logger.isDebugEnabled()) {
+        Transaction.execute(new TransactionCallbackNoReturn() {
-                    logger.debug("Processing affinity group " + group.getName() + " for VM Id: " + vm.getId());
+            @Override
+            public void doInTransactionWithoutResult(TransactionStatus status) {
+                _affinityGroupDao.listByIds(affinityGroupIds, true);
+                for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
+                    processAffinityGroup(vmGroupMapping, avoid, vm);
                 }
+            }
+        });
 
-                List<Long> groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(group.getId());
+    }
-                groupVMIds.remove(vm.getId());
 
-                for (Long groupVMId : groupVMIds) {
+    protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, ExcludeList avoid, VirtualMachine vm) {
-                    VMInstanceVO groupVM = _vmInstanceDao.findById(groupVMId);
+        if (vmGroupMapping != null) {
-                    if (groupVM != null && !groupVM.isRemoved()) {
+            AffinityGroupVO group = _affinityGroupDao.findById(vmGroupMapping.getAffinityGroupId());
-                        if (groupVM.getHostId() != null) {
+
-                            avoid.addHost(groupVM.getHostId());
+            if (logger.isDebugEnabled()) {
-                            if (logger.isDebugEnabled()) {
+                logger.debug("Processing affinity group " + group.getName() + " for VM Id: " + vm.getId());
-                                logger.debug("Added host " + groupVM.getHostId() + " to avoid set, since VM " + groupVM.getId() + " is present on the host");
+            }
-                            }
+
-                        } else if (Arrays.asList(VirtualMachine.State.Starting, VirtualMachine.State.Stopped).contains(groupVM.getState()) && groupVM.getLastHostId() != null) {
+            List<Long> groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(group.getId());
-                            long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - groupVM.getUpdateTime().getTime()) / 1000;
+            groupVMIds.remove(vm.getId());
-                            if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
+
-                                avoid.addHost(groupVM.getLastHostId());
+            for (Long groupVMId : groupVMIds) {
-                                if (logger.isDebugEnabled()) {
+                VMInstanceVO groupVM = _vmInstanceDao.findById(groupVMId);
-                                    logger.debug("Added host " + groupVM.getLastHostId() + " to avoid set, since VM " + groupVM.getId() +
+                if (groupVM != null && !groupVM.isRemoved()) {
-                                        " is present on the host, in Stopped state but has reserved capacity");
+                    if (groupVM.getHostId() != null) {
-                                }
+                        avoid.addHost(groupVM.getHostId());
-                            }
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("Added host " + groupVM.getHostId() + " to avoid set, since VM " + groupVM.getId() + " is present on the host");
+                        }
+                    }
+                } else if (Arrays.asList(VirtualMachine.State.Starting, VirtualMachine.State.Stopped).contains(groupVM.getState()) && groupVM.getLastHostId() != null) {
+                    long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - groupVM.getUpdateTime().getTime()) / 1000;
+                    if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
+                        avoid.addHost(groupVM.getLastHostId());
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("Added host " + groupVM.getLastHostId() + " to avoid set, since VM " + groupVM.getId() +
+                                    " is present on the host, in Stopped state but has reserved capacity");
                         }
                     }
                 }
             }
         }
-
     }
 
     @Override
@@ -137,7 +157,32 @@ public class HostAntiAffinityProcessor extends AffinityProcessorBase implements
                 }
             }
         }
-        return true;
+
+        List<Long> affinityGroupIds = vmGroupMappings.stream().map(AffinityGroupVMMapVO::getAffinityGroupId).collect(Collectors.toList());
+        return Transaction.execute(new TransactionCallback<Boolean>() {
+            @Override
+            public Boolean doInTransaction(TransactionStatus status) {
+                _affinityGroupDao.listByIds(affinityGroupIds, true);
+                for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
+                    // if more than 1 VM's are present in the group then check for
+                    // conflict due to parallel deployment
+                    List<Long> groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(vmGroupMapping.getAffinityGroupId());
+                    groupVMIds.remove(vm.getId());
+
+                    for (Long groupVMId : groupVMIds) {
+                        VMReservationVO vmReservation = _reservationDao.findByVmId(groupVMId);
+                        if (vmReservation != null && vmReservation.getHostId() != null && vmReservation.getHostId().equals(plannedHostId)) {
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("Planned destination for VM " + vm.getId() + " conflicts with an existing VM " + vmReservation.getVmId() +
+                                        " reserved on the same host " + plannedHostId);
+                            }
+                            return false;
+                        }
+                    }
+                }
+                return true;
+            }
+        });
     }
 
 }