Merge release branch 4.20 to main

* 4.20:
  UI: Fix userdata and load balancer selection (#10016)
  Prevent password updates for SAML and LDAP users (#9999)
  cloudstack-migrate-databases: sql AND added (#10033)
  engine/schema: move SQLs to 4.20.0 to 4.20.1 upgrade (#10018)
  Remove user from project before deletion (#10008)
  Simplify validation for creating volume templates via UI (#9828)

api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java

@@ -42,7 +42,7 @@ public class ListMgmtsCmd extends BaseListCmd {
 
     @Parameter(name = ApiConstants.PEERS, type = CommandType.BOOLEAN,
             description = "Whether to return the management server peers or not. By default, the management server peers will not be returned.",
-            since = "4.20.0.0")
+            since = "4.20.1.0")
     private Boolean peers;
 
     /////////////////////////////////////////////////////

engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDao.java

@@ -47,6 +47,8 @@ public interface ProjectAccountDao extends GenericDao<ProjectAccountVO, Long> {
 
     void removeAccountFromProjects(long accountId);
 
+    void removeUserFromProjects(long userId);
+
     boolean canUserModifyProject(long projectId, long accountId, long userId);
 
     List<ProjectAccountVO> listUsersOrAccountsByRole(long id);

engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDaoImpl.java

@@ -192,6 +192,17 @@ public class ProjectAccountDaoImpl extends GenericDaoBase<ProjectAccountVO, Long
         }
     }
 
+    @Override
+    public void removeUserFromProjects(long userId) {
+        SearchCriteria<ProjectAccountVO> sc = AllFieldsSearch.create();
+        sc.setParameters("userId", userId);
+
+        int removedCount = remove(sc);
+        if (removedCount > 0) {
+            logger.debug(String.format("Removed user [%s] from %s project(s).", userId, removedCount));
+        }
+    }
+
     @Override
     public boolean canUserModifyProject(long projectId, long accountId, long userId) {
         SearchCriteria<ProjectAccountVO> sc = AllFieldsSearch.create();

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to41920.java

@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.upgrade.dao;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+
+import java.io.InputStream;
+import java.sql.Connection;
+
+public class Upgrade41910to41920 implements DbUpgrade {
+
+    @Override
+    public String[] getUpgradableVersionRange() {
+        return new String[]{"4.19.1.0", "4.19.2.0"};
+    }
+
+    @Override
+    public String getUpgradedVersion() {
+        return "4.19.2.0";
+    }
+
+    @Override
+    public boolean supportsRollingUpgrade() {
+        return false;
+    }
+
+    @Override
+    public InputStream[] getPrepareScripts() {
+        final String scriptFile = "META-INF/db/schema-41910to41920.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[]{script};
+    }
+
+    @Override
+    public void performDataMigration(Connection conn) {
+    }
+
+    @Override
+    public InputStream[] getCleanupScripts() {
+        final String scriptFile = "META-INF/db/schema-41910to41920-cleanup.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[]{script};
+    }
+}

engine/schema/src/main/resources/META-INF/db/schema-41910to41920-cleanup.sql

@@ -0,0 +1,23 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade cleanup from 4.19.1.0 to 4.19.2.0
+--;
+
+-- Delete `project_account` entries for users that were removed
+DELETE FROM `cloud`.`project_account` WHERE `user_id` IN (SELECT `id` FROM `cloud`.`user` WHERE `removed`);

engine/schema/src/main/resources/META-INF/db/schema-41910to41920.sql

@@ -0,0 +1,20 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade from 4.19.1.0 to 4.19.2.0
+--;

engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql

@@ -425,10 +425,3 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervi
 
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_instance', 'delete_protection', 'boolean DEFAULT FALSE COMMENT "delete protection for vm" ');
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'delete_protection', 'boolean DEFAULT FALSE COMMENT "delete protection for volumes" ');
-
--- Modify index for mshost_peer
-DELETE FROM `cloud`.`mshost_peer`;
-CALL `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`('cloud.mshost_peer','fk_mshost_peer__owner_mshost');
-CALL `cloud`.`IDEMPOTENT_DROP_INDEX`('i_mshost_peer__owner_peer_runid','mshost_peer');
-CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.mshost_peer', 'i_mshost_peer__owner_peer', '(owner_mshost, peer_mshost)');
-CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.mshost_peer', 'fk_mshost_peer__owner_mshost', '(owner_mshost)', '`mshost`(`id`)');

engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql

@@ -22,3 +22,10 @@
 -- Add column api_key_access to user and account tables
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the user" AFTER `secret_key`');
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" ');
+
+-- Modify index for mshost_peer
+DELETE FROM `cloud`.`mshost_peer`;
+CALL `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`('cloud.mshost_peer','fk_mshost_peer__owner_mshost');
+CALL `cloud`.`IDEMPOTENT_DROP_INDEX`('i_mshost_peer__owner_peer_runid','mshost_peer');
+CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.mshost_peer', 'i_mshost_peer__owner_peer', '(owner_mshost, peer_mshost)');
+CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.mshost_peer', 'fk_mshost_peer__owner_mshost', '(owner_mshost)', '`mshost`(`id`)');

framework/db/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChanger.java

@@ -656,7 +656,7 @@ public class EncryptionSecretKeyChanger {
         String sqlTemplateDeployAsIsDetails = "SELECT template_deploy_as_is_details.value " +
                 "FROM template_deploy_as_is_details JOIN vm_instance " +
                 "WHERE template_deploy_as_is_details.template_id = vm_instance.vm_template_id " +
-                "vm_instance.id = %s AND template_deploy_as_is_details.name = '%s' LIMIT 1";
+                "AND vm_instance.id = %s AND template_deploy_as_is_details.name = '%s' LIMIT 1";
         try (PreparedStatement selectPstmt = conn.prepareStatement("SELECT id, vm_id, name, value FROM user_vm_deploy_as_is_details");
              ResultSet rs = selectPstmt.executeQuery();
              PreparedStatement updatePstmt = conn.prepareStatement("UPDATE user_vm_deploy_as_is_details SET value=? WHERE id=?")

server/src/main/java/com/cloud/user/AccountManagerImpl.java

@@ -1500,6 +1500,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
      * <ul>
      *  <li> If 'password' is blank, we throw an {@link InvalidParameterValueException};
      *  <li> If 'current password' is not provided and user is not an Admin, we throw an {@link InvalidParameterValueException};
+     *  <li> If the user whose password is being changed has a source equal to {@link User.Source#SAML2}, {@link User.Source#SAML2DISABLED} or {@link User.Source#LDAP},
+     *      we throw an {@link InvalidParameterValueException};
      *  <li> If a normal user is calling this method, we use {@link #validateCurrentPassword(UserVO, String)} to check if the provided old password matches the database one;
      * </ul>
      *
@@ -1514,6 +1516,12 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
             throw new InvalidParameterValueException("Password cannot be empty or blank.");
         }
 
+        User.Source userSource = user.getSource();
+        if (userSource == User.Source.SAML2 || userSource == User.Source.SAML2DISABLED || userSource == User.Source.LDAP) {
+            logger.warn(String.format("Unable to update the password for user [%d], as its source is [%s].", user.getId(), user.getSource().toString()));
+            throw new InvalidParameterValueException("CloudStack does not support updating passwords for SAML or LDAP users. Please contact your cloud administrator for assistance.");
+        }
+
         passwordPolicy.verifyIfPasswordCompliesWithPasswordPolicies(newPassword, user.getUsername(), getAccount(user.getAccountId()).getDomainId());
 
         Account callingAccount = getCurrentCallingAccount();

server/src/test/java/com/cloud/user/AccountManagerImplTest.java

@@ -874,6 +874,36 @@ public class AccountManagerImplTest extends AccountManagetImplTestBase {
         accountManagerImpl.validateUserPasswordAndUpdateIfNeeded(newPassword, userVoMock, currentPassword, false);
     }
 
+    @Test(expected = InvalidParameterValueException.class)
+    public void validateUserPasswordAndUpdateIfNeededTestSaml2UserShouldNotBeAllowedToUpdateTheirPassword() {
+        String newPassword = "newPassword";
+        String currentPassword = "theCurrentPassword";
+
+        Mockito.when(userVoMock.getSource()).thenReturn(User.Source.SAML2);
+
+        accountManagerImpl.validateUserPasswordAndUpdateIfNeeded(newPassword, userVoMock, currentPassword, false);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void validateUserPasswordAndUpdateIfNeededTestSaml2DisabledUserShouldNotBeAllowedToUpdateTheirPassword() {
+        String newPassword = "newPassword";
+        String currentPassword = "theCurrentPassword";
+
+        Mockito.when(userVoMock.getSource()).thenReturn(User.Source.SAML2DISABLED);
+
+        accountManagerImpl.validateUserPasswordAndUpdateIfNeeded(newPassword, userVoMock, currentPassword, false);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void validateUserPasswordAndUpdateIfNeededTestLdapUserShouldNotBeAllowedToUpdateTheirPassword() {
+        String newPassword = "newPassword";
+        String currentPassword = "theCurrentPassword";
+
+        Mockito.when(userVoMock.getSource()).thenReturn(User.Source.LDAP);
+
+        accountManagerImpl.validateUserPasswordAndUpdateIfNeeded(newPassword, userVoMock, currentPassword, false);
+    }
+
     private String configureUserMockAuthenticators(String newPassword) {
         accountManagerImpl._userPasswordEncoders = new ArrayList<>();
         UserAuthenticator authenticatorMock1 = Mockito.mock(UserAuthenticator.class);

ui/src/config/section/storage.js

@@ -251,9 +251,7 @@ export default {
           label: 'label.action.create.template.from.volume',
           dataView: true,
           show: (record) => {
-            return !['Destroy', 'Destroyed', 'Expunging', 'Expunged', 'Migrating', 'Uploading', 'UploadError', 'Creating'].includes(record.state) &&
+            return record.state === 'Ready' && (record.vmstate === 'Stopped' || !record.virtualmachineid)
-                ((record.type === 'ROOT' && record.vmstate === 'Stopped') ||
-                    (record.type !== 'ROOT' && !record.virtualmachineid && !['Allocated', 'Uploaded'].includes(record.state)))
           },
           args: (record, store) => {
             var fields = ['volumeid', 'name', 'displaytext', 'ostypeid', 'isdynamicallyscalable', 'requireshvm', 'passwordenabled']

ui/src/views/compute/DeployVM.vue

@@ -1940,18 +1940,16 @@ export default {
         this.form.userdataid = undefined
         return
       }
+
       this.form.userdataid = id
       this.userDataParams = []
       api('listUserData', { id: id }).then(json => {
         const resp = json?.listuserdataresponse?.userdata || []
         if (resp[0]) {
-          var params = resp[0].params
+          const params = resp[0].params
-          if (params) {
+          const dataParams = params ? params.split(',') : []
-            var dataParams = params.split(',')
+          dataParams.forEach((val, index) => {
-          }
+            this.userDataParams.push({
-          var that = this
-          dataParams.forEach(function (val, index) {
-            that.userDataParams.push({
               id: index,
               key: val
             })

ui/src/views/compute/wizard/LoadBalancerSelection.vue

@@ -30,6 +30,7 @@
       :rowKey="record => record.id"
       :pagination="false"
       :rowSelection="rowSelection"
+      :customRow="onClickRow"
       size="middle"
       :scroll="{ y: 225 }">
       <template #headerCell="{ column }">
@@ -197,6 +198,14 @@ export default {
       this.options.page = page
       this.options.pageSize = pageSize
       this.$emit('handle-search-filter', this.options)
+    },
+    onClickRow (record) {
+      return {
+        onClick: () => {
+          this.selectedRowKeys = [record.id]
+          this.$emit('select-load-balancer-item', record.id)
+        }
+      }
     }
   }
 }

ui/src/views/compute/wizard/UserDataSelection.vue

@@ -33,6 +33,7 @@
       :scroll="{ y: 225 }"
     >
       <template #headerCell="{ column }">
+        <template v-if="column.key === 'name'"><solution-outlined /> {{ $t('label.userdata') }}</template>
         <template v-if="column.key === 'account'"><user-outlined /> {{ $t('label.account') }}</template>
         <template v-if="column.key === 'domain'"><block-outlined /> {{ $t('label.domain') }}</template>
       </template>
@@ -78,6 +79,7 @@ export default {
       filter: '',
       columns: [
         {
+          key: 'name',
           dataIndex: 'name',
           title: this.$t('label.userdata'),
           width: '40%'
@@ -181,11 +183,9 @@ export default {
     },
     onClickRow (record) {
       return {
-        on: {
+        onClick: () => {
-          click: () => {
+          this.selectedRowKeys = [record.key]
-            this.selectedRowKeys = [record.key]
+          this.$emit('select-user-data-item', record.key)
-            this.$emit('select-user-data-item', record.key)
-          }
         }
       }
     }