From 3321971adc1b47be0c367f8c4026c2478f309b82 Mon Sep 17 00:00:00 2001
From: Vishesh <vishesh92@gmail.com>
Date: Tue, 11 Jun 2024 11:18:03 +0530
Subject: [PATCH 01/20] UI: Update AddAccountOrUserToProject to show a drop
 down for account & user (#9072)

---
 .../project/AddAccountOrUserToProject.vue     | 70 +++++++++++++++----
 1 file changed, 56 insertions(+), 14 deletions(-)

diff --git a/ui/src/views/project/AddAccountOrUserToProject.vue b/ui/src/views/project/AddAccountOrUserToProject.vue
index e65c6a7492d..d85dc785729 100644
--- a/ui/src/views/project/AddAccountOrUserToProject.vue
+++ b/ui/src/views/project/AddAccountOrUserToProject.vue
@@ -31,10 +31,27 @@
             <template #label>
               <tooltip-label :title="$t('label.account')" :tooltip="apiParams.addAccountToProject.account.description"/>
             </template>
-            <a-input
-              v-model:value="form.account"
-              :placeholder="apiParams.addAccountToProject.account.description"
-              v-focus="true" />
+            <a-select
+                show-search
+                v-model:value="form.account"
+                :placeholder="apiParams.addAccountToProject.account.description"
+                v-focus="true"
+                :filterOption="false"
+                @search="fetchAccounts"
+              >
+                <template v-if="load.accounts" #notFoundContent>
+                  <a-spin size="small" />
+                </template>
+                <template v-if="!load.accounts">
+                  <a-select-option v-for="account in accounts" :key="account.name" :value="account.name">
+                    <span v-if="account.icon">
+                      <resource-icon :image="account.icon.base64image" size="1x" style="margin-right: 5px"/>
+                    </span>
+                    <block-outlined v-else style="margin-right: 5px" />
+                    {{ account.name }}
+                  </a-select-option>
+                </template>
+              </a-select>
           </a-form-item>
           <a-form-item name="email" ref="email">
             <template #label>
@@ -104,10 +121,27 @@
             <template #label>
               <tooltip-label :title="$t('label.name')" :tooltip="apiParams.addUserToProject.username.description"/>
             </template>
-            <a-input
-              v-model:value="form.username"
-              :placeholder="apiParams.addUserToProject.username.description"
-              v-focus="true" />
+              <a-select
+                show-search
+                v-model:value="form.username"
+                :placeholder="apiParams.addUserToProject.username.description"
+                v-focus="true"
+                :filterOption="false"
+                @search="fetchUsers"
+              >
+                <template v-if="load.users" #notFoundContent>
+                  <a-spin size="small" />
+                </template>
+                <template v-if="!load.users">
+                  <a-select-option v-for="user in users" :key="user.username" :value="user.username">
+                    <span v-if="user.icon">
+                      <resource-icon :image="user.icon.base64image" size="1x" style="margin-right: 5px"/>
+                    </span>
+                    <block-outlined v-else style="margin-right: 5px" />
+                    {{ user.firstname + ' ' + user.lastname + " (" + user.username + ")" }}
+                  </a-select-option>
+                </template>
+              </a-select>
           </a-form-item>
           <a-form-item name="email" ref="email">
             <template #label>
@@ -163,11 +197,13 @@
 <script>
 import { ref, reactive, toRaw } from 'vue'
 import { api } from '@/api'
+import ResourceIcon from '@/components/view/ResourceIcon'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
 
 export default {
   name: 'AddAccountOrUserToProject',
   components: {
+    ResourceIcon,
     TooltipLabel
   },
   props: {
@@ -218,9 +254,13 @@ export default {
         this.fetchProjectRoles()
       }
     },
-    fetchUsers () {
+    fetchUsers (keyword) {
       this.load.users = true
-      api('listUsers', { listall: true }).then(response => {
+      const params = { listall: true, showicon: true }
+      if (keyword) {
+        params.keyword = keyword
+      }
+      api('listUsers', params).then(response => {
         this.users = response.listusersresponse.user ? response.listusersresponse.user : []
       }).catch(error => {
         this.$notifyError(error)
@@ -228,11 +268,13 @@ export default {
         this.load.users = false
       })
     },
-    fetchAccounts () {
+    fetchAccounts (keyword) {
       this.load.accounts = true
-      api('listAccounts', {
-        domainid: this.resource.domainid
-      }).then(response => {
+      const params = { domainid: this.resource.domainid, showicon: true }
+      if (keyword) {
+        params.keyword = keyword
+      }
+      api('listAccounts', params).then(response => {
         this.accounts = response.listaccountsresponse.account || []
       }).catch(error => {
         this.$notifyError(error)

From fcca3e8f39185122771495103c8ec0d68cf2b958 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek.mrt22@gmail.com>
Date: Tue, 11 Jun 2024 11:20:50 +0530
Subject: [PATCH 02/20] api,server: encryptformat in volume response (#9106)

Fixes #9014

* api,server: encryptformat in volume response

* Apply suggestions from code review

* remove passphrase db change
---
 .../cloudstack/api/response/VolumeResponse.java      | 12 ++++++++++--
 .../META-INF/db/views/cloud.volume_view.sql          |  1 +
 .../com/cloud/api/query/dao/VolumeJoinDaoImpl.java   |  1 +
 .../java/com/cloud/api/query/vo/VolumeJoinVO.java    |  7 +++++++
 4 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java
index 0d502a6d7a7..2ddc5fd097e 100644
--- a/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java
+++ b/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java
@@ -290,13 +290,17 @@ public class VolumeResponse extends BaseResponseWithTagInformation implements Co
     private String externalUuid;
 
     @SerializedName(ApiConstants.VOLUME_CHECK_RESULT)
-    @Param(description = "details for the volume check result, they may vary for different hypervisors, since = 4.19.1")
+    @Param(description = "details for the volume check result, they may vary for different hypervisors", since = "4.19.1")
     private Map<String, String> volumeCheckResult;
 
     @SerializedName(ApiConstants.VOLUME_REPAIR_RESULT)
-    @Param(description = "details for the volume repair result, they may vary for different hypervisors, since = 4.19.1")
+    @Param(description = "details for the volume repair result, they may vary for different hypervisors", since = "4.19.1")
     private Map<String, String> volumeRepairResult;
 
+    @SerializedName(ApiConstants.ENCRYPT_FORMAT)
+    @Param(description = "the encrypt format of the volume", since = "4.19.1")
+    private String encryptFormat;
+
     public String getPath() {
         return path;
     }
@@ -842,4 +846,8 @@ public class VolumeResponse extends BaseResponseWithTagInformation implements Co
     public void setVolumeRepairResult(Map<String, String> volumeRepairResult) {
         this.volumeRepairResult = volumeRepairResult;
     }
+
+    public void setEncryptFormat(String encryptFormat) {
+        this.encryptFormat = encryptFormat;
+    }
 }
diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.volume_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.volume_view.sql
index fd21fff1494..950dcddf4c7 100644
--- a/engine/schema/src/main/resources/META-INF/db/views/cloud.volume_view.sql
+++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.volume_view.sql
@@ -39,6 +39,7 @@ SELECT
     `volumes`.`path` AS `path`,
     `volumes`.`chain_info` AS `chain_info`,
     `volumes`.`external_uuid` AS `external_uuid`,
+    `volumes`.`encrypt_format` AS `encrypt_format`,
     `account`.`id` AS `account_id`,
     `account`.`uuid` AS `account_uuid`,
     `account`.`account_name` AS `account_name`,
diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java
index 8fcad6ed45a..c4e4ba223d8 100644
--- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java
+++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java
@@ -180,6 +180,7 @@ public class VolumeJoinDaoImpl extends GenericDaoBaseWithTagInformation<VolumeJo
 
         if (view == ResponseView.Full) {
             volResponse.setPath(volume.getPath());
+            volResponse.setEncryptFormat(volume.getEncryptFormat());
         }
 
         // populate owner.
diff --git a/server/src/main/java/com/cloud/api/query/vo/VolumeJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/VolumeJoinVO.java
index a8d568ff245..a8fcbfe824e 100644
--- a/server/src/main/java/com/cloud/api/query/vo/VolumeJoinVO.java
+++ b/server/src/main/java/com/cloud/api/query/vo/VolumeJoinVO.java
@@ -275,6 +275,9 @@ public class VolumeJoinVO extends BaseViewWithTagInformationVO implements Contro
     @Column(name = "external_uuid")
     private String externalUuid = null;
 
+    @Column(name = "encrypt_format")
+    private String encryptFormat;
+
     public VolumeJoinVO() {
     }
 
@@ -610,6 +613,10 @@ public class VolumeJoinVO extends BaseViewWithTagInformationVO implements Contro
         this.externalUuid = externalUuid;
     }
 
+    public String getEncryptFormat() {
+        return encryptFormat;
+    }
+
     @Override
     public Class<?> getEntityType() {
         return Volume.class;

From 7aacbcb559e976efb9b443b5dcc1500c2170b6ad Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek.mrt22@gmail.com>
Date: Tue, 11 Jun 2024 11:28:08 +0530
Subject: [PATCH 03/20] api: listApis should return params based on caller
 (#8973)

---
 .../api/response/ApiDiscoveryResponse.java    | 25 ++++++++++--
 .../api/response/ApiParameterResponse.java    | 13 ++++++-
 .../discovery/ApiDiscoveryServiceImpl.java    | 38 ++++++++++++++++---
 3 files changed, 65 insertions(+), 11 deletions(-)

diff --git a/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java b/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java
index dccf5a68e11..81a9df750cb 100644
--- a/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java
+++ b/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java
@@ -16,13 +16,14 @@
 // under the License.
 package org.apache.cloudstack.api.response;
 
-import com.cloud.serializer.Param;
-import com.google.gson.annotations.SerializedName;
+import java.util.HashSet;
+import java.util.Set;
+
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponse;
 
-import java.util.HashSet;
-import java.util.Set;
+import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
 
 @SuppressWarnings("unused")
 public class ApiDiscoveryResponse extends BaseResponse {
@@ -64,6 +65,18 @@ public class ApiDiscoveryResponse extends BaseResponse {
         isAsync = false;
     }
 
+    public ApiDiscoveryResponse(ApiDiscoveryResponse another) {
+        this.name = another.getName();
+        this.description = another.getDescription();
+        this.since = another.getSince();
+        this.isAsync = another.getAsync();
+        this.related = another.getRelated();
+        this.params = new HashSet<>(another.getParams());
+        this.apiResponse = new HashSet<>(another.getApiResponse());
+        this.type = another.getType();
+        this.setObjectName(another.getObjectName());
+    }
+
     public void setName(String name) {
         this.name = name;
     }
@@ -123,4 +136,8 @@ public class ApiDiscoveryResponse extends BaseResponse {
     public Set<ApiResponseResponse> getApiResponse() {
         return apiResponse;
     }
+
+    public String getType() {
+        return type;
+    }
 }
diff --git a/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiParameterResponse.java b/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiParameterResponse.java
index 7713f6b5d69..75f0aacd504 100644
--- a/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiParameterResponse.java
+++ b/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiParameterResponse.java
@@ -16,12 +16,14 @@
 // under the License.
 package org.apache.cloudstack.api.response;
 
-import com.google.gson.annotations.SerializedName;
+import java.util.List;
 
+import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponse;
 
 import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
 
 public class ApiParameterResponse extends BaseResponse {
     @SerializedName(ApiConstants.NAME)
@@ -52,6 +54,8 @@ public class ApiParameterResponse extends BaseResponse {
     @Param(description = "comma separated related apis to get the parameter")
     private String related;
 
+    private transient List<RoleType> authorizedRoleTypes = null;
+
     public ApiParameterResponse() {
     }
 
@@ -87,4 +91,11 @@ public class ApiParameterResponse extends BaseResponse {
         this.related = related;
     }
 
+    public void setAuthorizedRoleTypes(List<RoleType> authorizedRoleTypes) {
+        this.authorizedRoleTypes = authorizedRoleTypes;
+    }
+
+    public List<RoleType> getAuthorizedRoleTypes() {
+        return authorizedRoleTypes;
+    }
 }
diff --git a/plugins/api/discovery/src/main/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java b/plugins/api/discovery/src/main/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java
index 3bdf2e9ce92..eedf1878ec8 100644
--- a/plugins/api/discovery/src/main/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java
+++ b/plugins/api/discovery/src/main/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java
@@ -18,8 +18,10 @@ package org.apache.cloudstack.discovery;
 
 import java.lang.reflect.Field;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -28,21 +30,22 @@ import java.util.Set;
 import javax.inject.Inject;
 
 import org.apache.cloudstack.acl.APIChecker;
+import org.apache.cloudstack.acl.Role;
+import org.apache.cloudstack.acl.RoleService;
+import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.BaseAsyncCmd;
 import org.apache.cloudstack.api.BaseAsyncCreateCmd;
 import org.apache.cloudstack.api.BaseCmd;
 import org.apache.cloudstack.api.BaseResponse;
 import org.apache.cloudstack.api.Parameter;
-import org.apache.cloudstack.acl.Role;
-import org.apache.cloudstack.acl.RoleService;
-import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.command.user.discovery.ListApisCmd;
 import org.apache.cloudstack.api.response.ApiDiscoveryResponse;
 import org.apache.cloudstack.api.response.ApiParameterResponse;
 import org.apache.cloudstack.api.response.ApiResponseResponse;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 import org.reflections.ReflectionUtils;
@@ -217,6 +220,9 @@ public class ApiDiscoveryServiceImpl extends ComponentLifecycleBase implements A
                     paramResponse.setSince(parameterAnnotation.since());
                 }
                 paramResponse.setRelated(parameterAnnotation.entityType()[0].getName());
+                if (parameterAnnotation.authorized() != null) {
+                    paramResponse.setAuthorizedRoleTypes(Arrays.asList(parameterAnnotation.authorized()));
+                }
                 response.addParam(paramResponse);
             }
         }
@@ -249,6 +255,7 @@ public class ApiDiscoveryServiceImpl extends ComponentLifecycleBase implements A
 
         if (user == null)
             return null;
+        Account account = accountService.getAccount(user.getAccountId());
 
         if (name != null) {
             if (!s_apiNameDiscoveryResponseMap.containsKey(name))
@@ -262,10 +269,9 @@ public class ApiDiscoveryServiceImpl extends ComponentLifecycleBase implements A
                     return null;
                 }
             }
-            responseList.add(s_apiNameDiscoveryResponseMap.get(name));
+            responseList.add(getApiDiscoveryResponseWithAccessibleParams(name, account));
 
         } else {
-            Account account = accountService.getAccount(user.getAccountId());
             if (account == null) {
                 throw new PermissionDeniedException(String.format("The account with id [%s] for user [%s] is null.", user.getAccountId(), user));
             }
@@ -286,13 +292,33 @@ public class ApiDiscoveryServiceImpl extends ComponentLifecycleBase implements A
             }
 
             for (String apiName: apisAllowed) {
-                responseList.add(s_apiNameDiscoveryResponseMap.get(apiName));
+                responseList.add(getApiDiscoveryResponseWithAccessibleParams(apiName, account));
             }
         }
         response.setResponses(responseList);
         return response;
     }
 
+    private static ApiDiscoveryResponse getApiDiscoveryResponseWithAccessibleParams(String name, Account account) {
+        if (Account.Type.ADMIN.equals(account.getType())) {
+            return s_apiNameDiscoveryResponseMap.get(name);
+        }
+        ApiDiscoveryResponse apiDiscoveryResponse =
+                new ApiDiscoveryResponse(s_apiNameDiscoveryResponseMap.get(name));
+        Iterator<ApiParameterResponse> iterator = apiDiscoveryResponse.getParams().iterator();
+        while (iterator.hasNext()) {
+            ApiParameterResponse parameterResponse = iterator.next();
+            List<RoleType> authorizedRoleTypes = parameterResponse.getAuthorizedRoleTypes();
+            RoleType accountRoleType = RoleType.getByAccountType(account.getType());
+            if (CollectionUtils.isNotEmpty(parameterResponse.getAuthorizedRoleTypes()) &&
+                    accountRoleType != null &&
+                    !authorizedRoleTypes.contains(accountRoleType)) {
+                iterator.remove();
+            }
+        }
+        return apiDiscoveryResponse;
+    }
+
     @Override
     public List<Class<?>> getCommands() {
         List<Class<?>> cmdList = new ArrayList<Class<?>>();

From 4bd0b1c24f12251c99b9fc3818add4bebd367258 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek.mrt22@gmail.com>
Date: Tue, 11 Jun 2024 13:03:37 +0530
Subject: [PATCH 04/20] ui: refactor global createmenu (#9139)

Refactors global create menu code to make it visible only when any menu
is accessible.

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
---
 ui/src/components/header/CreateMenu.vue | 181 ++++++++----------------
 1 file changed, 60 insertions(+), 121 deletions(-)

diff --git a/ui/src/components/header/CreateMenu.vue b/ui/src/components/header/CreateMenu.vue
index a1c40f38e1a..e80120d7a1c 100644
--- a/ui/src/components/header/CreateMenu.vue
+++ b/ui/src/components/header/CreateMenu.vue
@@ -16,138 +16,25 @@
 // under the License.
 
 <template>
-  <a-dropdown>
+  <a-dropdown v-if="accessibleCreateActions && accessibleCreateActions.length > 0">
     <template #overlay>
       <a-menu>
-        <a-menu-item style="width: 100%; padding: 12px" v-if="'deployVirtualMachine' in $store.getters.apis">
-          <router-link :to="{ path: '/action/deployVirtualMachine'}">
+        <a-menu-item style="width: 100%; padding: 12px" v-for="menuItem in accessibleCreateActions" :key="menuItem.api">
+          <router-link :to="menuItem.route">
             <a-row>
               <a-col style="margin-right: 12px">
                 <a-avatar :style="{ backgroundColor: $config.theme['@primary-color'] }">
                   <template #icon>
-                    <cloud-server-outlined/>
+                    <render-icon v-if="(typeof menuItem.icon === 'string')" :icon="menuItem.icon" />
+                    <font-awesome-icon v-else :icon="menuItem.icon" />
                   </template>
                 </a-avatar>
               </a-col>
               <a-col>
                 <h3 style="margin-bottom: 0px;">
-                  {{ $t('label.instance') }}
+                  {{ menuItem.title }}
                 </h3>
-                <small>{{ $t('label.create.instance') }}</small>
-              </a-col>
-            </a-row>
-          </router-link>
-        </a-menu-item>
-        <a-menu-item style="width: 100%; padding: 12px" v-if="'createKubernetesCluster' in $store.getters.apis">
-          <router-link :to="{ path: '/kubernetes', query: { action: 'createKubernetesCluster' } }">
-            <a-row>
-              <a-col style="margin-right: 12px">
-                <a-avatar :style="{ backgroundColor: $config.theme['@primary-color'] }">
-                  <template #icon>
-                    <font-awesome-icon :icon="['fa-solid', 'fa-dharmachakra']" />
-                  </template>
-                </a-avatar>
-              </a-col>
-              <a-col>
-                <h3 style="margin-bottom: 0px;">
-                  {{ $t('label.kubernetes') }}
-                </h3>
-                <small>{{ $t('label.kubernetes.cluster.create') }}</small>
-              </a-col>
-            </a-row>
-          </router-link>
-        </a-menu-item>
-        <a-menu-item style="width: 100%; padding: 12px" v-if="'createVolume' in $store.getters.apis">
-          <router-link :to="{ path: '/volume', query: { action: 'createVolume' } }">
-            <a-row>
-              <a-col style="margin-right: 12px">
-                <a-avatar :style="{ backgroundColor: $config.theme['@primary-color'] }">
-                  <template #icon>
-                    <hdd-outlined />
-                  </template>
-                </a-avatar>
-              </a-col>
-              <a-col>
-                <h3 style="margin-bottom: 0px;">
-                  {{ $t('label.volume') }}
-                </h3>
-                <small>{{ $t('label.action.create.volume') }}</small>
-              </a-col>
-            </a-row>
-          </router-link>
-        </a-menu-item>
-        <a-menu-item style="width: 100%; padding: 12px" v-if="'createNetwork' in $store.getters.apis">
-          <router-link :to="{ path: '/guestnetwork', query: { action: 'createNetwork' } }">
-            <a-row>
-              <a-col style="margin-right: 12px">
-                <a-avatar :style="{ backgroundColor: $config.theme['@primary-color'] }">
-                  <template #icon>
-                    <apartment-outlined />
-                  </template>
-                </a-avatar>
-              </a-col>
-              <a-col>
-                <h3 style="margin-bottom: 0px;">
-                  {{ $t('label.network') }}
-                </h3>
-                <small>{{ $t('label.add.network') }}</small>
-              </a-col>
-            </a-row>
-          </router-link>
-        </a-menu-item>
-        <a-menu-item style="width: 100%; padding: 12px" v-if="'createVPC' in $store.getters.apis">
-          <router-link :to="{ path: '/vpc', query: { action: 'createVPC' } }">
-            <a-row>
-              <a-col style="margin-right: 12px">
-                <a-avatar :style="{ backgroundColor: $config.theme['@primary-color'] }">
-                  <template #icon>
-                    <deployment-unit-outlined />
-                  </template>
-                </a-avatar>
-              </a-col>
-              <a-col>
-                <h3 style="margin-bottom: 0px;">
-                  {{ $t('label.vpc') }}
-                </h3>
-                <small>{{ $t('label.add.vpc') }}</small>
-              </a-col>
-            </a-row>
-          </router-link>
-        </a-menu-item>
-        <a-menu-item style="width: 100%; padding: 12px" v-if="'registerTemplate' in $store.getters.apis">
-          <router-link :to="{ path: '/template', query: { action: 'registerTemplate' } }">
-            <a-row>
-              <a-col style="margin-right: 12px">
-                <a-avatar :style="{ backgroundColor: $config.theme['@primary-color'] }">
-                  <template #icon>
-                    <picture-outlined />
-                  </template>
-                </a-avatar>
-              </a-col>
-              <a-col>
-                <h3 style="margin-bottom: 0px;">
-                  {{ $t('label.templatename') }}
-                </h3>
-                <small>{{ $t('label.action.register.template') }}</small>
-              </a-col>
-            </a-row>
-          </router-link>
-        </a-menu-item>
-        <a-menu-item style="width: 100%; padding: 12px" v-if="'deployVnfAppliance' in $store.getters.apis">
-          <router-link :to="{ path: '/action/deployVnfAppliance'}">
-            <a-row>
-              <a-col style="margin-right: 12px">
-                <a-avatar :style="{ backgroundColor: $config.theme['@primary-color'] }">
-                  <template #icon>
-                    <font-awesome-icon :icon="['fa-solid', 'fa-dharmachakra']" />
-                  </template>
-                </a-avatar>
-              </a-col>
-              <a-col>
-                <h3 style="margin-bottom: 0px;">
-                  {{ $t('label.vnf.appliance') }}
-                </h3>
-                <small>{{ $t('label.vnf.appliance.add') }}</small>
+                <small>{{ menuItem.subtitle }}</small>
               </a-col>
             </a-row>
           </router-link>
@@ -165,7 +52,59 @@
 
 export default {
   name: 'CreateMenu',
-  components: {
+  beforeCreate () {
+    const menuItems = [
+      {
+        api: 'deployVirtualMachine',
+        title: this.$t('label.instance'),
+        subtitle: this.$t('label.create.instance'),
+        icon: 'cloud-server-outlined',
+        route: { path: '/action/deployVirtualMachine' }
+      },
+      {
+        api: 'createKubernetesCluster',
+        title: this.$t('label.kubernetes'),
+        subtitle: this.$t('label.kubernetes.cluster.create'),
+        icon: ['fa-solid', 'fa-dharmachakra'],
+        route: { path: '/kubernetes', query: { action: 'createKubernetesCluster' } }
+      },
+      {
+        api: 'createVolume',
+        title: this.$t('label.volume'),
+        subtitle: this.$t('label.action.create.volume'),
+        icon: 'hdd-outlined',
+        route: { path: '/volume', query: { action: 'createVolume' } }
+      },
+      {
+        api: 'createNetwork',
+        title: this.$t('label.network'),
+        subtitle: this.$t('label.add.network'),
+        icon: 'apartment-outlined',
+        route: { path: '/guestnetwork', query: { action: 'createNetwork' } }
+      },
+      {
+        api: 'createVPC',
+        title: this.$t('label.vpc'),
+        subtitle: this.$t('label.add.vpc'),
+        icon: 'deployment-unit-outlined',
+        route: { path: '/vpc', query: { action: 'createVPC' } }
+      },
+      {
+        api: 'registerTemplate',
+        title: this.$t('label.templatename'),
+        subtitle: this.$t('label.action.register.template'),
+        icon: 'picture-outlined',
+        route: { path: '/template', query: { action: 'registerTemplate' } }
+      },
+      {
+        api: 'deployVnfAppliance',
+        title: this.$t('label.vnf.appliance'),
+        subtitle: this.$t('label.vnf.appliance.add'),
+        icon: 'gateway-outlined',
+        route: { path: '/action/deployVnfAppliance' }
+      }
+    ]
+    this.accessibleCreateActions = menuItems.filter(m => m.api in this.$store.getters.apis)
   }
 }
 </script>

From 43ab8a936747fbd34d3e28fece69a78a97be0ea6 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek.mrt22@gmail.com>
Date: Tue, 11 Jun 2024 14:36:11 +0530
Subject: [PATCH 05/20] cks,ui: fix npe and check for disable zone (#9105)

Fixes #8962
---
 .../version/KubernetesVersionManagerImpl.java | 22 ++++--
 .../KubernetesVersionManagerImplTest.java     | 73 +++++++++++++++++++
 .../views/compute/CreateKubernetesCluster.vue |  3 +-
 3 files changed, 90 insertions(+), 8 deletions(-)
 create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java

diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java
index 3ea30291f43..b2dafbf70bc 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java
@@ -81,6 +81,20 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne
 
     public static final String MINIMUN_AUTOSCALER_SUPPORTED_VERSION = "1.15.0";
 
+    protected void updateTemplateDetailsInKubernetesSupportedVersionResponse(
+            final KubernetesSupportedVersion kubernetesSupportedVersion, KubernetesSupportedVersionResponse response) {
+        TemplateJoinVO template = templateJoinDao.findById(kubernetesSupportedVersion.getIsoId());
+        if (template == null) {
+            return;
+        }
+        response.setIsoId(template.getUuid());
+        response.setIsoName(template.getName());
+        if (template.getState() != null) {
+            response.setIsoState(template.getState().toString());
+        }
+        response.setDirectDownload(template.isDirectDownload());
+    }
+
     private KubernetesSupportedVersionResponse createKubernetesSupportedVersionResponse(final KubernetesSupportedVersion kubernetesSupportedVersion) {
         KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse();
         response.setObjectName("kubernetessupportedversion");
@@ -100,13 +114,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne
         response.setSupportsHA(compareSemanticVersions(kubernetesSupportedVersion.getSemanticVersion(),
             KubernetesClusterService.MIN_KUBERNETES_VERSION_HA_SUPPORT)>=0);
         response.setSupportsAutoscaling(versionSupportsAutoscaling(kubernetesSupportedVersion));
-        TemplateJoinVO template = templateJoinDao.findById(kubernetesSupportedVersion.getIsoId());
-        if (template != null) {
-            response.setIsoId(template.getUuid());
-            response.setIsoName(template.getName());
-            response.setIsoState(template.getState().toString());
-            response.setDirectDownload(template.isDirectDownload());
-        }
+        updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, response);
         response.setCreated(kubernetesSupportedVersion.getCreated());
         return response;
     }
diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java
new file mode 100644
index 00000000000..ee371df3b72
--- /dev/null
+++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.kubernetes.version;
+
+import java.util.UUID;
+
+import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import com.cloud.api.query.dao.TemplateJoinDao;
+import com.cloud.api.query.vo.TemplateJoinVO;
+
+@RunWith(MockitoJUnitRunner.class)
+public class KubernetesVersionManagerImplTest {
+
+    @Mock
+    TemplateJoinDao templateJoinDao;
+
+    @InjectMocks
+    KubernetesVersionManagerImpl kubernetesVersionManager = new KubernetesVersionManagerImpl();
+
+    @Test
+    public void testUpdateTemplateDetailsInKubernetesSupportedVersionResponseNullTemplate() {
+        KubernetesSupportedVersion kubernetesSupportedVersion = Mockito.mock(KubernetesSupportedVersion.class);
+        Mockito.when(kubernetesSupportedVersion.getIsoId()).thenReturn(1L);
+        KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse();
+        kubernetesVersionManager.updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion,
+                response);
+        Assert.assertNull(ReflectionTestUtils.getField(response, "isoId"));
+    }
+
+    @Test
+    public void testUpdateTemplateDetailsInKubernetesSupportedVersionResponseValidTemplate() {
+        KubernetesSupportedVersion kubernetesSupportedVersion = Mockito.mock(KubernetesSupportedVersion.class);
+        Mockito.when(kubernetesSupportedVersion.getIsoId()).thenReturn(1L);
+        KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse();
+        TemplateJoinVO templateJoinVO = Mockito.mock(TemplateJoinVO.class);
+        String uuid = UUID.randomUUID().toString();
+        Mockito.when(templateJoinVO.getUuid()).thenReturn(uuid);
+        Mockito.when(templateJoinDao.findById(1L)).thenReturn(templateJoinVO);
+        kubernetesVersionManager.updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion,
+                response);
+        Assert.assertEquals(uuid, ReflectionTestUtils.getField(response, "isoId"));
+        Assert.assertNull(ReflectionTestUtils.getField(response, "isoState"));
+        ObjectInDataStoreStateMachine.State state = ObjectInDataStoreStateMachine.State.Ready;
+        Mockito.when(templateJoinVO.getState()).thenReturn(state);
+        kubernetesVersionManager.updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion,
+                response);
+        Assert.assertEquals(state.toString(), ReflectionTestUtils.getField(response, "isoState"));
+    }
+}
diff --git a/ui/src/views/compute/CreateKubernetesCluster.vue b/ui/src/views/compute/CreateKubernetesCluster.vue
index ca1e424cca4..e701ef4d507 100644
--- a/ui/src/views/compute/CreateKubernetesCluster.vue
+++ b/ui/src/views/compute/CreateKubernetesCluster.vue
@@ -332,8 +332,9 @@ export default {
       this.zoneLoading = true
       params.showicon = true
       api('listZones', params).then(json => {
-        const listZones = json.listzonesresponse.zone
+        var listZones = json.listzonesresponse.zone
         if (listZones) {
+          listZones = listZones.filter(x => x.allocationstate === 'Enabled')
           this.zones = this.zones.concat(listZones)
         }
       }).finally(() => {

From acae5c5b9ebf2961b05f4b092e8b6aaa7cca9fb4 Mon Sep 17 00:00:00 2001
From: Harikrishna <harikrishna.patnala@gmail.com>
Date: Tue, 11 Jun 2024 14:44:46 +0530
Subject: [PATCH 06/20] kvm: Update the java doc for the method
 disconnectPhysicalDiskByPath (#9210)

This PR addresses the issue #8789

The original issue is disconnectPhysicalDiskByPath() implementation in FibreChannelAdaptor always returns true irrespective of the success of the operation. This was already fixed in the PR #8889 .

Ideally this method has to be called after choosing the right adapter based on the storage pool type of the volume path, but currently it is just called in a loop.
https://github.com/shapeblue/cloudstack/blob/05b9b6e2e77b9b633f0fe22aff5f9f3c047b2303/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java#L200-L212

while trying to fix the case of running into the loop of all adapters by somehow passing the storage pool type to that caller cleanup() method but this is touching all over the code (which I fear it creates other regressions), instead I feel we can keep it the current way only since Fibrechannel adapter has already fixed.

In this PR I've added the java doc explaining the method and situation.
---
 .../hypervisor/kvm/storage/StorageAdaptor.java      | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java
index ecf8691c6ed..efb2821b9d9 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java
@@ -50,8 +50,17 @@ public interface StorageAdaptor {
 
     public boolean disconnectPhysicalDisk(Map<String, String> volumeToDisconnect);
 
-    // given local path to file/device (per Libvirt XML), 1) check that device is
-    // handled by your adaptor, return false if not. 2) clean up device, return true
+    /**
+     * Given local path to file/device (per Libvirt XML),
+     * 1) Make sure to check that device is handled by your adaptor, return false if not.
+     * 2) clean up device, return true
+     * 3) if clean up fails, then return false
+     *
+     * If the method wrongly returns true, then there are chances that disconnect will not reach the right storage adapter
+     *
+     * @param localPath path for the file/device from the disk definition per Libvirt XML.
+     * @return true if the operation is successful; false if the operation fails or the adapter fails to handle the path.
+     */
     public boolean disconnectPhysicalDiskByPath(String localPath);
 
     public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.ImageFormat format);

From 6edcf32c5eecd4a2c4289489b912a692d5f2043e Mon Sep 17 00:00:00 2001
From: Harikrishna <harikrishna.patnala@gmail.com>
Date: Tue, 11 Jun 2024 17:19:58 +0530
Subject: [PATCH 07/20] Fix vsphere storage policy when compute offering uses
 the disk offering attached to it (#9203)

---
 .../ConfigurationManagerImpl.java             | 26 ++++++++++++-------
 .../com/cloud/storage/StorageManagerImpl.java | 11 +-------
 2 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
index b59c8d018ee..170f092ecc3 100644
--- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
+++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
@@ -3269,10 +3269,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
 
         DiskOfferingVO diskOffering = null;
         if (diskOfferingId == null) {
-            diskOffering = createDiskOfferingInternal(userId, isSystem, vmType,
-                    name, cpu, ramSize, speed, displayText, typedProvisioningType, localStorageRequired,
-                    offerHA, limitResourceUse, volatileVm, tags, domainIds, zoneIds, hostTag,
-                    networkRate, deploymentPlanner, details, rootDiskSizeInGiB, isCustomizedIops, minIops, maxIops,
+            diskOffering = createDiskOfferingInternal(
+                    name, displayText, typedProvisioningType, localStorageRequired,
+                    tags, details, rootDiskSizeInGiB, isCustomizedIops, minIops, maxIops,
                     bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength,
                     bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength,
                     iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength,
@@ -3280,6 +3279,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                     hypervisorSnapshotReserve, cacheMode, storagePolicyID, encryptRoot);
         } else {
             diskOffering = _diskOfferingDao.findById(diskOfferingId);
+            String diskStoragePolicyId = diskOfferingDetailsDao.getDetail(diskOfferingId, ApiConstants.STORAGE_POLICY);
+            if (storagePolicyID != null && diskStoragePolicyId != null) {
+                throw new InvalidParameterValueException("Storage policy cannot be defined on both compute and disk offering");
+            }
         }
         if (diskOffering != null) {
             serviceOffering.setDiskOfferingId(diskOffering.getId());
@@ -3319,10 +3322,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         }
     }
 
-    private DiskOfferingVO createDiskOfferingInternal(final long userId, final boolean isSystem, final VirtualMachine.Type vmType,
-                                                      final String name, final Integer cpu, final Integer ramSize, final Integer speed, final String displayText, final ProvisioningType typedProvisioningType, final boolean localStorageRequired,
-                                                      final boolean offerHA, final boolean limitResourceUse, final boolean volatileVm, String tags, final List<Long> domainIds, List<Long> zoneIds, final String hostTag,
-                                                      final Integer networkRate, final String deploymentPlanner, final Map<String, String> details, Long rootDiskSizeInGiB, final Boolean isCustomizedIops, Long minIops, Long maxIops,
+    private DiskOfferingVO createDiskOfferingInternal(final String name, final String displayText, final ProvisioningType typedProvisioningType, final boolean localStorageRequired,
+                                                      String tags, final Map<String, String> details, Long rootDiskSizeInGiB, final Boolean isCustomizedIops, Long minIops, Long maxIops,
                                                       Long bytesReadRate, Long bytesReadRateMax, Long bytesReadRateMaxLength,
                                                       Long bytesWriteRate, Long bytesWriteRateMax, Long bytesWriteRateMaxLength,
                                                       Long iopsReadRate, Long iopsReadRateMax, Long iopsReadRateMaxLength,
@@ -3383,8 +3384,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         diskOffering.setHypervisorSnapshotReserve(hypervisorSnapshotReserve);
 
         if ((diskOffering = _diskOfferingDao.persist(diskOffering)) != null) {
-            if (details != null && !details.isEmpty()) {
-                List<DiskOfferingDetailVO> diskDetailsVO = new ArrayList<DiskOfferingDetailVO>();
+            if ((details != null && !details.isEmpty()) || (storagePolicyID != null)) {
+                List<DiskOfferingDetailVO> diskDetailsVO = new ArrayList<>();
                 // Support disk offering details for below parameters
                 if (details.containsKey(Volume.BANDWIDTH_LIMIT_IN_MBPS)) {
                     diskDetailsVO.add(new DiskOfferingDetailVO(diskOffering.getId(), Volume.BANDWIDTH_LIMIT_IN_MBPS, details.get(Volume.BANDWIDTH_LIMIT_IN_MBPS), false));
@@ -3392,6 +3393,11 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                 if (details.containsKey(Volume.IOPS_LIMIT)) {
                     diskDetailsVO.add(new DiskOfferingDetailVO(diskOffering.getId(), Volume.IOPS_LIMIT, details.get(Volume.IOPS_LIMIT), false));
                 }
+
+                if (storagePolicyID != null) {
+                    diskDetailsVO.add(new DiskOfferingDetailVO(diskOffering.getId(), ApiConstants.STORAGE_POLICY, String.valueOf(storagePolicyID), false));
+                }
+
                 if (!diskDetailsVO.isEmpty()) {
                     diskOfferingDetailsDao.saveDetails(diskDetailsVO);
                 }
diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java
index 3f5054bc0e8..fd6813d9d38 100644
--- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java
+++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java
@@ -2824,18 +2824,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         List<Pair<Volume, Answer>> answers = new ArrayList<Pair<Volume, Answer>>();
 
         for (Pair<Volume, DiskProfile> volumeDiskProfilePair : volumes) {
-            String storagePolicyId = null;
             Volume volume = volumeDiskProfilePair.first();
             DiskProfile diskProfile = volumeDiskProfilePair.second();
-            if (volume.getVolumeType() == Type.ROOT) {
-                Long vmId = volume.getInstanceId();
-                if (vmId != null) {
-                    VMInstanceVO vm = _vmInstanceDao.findByIdIncludingRemoved(vmId);
-                    storagePolicyId = _serviceOfferingDetailsDao.getDetail(vm.getServiceOfferingId(), ApiConstants.STORAGE_POLICY);
-                }
-            } else {
-                storagePolicyId = _diskOfferingDetailsDao.getDetail(diskProfile.getDiskOfferingId(), ApiConstants.STORAGE_POLICY);
-            }
+            String storagePolicyId = _diskOfferingDetailsDao.getDetail(diskProfile.getDiskOfferingId(), ApiConstants.STORAGE_POLICY);
             if (StringUtils.isNotEmpty(storagePolicyId)) {
                 VsphereStoragePolicyVO storagePolicyVO = _vsphereStoragePolicyDao.findById(Long.parseLong(storagePolicyId));
                 List<Long> hostIds = getUpHostsInPool(pool.getId());

From 2e3f76ec031e732648b75781e15b030d12533a2b Mon Sep 17 00:00:00 2001
From: Suresh Kumar Anaparti <sureshkumar.anaparti@gmail.com>
Date: Tue, 11 Jun 2024 18:18:21 +0530
Subject: [PATCH 08/20] Improve error messaging / logs when listing VMs on the
 remote KVM host (for import) (#9204)

---
 .../LibvirtGetRemoteVmsCommandWrapper.java    | 27 +++++++++----------
 .../vm/UnmanagedVMsManagerImpl.java           | 14 +++++-----
 2 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java
index 700f058b59b..69802bb845f 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java
@@ -50,9 +50,7 @@ public final class LibvirtGetRemoteVmsCommandWrapper extends CommandWrapper<GetR
 
     @Override
     public Answer execute(final GetRemoteVmsCommand command, final LibvirtComputingResource libvirtComputingResource) {
-        String result = null;
-        String hypervisorURI = "qemu+tcp://" + command.getRemoteIp() +
-                "/system";
+        String hypervisorURI = "qemu+tcp://" + command.getRemoteIp() + "/system";
         HashMap<String, UnmanagedInstanceTO> unmanagedInstances = new HashMap<>();
         try {
             Connect conn = LibvirtConnection.getConnection(hypervisorURI);
@@ -61,26 +59,27 @@ public final class LibvirtGetRemoteVmsCommandWrapper extends CommandWrapper<GetR
                 final Domain domain = libvirtComputingResource.getDomain(conn, name);
 
                 final DomainInfo.DomainState ps = domain.getInfo().state;
-
                 final VirtualMachine.PowerState state = libvirtComputingResource.convertToPowerState(ps);
-
-                s_logger.debug("VM " + domain.getName() + ": powerstate = " + ps + "; vm state=" + state.toString());
+                s_logger.debug("VM " + domain.getName() + " - powerstate: " + ps + ", state: " + state.toString());
 
                 if (state == VirtualMachine.PowerState.PowerOff) {
                     try {
                         UnmanagedInstanceTO instance = getUnmanagedInstance(libvirtComputingResource, domain, conn);
                         unmanagedInstances.put(instance.getName(), instance);
                     } catch (Exception e) {
-                        s_logger.error("Error while fetching instance details", e);
+                        s_logger.error("Couldn't fetch VM " + domain.getName() + " details, due to: " + e.getMessage(), e);
                     }
                 }
                 domain.free();
             }
-            s_logger.debug("Found Vms: "+ unmanagedInstances.size());
-            return  new GetRemoteVmsAnswer(command, "", unmanagedInstances);
+            s_logger.debug("Found " + unmanagedInstances.size() + " stopped VMs on host " + command.getRemoteIp());
+            return new GetRemoteVmsAnswer(command, "", unmanagedInstances);
         } catch (final LibvirtException e) {
-            s_logger.error("Error while listing stopped Vms on remote host: "+ e.getMessage());
-            return new Answer(command, false, result);
+            s_logger.error("Failed to list stopped VMs on remote host " + command.getRemoteIp() + ", due to: " + e.getMessage(), e);
+            if (e.getMessage().toLowerCase().contains("connection refused")) {
+                return new Answer(command, false, "Unable to connect to remote host " + command.getRemoteIp() + ", please check the libvirtd tcp connectivity and retry");
+            }
+            return new Answer(command, false, "Unable to list stopped VMs on remote host " + command.getRemoteIp() + ", due to: " + e.getMessage());
         }
     }
 
@@ -106,8 +105,8 @@ public final class LibvirtGetRemoteVmsCommandWrapper extends CommandWrapper<GetR
 
             return instance;
         } catch (Exception e) {
-            s_logger.debug("Unable to retrieve unmanaged instance info. ", e);
-            throw new CloudRuntimeException("Unable to retrieve unmanaged instance info. " + e.getMessage());
+            s_logger.debug("Unable to retrieve unmanaged instance info,  due to: " + e.getMessage(), e);
+            throw new CloudRuntimeException("Unable to retrieve unmanaged instance info, due to: " + e.getMessage());
         }
     }
 
@@ -119,7 +118,6 @@ public final class LibvirtGetRemoteVmsCommandWrapper extends CommandWrapper<GetR
                 return UnmanagedInstanceTO.PowerState.PowerOff;
             default:
                 return UnmanagedInstanceTO.PowerState.PowerUnknown;
-
         }
     }
 
@@ -166,7 +164,6 @@ public final class LibvirtGetRemoteVmsCommandWrapper extends CommandWrapper<GetR
             disk.setLabel(diskDef.getDiskLabel());
             disk.setController(diskDef.getBusType().toString());
 
-
             Pair<String, String> sourceHostPath = getSourceHostPath(libvirtComputingResource, diskDef.getSourcePath());
             if (sourceHostPath != null) {
                 disk.setDatastoreHost(sourceHostPath.first());
diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java
index 7dc000c7375..2a177b2b042 100644
--- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java
+++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java
@@ -2046,10 +2046,10 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
                 throw new InvalidParameterValueException("Username need to be provided.");
             }
 
-            HashMap<String, UnmanagedInstanceTO> instancesMap = getRemoteVms(zoneId, remoteUrl, cmd.getUsername(), cmd.getPassword());
+            HashMap<String, UnmanagedInstanceTO> instancesMap = getRemoteVmsOnKVMHost(zoneId, remoteUrl, cmd.getUsername(), cmd.getPassword());
             unmanagedInstanceTO = instancesMap.get(cmd.getName());
             if (unmanagedInstanceTO == null) {
-                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Vm with name: %s not found on remote host", instanceName));
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("VM with name: %s not found on remote host %s", instanceName, remoteUrl));
             }
         }
 
@@ -2460,7 +2460,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
         }
 
         List<UnmanagedInstanceResponse> responses = new ArrayList<>();
-        HashMap<String, UnmanagedInstanceTO> vmMap = getRemoteVms(zoneId, cmd.getHost(), cmd.getUsername(), cmd.getPassword());
+        HashMap<String, UnmanagedInstanceTO> vmMap = getRemoteVmsOnKVMHost(zoneId, cmd.getHost(), cmd.getUsername(), cmd.getPassword());
         for (String key : vmMap.keySet()) {
             UnmanagedInstanceTO instance = vmMap.get(key);
             if (StringUtils.isNotEmpty(keyword) &&
@@ -2475,17 +2475,17 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
         return listResponses;
     }
 
-    private HashMap<String, UnmanagedInstanceTO> getRemoteVms(long zoneId, String remoteUrl, String username, String password) {
+    private HashMap<String, UnmanagedInstanceTO> getRemoteVmsOnKVMHost(long zoneId, String remoteHostUrl, String username, String password) {
         //ToDo: add option to list one Vm by name
         List<HostVO> hosts = resourceManager.listAllUpAndEnabledHostsInOneZoneByHypervisor(Hypervisor.HypervisorType.KVM, zoneId);
         if(hosts.size() < 1) {
-            throw new CloudRuntimeException("No hosts available for Vm Import");
+            throw new CloudRuntimeException("No hosts available for VM import");
         }
         HostVO host = hosts.get(0);
-        GetRemoteVmsCommand getRemoteVmsCommand = new GetRemoteVmsCommand(remoteUrl, username, password);
+        GetRemoteVmsCommand getRemoteVmsCommand = new GetRemoteVmsCommand(remoteHostUrl, username, password);
         Answer answer = agentManager.easySend(host.getId(), getRemoteVmsCommand);
         if (!(answer instanceof GetRemoteVmsAnswer)) {
-            throw new CloudRuntimeException("Error while listing remote Vms");
+            throw new CloudRuntimeException("Failed to list VMs, due to: " + answer.getDetails());
         }
         GetRemoteVmsAnswer getRemoteVmsAnswer = (GetRemoteVmsAnswer) answer;
         return getRemoteVmsAnswer.getUnmanagedInstances();

From edf7394919fa29b2e0c2ebec5b624b5dde7dedf6 Mon Sep 17 00:00:00 2001
From: Wei Zhou <weizhou@apache.org>
Date: Tue, 11 Jun 2024 16:05:43 +0200
Subject: [PATCH 09/20] server: search template name and display text if
 keyword is passed (#9215)

---
 .../main/java/com/cloud/api/query/QueryManagerImpl.java   | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
index dc591e8f30c..bba5c02151a 100644
--- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
+++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
@@ -4637,8 +4637,12 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         // other criteria
 
         if (keyword != null) {
-            sc.addAnd("name", SearchCriteria.Op.LIKE, "%" + keyword + "%");
-        } else if (name != null) {
+            SearchCriteria<TemplateJoinVO> scc = _templateJoinDao.createSearchCriteria();
+            scc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%");
+            scc.addOr("displayText", SearchCriteria.Op.LIKE, "%" + keyword + "%");
+            sc.addAnd("name", SearchCriteria.Op.SC, scc);
+        }
+        if (name != null) {
             sc.addAnd("name", SearchCriteria.Op.EQ, name);
         }
 

From ca59720015661f21045a25cb3f4e152dd8721a59 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek.mrt22@gmail.com>
Date: Wed, 12 Jun 2024 12:56:08 +0530
Subject: [PATCH 10/20] vpc,event: fix events for createVpc (#9055)

Fixes #8496

* vpc,event: fix events for createVpc
---
 .../main/java/com/cloud/network/vpc/VpcService.java   |  2 ++
 .../cloudstack/api/command/user/vpc/CreateVPCCmd.java |  6 +-----
 .../api/command/user/vpc/CreateVPCCmdTest.java        |  5 ++---
 .../java/com/cloud/network/vpc/VpcManagerImpl.java    | 11 +++++++++++
 4 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/api/src/main/java/com/cloud/network/vpc/VpcService.java b/api/src/main/java/com/cloud/network/vpc/VpcService.java
index 2cdc034a16e..0f0d29f4082 100644
--- a/api/src/main/java/com/cloud/network/vpc/VpcService.java
+++ b/api/src/main/java/com/cloud/network/vpc/VpcService.java
@@ -132,6 +132,8 @@ public interface VpcService {
      */
     boolean startVpc(long vpcId, boolean destroyOnFailure) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;
 
+    void startVpc(CreateVPCCmd cmd) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException;
+
     /**
      * Shuts down the VPC which includes shutting down all VPC provider and rules cleanup on the backend
      *
diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java
index 7ca66b2a471..39f2b32751d 100644
--- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java
@@ -210,11 +210,7 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd {
     public void execute() {
         Vpc vpc = null;
         try {
-            if (isStart()) {
-                _vpcService.startVpc(getEntityId(), true);
-            } else {
-                s_logger.debug("Not starting VPC as " + ApiConstants.START + "=false was passed to the API");
-             }
+            _vpcService.startVpc(this);
             vpc = _entityMgr.findById(Vpc.class, getEntityId());
         } catch (ResourceUnavailableException ex) {
             s_logger.warn("Exception: ", ex);
diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmdTest.java
index 79f27fd6687..a28e9e9fd04 100644
--- a/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmdTest.java
+++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmdTest.java
@@ -167,17 +167,16 @@ public class CreateVPCCmdTest extends TestCase {
 
     @Test
     public void testExecute() throws ResourceUnavailableException, InsufficientCapacityException {
-        ReflectionTestUtils.setField(cmd, "start", true);
         Vpc vpc = Mockito.mock(Vpc.class);
         VpcResponse response = Mockito.mock(VpcResponse.class);
 
         ReflectionTestUtils.setField(cmd, "id", 1L);
         responseGenerator = Mockito.mock(ResponseGenerator.class);
-        Mockito.when(_vpcService.startVpc(1L, true)).thenReturn(true);
+        Mockito.doNothing().when(_vpcService).startVpc(cmd);
         Mockito.when(_entityMgr.findById(Mockito.eq(Vpc.class), Mockito.any(Long.class))).thenReturn(vpc);
         cmd._responseGenerator = responseGenerator;
         Mockito.when(responseGenerator.createVpcResponse(ResponseObject.ResponseView.Restricted, vpc)).thenReturn(response);
         cmd.execute();
-        Mockito.verify(_vpcService, Mockito.times(1)).startVpc(Mockito.anyLong(), Mockito.anyBoolean());
+        Mockito.verify(_vpcService, Mockito.times(1)).startVpc(cmd);
     }
 }
diff --git a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java
index 58098ee3832..c9391a32167 100644
--- a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java
@@ -1677,6 +1677,17 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
         return result;
     }
 
+
+    @Override
+    @ActionEvent(eventType = EventTypes.EVENT_VPC_CREATE, eventDescription = "creating vpc", async = true)
+    public void startVpc(final CreateVPCCmd cmd) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException {
+        if (!cmd.isStart()) {
+            s_logger.debug("Not starting VPC as " + ApiConstants.START + "=false was passed to the API");
+            return;
+        }
+        startVpc(cmd.getEntityId(), true);
+    }
+
     protected boolean startVpc(final Vpc vpc, final DeployDestination dest, final ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException,
             InsufficientCapacityException {
         // deploy provider

From 15772189990b2f65923178626ac0b57829714aab Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek.mrt22@gmail.com>
Date: Wed, 12 Jun 2024 13:13:13 +0530
Subject: [PATCH 11/20] server,event: fix events for storage maintenance
 (#9056)

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
---
 .../src/main/java/com/cloud/storage/StorageManagerImpl.java   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java
index fd6813d9d38..adece636501 100644
--- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java
+++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java
@@ -1924,6 +1924,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
     @Override
     @DB
+    @ActionEvent(eventType = EventTypes.EVENT_MAINTENANCE_PREPARE_PRIMARY_STORAGE,
+            eventDescription = "preparing storage pool for maintenance", async = true)
     public PrimaryDataStoreInfo preparePrimaryStorageForMaintenance(Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException {
         StoragePoolVO primaryStorage = null;
         primaryStorage = _storagePoolDao.findById(primaryStorageId);
@@ -1992,6 +1994,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
     @Override
     @DB
+    @ActionEvent(eventType = EventTypes.EVENT_MAINTENANCE_CANCEL_PRIMARY_STORAGE,
+            eventDescription = "canceling maintenance for primary storage pool", async = true)
     public PrimaryDataStoreInfo cancelPrimaryStorageForMaintenance(CancelPrimaryStorageMaintenanceCmd cmd) throws ResourceUnavailableException {
         Long primaryStorageId = cmd.getId();
         StoragePoolVO primaryStorage = null;

From 4ec0f823cf39c0b00cb935c8dab183283b7e7b9e Mon Sep 17 00:00:00 2001
From: Suresh Kumar Anaparti <suresh.anaparti@shapeblue.com>
Date: Wed, 12 Jun 2024 13:58:10 +0530
Subject: [PATCH 12/20] ScaleIO volume live migration - use usable bytes from
 source disk to format the destination disk (#9174)

---
 .../cloud/agent/api/GetVolumeStatAnswer.java  | 85 +++++++++++++++++++
 .../cloud/agent/api/GetVolumeStatCommand.java | 72 ++++++++++++++++
 .../cloudstack/storage/to/VolumeObjectTO.java |  9 ++
 .../LibvirtGetVolumeStatCommandWrapper.java   | 66 ++++++++++++++
 .../LibvirtMigrateVolumeCommandWrapper.java   | 23 +++--
 .../kvm/storage/KVMStoragePool.java           |  3 +
 .../kvm/storage/KVMStorageProcessor.java      |  2 +-
 .../kvm/storage/ScaleIOStorageAdaptor.java    | 15 +++-
 .../kvm/storage/ScaleIOStoragePool.java       |  5 ++
 .../kvm/storage/StorageAdaptor.java           |  5 ++
 .../driver/ScaleIOPrimaryDataStoreDriver.java | 23 ++++-
 .../ScaleIOPrimaryDataStoreDriverTest.java    | 31 +++++--
 12 files changed, 317 insertions(+), 22 deletions(-)
 create mode 100644 core/src/main/java/com/cloud/agent/api/GetVolumeStatAnswer.java
 create mode 100644 core/src/main/java/com/cloud/agent/api/GetVolumeStatCommand.java
 create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumeStatCommandWrapper.java

diff --git a/core/src/main/java/com/cloud/agent/api/GetVolumeStatAnswer.java b/core/src/main/java/com/cloud/agent/api/GetVolumeStatAnswer.java
new file mode 100644
index 00000000000..8352c97c108
--- /dev/null
+++ b/core/src/main/java/com/cloud/agent/api/GetVolumeStatAnswer.java
@@ -0,0 +1,85 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.LogLevel.Log4jLevel;
+import com.cloud.storage.Storage.StoragePoolType;
+
+@LogLevel(Log4jLevel.Trace)
+public class GetVolumeStatAnswer extends Answer {
+    String poolUuid;
+    StoragePoolType poolType;
+    String volumePath;
+    long size = 0;
+    long virtualSize = 0;
+
+    public GetVolumeStatAnswer(GetVolumeStatCommand cmd, long size, long virtualSize) {
+        super(cmd, true, "");
+        this.poolUuid = cmd.getPoolUuid();
+        this.poolType = cmd.getPoolType();
+        this.volumePath = cmd.getVolumePath();
+        this.size = size;
+        this.virtualSize = virtualSize;
+    }
+
+    public GetVolumeStatAnswer(GetVolumeStatCommand cmd, boolean success, String details) {
+        super(cmd, success, details);
+    }
+
+    protected GetVolumeStatAnswer() {
+        //no-args constructor for json serialization-deserialization
+    }
+
+    public String getPoolUuid() {
+        return poolUuid;
+    }
+
+    public void setPoolUuid(String poolUuid) {
+        this.poolUuid = poolUuid;
+    }
+
+    public StoragePoolType getPoolType() {
+        return poolType;
+    }
+
+    public void setPoolType(StoragePoolType poolType) {
+        this.poolType = poolType;
+    }
+
+    public long getSize() {
+        return size;
+    }
+
+    public void setSize(long size) {
+        this.size = size;
+    }
+
+    public long getVirtualSize() {
+        return virtualSize;
+    }
+
+    public void setVirtualSize(long virtualSize) {
+        this.virtualSize = virtualSize;
+    }
+
+    public String getString() {
+        return "GetVolumeStatAnswer [poolUuid=" + poolUuid + ", poolType=" + poolType + ", volumePath=" + volumePath + ", size=" + size + ", virtualSize=" + virtualSize + "]";
+    }
+}
diff --git a/core/src/main/java/com/cloud/agent/api/GetVolumeStatCommand.java b/core/src/main/java/com/cloud/agent/api/GetVolumeStatCommand.java
new file mode 100644
index 00000000000..1be3d6a0419
--- /dev/null
+++ b/core/src/main/java/com/cloud/agent/api/GetVolumeStatCommand.java
@@ -0,0 +1,72 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.LogLevel.Log4jLevel;
+import com.cloud.storage.Storage.StoragePoolType;
+
+@LogLevel(Log4jLevel.Trace)
+public class GetVolumeStatCommand extends Command {
+    String volumePath;
+    StoragePoolType poolType;
+    String poolUuid;
+
+    protected GetVolumeStatCommand() {
+    }
+
+    public GetVolumeStatCommand(String volumePath, StoragePoolType poolType, String poolUuid) {
+        this.volumePath = volumePath;
+        this.poolType = poolType;
+        this.poolUuid = poolUuid;
+    }
+
+    public String getVolumePath() {
+        return volumePath;
+    }
+
+    public void setVolumePath(String volumePath) {
+        this.volumePath = volumePath;
+    }
+
+    public StoragePoolType getPoolType() {
+        return poolType;
+    }
+
+    public void setPoolType(StoragePoolType poolType) {
+        this.poolType = poolType;
+    }
+
+    public String getPoolUuid() {
+        return poolUuid;
+    }
+
+    public void setPoolUuid(String storeUuid) {
+        this.poolUuid = storeUuid;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+
+    public String getString() {
+        return "GetVolumeStatCommand [volumePath=" + volumePath + ", poolType=" + poolType + ", poolUuid=" + poolUuid + "]";
+    }
+}
diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
index 2bb67c80ce4..6514038ac62 100644
--- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
+++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
@@ -39,6 +39,7 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
     private DataStoreTO dataStore;
     private String name;
     private Long size;
+    private Long usableSize;
     private String path;
     private Long volumeId;
     private String vmName;
@@ -161,6 +162,10 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
         return size;
     }
 
+    public Long getUsableSize() {
+        return usableSize;
+    }
+
     @Override
     public DataObjectType getObjectType() {
         return DataObjectType.VOLUME;
@@ -178,6 +183,10 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
         this.size = size;
     }
 
+    public void setUsableSize(Long usableSize) {
+        this.usableSize = usableSize;
+    }
+
     public void setPath(String path) {
         this.path = path;
     }
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumeStatCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumeStatCommandWrapper.java
new file mode 100644
index 00000000000..2a9c886f449
--- /dev/null
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumeStatCommandWrapper.java
@@ -0,0 +1,66 @@
+//
+//Licensed to the Apache Software Foundation (ASF) under one
+//or more contributor license agreements.  See the NOTICE file
+//distributed with this work for additional information
+//regarding copyright ownership.  The ASF licenses this file
+//to you under the Apache License, Version 2.0 (the
+//"License"); you may not use this file except in compliance
+//with the License.  You may obtain a copy of the License at
+//
+//http://www.apache.org/licenses/LICENSE-2.0
+//
+//Unless required by applicable law or agreed to in writing,
+//software distributed under the License is distributed on an
+//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+//KIND, either express or implied.  See the License for the
+//specific language governing permissions and limitations
+//under the License.
+//
+
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.GetVolumeStatAnswer;
+import com.cloud.agent.api.GetVolumeStatCommand;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+import com.cloud.storage.Storage.StoragePoolType;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+@ResourceWrapper(handles = GetVolumeStatCommand.class)
+public final class LibvirtGetVolumeStatCommandWrapper extends CommandWrapper<GetVolumeStatCommand, Answer, LibvirtComputingResource> {
+    private static final Logger s_logger = Logger.getLogger(LibvirtGetVolumeStatCommandWrapper.class);
+
+    @Override
+    public Answer execute(final GetVolumeStatCommand cmd, final LibvirtComputingResource libvirtComputingResource) {
+        try {
+            String volumePath = cmd.getVolumePath();
+            StoragePoolType poolType = cmd.getPoolType();
+            String poolUuid = cmd.getPoolUuid();
+
+            KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
+            KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(poolType, poolUuid);
+            if (primaryPool == null) {
+                String msg = "Can't get volume stats as pool details unavailable for volume: " + volumePath + " on the storage pool: " + poolUuid;
+                return new GetVolumeStatAnswer(cmd, false, msg);
+            }
+
+            KVMPhysicalDisk disk = primaryPool.getPhysicalDisk(volumePath);
+            if (disk == null) {
+                String msg = "Can't get volume stats as disk details unavailable for volume: " + volumePath + " on the storage pool: " + poolUuid;
+                return new GetVolumeStatAnswer(cmd, false, msg);
+            }
+
+            return new GetVolumeStatAnswer(cmd, disk.getSize(), disk.getVirtualSize());
+        } catch (CloudRuntimeException e) {
+            s_logger.error("Can't get volume stats, due to: " + e.getMessage(), e);
+            return new GetVolumeStatAnswer(cmd, false, "Can't get volume stats, due to: " + e.getMessage());
+        }
+    }
+}
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java
index 2a09c340891..0b9b951122a 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java
@@ -142,7 +142,6 @@ public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<MigrateVo
             LOGGER.info(String.format("Block copy has started for the volume %s : %s ", destDiskLabel, srcPath));
 
             return checkBlockJobStatus(command, dm, destDiskLabel, srcPath, destPath, libvirtComputingResource, conn, srcSecretUUID);
-
         } catch (Exception e) {
             String msg = "Migrate volume failed due to " + e.toString();
             LOGGER.warn(msg, e);
@@ -168,17 +167,27 @@ public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<MigrateVo
     protected MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath, LibvirtComputingResource libvirtComputingResource, Connect conn, String srcSecretUUID) throws LibvirtException {
         int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found
         int waitTimeInSec = command.getWait();
+        double blockCopyProgress = 0;
         while (waitTimeInSec > 0) {
             DomainBlockJobInfo blockJobInfo = dm.getBlockJobInfo(diskLabel, 0);
             if (blockJobInfo != null) {
-                LOGGER.debug(String.format("Volume %s : %s block copy progress: %s%% current value:%s end value:%s", diskLabel, srcPath, (blockJobInfo.end == 0)? 0 : 100*(blockJobInfo.cur / (double) blockJobInfo.end), blockJobInfo.cur, blockJobInfo.end));
+                blockCopyProgress = (blockJobInfo.end == 0)? blockCopyProgress : 100 * (blockJobInfo.cur / (double) blockJobInfo.end);
+                LOGGER.debug(String.format("Volume %s : %s, block copy progress: %s%%, current value: %s end value: %s, job info - type: %s, bandwidth: %s",
+                        diskLabel, srcPath, blockCopyProgress, blockJobInfo.cur, blockJobInfo.end, blockJobInfo.type, blockJobInfo.bandwidth));
                 if (blockJobInfo.cur == blockJobInfo.end) {
-                    LOGGER.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath));
-                    dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT);
-                    if (StringUtils.isNotEmpty(srcSecretUUID)) {
-                        libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID);
+                    if (blockJobInfo.end > 0) {
+                        LOGGER.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath));
+                        dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT);
+                        if (StringUtils.isNotEmpty(srcSecretUUID)) {
+                            libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID);
+                        }
+                        break;
+                    } else {
+                        // cur = 0, end = 0 - at this point, disk does not have an active block job (so, no need to abort job)
+                        String msg = String.format("No active block copy job for the volume %s : %s - job stopped at %s progress", diskLabel, srcPath, blockCopyProgress);
+                        LOGGER.warn(msg);
+                        return new MigrateVolumeAnswer(command, false, msg, null);
                     }
-                    break;
                 }
             } else {
                 LOGGER.info("Failed to get the block copy status, trying to abort the job");
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java
index 43a09ccf2bf..e801142eb0b 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java
@@ -37,6 +37,9 @@ public interface KVMStoragePool {
     public static final long HeartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP);
     public static final long HeartBeatCheckerTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_CHECKER_TIMEOUT);
 
+    public default KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) {
+        return createPhysicalDisk(volumeUuid, format, provisioningType, size, passphrase);
+    }
 
     public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase);
 
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
index f0ce56e0f3b..fc86ddb0c1c 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
@@ -1629,7 +1629,7 @@ public class KVMStorageProcessor implements StorageProcessor {
                 }
             } else {
                 vol = primaryPool.createPhysicalDisk(volume.getUuid(), format,
-                        volume.getProvisioningType(), disksize, volume.getPassphrase());
+                        volume.getProvisioningType(), disksize, volume.getUsableSize(), volume.getPassphrase());
             }
 
             final VolumeObjectTO newVol = new VolumeObjectTO();
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
index b85be041e48..7a98e3fb11f 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
@@ -151,6 +151,11 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
         return MapStorageUuidToStoragePool.remove(uuid) != null;
     }
 
+    @Override
+    public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
+        return createPhysicalDisk(name, pool, format, provisioningType, size, null, passphrase);
+    }
+
     /**
      * ScaleIO doesn't need to communicate with the hypervisor normally to create a volume. This is used only to prepare a ScaleIO data disk for encryption.
      * Thin encrypted volumes are provisioned in QCOW2 format, which insulates the guest from zeroes/unallocated blocks in the block device that would
@@ -160,11 +165,12 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
      * @param format disk format
      * @param provisioningType provisioning type
      * @param size disk size
+     * @param usableSize usage disk size
      * @param passphrase passphrase
      * @return the disk object
      */
     @Override
-    public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
+    public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) {
         if (passphrase == null || passphrase.length == 0) {
             return null;
         }
@@ -182,7 +188,12 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
                 QemuImg qemuImg = new QemuImg(0, true, false);
                 Map<String, String> options = new HashMap<>();
                 List<QemuObject> qemuObjects = new ArrayList<>();
-                long formattedSize = getUsableBytesFromRawBytes(disk.getSize());
+                long formattedSize;
+                if (usableSize != null && usableSize > 0) {
+                    formattedSize = usableSize;
+                } else {
+                    formattedSize = getUsableBytesFromRawBytes(disk.getSize());
+                }
 
                 options.put("preallocation", QemuImg.PreallocationType.Metadata.toString());
                 qemuObjects.add(QemuObject.prepareSecretForQemuImg(disk.getFormat(), disk.getQemuEncryptFormat(), keyFile.toString(), "sec0", options));
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java
index 293ff29f984..77f21910da6 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java
@@ -72,6 +72,11 @@ public class ScaleIOStoragePool implements KVMStoragePool {
         }
     }
 
+    @Override
+    public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) {
+        return this.storageAdaptor.createPhysicalDisk(volumeUuid, this, format, provisioningType, size, usableSize, passphrase);
+    }
+
     @Override
     public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
         return this.storageAdaptor.createPhysicalDisk(volumeUuid, this, format, provisioningType, size, passphrase);
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java
index efb2821b9d9..5cfdf6d1a8b 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java
@@ -39,6 +39,11 @@ public interface StorageAdaptor {
 
     public boolean deleteStoragePool(String uuid);
 
+    public default KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool,
+                                                      PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) {
+        return createPhysicalDisk(name, pool, format, provisioningType, size, passphrase);
+    }
+
     public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool,
             PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase);
 
diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
index 1d2cace8d21..a359113ab08 100644
--- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
+++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
@@ -65,6 +65,8 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.GetVolumeStatAnswer;
+import com.cloud.agent.api.GetVolumeStatCommand;
 import com.cloud.agent.api.storage.MigrateVolumeCommand;
 import com.cloud.agent.api.storage.ResizeVolumeCommand;
 import com.cloud.agent.api.to.DataObjectType;
@@ -490,10 +492,10 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
     }
 
     public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId) {
-        return createVolume(volumeInfo, storagePoolId, false);
+        return createVolume(volumeInfo, storagePoolId, false, null);
     }
 
-    public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId, boolean migrationInvolved) {
+    public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId, boolean migrationInvolved, Long usageSize) {
         LOGGER.debug("Creating PowerFlex volume");
 
         StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId);
@@ -543,6 +545,9 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
                 VolumeObjectTO prepVolume = (VolumeObjectTO) createdObject.getTO();
                 prepVolume.setPath(volumePath);
                 prepVolume.setUuid(volumePath);
+                if (usageSize != null) {
+                    prepVolume.setUsableSize(usageSize);
+                }
                 CreateObjectCommand cmd = new CreateObjectCommand(prepVolume);
                 EndPoint ep = selector.select(volumeInfo, true);
                 if (ep == null) {
@@ -845,7 +850,8 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
         // Volume migration across different PowerFlex/ScaleIO clusters
         final long srcVolumeId = srcData.getId();
         DataStore srcStore = srcData.getDataStore();
-        Map<String, String> srcDetails = getVolumeDetails((VolumeInfo) srcData, srcStore);
+        VolumeInfo srcVolumeInfo = (VolumeInfo) srcData;
+        Map<String, String> srcDetails = getVolumeDetails(srcVolumeInfo, srcStore);
 
         DataStore destStore = destData.getDataStore();
         final long destPoolId = destStore.getId();
@@ -857,8 +863,17 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
         EndPoint ep = RemoteHostEndPoint.getHypervisorHostEndPoint(host);
 
         Answer answer = null;
+        Long srcVolumeUsableSize = null;
         try {
-            CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId(), true);
+            GetVolumeStatCommand statCmd = new GetVolumeStatCommand(srcVolumeInfo.getPath(), srcVolumeInfo.getStoragePoolType(), srcStore.getUuid());
+            GetVolumeStatAnswer statAnswer = (GetVolumeStatAnswer) ep.sendMessage(statCmd);
+            if (!statAnswer.getResult() ) {
+                LOGGER.warn(String.format("Unable to get volume %s stats", srcVolumeInfo.getId()));
+            } else if (statAnswer.getVirtualSize() > 0) {
+                srcVolumeUsableSize = statAnswer.getVirtualSize();
+            }
+
+            CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId(), true, srcVolumeUsableSize);
             destVolumePath = createAnswer.getData().getPath();
             destVolTO.setPath(destVolumePath);
 
diff --git a/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriverTest.java b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriverTest.java
index 1852ec1c0bd..cc1823dd782 100644
--- a/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriverTest.java
+++ b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriverTest.java
@@ -20,7 +20,10 @@
 package org.apache.cloudstack.storage.datastore.driver;
 
 import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.GetVolumeStatAnswer;
+import com.cloud.agent.api.GetVolumeStatCommand;
 import com.cloud.agent.api.storage.MigrateVolumeAnswer;
+import com.cloud.agent.api.storage.MigrateVolumeCommand;
 import com.cloud.agent.api.to.DataTO;
 import com.cloud.agent.api.to.DiskTO;
 import com.cloud.configuration.Config;
@@ -196,16 +199,23 @@ public class ScaleIOPrimaryDataStoreDriverTest {
         RemoteHostEndPoint ep = Mockito.mock(RemoteHostEndPoint.class);
         remoteHostEndPointMock.when(() -> RemoteHostEndPoint.getHypervisorHostEndPoint(host)).thenReturn(ep);
 
+        long volumeVirtualSize = 68673077248L;
         DataTO dataTO = Mockito.mock(DataTO.class);
         CreateObjectAnswer createAnswer = new CreateObjectAnswer(dataTO);
-        doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true);
+        doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true, volumeVirtualSize);
         when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee");
         doReturn(true).when(scaleIOPrimaryDataStoreDriver)
                 .grantAccess(any(), any(), any());
 
         when(configDao.getValue(Config.MigrateWait.key())).thenReturn("3600");
+
+        GetVolumeStatAnswer getVolumeStatAnswer = Mockito.mock(GetVolumeStatAnswer.class);
+        when(ep.sendMessage(any(GetVolumeStatCommand.class))).thenReturn(getVolumeStatAnswer);
+        when(getVolumeStatAnswer.getResult()).thenReturn(true);
+        when(getVolumeStatAnswer.getVirtualSize()).thenReturn(volumeVirtualSize);
+
         MigrateVolumeAnswer migrateVolumeAnswer = Mockito.mock(MigrateVolumeAnswer.class);
-        when(ep.sendMessage(any())).thenReturn(migrateVolumeAnswer);
+        when(ep.sendMessage(any(MigrateVolumeCommand.class))).thenReturn(migrateVolumeAnswer);
         when(migrateVolumeAnswer.getResult()).thenReturn(true);
 
         Mockito.doNothing().when(scaleIOPrimaryDataStoreDriver)
@@ -242,16 +252,21 @@ public class ScaleIOPrimaryDataStoreDriverTest {
 
         DataTO dataTO = Mockito.mock(DataTO.class);
         CreateObjectAnswer createAnswer = new CreateObjectAnswer(dataTO);
-        doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true);
-        when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee");
-        doReturn(true).when(scaleIOPrimaryDataStoreDriver)
+        Mockito.lenient().doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true, null);
+        Mockito.lenient().when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee");
+        Mockito.lenient().doReturn(true).when(scaleIOPrimaryDataStoreDriver)
                 .grantAccess(any(), any(), any());
 
-        when(configDao.getValue(Config.MigrateWait.key())).thenReturn("3600");
+        Mockito.lenient().when(configDao.getValue(Config.MigrateWait.key())).thenReturn("3600");
+
+        GetVolumeStatAnswer getVolumeStatAnswer = Mockito.mock(GetVolumeStatAnswer.class);
+        Mockito.lenient().when(ep.sendMessage(any(GetVolumeStatCommand.class))).thenReturn(getVolumeStatAnswer);
+        Mockito.lenient().when(getVolumeStatAnswer.getResult()).thenReturn(false);
+
         MigrateVolumeAnswer migrateVolumeAnswer = Mockito.mock(MigrateVolumeAnswer.class);
         when(ep.sendMessage(any())).thenReturn(migrateVolumeAnswer);
-        when(migrateVolumeAnswer.getResult()).thenReturn(false);
-        Mockito.doNothing().when(scaleIOPrimaryDataStoreDriver)
+        Mockito.lenient().when(migrateVolumeAnswer.getResult()).thenReturn(false);
+        Mockito.lenient().doNothing().when(scaleIOPrimaryDataStoreDriver)
                 .revertBlockCopyVolumeOperations(any(), any(), any(), any());
 
         Answer answer = scaleIOPrimaryDataStoreDriver.liveMigrateVolume(srcData, destData);

From 0f770194df7962c5fbd66c06d57a120e7f591e04 Mon Sep 17 00:00:00 2001
From: Pearl Dsilva <pearl1594@gmail.com>
Date: Wed, 12 Jun 2024 05:00:08 -0400
Subject: [PATCH 13/20] Configure config drive ISO with password file on
 reinstallation of VM  with password-enabled template (#9113)

* Configure config drive ISO with password file on reinstallation of VM with password-enabled template

* address comment
---
 server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
index d9484761998..4d966d8e745 100644
--- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
+++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
@@ -7957,7 +7957,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         }
         if (needRestart) {
             try {
-                if (vm.getDetail(VmDetailConstants.PASSWORD) != null) {
+                if (Objects.nonNull(password)) {
                     params = new HashMap<>();
                     params.put(VirtualMachineProfile.Param.VmPassword, password);
                 }

From 503ae64a3db711c8cffc5e41251f46706a83beb5 Mon Sep 17 00:00:00 2001
From: Suresh Kumar Anaparti <sureshkumar.anaparti@gmail.com>
Date: Wed, 12 Jun 2024 15:10:29 +0530
Subject: [PATCH 14/20] Upload volume format check with the supported
 hypervisor, fail if supported hypervisor not found on zone (#9224)

---
 .../com/cloud/storage/VolumeApiServiceImpl.java    | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
index 02add0db174..1cf069feae8 100644
--- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
+++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
@@ -158,6 +158,7 @@ import com.cloud.offering.DiskOffering;
 import com.cloud.org.Grouping;
 import com.cloud.projects.Project;
 import com.cloud.projects.ProjectManager;
+import com.cloud.resource.ResourceManager;
 import com.cloud.resource.ResourceState;
 import com.cloud.serializer.GsonHelper;
 import com.cloud.server.ManagementService;
@@ -258,6 +259,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
     @Inject
     private ConfigurationManager _configMgr;
     @Inject
+    private ResourceManager _resourceMgr;
+    @Inject
     private VolumeDao _volsDao;
     @Inject
     private VolumeDetailsDao _volsDetailsDao;
@@ -564,7 +567,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
             _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage);
         }
 
-        sanitizeFormat(format);
+        checkFormatWithSupportedHypervisorsInZone(format, zoneId);
 
         // Check that the disk offering specified is valid
         if (diskOfferingId != null) {
@@ -581,6 +584,15 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
         return false;
     }
 
+    private void checkFormatWithSupportedHypervisorsInZone(String format, Long zoneId) {
+        ImageFormat imageformat = ImageFormat.valueOf(format);
+        final List<HypervisorType> supportedHypervisorTypesInZone = _resourceMgr.getSupportedHypervisorTypes(zoneId, false, null);
+        final HypervisorType hypervisorTypeFromFormat = ApiDBUtils.getHypervisorTypeFromFormat(zoneId, imageformat);
+        if (!(supportedHypervisorTypesInZone.contains(hypervisorTypeFromFormat))) {
+            throw new InvalidParameterValueException(String.format("The %s hypervisor supported for %s file format, is not found on the zone", hypervisorTypeFromFormat.toString(), format));
+        }
+    }
+
     public String getRandomVolumeName() {
         return UUID.randomUUID().toString();
     }

From 4de975ff2526bc9d1ff3b5fc7c23b0d94c690d53 Mon Sep 17 00:00:00 2001
From: dahn <daan@onecht.net>
Date: Wed, 12 Jun 2024 11:41:00 +0200
Subject: [PATCH 15/20] list by isEncrypted (#8643)

* list by isEncrypted

* use filter on VO and cleanup

* add encryption type to volume response

* Update api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java

Co-authored-by: Suresh Kumar Anaparti <sureshkumar.anaparti@gmail.com>
---
 .../java/org/apache/cloudstack/api/ApiConstants.java  |  1 +
 .../api/command/user/volume/ListVolumesCmd.java       |  7 +++++++
 .../cloudstack/api/response/VolumeResponse.java       |  8 ++++----
 .../main/java/com/cloud/storage/dao/VolumeDao.java    |  3 ++-
 .../java/com/cloud/storage/dao/VolumeDaoImpl.java     |  1 -
 .../java/com/cloud/api/query/QueryManagerImpl.java    | 11 +++++++++--
 .../com/cloud/api/query/dao/VolumeJoinDaoImpl.java    |  3 ++-
 .../java/com/cloud/api/query/vo/VolumeJoinVO.java     |  7 +++----
 8 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
index 7565b679e58..7512120cc00 100644
--- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
+++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
@@ -263,6 +263,7 @@ public class ApiConstants {
     public static final String IS_CLEANUP_REQUIRED = "iscleanuprequired";
     public static final String IS_DYNAMIC = "isdynamic";
     public static final String IS_EDGE = "isedge";
+    public static final String IS_ENCRYPTED = "isencrypted";
     public static final String IS_EXTRACTABLE = "isextractable";
     public static final String IS_FEATURED = "isfeatured";
     public static final String IS_PORTABLE = "isportable";
diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java
index b62a909d71f..18a32e324f7 100644
--- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java
@@ -96,6 +96,9 @@ public class ListVolumesCmd extends BaseListRetrieveOnlyResourceCountCmd impleme
     @Parameter(name = ApiConstants.STATE, type = CommandType.STRING, description = "state of the volume. Possible values are: Ready, Allocated, Destroy, Expunging, Expunged.")
     private String state;
 
+    @Parameter(name = ApiConstants.IS_ENCRYPTED, type = CommandType.BOOLEAN, description = "list only volumes that are encrypted", since = "4.19.1",
+            authorized = { RoleType.Admin })
+    private Boolean encrypted;
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -153,6 +156,10 @@ public class ListVolumesCmd extends BaseListRetrieveOnlyResourceCountCmd impleme
         return state;
     }
 
+    public Boolean isEncrypted() {
+        return encrypted;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java
index 2ddc5fd097e..726c9adf8a3 100644
--- a/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java
+++ b/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java
@@ -298,8 +298,8 @@ public class VolumeResponse extends BaseResponseWithTagInformation implements Co
     private Map<String, String> volumeRepairResult;
 
     @SerializedName(ApiConstants.ENCRYPT_FORMAT)
-    @Param(description = "the encrypt format of the volume", since = "4.19.1")
-    private String encryptFormat;
+    @Param(description = "the format of the disk encryption if applicable", since = "4.19.1")
+    private String encryptionFormat;
 
     public String getPath() {
         return path;
@@ -847,7 +847,7 @@ public class VolumeResponse extends BaseResponseWithTagInformation implements Co
         this.volumeRepairResult = volumeRepairResult;
     }
 
-    public void setEncryptFormat(String encryptFormat) {
-        this.encryptFormat = encryptFormat;
+    public void setEncryptionFormat(String encryptionFormat) {
+        this.encryptionFormat = encryptionFormat;
     }
 }
diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java
index be6588e3189..7aec8f02c76 100644
--- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java
+++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java
@@ -112,7 +112,8 @@ public interface VolumeDao extends GenericDao<VolumeVO, Long>, StateDao<Volume.S
     /**
      * Gets the Total Primary Storage space allocated for an account
      *
-     * @param list of ids of virtual router VMs under this account
+     * @param accountId
+     * @param virtualRouters list of ids of virtual router VMs under this account
      * @return total Primary Storage space (in bytes) used
      */
     long primaryStorageUsedForAccount(long accountId, List<Long> virtualRouters);
diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java
index a773a9502ce..be1a7e01b4e 100644
--- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java
@@ -75,7 +75,6 @@ public class VolumeDaoImpl extends GenericDaoBase<VolumeVO, Long> implements Vol
     @Inject
     ResourceTagDao _tagsDao;
 
-    protected static final String SELECT_VM_SQL = "SELECT DISTINCT instance_id from volumes v where v.host_id = ? and v.mirror_state = ?";
     // need to account for zone-wide primary storage where storage_pool has
     // null-value pod and cluster, where hypervisor information is stored in
     // storage_pool
diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
index bba5c02151a..f83192350db 100644
--- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
+++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
@@ -2396,7 +2396,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
     private Pair<List<Long>, Integer> searchForVolumeIdsAndCount(ListVolumesCmd cmd) {
 
         Account caller = CallContext.current().getCallingAccount();
-        List<Long> permittedAccounts = new ArrayList<Long>();
+        List<Long> permittedAccounts = new ArrayList<>();
 
         Long id = cmd.getId();
         Long vmInstanceId = cmd.getVirtualMachineId();
@@ -2416,7 +2416,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
 
         List<Long> ids = getIdsListFromCmd(cmd.getId(), cmd.getIds());
 
-        Ternary<Long, Boolean, ListProjectResourcesCriteria> domainIdRecursiveListProject = new Ternary<Long, Boolean, ListProjectResourcesCriteria>(cmd.getDomainId(), cmd.isRecursive(), null);
+        Ternary<Long, Boolean, ListProjectResourcesCriteria> domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null);
         accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false);
         Long domainId = domainIdRecursiveListProject.first();
         Boolean isRecursive = domainIdRecursiveListProject.second();
@@ -2436,6 +2436,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         volumeSearchBuilder.and("uuid", volumeSearchBuilder.entity().getUuid(), SearchCriteria.Op.NNULL);
         volumeSearchBuilder.and("instanceId", volumeSearchBuilder.entity().getInstanceId(), SearchCriteria.Op.EQ);
         volumeSearchBuilder.and("dataCenterId", volumeSearchBuilder.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        if (cmd.isEncrypted() != null) {
+            if (cmd.isEncrypted()) {
+                volumeSearchBuilder.and("encryptFormat", volumeSearchBuilder.entity().getEncryptFormat(), SearchCriteria.Op.NNULL);
+            } else {
+                volumeSearchBuilder.and("encryptFormat", volumeSearchBuilder.entity().getEncryptFormat(), SearchCriteria.Op.NULL);
+            }
+        }
 
         if (keyword != null) {
             volumeSearchBuilder.and().op("keywordName", volumeSearchBuilder.entity().getName(), SearchCriteria.Op.LIKE);
diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java
index c4e4ba223d8..e309a7534a3 100644
--- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java
+++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java
@@ -180,7 +180,7 @@ public class VolumeJoinDaoImpl extends GenericDaoBaseWithTagInformation<VolumeJo
 
         if (view == ResponseView.Full) {
             volResponse.setPath(volume.getPath());
-            volResponse.setEncryptFormat(volume.getEncryptFormat());
+            volResponse.setEncryptionFormat(volume.getEncryptionFormat());
         }
 
         // populate owner.
@@ -279,6 +279,7 @@ public class VolumeJoinDaoImpl extends GenericDaoBaseWithTagInformation<VolumeJo
 
         volResponse.setObjectName("volume");
         volResponse.setExternalUuid(volume.getExternalUuid());
+        volResponse.setEncryptionFormat(volume.getEncryptionFormat());
         return volResponse;
     }
 
diff --git a/server/src/main/java/com/cloud/api/query/vo/VolumeJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/VolumeJoinVO.java
index a8fcbfe824e..ee82f809aaf 100644
--- a/server/src/main/java/com/cloud/api/query/vo/VolumeJoinVO.java
+++ b/server/src/main/java/com/cloud/api/query/vo/VolumeJoinVO.java
@@ -276,7 +276,7 @@ public class VolumeJoinVO extends BaseViewWithTagInformationVO implements Contro
     private String externalUuid = null;
 
     @Column(name = "encrypt_format")
-    private String encryptFormat;
+    private String encryptionFormat = null;
 
     public VolumeJoinVO() {
     }
@@ -613,13 +613,12 @@ public class VolumeJoinVO extends BaseViewWithTagInformationVO implements Contro
         this.externalUuid = externalUuid;
     }
 
-    public String getEncryptFormat() {
-        return encryptFormat;
+    public String getEncryptionFormat() {
+        return encryptionFormat;
     }
 
     @Override
     public Class<?> getEntityType() {
         return Volume.class;
     }
-
 }

From c756e4370adb3d7a3d1ba85fd516691f29562e87 Mon Sep 17 00:00:00 2001
From: Vishesh <vishesh92@gmail.com>
Date: Wed, 12 Jun 2024 16:13:56 +0530
Subject: [PATCH 16/20] List volumes by service offering id (#9211)

* Allow listing of volumes by service offering id

* Address comments
---
 .../command/user/volume/ListVolumesCmd.java   | 11 ++++
 .../com/cloud/api/query/QueryManagerImpl.java | 14 ++++-
 ui/src/components/view/SearchView.vue         | 62 ++++++++++++++++++-
 ui/src/config/section/storage.js              |  2 +-
 4 files changed, 84 insertions(+), 5 deletions(-)

diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java
index 18a32e324f7..6510106557b 100644
--- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ListVolumesCmd.java
@@ -31,6 +31,7 @@ import org.apache.cloudstack.api.response.DiskOfferingResponse;
 import org.apache.cloudstack.api.response.HostResponse;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.PodResponse;
+import org.apache.cloudstack.api.response.ServiceOfferingResponse;
 import org.apache.cloudstack.api.response.StoragePoolResponse;
 import org.apache.cloudstack.api.response.UserVmResponse;
 import org.apache.cloudstack.api.response.VolumeResponse;
@@ -82,6 +83,12 @@ public class ListVolumesCmd extends BaseListRetrieveOnlyResourceCountCmd impleme
             RoleType.Admin})
     private String storageId;
 
+    @Parameter(name = ApiConstants.SERVICE_OFFERING_ID, type = CommandType.UUID,
+               entityType = ServiceOfferingResponse.class,
+               description = "list volumes by disk offering of a service offering. If both service offering and " +
+                       "disk offering are passed, service offering is ignored", since = "4.19.1")
+    private Long serviceOfferingId;
+
     @Parameter(name = ApiConstants.DISK_OFFERING_ID, type = CommandType.UUID, entityType = DiskOfferingResponse.class, description = "list volumes by disk offering", since = "4.4")
     private Long diskOfferingId;
 
@@ -123,6 +130,10 @@ public class ListVolumesCmd extends BaseListRetrieveOnlyResourceCountCmd impleme
         return podId;
     }
 
+    public Long getServiceOfferingId() {
+        return serviceOfferingId;
+    }
+
     public Long getDiskOfferingId() {
         return diskOfferingId;
     }
diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
index f83192350db..320432c5f9f 100644
--- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
+++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
@@ -2406,7 +2406,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         Map<String, String> tags = cmd.getTags();
         String storageId = cmd.getStorageId();
         Long clusterId = cmd.getClusterId();
-        Long diskOffId = cmd.getDiskOfferingId();
+        Long serviceOfferingId = cmd.getServiceOfferingId();
+        Long diskOfferingId = cmd.getDiskOfferingId();
         Boolean display = cmd.getDisplay();
         String state = cmd.getState();
         boolean shouldListSystemVms = shouldListSystemVms(cmd, caller.getId());
@@ -2416,6 +2417,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
 
         List<Long> ids = getIdsListFromCmd(cmd.getId(), cmd.getIds());
 
+        if (diskOfferingId == null && serviceOfferingId != null) {
+            ServiceOfferingVO serviceOffering = _srvOfferingDao.findById(serviceOfferingId);
+            if (serviceOffering != null) {
+                diskOfferingId = serviceOffering.getDiskOfferingId();
+            }
+        }
+
         Ternary<Long, Boolean, ListProjectResourcesCriteria> domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null);
         accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false);
         Long domainId = domainIdRecursiveListProject.first();
@@ -2546,8 +2554,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
             }
         }
 
-        if (diskOffId != null) {
-            sc.setParameters("diskOfferingId", diskOffId);
+        if (diskOfferingId != null) {
+            sc.setParameters("diskOfferingId", diskOfferingId);
         }
 
         if (id != null) {
diff --git a/ui/src/components/view/SearchView.vue b/ui/src/components/view/SearchView.vue
index 43f07c7456b..c284b8a635a 100644
--- a/ui/src/components/view/SearchView.vue
+++ b/ui/src/components/view/SearchView.vue
@@ -285,7 +285,7 @@ export default {
         }
         if (['zoneid', 'domainid', 'imagestoreid', 'storageid', 'state', 'account', 'hypervisor', 'level',
           'clusterid', 'podid', 'groupid', 'entitytype', 'accounttype', 'systemvmtype', 'scope', 'provider',
-          'type'].includes(item)
+          'type', 'serviceofferingid', 'diskofferingid'].includes(item)
         ) {
           type = 'list'
         } else if (item === 'tags') {
@@ -397,6 +397,8 @@ export default {
       let podIndex = -1
       let clusterIndex = -1
       let groupIndex = -1
+      let serviceOfferingIndex = -1
+      let diskOfferingIndex = -1
 
       if (arrayField.includes('type')) {
         if (this.$route.path === '/alert') {
@@ -464,6 +466,18 @@ export default {
         promises.push(await this.fetchInstanceGroups(searchKeyword))
       }
 
+      if (arrayField.includes('serviceofferingid')) {
+        serviceOfferingIndex = this.fields.findIndex(item => item.name === 'serviceofferingid')
+        this.fields[serviceOfferingIndex].loading = true
+        promises.push(await this.fetchServiceOfferings(searchKeyword))
+      }
+
+      if (arrayField.includes('diskofferingid')) {
+        diskOfferingIndex = this.fields.findIndex(item => item.name === 'diskofferingid')
+        this.fields[diskOfferingIndex].loading = true
+        promises.push(await this.fetchDiskOfferings(searchKeyword))
+      }
+
       Promise.all(promises).then(response => {
         if (typeIndex > -1) {
           const types = response.filter(item => item.type === 'type')
@@ -525,6 +539,20 @@ export default {
             this.fields[groupIndex].opts = this.sortArray(groups[0].data)
           }
         }
+
+        if (serviceOfferingIndex > -1) {
+          const serviceOfferings = response.filter(item => item.type === 'serviceofferingid')
+          if (serviceOfferings && serviceOfferings.length > 0) {
+            this.fields[serviceOfferingIndex].opts = this.sortArray(serviceOfferings[0].data)
+          }
+        }
+
+        if (diskOfferingIndex > -1) {
+          const diskOfferings = response.filter(item => item.type === 'diskofferingid')
+          if (diskOfferings && diskOfferings.length > 0) {
+            this.fields[diskOfferingIndex].opts = this.sortArray(diskOfferings[0].data)
+          }
+        }
       }).finally(() => {
         if (typeIndex > -1) {
           this.fields[typeIndex].loading = false
@@ -550,6 +578,12 @@ export default {
         if (groupIndex > -1) {
           this.fields[groupIndex].loading = false
         }
+        if (serviceOfferingIndex > -1) {
+          this.fields[serviceOfferingIndex].loading = false
+        }
+        if (diskOfferingIndex > -1) {
+          this.fields[diskOfferingIndex].loading = false
+        }
         this.fillFormFieldValues()
       })
     },
@@ -699,6 +733,32 @@ export default {
         })
       })
     },
+    fetchServiceOfferings (searchKeyword) {
+      return new Promise((resolve, reject) => {
+        api('listServiceOfferings', { listAll: true, keyword: searchKeyword }).then(json => {
+          const serviceOfferings = json.listserviceofferingsresponse.serviceoffering
+          resolve({
+            type: 'serviceofferingid',
+            data: serviceOfferings
+          })
+        }).catch(error => {
+          reject(error.response.headers['x-description'])
+        })
+      })
+    },
+    fetchDiskOfferings (searchKeyword) {
+      return new Promise((resolve, reject) => {
+        api('listDiskOfferings', { listAll: true, keyword: searchKeyword }).then(json => {
+          const diskOfferings = json.listdiskofferingsresponse.diskoffering
+          resolve({
+            type: 'diskofferingid',
+            data: diskOfferings
+          })
+        }).catch(error => {
+          reject(error.response.headers['x-description'])
+        })
+      })
+    },
     fetchAlertTypes () {
       if (this.alertTypes.length > 0) {
         return new Promise((resolve, reject) => {
diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js
index ff0fd99242d..772f2a3fd3b 100644
--- a/ui/src/config/section/storage.js
+++ b/ui/src/config/section/storage.js
@@ -91,7 +91,7 @@ export default {
         }
       ],
       searchFilters: () => {
-        var filters = ['name', 'zoneid', 'domainid', 'account', 'state', 'tags']
+        var filters = ['name', 'zoneid', 'domainid', 'account', 'state', 'tags', 'serviceofferingid', 'diskofferingid']
         if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) {
           filters.push('storageid')
         }

From 10dc5ad7744b64be4ba967589797d70f9f83548c Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek.mrt22@gmail.com>
Date: Wed, 12 Jun 2024 17:01:52 +0530
Subject: [PATCH 17/20] ui; fix admin dashboard vue warnings (#9140)

Fixes warnings on admin dashboard in console.

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
---
 ui/src/views/dashboard/CapacityDashboard.vue | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ui/src/views/dashboard/CapacityDashboard.vue b/ui/src/views/dashboard/CapacityDashboard.vue
index 2fc41b6c1be..dae53cf0015 100644
--- a/ui/src/views/dashboard/CapacityDashboard.vue
+++ b/ui/src/views/dashboard/CapacityDashboard.vue
@@ -184,7 +184,7 @@
               </div>
               <a-progress
               status="active"
-              :percent="statsMap[ctype]?.capacitytotal > 0 ? parseFloat(100.0 * statsMap[ctype]?.capacityused / statsMap[ctype]?.capacitytotal).toFixed(2) : 0"
+              :percent="statsMap[ctype]?.capacitytotal > 0 ? parseFloat(100.0 * statsMap[ctype]?.capacityused / statsMap[ctype]?.capacitytotal) : 0"
               :format="p => statsMap[ctype]?.capacitytotal > 0 ? parseFloat(100.0 * statsMap[ctype]?.capacityused / statsMap[ctype]?.capacitytotal).toFixed(2) + '%' : '0%'"
               stroke-color="#52c41a"
               size="small"
@@ -214,7 +214,7 @@
               </div>
               <a-progress
               status="active"
-              :percent="statsMap[ctype]?.capacitytotal > 0 ? parseFloat(100.0 * statsMap[ctype]?.capacityused / statsMap[ctype]?.capacitytotal).toFixed(2) : 0"
+              :percent="statsMap[ctype]?.capacitytotal > 0 ? parseFloat(100.0 * statsMap[ctype]?.capacityused / statsMap[ctype]?.capacitytotal) : 0"
               :format="p => statsMap[ctype]?.capacitytotal > 0 ? parseFloat(100.0 * statsMap[ctype]?.capacityused / statsMap[ctype]?.capacitytotal).toFixed(2) + '%' : '0%'"
               stroke-color="#52c41a"
               size="small"
@@ -244,7 +244,7 @@
               </div>
               <a-progress
               status="active"
-              :percent="statsMap[ctype]?.capacitytotal > 0 ? parseFloat(100.0 * statsMap[ctype]?.capacityused / statsMap[ctype]?.capacitytotal).toFixed(2) : 0"
+              :percent="statsMap[ctype]?.capacitytotal > 0 ? parseFloat(100.0 * statsMap[ctype]?.capacityused / statsMap[ctype]?.capacitytotal) : 0"
               :format="p => statsMap[ctype]?.capacitytotal > 0 ? parseFloat(100.0 * statsMap[ctype]?.capacityused / statsMap[ctype]?.capacitytotal).toFixed(2) + '%' : '0%'"
               stroke-color="#52c41a"
               size="small"

From e2e46e31087ea6c5993758ac0edf734267fcbfb7 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek.mrt22@gmail.com>
Date: Wed, 12 Jun 2024 17:19:21 +0530
Subject: [PATCH 18/20] ui: fix dashboard on saml user login (#9097)

* ui: fix dashboard on saml user login

Fixes #8950
Fixes #9045

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

* Update ui/src/permission.js

---------

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
---
 ui/src/permission.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ui/src/permission.js b/ui/src/permission.js
index 6db690f6e6a..4380c7660d8 100644
--- a/ui/src/permission.js
+++ b/ui/src/permission.js
@@ -112,7 +112,11 @@ router.beforeEach((to, from, next) => {
               } else {
                 next({ path: redirect })
               }
-              const project = vueProps.$localStorage.get(CURRENT_PROJECT)
+              var project = vueProps.$localStorage.get(CURRENT_PROJECT)
+              if (project == null) {
+                project = {}
+                store.commit('SET_PROJECT', project)
+              }
               store.dispatch('ToggleTheme', project.id === undefined ? 'light' : 'dark')
             })
           })

From 5c749eced26186694e4136f3847fcc24ad9010e1 Mon Sep 17 00:00:00 2001
From: Abhishek Kumar <abhishek.mrt22@gmail.com>
Date: Wed, 12 Jun 2024 17:27:41 +0530
Subject: [PATCH 19/20] logging: httpupload do not warn with exception trace
 (#9220)

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
Co-authored-by: Rohit Yadav <rohityadav89@gmail.com>
---
 .../cloudstack/storage/resource/HttpUploadServerHandler.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java
index aec1560f411..9b1c81284f4 100644
--- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java
+++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java
@@ -295,7 +295,8 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
 
     @Override
     public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
-        logger.warn(responseContent.toString(), cause);
+        logger.warn(String.format("%s. Exception occurred: %s", responseContent.toString(), cause.getMessage()));
+        logger.debug("Exception caught by HTTP upload handler, caused due to: ", cause);
         responseContent.append("\r\nException occurred: ").append(cause.getMessage());
         writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR);
         ctx.channel().close();

From b2ef53b8a2d95c0ab59abd7e8e2ec5b7343a8615 Mon Sep 17 00:00:00 2001
From: Wei Zhou <weizhou@apache.org>
Date: Wed, 12 Jun 2024 16:01:23 +0200
Subject: [PATCH 20/20] kvm: replace ISO path in vm XML configuration during vm
 migration (#9212)

* kvm: replace ISO path in vm XML configuration during vm migration

* Update 9212: address comments

* kvm: fix vm migration if there are multiple image stores
---
 .../datastore/db/ImageStoreDaoImpl.java       |  2 +-
 .../wrapper/LibvirtMigrateCommandWrapper.java | 86 +++++++++++++++++++
 .../LibvirtMigrateCommandWrapperTest.java     | 12 +++
 3 files changed, 99 insertions(+), 1 deletion(-)

diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java
index 84b88c215ca..21c5dc76d96 100644
--- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java
+++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java
@@ -130,7 +130,7 @@ public class ImageStoreDaoImpl extends GenericDaoBase<ImageStoreVO, Long> implem
         }
         if (scope.getScopeId() != null) {
             SearchCriteria<ImageStoreVO> scc = createSearchCriteria();
-            scc.addOr("scope", SearchCriteria.Op.EQ, ScopeType.REGION);
+            scc.addOr("scope", SearchCriteria.Op.EQ, ScopeType.ZONE);
             scc.addOr("dcId", SearchCriteria.Op.EQ, scope.getScopeId());
             sc.addAnd("scope", SearchCriteria.Op.SC, scc);
         }
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java
index aebbaa4119d..5ba174acd39 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java
@@ -68,6 +68,7 @@ import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.MigrateAnswer;
 import com.cloud.agent.api.MigrateCommand;
 import com.cloud.agent.api.MigrateCommand.MigrateDiskInfo;
+import com.cloud.agent.api.to.DataTO;
 import com.cloud.agent.api.to.DiskTO;
 import com.cloud.agent.api.to.DpdkTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
@@ -91,6 +92,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
     private static final String GRAPHICS_ELEM_END = "/graphics>";
     private static final String GRAPHICS_ELEM_START = "<graphics";
     private static final String CONTENTS_WILDCARD = "(?s).*";
+    private static final String CDROM_LABEL = "hdc";
     private static final Logger s_logger = Logger.getLogger(LibvirtMigrateCommandWrapper.class);
 
     protected String createMigrationURI(final String destinationIp, final LibvirtComputingResource libvirtComputingResource) {
@@ -166,6 +168,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
             String vncPassword = org.apache.commons.lang3.StringUtils.truncate(to.getVncPassword(), 8);
             xmlDesc = replaceIpForVNCInDescFileAndNormalizePassword(xmlDesc, target, vncPassword, vmName);
 
+            // Replace Config Drive ISO path
             String oldIsoVolumePath = getOldVolumePath(disks, vmName);
             String newIsoVolumePath = getNewVolumePathIfDatastoreHasChanged(libvirtComputingResource, conn, to);
             if (newIsoVolumePath != null && !newIsoVolumePath.equals(oldIsoVolumePath)) {
@@ -175,6 +178,14 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
                     s_logger.debug(String.format("Replaced disk mount point [%s] with [%s] in VM [%s] XML configuration. New XML configuration is [%s].", oldIsoVolumePath, newIsoVolumePath, vmName, xmlDesc));
                 }
             }
+
+            // Replace CDROM ISO path
+            String oldCdromIsoPath = getOldVolumePathForCdrom(disks, vmName);
+            String newCdromIsoPath = getNewVolumePathForCdrom(libvirtComputingResource, conn, to);
+            if (newCdromIsoPath != null && !newCdromIsoPath.equals(oldCdromIsoPath)) {
+                xmlDesc = replaceCdromIsoPath(xmlDesc, vmName, oldCdromIsoPath, newCdromIsoPath);
+            }
+
             // delete the metadata of vm snapshots before migration
             vmsnapshots = libvirtComputingResource.cleanVMSnapshotMetadata(dm);
 
@@ -702,6 +713,81 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
         return newIsoVolumePath;
     }
 
+    private String getOldVolumePathForCdrom(List<DiskDef> disks, String vmName) {
+        String oldIsoVolumePath = null;
+        for (DiskDef disk : disks) {
+            if (DiskDef.DeviceType.CDROM.equals(disk.getDeviceType())
+                    && CDROM_LABEL.equals(disk.getDiskLabel())
+                    && disk.getDiskPath() != null) {
+                oldIsoVolumePath = disk.getDiskPath();
+                break;
+            }
+        }
+        return oldIsoVolumePath;
+    }
+
+    private String getNewVolumePathForCdrom(LibvirtComputingResource libvirtComputingResource, Connect conn, VirtualMachineTO to) throws LibvirtException, URISyntaxException {
+        DiskTO newDisk = null;
+        for (DiskTO disk : to.getDisks()) {
+            DataTO data = disk.getData();
+            if (disk.getDiskSeq() == 3 && data != null && data.getPath() != null) {
+                newDisk = disk;
+                break;
+            }
+        }
+
+        String newIsoVolumePath = null;
+        if (newDisk != null) {
+            newIsoVolumePath = libvirtComputingResource.getVolumePath(conn, newDisk);
+        }
+        return newIsoVolumePath;
+    }
+
+    protected String replaceCdromIsoPath(String xmlDesc, String vmName, String oldIsoVolumePath, String newIsoVolumePath) throws IOException, ParserConfigurationException, TransformerException, SAXException {
+        InputStream in = IOUtils.toInputStream(xmlDesc);
+
+        DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory();
+        DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
+        Document doc = docBuilder.parse(in);
+
+        // Get the root element
+        Node domainNode = doc.getFirstChild();
+
+        NodeList domainChildNodes = domainNode.getChildNodes();
+
+        for (int i = 0; i < domainChildNodes.getLength(); i++) {
+            Node domainChildNode = domainChildNodes.item(i);
+            if ("devices".equals(domainChildNode.getNodeName())) {
+                NodeList devicesChildNodes = domainChildNode.getChildNodes();
+                for (int x = 0; x < devicesChildNodes.getLength(); x++) {
+                    Node deviceChildNode = devicesChildNodes.item(x);
+                    if ("disk".equals(deviceChildNode.getNodeName())) {
+                        Node diskNode = deviceChildNode;
+                        NodeList diskChildNodes = diskNode.getChildNodes();
+                        for (int z = 0; z < diskChildNodes.getLength(); z++) {
+                            Node diskChildNode = diskChildNodes.item(z);
+                            if ("source".equals(diskChildNode.getNodeName())) {
+                                NamedNodeMap sourceNodeAttributes = diskChildNode.getAttributes();
+                                Node sourceNodeAttribute = sourceNodeAttributes.getNamedItem("file");
+                                if (oldIsoVolumePath != null && sourceNodeAttribute != null
+                                        && oldIsoVolumePath.equals(sourceNodeAttribute.getNodeValue())) {
+                                    diskNode.removeChild(diskChildNode);
+                                    Element newChildSourceNode = doc.createElement("source");
+                                    newChildSourceNode.setAttribute("file", newIsoVolumePath);
+                                    diskNode.appendChild(newChildSourceNode);
+                                    s_logger.debug(String.format("Replaced ISO path [%s] with [%s] in VM [%s] XML configuration.", oldIsoVolumePath, newIsoVolumePath, vmName));
+                                    return getXml(doc);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        return getXml(doc);
+    }
+
     private String getPathFromSourceText(Set<String> paths, String sourceText) {
         if (paths != null && StringUtils.isNotBlank(sourceText)) {
             for (String path : paths) {
diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java
index 29a5e1f934b..3c5e54e2ba8 100644
--- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java
+++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java
@@ -132,6 +132,7 @@ public class LibvirtMigrateCommandWrapperTest {
 "    </disk>\n" +
 "    <disk type='file' device='cdrom'>\n" +
 "      <driver name='qemu' type='raw' cache='none'/>\n" +
+"      <source file='/mnt/ec8dfdd8-f341-3b0a-988f-cfbc93e46fc4/251-2-2b2071a4-21c7-340e-a861-1bd30fb5cbed.iso'/>\n" +
 "      <backingStore/>\n" +
 "      <target dev='hdc' bus='ide'/>\n" +
 "      <readonly/>\n" +
@@ -249,6 +250,7 @@ public class LibvirtMigrateCommandWrapperTest {
 "    </disk>\n" +
 "    <disk type='file' device='cdrom'>\n" +
 "      <driver name='qemu' type='raw' cache='none'/>\n" +
+"      <source file='/mnt/ec8dfdd8-f341-3b0a-988f-cfbc93e46fc4/251-2-2b2071a4-21c7-340e-a861-1bd30fb5cbed.iso'/>\n" +
 "      <backingStore/>\n" +
 "      <target dev='hdc' bus='ide'/>\n" +
 "      <readonly/>\n" +
@@ -1007,4 +1009,14 @@ public class LibvirtMigrateCommandWrapperTest {
         assertTrue(result.containsAll(Arrays.asList("vda", "vdb")));
     }
 
+    @Test
+    public void replaceCdromIsoPathTest() throws ParserConfigurationException, IOException, TransformerException,
+            SAXException {
+        String oldIsoVolumePath = "/mnt/ec8dfdd8-f341-3b0a-988f-cfbc93e46fc4/251-2-2b2071a4-21c7-340e-a861-1bd30fb5cbed.iso";
+        String newIsoVolumePath = "/mnt/50bf9d15-1b0f-3cc1-9e8a-55df3e17e0c4/251-2-2b2071a4-21c7-340e-a861-1bd30fb5cbed.iso";
+
+        String finalXml = libvirtMigrateCmdWrapper.replaceCdromIsoPath(fullfile, null, oldIsoVolumePath, newIsoVolumePath);
+
+        Assert.assertTrue(finalXml.contains(newIsoVolumePath));
+    }
 }