Add new API to check storage policy compatible pools and fix marvin test for storage policies

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java

@@ -20,12 +20,14 @@ package com.cloud.hypervisor.vmware;
 import com.cloud.dc.VsphereStoragePolicy;
 import com.cloud.exception.DiscoveryException;
 import com.cloud.exception.ResourceInUseException;
+import com.cloud.storage.StoragePool;
 import com.cloud.utils.component.PluggableService;
 import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd;
 import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd;
 import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd;
 import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd;
+import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePolicyCompatiblePoolsCmd;
 import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd;
 import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd;
 
@@ -45,4 +47,5 @@ public interface VmwareDatacenterService extends PluggableService {
 
     List<? extends VsphereStoragePolicy> listVsphereStoragePolicies(ListVsphereStoragePoliciesCmd cmd);
 
+    List<StoragePool> listVsphereStoragePolicyCompatibleStoragePools(ListVsphereStoragePolicyCompatiblePoolsCmd cmd);
 }

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java

@@ -25,6 +25,7 @@ import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.StartupCommand;
 import com.cloud.agent.api.StartupRoutingCommand;
+import com.cloud.agent.api.to.StorageFilerTO;
 import com.cloud.api.query.dao.TemplateJoinDao;
 import com.cloud.cluster.ClusterManager;
 import com.cloud.cluster.dao.ManagementServerHostPeerDao;
@@ -41,9 +42,11 @@ import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.dc.dao.VsphereStoragePolicyDao;
 import com.cloud.event.ActionEvent;
 import com.cloud.event.EventTypes;
+import com.cloud.exception.AgentUnavailableException;
 import com.cloud.exception.DiscoveredWithErrorException;
 import com.cloud.exception.DiscoveryException;
 import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.exception.OperationTimedoutException;
 import com.cloud.exception.ResourceInUseException;
 import com.cloud.host.Host;
 import com.cloud.host.Status;
@@ -51,6 +54,7 @@ import com.cloud.host.dao.HostDao;
 import com.cloud.host.dao.HostDetailsDao;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.hypervisor.HypervisorGuruManager;
 import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao;
 import com.cloud.hypervisor.vmware.LegacyZoneVO;
 import com.cloud.hypervisor.vmware.VmwareCleanupMaid;
@@ -89,6 +93,8 @@ import com.cloud.storage.ImageStoreDetailsUtil;
 import com.cloud.storage.JavaStorageLayer;
 import com.cloud.storage.StorageLayer;
 import com.cloud.storage.StorageManager;
+import com.cloud.storage.StoragePool;
+import com.cloud.storage.StoragePoolStatus;
 import com.cloud.storage.dao.VMTemplatePoolDao;
 import com.cloud.template.TemplateManager;
 import com.cloud.utils.FileUtil;
@@ -116,6 +122,7 @@ import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd;
 import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd;
 import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd;
 import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd;
+import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePolicyCompatiblePoolsCmd;
 import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd;
 import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@@ -125,7 +132,9 @@ import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl;
 import org.apache.cloudstack.management.ManagementServerHost;
+import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.log4j.Logger;
 
@@ -217,6 +226,10 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
     private TemplateManager templateManager;
     @Inject
     private VsphereStoragePolicyDao vsphereStoragePolicyDao;
+    @Inject
+    private StorageManager storageManager;
+    @Inject
+    private HypervisorGuruManager hypervisorGuruManager;
 
     private String _mountParent;
     private StorageLayer _storage;
@@ -1057,6 +1070,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
         cmdList.add(ListVmwareDcsCmd.class);
         cmdList.add(ImportVsphereStoragePoliciesCmd.class);
         cmdList.add(ListVsphereStoragePoliciesCmd.class);
+        cmdList.add(ListVsphereStoragePolicyCompatiblePoolsCmd.class);
         return cmdList;
     }
 
@@ -1469,6 +1483,35 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
         return Collections.emptyList();
     }
 
+    @Override
+    public List<StoragePool> listVsphereStoragePolicyCompatibleStoragePools(ListVsphereStoragePolicyCompatiblePoolsCmd cmd) {
+        Long policyId = cmd.getPolicyId();
+        VsphereStoragePolicyVO storagePolicy = vsphereStoragePolicyDao.findById(policyId);
+        if (storagePolicy == null) {
+            throw new CloudRuntimeException("Storage policy with ID = " + policyId + " was not found");
+        }
+        long zoneId = storagePolicy.getZoneId();
+        List<StoragePoolVO> poolsInZone = primaryStorageDao.listByStatusInZone(zoneId, StoragePoolStatus.Up);
+        List<StoragePool> compatiblePools = new ArrayList<>();
+        for (StoragePoolVO pool : poolsInZone) {
+            StorageFilerTO storageFilerTO = new StorageFilerTO(pool);
+            List<Long> hostIds = storageManager.getUpHostsInPool(pool.getId());
+            Collections.shuffle(hostIds);
+            CheckDataStoreStoragePolicyComplainceCommand command = new CheckDataStoreStoragePolicyComplainceCommand(storagePolicy.getPolicyId(), storageFilerTO);
+            long targetHostId = hypervisorGuruManager.getGuruProcessedCommandTargetHost(hostIds.get(0), command);
+            try {
+                Answer answer = _agentMgr.send(targetHostId, command);
+                boolean result = answer != null && answer.getResult();
+                if (result) {
+                    compatiblePools.add(pool);
+                }
+            } catch (AgentUnavailableException | OperationTimedoutException e) {
+                s_logger.error("Could not verify if storage policy " + storagePolicy.getName() + " is compatible with storage pool " + pool.getName());
+            }
+        }
+        return compatiblePools;
+    }
+
     @Override
     public boolean hasNexusVSM(Long clusterId) {
         ClusterVSMMapVO vsmMapVo = null;

plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java

@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.zone;
+
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.hypervisor.vmware.VmwareDatacenterService;
+import com.cloud.storage.StoragePool;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.BaseListCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.api.response.StoragePoolResponse;
+import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse;
+import org.apache.cloudstack.api.response.ZoneResponse;
+import org.apache.cloudstack.context.CallContext;
+
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.List;
+
+@APICommand(name = ListVsphereStoragePolicyCompatiblePoolsCmd.APINAME, description = "List storage pools compatible with a vSphere storage policy",
+        responseObject = StoragePoolResponse.class,
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        authorized = {RoleType.Admin})
+public class ListVsphereStoragePolicyCompatiblePoolsCmd extends BaseListCmd {
+    public static final String APINAME = "listVsphereStoragePolicyCompatiblePools";
+
+    @Inject
+    public VmwareDatacenterService vmwareDatacenterService;
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class,
+            description = "ID of the zone")
+    private Long zoneId;
+
+    @Parameter(name = ApiConstants.POLICY_ID, type = BaseCmd.CommandType.UUID, entityType = VsphereStoragePoliciesResponse.class,
+            description = "ID of the storage policy")
+    private Long policyId;
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    public void execute() throws ServerApiException, ConcurrentOperationException {
+        List<StoragePool> pools = vmwareDatacenterService.listVsphereStoragePolicyCompatibleStoragePools(this);
+        ListResponse<StoragePoolResponse> response = new ListResponse<>();
+        List<StoragePoolResponse> poolResponses = new ArrayList<>();
+        for (StoragePool pool : pools) {
+            StoragePoolResponse poolResponse = _responseGenerator.createStoragePoolForMigrationResponse(pool);
+            poolResponse.setObjectName("storagepool");
+            poolResponses.add(poolResponse);
+        }
+        response.setResponses(poolResponses);
+        response.setResponseName(getCommandName());
+        this.setResponseObject(response);
+    }
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+
+    public Long getZoneId() {
+        return zoneId;
+    }
+
+    public Long getPolicyId() {
+        return policyId;
+    }
+}

test/integration/smoke/test_storage_policy.py

@@ -31,9 +31,9 @@ from marvin.lib.base import (Account,
 from marvin.lib.common import (get_zone,
                                get_domain,
                                get_test_template)
-from marvin.codes import PASS
+from marvin.cloudstackAPI import (importVsphereStoragePolicies,
-from marvin.cloudstackAPI import (importVsphereStoragePolicies)
+                                  listVsphereStoragePolicies,
-from marvin.cloudstackAPI import (listVsphereStoragePolicies)
+                                  listVsphereStoragePolicyCompatiblePools)
 
 
 class TestVMWareStoragePolicies(cloudstackTestCase):
@@ -103,6 +103,12 @@ class TestVMWareStoragePolicies(cloudstackTestCase):
         cmd.zoneid = self.zone.id
         return apiclient.listVsphereStoragePolicies(cmd)
 
+    def list_storage_policy_compatible_pools(self, apiclient, policyid):
+        cmd = listVsphereStoragePolicyCompatiblePools.listVsphereStoragePolicyCompatiblePoolsCmd()
+        cmd.zoneid = self.zone.id
+        cmd.policyid = policyid
+        return apiclient.listVsphereStoragePolicyCompatiblePools(cmd)
+
     def create_volume(self, apiclient):
         cmd = create
 
@@ -155,11 +161,23 @@ class TestVMWareStoragePolicies(cloudstackTestCase):
             "Check if the number of imported policies is identical to the number of listed policies"
         )
 
+        are_compatible_pools = False
+        selected_policy = None
+        for imported_policy in imported_policies:
+            compatible_pools = self.list_storage_policy_compatible_pools(self.apiclient, imported_policy.id)
+            if len(compatible_pools) > 0:
+                are_compatible_pools = True
+                selected_policy = imported_policy
+                break
+
+        if not are_compatible_pools:
+            self.skipTest("There are no compatible storage pools with the imported policies")
+
         # Create service offering with the first storage policy from the list
         service_offering = ServiceOffering.create(
             self.apiclient,
             self.testdata["service_offering"],
-            storagepolicy=listed_policies[0].id
+            storagepolicy=selected_policy.id
         )
         self.cleanup.append(service_offering)
 
@@ -167,7 +185,7 @@ class TestVMWareStoragePolicies(cloudstackTestCase):
         disk_offering = DiskOffering.create(
             self.apiclient,
             self.testdata["disk_offering"],
-            storagepolicy=listed_policies[0].id
+            storagepolicy=selected_policy.id
         )
         self.cleanup.append(disk_offering)