scaleio: Updated PowerFlex/ScaleIO gateway client with some improvements. (#5037)

- Added connection manager to the gateway client.
 - Renew the client session on '401 Unauthorized' response.
 - Refactored the gateway client calls, for GET and POST methods.
 - Consume the http entity content after login/(re)authentication and close the content stream if exists.
 - Updated storage pool client connection timeout configuration 'storage.pool.client.timeout' to non-dynamic.
 - Added storage pool client max connections configuration 'storage.pool.client.max.connections' (default: 100) to specify the maximum connections for the ScaleIO storage pool client.
 - Updated unit tests.
and blocked the attach volume operation for uploaded volume on ScaleIO/PowerFlex storage pool

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -124,7 +124,16 @@ public interface StorageManager extends StorageService {
             "Storage",
             "60",
             "Timeout (in secs) for the storage pool client connection timeout (for managed pools). Currently only supported for PowerFlex.",
-            true,
+            false,
+            ConfigKey.Scope.StoragePool,
+            null);
+
+    ConfigKey<Integer> STORAGE_POOL_CLIENT_MAX_CONNECTIONS = new ConfigKey<>(Integer.class,
+            "storage.pool.client.max.connections",
+            "Storage",
+            "100",
+            "Maximum connections for the storage pool client (for managed pools). Currently only supported for PowerFlex.",
+            false,
             ConfigKey.Scope.StoragePool,
             null);
 

plugins/storage/volume/scaleio/pom.xml

@@ -33,6 +33,12 @@
             <artifactId>cloud-engine-storage-volume</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.github.tomakehurst</groupId>
+            <artifactId>wiremock-standalone</artifactId>
+            <version>${cs.wiremock.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java

@@ -40,8 +40,8 @@ public interface ScaleIOGatewayClient {
     String STORAGE_POOL_SYSTEM_ID = "powerflex.storagepool.system.id";
 
     static ScaleIOGatewayClient getClient(final String url, final String username, final String password,
-                                          final boolean validateCertificate, final int timeout) throws NoSuchAlgorithmException, KeyManagementException, URISyntaxException {
+                                          final boolean validateCertificate, final int timeout, final int maxConnections) throws NoSuchAlgorithmException, KeyManagementException, URISyntaxException {
-        return new ScaleIOGatewayClientImpl(url, username, password, validateCertificate, timeout);
+        return new ScaleIOGatewayClientImpl(url, username, password, validateCertificate, timeout, maxConnections);
     }
 
     // Volume APIs

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientConnectionPool.java

@@ -62,8 +62,9 @@ public class ScaleIOGatewayClientConnectionPool {
                 final String encryptedPassword = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_PASSWORD).getValue();
                 final String password = DBEncryptionUtil.decrypt(encryptedPassword);
                 final int clientTimeout = StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.valueIn(storagePoolId);
+                final int clientMaxConnections = StorageManager.STORAGE_POOL_CLIENT_MAX_CONNECTIONS.valueIn(storagePoolId);
 
-                client = new ScaleIOGatewayClientImpl(url, username, password, false, clientTimeout);
+                client = new ScaleIOGatewayClientImpl(url, username, password, false, clientTimeout, clientMaxConnections);
                 gatewayClients.put(storagePoolId, client);
                 LOGGER.debug("Added gateway client for the storage pool: " + storagePoolId);
             }

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java

@@ -106,7 +106,8 @@ public class ScaleIOPrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeCyc
     private org.apache.cloudstack.storage.datastore.api.StoragePool findStoragePool(String url, String username, String password, String storagePoolName) {
         try {
             final int clientTimeout = StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.value();
-            ScaleIOGatewayClient client = ScaleIOGatewayClient.getClient(url, username, password, false, clientTimeout);
+            final int clientMaxConnections = StorageManager.STORAGE_POOL_CLIENT_MAX_CONNECTIONS.value();
+            ScaleIOGatewayClient client = ScaleIOGatewayClient.getClient(url, username, password, false, clientTimeout, clientMaxConnections);
             List<org.apache.cloudstack.storage.datastore.api.StoragePool> storagePools = client.listStoragePools();
             for (org.apache.cloudstack.storage.datastore.api.StoragePool pool : storagePools) {
                 if (pool.getName().equals(storagePoolName)) {
@@ -121,9 +122,9 @@ public class ScaleIOPrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeCyc
             }
         } catch (NoSuchAlgorithmException | KeyManagementException | URISyntaxException e) {
             LOGGER.error("Failed to add storage pool", e);
-            throw new CloudRuntimeException("Failed to establish connection with PowerFlex Gateway to validate storage pool");
+            throw new CloudRuntimeException("Failed to establish connection with PowerFlex Gateway to find and validate storage pool: " + storagePoolName);
         }
-        throw new CloudRuntimeException("Failed to find the provided storage pool name in discovered PowerFlex storage pools");
+        throw new CloudRuntimeException("Failed to find the provided storage pool name: " + storagePoolName + " in the discovered PowerFlex storage pools");
     }
 
     @SuppressWarnings("unchecked")

plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImplTest.java

@@ -19,30 +19,179 @@
 
 package org.apache.cloudstack.storage.datastore.client;
 
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.containing;
+import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.ok;
+import static com.github.tomakehurst.wiremock.client.WireMock.post;
+import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.unauthorized;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
+
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.storage.datastore.api.Volume;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import com.cloud.storage.Storage;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.github.tomakehurst.wiremock.client.BasicCredentials;
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
 
 @RunWith(MockitoJUnitRunner.class)
 public class ScaleIOGatewayClientImplTest {
+    private final int port = 443;
+    private final int timeout = 30;
+    private final int maxConnections = 50;
+    private final String username = "admin";
+    private final String password = "P@ssword123";
+    private final String sessionKey = "YWRtaW46MTYyMzM0OTc4NDk0MTo2MWQ2NGQzZWJhMTVmYTVkNDIwNjZmOWMwZDg0ZGZmOQ";
+    private ScaleIOGatewayClient client = null;
 
-    ScaleIOGatewayClientImpl client;
+    @Rule
+    public WireMockRule wireMockRule = new WireMockRule(wireMockConfig()
+            .httpsPort(port)
+            .needClientAuth(false)
+            .basicAdminAuthenticator(username, password)
+            .bindAddress("localhost"));
 
     @Before
     public void setUp() throws Exception {
+        wireMockRule.stubFor(get("/api/login")
+                .willReturn(ok()
+                        .withHeader("Content-Type", "application/json;charset=UTF-8")
+                        .withBody(sessionKey)));
+
+        client = new ScaleIOGatewayClientImpl("https://localhost/api", username, password, false, timeout, maxConnections);
+
+        wireMockRule.stubFor(post("/api/types/Volume/instances")
+                .willReturn(aResponse()
+                        .withHeader("Content-Type", "application/json;charset=UTF-8")
+                        .withStatus(200)
+                        .withBody("{\"id\":\"c948d0b10000000a\"}")));
+
+        wireMockRule.stubFor(get("/api/instances/Volume::c948d0b10000000a")
+                .willReturn(aResponse()
+                        .withHeader("Content-Type", "application/json;charset=UTF-8")
+                        .withStatus(200)
+                        .withBody("{\"storagePoolId\":\"4daaa55e00000000\",\"dataLayout\":\"MediumGranularity\",\"vtreeId\":\"657e289500000009\","
+                                + "\"sizeInKb\":8388608,\"snplIdOfAutoSnapshot\":null,\"volumeType\":\"ThinProvisioned\",\"consistencyGroupId\":null,"
+                                + "\"ancestorVolumeId\":null,\"notGenuineSnapshot\":false,\"accessModeLimit\":\"ReadWrite\",\"secureSnapshotExpTime\":0,"
+                                + "\"useRmcache\":false,\"managedBy\":\"ScaleIO\",\"lockedAutoSnapshot\":false,\"lockedAutoSnapshotMarkedForRemoval\":false,"
+                                + "\"autoSnapshotGroupId\":null,\"compressionMethod\":\"Invalid\",\"pairIds\":null,\"timeStampIsAccurate\":false,\"mappedSdcInfo\":null,"
+                                + "\"retentionLevels\":[],\"snplIdOfSourceVolume\":null,\"volumeReplicationState\":\"UnmarkedForReplication\",\"replicationJournalVolume\":false,"
+                                + "\"replicationTimeStamp\":0,\"originalExpiryTime\":0,\"creationTime\":1623335880,\"name\":\"testvolume\",\"id\":\"c948d0b10000000a\"}")));
     }
 
     @After
     public void tearDown() throws Exception {
     }
 
+    @Test
+    public void testClientAuthSuccess() {
+        Assert.assertNotNull(client);
+        wireMockRule.verify(getRequestedFor(urlEqualTo("/api/login"))
+                .withBasicAuth(new BasicCredentials(username, password)));
+
+        wireMockRule.stubFor(get("/api/types/StoragePool/instances")
+                .willReturn(aResponse()
+                        .withHeader("Content-Type", "application/json;charset=UTF-8")
+                        .withStatus(200)
+                        .withBody("")));
+
+        client.listStoragePools();
+
+        wireMockRule.verify(getRequestedFor(urlEqualTo("/api/types/StoragePool/instances"))
+                .withBasicAuth(new BasicCredentials(username, sessionKey)));
+    }
+
     @Test(expected = CloudRuntimeException.class)
-    public void testClient() throws Exception {
+    public void testClientAuthFailure() throws Exception {
-        client = (ScaleIOGatewayClientImpl) ScaleIOGatewayClient.getClient("https://10.2.3.149/api",
+        wireMockRule.stubFor(get("/api/login")
-                "admin", "P@ssword123", false, 60);
+                .willReturn(unauthorized()
+                        .withHeader("Content-Type", "application/json;charset=UTF-8")
+                        .withBody("")));
+
+        new ScaleIOGatewayClientImpl("https://localhost/api", username, password, false, timeout, maxConnections);
+    }
+
+    @Test(expected = ServerApiException.class)
+    public void testRequestTimeout() {
+        Assert.assertNotNull(client);
+        wireMockRule.verify(getRequestedFor(urlEqualTo("/api/login"))
+                .withBasicAuth(new BasicCredentials(username, password)));
+
+        wireMockRule.stubFor(get("/api/types/StoragePool/instances")
+                .willReturn(aResponse()
+                        .withHeader("Content-Type", "application/json;charset=UTF-8")
+                        .withStatus(200)
+                        .withFixedDelay(2 * timeout * 1000)
+                        .withBody("")));
+
+        client.listStoragePools();
+    }
+
+    @Test
+    public void testCreateSingleVolume() {
+        Assert.assertNotNull(client);
+        wireMockRule.verify(getRequestedFor(urlEqualTo("/api/login"))
+                .withBasicAuth(new BasicCredentials(username, password)));
+
+        final String volumeName = "testvolume";
+        final String scaleIOStoragePoolId = "4daaa55e00000000";
+        final int sizeInGb = 8;
+        Volume scaleIOVolume = client.createVolume(volumeName, scaleIOStoragePoolId, sizeInGb, Storage.ProvisioningType.THIN);
+
+        wireMockRule.verify(postRequestedFor(urlEqualTo("/api/types/Volume/instances"))
+                .withBasicAuth(new BasicCredentials(username, sessionKey))
+                .withRequestBody(containing("\"name\":\"" + volumeName + "\""))
+                .withHeader("Content-Type", equalTo("application/json")));
+        wireMockRule.verify(getRequestedFor(urlEqualTo("/api/instances/Volume::c948d0b10000000a"))
+                .withBasicAuth(new BasicCredentials(username, sessionKey)));
+
+        Assert.assertNotNull(scaleIOVolume);
+        Assert.assertEquals(scaleIOVolume.getId(), "c948d0b10000000a");
+        Assert.assertEquals(scaleIOVolume.getName(), volumeName);
+        Assert.assertEquals(scaleIOVolume.getStoragePoolId(), scaleIOStoragePoolId);
+        Assert.assertEquals(scaleIOVolume.getSizeInKb(), Long.valueOf(sizeInGb * 1024 * 1024));
+        Assert.assertEquals(scaleIOVolume.getVolumeType(), Volume.VolumeType.ThinProvisioned);
+    }
+
+    @Test
+    public void testCreateMultipleVolumes() {
+        Assert.assertNotNull(client);
+        wireMockRule.verify(getRequestedFor(urlEqualTo("/api/login"))
+                .withBasicAuth(new BasicCredentials(username, password)));
+
+        final String volumeNamePrefix = "testvolume_";
+        final String scaleIOStoragePoolId = "4daaa55e00000000";
+        final int sizeInGb = 8;
+        final int volumesCount = 1000;
+
+        for (int i = 1; i <= volumesCount; i++) {
+            String volumeName = volumeNamePrefix + i;
+            Volume scaleIOVolume = client.createVolume(volumeName, scaleIOStoragePoolId, sizeInGb, Storage.ProvisioningType.THIN);
+
+            Assert.assertNotNull(scaleIOVolume);
+            Assert.assertEquals(scaleIOVolume.getId(), "c948d0b10000000a");
+            Assert.assertEquals(scaleIOVolume.getStoragePoolId(), scaleIOStoragePoolId);
+            Assert.assertEquals(scaleIOVolume.getSizeInKb(), Long.valueOf(sizeInGb * 1024 * 1024));
+            Assert.assertEquals(scaleIOVolume.getVolumeType(), Volume.VolumeType.ThinProvisioned);
+        }
+
+        wireMockRule.verify(volumesCount, postRequestedFor(urlEqualTo("/api/types/Volume/instances"))
+                .withBasicAuth(new BasicCredentials(username, sessionKey))
+                .withRequestBody(containing("\"name\":\"" + volumeNamePrefix))
+                .withHeader("Content-Type", equalTo("application/json")));
+        wireMockRule.verify(volumesCount, getRequestedFor(urlEqualTo("/api/instances/Volume::c948d0b10000000a"))
+                .withBasicAuth(new BasicCredentials(username, sessionKey)));
     }
 }

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -474,6 +474,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         configValuesForValidation.add("externaldhcp.vmip.max.retry");
         configValuesForValidation.add("externaldhcp.vmipFetch.threadPool.max");
         configValuesForValidation.add("remote.access.vpn.psk.length");
+        configValuesForValidation.add(StorageManager.STORAGE_POOL_DISK_WAIT.key());
+        configValuesForValidation.add(StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.key());
+        configValuesForValidation.add(StorageManager.STORAGE_POOL_CLIENT_MAX_CONNECTIONS.key());
     }
 
     private void weightBasedParametersForValidation() {

server/src/main/java/com/cloud/storage/StorageManagerImpl.java

@@ -3130,6 +3130,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
                 MaxNumberOfManagedClusteredFileSystems,
                 STORAGE_POOL_DISK_WAIT,
                 STORAGE_POOL_CLIENT_TIMEOUT,
+                STORAGE_POOL_CLIENT_MAX_CONNECTIONS,
                 PRIMARY_STORAGE_DOWNLOAD_WAIT,
                 SecStorageMaxMigrateSessions,
                 MaxDataMigrationWaitTime

server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java

@@ -1485,6 +1485,13 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
         }
     }
 
+    private void removeVolume(long volumeId) {
+        final VolumeVO volume = _volsDao.findById(volumeId);
+        if (volume != null) {
+            _volsDao.remove(volumeId);
+        }
+    }
+
     protected boolean stateTransitTo(Volume vol, Volume.Event event) throws NoTransitionException {
         return _volStateMachine.transitTo(vol, event, null, _volsDao);
     }
@@ -1526,6 +1533,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
             }
         }
 
+        removeVolume(volume.getId());
         return volume;
     }
 
@@ -1621,6 +1629,9 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
 
         if (destPrimaryStorage != null && (volumeToAttach.getState() == Volume.State.Allocated || volumeOnSecondary)) {
             try {
+                if (volumeOnSecondary && destPrimaryStorage.getPoolType() == Storage.StoragePoolType.PowerFlex) {
+                    throw new InvalidParameterValueException("Cannot attach uploaded volume, this operation is unsupported on storage pool type " + destPrimaryStorage.getPoolType());
+                }
                 newVolumeOnPrimaryStorage = _volumeMgr.createVolumeOnPrimaryStorage(vm, volumeToAttach, rootDiskHyperType, destPrimaryStorage);
             } catch (NoTransitionException e) {
                 s_logger.debug("Failed to create volume on primary storage", e);