From 1ba6a49fa484241833d75350ca5d6975c9d67952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Beims=20Br=C3=A4scher?= Date: Tue, 20 Oct 2020 08:19:55 -0300 Subject: [PATCH 1/9] Fix JsonSyntaxException when creating API command response #4355 (#4387) * Add missing \" on obfuscatePassword * Add tests for AsyncJobManagerImpl.obfuscatePassword * fix for "password}," --- .../jobs/impl/AsyncJobManagerImpl.java | 9 +- .../framework/jobs/AsyncJobManagerTest.java | 157 ++++++------------ 2 files changed, 61 insertions(+), 105 deletions(-) diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index a16fc62f869..f1398a344a5 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -475,14 +475,19 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, return job; } - private String obfuscatePassword(String result, boolean hidePassword) { + public String obfuscatePassword(String result, boolean hidePassword) { if (hidePassword) { String pattern = "\"password\":"; if (result != null) { if (result.contains(pattern)) { String[] resp = result.split(pattern); String psswd = resp[1].toString().split(",")[0]; - result = resp[0] + pattern + psswd.replace(psswd.substring(2, psswd.length() - 1), "*****") + "," + resp[1].split(",", 2)[1]; + if (psswd.endsWith("}")) { + psswd = psswd.substring(0, psswd.length() - 1); + result = resp[0] + pattern + psswd.replace(psswd.substring(2, psswd.length() - 1), "*****") + "}," + resp[1].split(",", 2)[1]; + } else { + result = resp[0] + pattern + psswd.replace(psswd.substring(2, psswd.length() - 1), "*****") + "," + resp[1].split(",", 2)[1]; + } } } } diff --git a/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerTest.java b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerTest.java index a49f28ef55d..7130873e4ee 100644 --- a/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerTest.java +++ b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerTest.java @@ -16,116 +16,67 @@ // under the License. package org.apache.cloudstack.framework.jobs; -/* - * This integration test requires real DB setup, it is not meant to run at per-build - * basis, it can only be opened in developer's run - * - * +import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(locations = "classpath:/AsyncJobManagerTestContext.xml") -public class AsyncJobManagerTest extends TestCase { - private static final Logger s_logger = - Logger.getLogger(AsyncJobManagerTest.class); +@RunWith (MockitoJUnitRunner.class) +public class AsyncJobManagerTest { - @Inject - AsyncJobManager _jobMgr; + @Spy + AsyncJobManagerImpl asyncJobManager; - @Inject - AsyncJobTestDashboard _testDashboard; + String input = "\"haprovider\":\"kvmhaprovider\"},\"outofbandmanagement\":{\"powerstate\":\"On\",\"enabled\":true,\"driver\":\"redfish\",\"address\":\"oob-address.com\",\"port\":\"80\",\"username\":\"root\",\"password\":\"password\"},\"resourcestate\":\"PrepareForMaintenance\",\"hahost\":false"; + String expected = "\"haprovider\":\"kvmhaprovider\"},\"outofbandmanagement\":{\"powerstate\":\"On\",\"enabled\":true,\"driver\":\"redfish\",\"address\":\"oob-address.com\",\"port\":\"80\",\"username\":\"root\",\"password\":\"p*****\"},\"resourcestate\":\"PrepareForMaintenance\",\"hahost\":false"; + String obfuscatedInput = "\"haprovider\":\"kvmhaprovider\"},\"outofbandmanagement\":{\"powerstate\":\"On\",\"enabled\":true,\"driver\":\"redfish\",\"address\":\"oob-address.com\",\"port\":\"80\",\"username\":\"root\",\"password\":\"p***\"},\"resourcestate\":\"PrepareForMaintenance\",\"hahost\":false"; + String noPassword = "\"haprovider\":\"kvmhaprovider\"},\"outofbandmanagement\":{\"powerstate\":\"On\",\"enabled\":true,\"driver\":\"redfish\",\"address\":\"oob-address.com\",\"port\":\"80\",\"username\":\"root\"},\"resourcestate\":\"PrepareForMaintenance\",\"hahost\":false"; - @Override - @Before - public void setUp() throws Exception { - try { - ComponentContext.initComponentsLifeCycle(); - } catch (Exception ex) { - ex.printStackTrace(); - s_logger.error(ex.getMessage()); - } - } + String inputNoBraces = "\"password\":\"password\"\",\"action\":\"OFF\""; + String expectedNoBraces = "\"password\":\"p*****\",\"action\":\"OFF\""; - @Override - @After - public void tearDown() throws Exception { - } - - public void testWaitBehave() { - - final Object me = this; - new Thread(new Runnable() { - - @Override - public void run() { - s_logger.info("Sleeping..."); - try { - Thread.sleep(3000); - } catch (InterruptedException e) { - s_logger.debug("[ignored] ."); - } - - s_logger.info("wakeup"); - synchronized (me) { - me.notifyAll(); - } - } - - }).start(); - - s_logger.info("First wait"); - synchronized (me) { - try { - wait(5000); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - s_logger.info("First wait done"); - - s_logger.info("Second wait"); - synchronized (me) { - try { - wait(5000); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - s_logger.info("Second wait done"); + @Test + public void obfuscatePasswordTest() { + String result = asyncJobManager.obfuscatePassword(input, true); + Assert.assertEquals(expected, result); } @Test - public void test() { - final int TOTAL_JOBS_PER_QUEUE = 5; - final int TOTAL_QUEUES = 100; - - for (int i = 0; i < TOTAL_QUEUES; i++) { - for (int j = 0; j < TOTAL_JOBS_PER_QUEUE; j++) { - AsyncJobVO job = new AsyncJobVO(); - job.setCmd("TestCmd"); - job.setDispatcher("TestJobDispatcher"); - job.setCmdInfo("TestCmd info"); - - _jobMgr.submitAsyncJob(job, "fakequeue", i); - - s_logger.info("Job submitted. job " + job.getId() + ", queue: " + i); - } - } - - while (true) { - if (_testDashboard.getCompletedJobCount() == TOTAL_JOBS_PER_QUEUE * TOTAL_QUEUES) - break; - - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - s_logger.debug("[ignored] ."); - } - } - - s_logger.info("Test done with " + _testDashboard.getCompletedJobCount() + " job executed"); + public void obfuscatePasswordTestNoBraces() { + String result = asyncJobManager.obfuscatePassword(inputNoBraces, true); + Assert.assertEquals(expectedNoBraces, result); } -} -*/ + @Test + public void obfuscatePasswordTestHidePasswordFalse() { + String result = asyncJobManager.obfuscatePassword(input, false); + Assert.assertEquals(input, result); + } + + @Test + public void obfuscatePasswordTestObfuscatedInput() { + String result = asyncJobManager.obfuscatePassword(obfuscatedInput, true); + Assert.assertEquals(expected, result); + } + + @Test + public void obfuscatePasswordTestHidePasswordFalseObfuscatedInput() { + String result = asyncJobManager.obfuscatePassword(obfuscatedInput, false); + Assert.assertEquals(obfuscatedInput, result); + } + + @Test + public void obfuscatePasswordTestNoPassword() { + String result = asyncJobManager.obfuscatePassword(noPassword, true); + Assert.assertEquals(noPassword, result); + } + + @Test + public void obfuscatePasswordTestHidePasswordNoPassword() { + String result = asyncJobManager.obfuscatePassword(noPassword, false); + Assert.assertEquals(noPassword, result); + } + +} From 2661ce81d48b46d87d21bf4a9edb9e967562c240 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 20 Oct 2020 15:17:00 +0200 Subject: [PATCH 2/9] server: add global configuration for default router service offering (#3946) --- .../VirtualNetworkApplianceManager.java | 4 +++ .../VirtualNetworkApplianceManagerImpl.java | 1 + .../RouterDeploymentDefinition.java | 29 ++++++++++++++++++- .../VpcRouterDeploymentDefinition.java | 3 ++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManager.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManager.java index 6edbb4400b1..26b2045e13a 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManager.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManager.java @@ -44,6 +44,7 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA static final String RouterTemplateOvm3CK = "router.template.ovm3"; static final String SetServiceMonitorCK = "network.router.EnableServiceMonitoring"; static final String RouterAlertsCheckIntervalCK = "router.alerts.check.interval"; + static final String VirtualRouterServiceOfferingCK = "router.service.offering"; static final String RouterHealthChecksConfigRefreshIntervalCK = "router.health.checks.config.refresh.interval"; static final String RouterHealthChecksResultFetchIntervalCK = "router.health.checks.results.fetch.interval"; @@ -74,6 +75,9 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA static final ConfigKey ExposeDnsAndBootpServer = new ConfigKey(Boolean.class, "expose.dns.externally", "Advanced", "true", "open dns, dhcp and bootp on the public interface", true, ConfigKey.Scope.Zone, null); + static final ConfigKey VirtualRouterServiceOffering = new ConfigKey(String.class, VirtualRouterServiceOfferingCK, "Advanced", "", + "Uuid of the service offering used by virtual routers; if NULL - system offering will be used", true, ConfigKey.Scope.Account, null); + // Health checks static final ConfigKey RouterHealthChecksEnabled = new ConfigKey(Boolean.class, "router.health.checks.enabled", "Advanced", "true", "If true, router health checks are allowed to be executed and read. If false, all scheduled checks and API calls for on demand checks are disabled.", diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 0e70ea8ab7c..1103ff9c058 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -3256,6 +3256,7 @@ Configurable, StateListener Date: Wed, 21 Oct 2020 13:18:05 +0530 Subject: [PATCH 3/9] Validating type parameter and including all types (#4412) --- api/src/main/java/com/cloud/network/Network.java | 16 +++++++++++++++- .../command/user/network/ListNetworksCmd.java | 11 +++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/com/cloud/network/Network.java b/api/src/main/java/com/cloud/network/Network.java index 28528f1b558..599bfeec49e 100644 --- a/api/src/main/java/com/cloud/network/Network.java +++ b/api/src/main/java/com/cloud/network/Network.java @@ -43,7 +43,21 @@ import com.cloud.utils.fsm.StateObject; public interface Network extends ControlledEntity, StateObject, InternalIdentity, Identity, Serializable, Displayable { enum GuestType { - Shared, Isolated, L2 + Shared, Isolated, L2; + + public static GuestType fromValue(String type) { + if (StringUtils.isBlank(type)) { + return null; + } else if (type.equalsIgnoreCase("Shared")) { + return Shared; + } else if (type.equalsIgnoreCase("Isolated")) { + return Isolated; + } else if (type.equalsIgnoreCase("L2")) { + return L2; + } else { + throw new InvalidParameterValueException("Unexpected Guest type : " + type); + } + } } enum PVlanType { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/ListNetworksCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/ListNetworksCmd.java index b737212e599..0127fac6af0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/ListNetworksCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/ListNetworksCmd.java @@ -36,6 +36,7 @@ import org.apache.cloudstack.api.response.ZoneResponse; import com.cloud.network.Network; import com.cloud.utils.Pair; +import com.google.common.base.Strings; @APICommand(name = "listNetworks", description = "Lists all available networks.", responseObject = NetworkResponse.class, responseView = ResponseView.Restricted, entityType = {Network.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -52,7 +53,7 @@ public class ListNetworksCmd extends BaseListTaggedResourcesCmd implements UserC @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "the zone ID of the network") private Long zoneId; - @Parameter(name = ApiConstants.TYPE, type = CommandType.STRING, description = "the type of the network. Supported values are: isolated and shared") + @Parameter(name = ApiConstants.TYPE, type = CommandType.STRING, description = "the type of the network. Supported values are: isolated, l2, shared and all") private String guestIpType; @Parameter(name = ApiConstants.IS_SYSTEM, type = CommandType.BOOLEAN, description = "true if network is system, false otherwise") @@ -101,7 +102,13 @@ public class ListNetworksCmd extends BaseListTaggedResourcesCmd implements UserC } public String getGuestIpType() { - return guestIpType; + if (!Strings.isNullOrEmpty(guestIpType)) { + if (guestIpType.equalsIgnoreCase("all")) { + return null; + } + return Network.GuestType.fromValue(guestIpType).toString(); + } + return null; } public Boolean getIsSystem() { From a5d4bd5f5a0a87509fe258ee4856c2b4d27ba746 Mon Sep 17 00:00:00 2001 From: Rakesh Date: Wed, 21 Oct 2020 14:01:56 +0200 Subject: [PATCH 4/9] Display Kubernetes cluster name instead of uuid (#4411) --- .../cluster/KubernetesClusterManagerImpl.java | 9 ++++----- .../actionworkers/KubernetesClusterStartWorker.java | 10 +++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 8896399ba4f..3ece789c01c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -1187,11 +1187,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne if (clusterDetailsVO != null && !Strings.isNullOrEmpty(clusterDetailsVO.getValue())) { configData = new String(Base64.decodeBase64(clusterDetailsVO.getValue())); } else { - if (KubernetesCluster.State.Starting.equals(kubernetesCluster.getState())) { - throw new CloudRuntimeException(String.format("Setup is in progress for Kubernetes cluster ID: %s, config not available at this moment", kubernetesCluster.getUuid())); - } else { - throw new CloudRuntimeException((String.format("Config not found for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()))); - } + String exceptionMessage = KubernetesCluster.State.Starting.equals(kubernetesCluster.getState()) ? + String.format("Setup is in progress for Kubernetes cluster : %s, config not available at this moment", kubernetesCluster.getName()) : + String.format("Config not found for Kubernetes cluster : %s", kubernetesCluster.getName()); + throw new CloudRuntimeException(exceptionMessage); } response.setConfigData(configData); response.setObjectName("clusterconfig"); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 90c03751279..20f509e3270 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -579,22 +579,22 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif try { InetAddress address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); } catch (MalformedURLException | UnknownHostException ex) { - logTransitStateAndThrow(Level.ERROR, String.format("Kubernetes cluster ID: %s has invalid API endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + logTransitStateAndThrow(Level.ERROR, String.format("Kubernetes cluster : %s has invalid API endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } Pair sshIpPort = getKubernetesClusterServerIpSshPort(null); publicIpAddress = sshIpPort.first(); sshPort = sshIpPort.second(); if (Strings.isNullOrEmpty(publicIpAddress)) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s as no public IP found for the cluster" , kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } if (!KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, CLUSTER_API_PORT, startTimeoutTime, 15000)) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s in usable state", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } if (!isKubernetesClusterKubeConfigAvailable(startTimeoutTime)) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } if (!isKubernetesClusterDashboardServiceRunning(false, startTimeoutTime)) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); if (LOGGER.isInfoEnabled()) { From 71c5dbcf492a023dbea5f8c34f8fd883c3ad653f Mon Sep 17 00:00:00 2001 From: Rakesh Date: Wed, 21 Oct 2020 19:18:03 +0200 Subject: [PATCH 5/9] server: Update use_bytes of storage pools (#4360) Update the used_bytes for all default primary storage pools Also get used_bytes of storage pool from database instead of memory --- .../storage/datastore/db/StoragePoolVO.java | 4 ++ .../java/com/cloud/server/StatsCollector.java | 8 +++- .../com/cloud/storage/StorageManagerImpl.java | 37 +++++++------------ 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java index 24fcaa03f56..b30b33db8fc 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java @@ -195,6 +195,10 @@ public class StoragePoolVO implements StoragePool { return updateTime; } + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + @Override public long getDataCenterId() { return dataCenterId; diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 43784daf9c8..fc1f39bccc6 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -36,6 +36,7 @@ import javax.inject.Inject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.framework.config.ConfigKey; @@ -1027,8 +1028,13 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc storagePoolStats.put(pool.getId(), (StorageStats)answer); // Seems like we have dynamically updated the pool size since the prev. size and the current do not match - if (_storagePoolStats.get(poolId) != null && _storagePoolStats.get(poolId).getCapacityBytes() != ((StorageStats)answer).getCapacityBytes()) { + if (pool.getCapacityBytes() != ((StorageStats)answer).getCapacityBytes() || + pool.getUsedBytes() != ((StorageStats)answer).getByteUsed()) { pool.setCapacityBytes(((StorageStats)answer).getCapacityBytes()); + if (pool.getStorageProviderName().equalsIgnoreCase(DataStoreProvider.DEFAULT_PRIMARY)) { + pool.setUsedBytes(((StorageStats) answer).getByteUsed()); + pool.setUpdateTime(new Date()); + } _storagePoolDao.update(pool.getId(), pool); } } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index c59a26d3bc6..3c707c37bac 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -151,7 +151,6 @@ import com.cloud.org.Grouping.AllocationState; import com.cloud.resource.ResourceState; import com.cloud.server.ConfigurationServer; import com.cloud.server.ManagementServer; -import com.cloud.server.StatsCollector; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Volume.Type; @@ -1730,31 +1729,21 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C return true; } - StatsCollector sc = StatsCollector.getInstance(); double storageUsedThreshold = CapacityManager.StorageCapacityDisableThreshold.valueIn(pool.getDataCenterId()); - if (sc != null) { - long totalSize = pool.getCapacityBytes(); - StorageStats stats = sc.getStoragePoolStats(pool.getId()); - if (stats == null) { - stats = sc.getStorageStats(pool.getId()); - } - if (stats != null) { - double usedPercentage = ((double)stats.getByteUsed() / (double)totalSize); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Checking pool " + pool.getId() + " for storage, totalSize: " + toHumanReadableSize(pool.getCapacityBytes()) + ", usedBytes: " + toHumanReadableSize(stats.getByteUsed()) + ", usedPct: " + usedPercentage - + ", disable threshold: " + storageUsedThreshold); - } - if (usedPercentage >= storageUsedThreshold) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Insufficient space on pool: " + pool.getId() + " since its usage percentage: " + usedPercentage + " has crossed the pool.storage.capacity.disablethreshold: " - + storageUsedThreshold); - } - return false; - } - } - return true; + long totalSize = pool.getCapacityBytes(); + double usedPercentage = ((double)pool.getUsedBytes() / (double)totalSize); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Checking pool " + pool.getId() + " for storage, totalSize: " + pool.getCapacityBytes() + ", usedBytes: " + pool.getUsedBytes() + + ", usedPct: " + usedPercentage + ", disable threshold: " + storageUsedThreshold); } - return false; + if (usedPercentage >= storageUsedThreshold) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Insufficient space on pool: " + pool.getId() + " since its usage percentage: " + usedPercentage + + " has crossed the pool.storage.capacity.disablethreshold: " + storageUsedThreshold); + } + return false; + } + return true; } @Override From ff8a84ee77f4f668cb9c8c5a3978f6fb4fb0867e Mon Sep 17 00:00:00 2001 From: Wei Zhou <57355700+weizhouapache@users.noreply.github.com> Date: Wed, 21 Oct 2020 20:21:54 +0300 Subject: [PATCH 6/9] systemvm: fix proc.find in CsProcess.py (#4413) Co-authored-by: Wei Zhou --- systemvm/debian/opt/cloud/bin/cs/CsApp.py | 1 - systemvm/debian/opt/cloud/bin/cs/CsProcess.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/systemvm/debian/opt/cloud/bin/cs/CsApp.py b/systemvm/debian/opt/cloud/bin/cs/CsApp.py index 9f3375f7980..a2292ae3069 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsApp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsApp.py @@ -17,7 +17,6 @@ # under the License. import os from CsFile import CsFile -from CsProcess import CsProcess import CsHelper diff --git a/systemvm/debian/opt/cloud/bin/cs/CsProcess.py b/systemvm/debian/opt/cloud/bin/cs/CsProcess.py index c3a42e5a8b3..4a64807c86f 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsProcess.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsProcess.py @@ -40,9 +40,9 @@ class CsProcess(object): def find_pid(self): self.pid = [] + items = len(self.search) for i in CsHelper.execute("ps aux"): - items = len(self.search) - proc = re.split(r"\s+", i)[items*-1:] + proc = re.split(r"\s+", i)[10:] matches = len([m for m in proc if m in self.search]) if matches == items: self.pid.append(re.split(r"\s+", i)[1]) From f4f35a8995a044446883e8e41c5a5de014e27581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Beims=20Br=C3=A4scher?= Date: Sat, 24 Oct 2020 08:00:11 -0300 Subject: [PATCH 7/9] Enhance UpdateDiskOfferingCmd (#4409) --- .../admin/offering/UpdateDiskOfferingCmd.java | 92 +++++++++- .../ConfigurationManagerImpl.java | 166 ++++++++++-------- .../ConfigurationManagerTest.java | 44 ++++- 3 files changed, 219 insertions(+), 83 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java index e1fd8783838..5ccdc06cc05 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java @@ -85,6 +85,44 @@ public class UpdateDiskOfferingCmd extends BaseCmd { since = "4.15") private String tags; + @Parameter(name = ApiConstants.BYTES_READ_RATE, type = CommandType.LONG, description = "bytes read rate of the disk offering", since = "4.15") + private Long bytesReadRate; + + @Parameter(name = ApiConstants.BYTES_READ_RATE_MAX, type = CommandType.LONG, description = "burst bytes read rate of the disk offering", since = "4.15") + private Long bytesReadRateMax; + + @Parameter(name = ApiConstants.BYTES_READ_RATE_MAX_LENGTH, type = CommandType.LONG, description = "length (in seconds) of the burst", since = "4.15") + private Long bytesReadRateMaxLength; + + @Parameter(name = ApiConstants.BYTES_WRITE_RATE, type = CommandType.LONG, description = "bytes write rate of the disk offering", since = "4.15") + private Long bytesWriteRate; + + @Parameter(name = ApiConstants.BYTES_WRITE_RATE_MAX, type = CommandType.LONG, description = "burst bytes write rate of the disk offering", since = "4.15") + private Long bytesWriteRateMax; + + @Parameter(name = ApiConstants.BYTES_WRITE_RATE_MAX_LENGTH, type = CommandType.LONG, description = "length (in seconds) of the burst", since = "4.15") + private Long bytesWriteRateMaxLength; + + @Parameter(name = ApiConstants.IOPS_READ_RATE, type = CommandType.LONG, description = "io requests read rate of the disk offering", since = "4.15") + private Long iopsReadRate; + + @Parameter(name = ApiConstants.IOPS_READ_RATE_MAX, type = CommandType.LONG, description = "burst requests read rate of the disk offering", since = "4.15") + private Long iopsReadRateMax; + + @Parameter(name = ApiConstants.IOPS_READ_RATE_MAX_LENGTH, type = CommandType.LONG, description = "length (in seconds) of the burst", since = "4.15") + private Long iopsReadRateMaxLength; + + @Parameter(name = ApiConstants.IOPS_WRITE_RATE, type = CommandType.LONG, description = "io requests write rate of the disk offering", since = "4.15") + private Long iopsWriteRate; + + @Parameter(name = ApiConstants.IOPS_WRITE_RATE_MAX, type = CommandType.LONG, description = "burst io requests write rate of the disk offering", since = "4.15") + private Long iopsWriteRateMax; + + @Parameter(name = ApiConstants.IOPS_WRITE_RATE_MAX_LENGTH, type = CommandType.LONG, description = "length (in seconds) of the burst", since = "4.15") + private Long iopsWriteRateMaxLength; + + @Parameter(name = ApiConstants.CACHE_MODE, type = CommandType.STRING, description = "the cache mode to use for this disk offering", since = "4.15") + private String cacheMode; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -174,7 +212,59 @@ public class UpdateDiskOfferingCmd extends BaseCmd { return tags; } -///////////////////////////////////////////////////// + public String getCacheMode() { + return cacheMode; + } + + public Long getBytesReadRate() { + return bytesReadRate; + } + + public Long getBytesReadRateMax() { + return bytesReadRateMax; + } + + public Long getBytesReadRateMaxLength() { + return bytesReadRateMaxLength; + } + + public Long getBytesWriteRate() { + return bytesWriteRate; + } + + public Long getBytesWriteRateMax() { + return bytesWriteRateMax; + } + + public Long getBytesWriteRateMaxLength() { + return bytesWriteRateMaxLength; + } + + public Long getIopsReadRate() { + return iopsReadRate; + } + + public Long getIopsReadRateMax() { + return iopsReadRateMax; + } + + public Long getIopsReadRateMaxLength() { + return iopsReadRateMaxLength; + } + + public Long getIopsWriteRate() { + return iopsWriteRate; + } + + public Long getIopsWriteRateMax() { + return iopsWriteRateMax; + } + + public Long getIopsWriteRateMaxLength() { + return iopsWriteRateMaxLength; + } + + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index ac2ff0762bc..cc24a45a08f 100755 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -2530,42 +2530,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati offering.setMinIops(minIops); offering.setMaxIops(maxIops); - if (bytesReadRate != null && bytesReadRate > 0) { - offering.setBytesReadRate(bytesReadRate); - } - if (bytesReadRateMax != null && bytesReadRateMax > 0) { - offering.setBytesReadRateMax(bytesReadRateMax); - } - if (bytesReadRateMaxLength != null && bytesReadRateMaxLength > 0) { - offering.setBytesReadRateMaxLength(bytesReadRateMaxLength); - } - if (bytesWriteRate != null && bytesWriteRate > 0) { - offering.setBytesWriteRate(bytesWriteRate); - } - if (bytesWriteRateMax != null && bytesWriteRateMax > 0) { - offering.setBytesWriteRateMax(bytesWriteRateMax); - } - if (bytesWriteRateMaxLength != null && bytesWriteRateMaxLength > 0) { - offering.setBytesWriteRateMaxLength(bytesWriteRateMaxLength); - } - if (iopsReadRate != null && iopsReadRate > 0) { - offering.setIopsReadRate(iopsReadRate); - } - if (iopsReadRateMax != null && iopsReadRateMax > 0) { - offering.setIopsReadRateMax(iopsReadRateMax); - } - if (iopsReadRateMaxLength != null && iopsReadRateMaxLength > 0) { - offering.setIopsReadRateMaxLength(iopsReadRateMaxLength); - } - if (iopsWriteRate != null && iopsWriteRate > 0) { - offering.setIopsWriteRate(iopsWriteRate); - } - if (iopsWriteRateMax != null && iopsWriteRateMax > 0) { - offering.setIopsWriteRateMax(iopsWriteRateMax); - } - if (iopsWriteRateMaxLength != null && iopsWriteRateMaxLength > 0) { - offering.setIopsWriteRateMaxLength(iopsWriteRateMaxLength); - } + setBytesRate(offering, bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength); + setIopsRate(offering, iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, iopsWriteRateMax, iopsWriteRateMaxLength); + if(cacheMode != null) { offering.setCacheMode(DiskOffering.DiskCacheMode.valueOf(cacheMode.toUpperCase())); } @@ -2631,6 +2598,48 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } + private void setIopsRate(DiskOffering offering, Long iopsReadRate, Long iopsReadRateMax, Long iopsReadRateMaxLength, Long iopsWriteRate, Long iopsWriteRateMax, Long iopsWriteRateMaxLength) { + if (iopsReadRate != null && iopsReadRate > 0) { + offering.setIopsReadRate(iopsReadRate); + } + if (iopsReadRateMax != null && iopsReadRateMax > 0) { + offering.setIopsReadRateMax(iopsReadRateMax); + } + if (iopsReadRateMaxLength != null && iopsReadRateMaxLength > 0) { + offering.setIopsReadRateMaxLength(iopsReadRateMaxLength); + } + if (iopsWriteRate != null && iopsWriteRate > 0) { + offering.setIopsWriteRate(iopsWriteRate); + } + if (iopsWriteRateMax != null && iopsWriteRateMax > 0) { + offering.setIopsWriteRateMax(iopsWriteRateMax); + } + if (iopsWriteRateMaxLength != null && iopsWriteRateMaxLength > 0) { + offering.setIopsWriteRateMaxLength(iopsWriteRateMaxLength); + } + } + + private void setBytesRate(DiskOffering offering, Long bytesReadRate, Long bytesReadRateMax, Long bytesReadRateMaxLength, Long bytesWriteRate, Long bytesWriteRateMax, Long bytesWriteRateMaxLength) { + if (bytesReadRate != null && bytesReadRate > 0) { + offering.setBytesReadRate(bytesReadRate); + } + if (bytesReadRateMax != null && bytesReadRateMax > 0) { + offering.setBytesReadRateMax(bytesReadRateMax); + } + if (bytesReadRateMaxLength != null && bytesReadRateMaxLength > 0) { + offering.setBytesReadRateMaxLength(bytesReadRateMaxLength); + } + if (bytesWriteRate != null && bytesWriteRate > 0) { + offering.setBytesWriteRate(bytesWriteRate); + } + if (bytesWriteRateMax != null && bytesWriteRateMax > 0) { + offering.setBytesWriteRateMax(bytesWriteRateMax); + } + if (bytesWriteRateMaxLength != null && bytesWriteRateMaxLength > 0) { + offering.setBytesWriteRateMaxLength(bytesWriteRateMaxLength); + } + } + @Override @ActionEvent(eventType = EventTypes.EVENT_SERVICE_OFFERING_EDIT, eventDescription = "updating service offering") public ServiceOffering updateServiceOffering(final UpdateServiceOfferingCmd cmd) { @@ -2897,42 +2906,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati newDiskOffering.setUseLocalStorage(localStorageRequired); newDiskOffering.setDisplayOffering(isDisplayOfferingEnabled); - if (bytesReadRate != null && bytesReadRate > 0) { - newDiskOffering.setBytesReadRate(bytesReadRate); - } - if (bytesReadRateMax != null && bytesReadRateMax > 0) { - newDiskOffering.setBytesReadRateMax(bytesReadRateMax); - } - if (bytesReadRateMaxLength != null && bytesReadRateMaxLength > 0) { - newDiskOffering.setBytesReadRateMaxLength(bytesReadRateMaxLength); - } - if (bytesWriteRate != null && bytesWriteRate > 0) { - newDiskOffering.setBytesWriteRate(bytesWriteRate); - } - if (bytesWriteRateMax != null && bytesWriteRateMax > 0) { - newDiskOffering.setBytesWriteRateMax(bytesWriteRateMax); - } - if (bytesWriteRateMaxLength != null && bytesWriteRateMaxLength > 0) { - newDiskOffering.setBytesWriteRateMaxLength(bytesWriteRateMaxLength); - } - if (iopsReadRate != null && iopsReadRate > 0) { - newDiskOffering.setIopsReadRate(iopsReadRate); - } - if (iopsReadRateMax != null && iopsReadRateMax > 0) { - newDiskOffering.setIopsReadRateMax(iopsReadRateMax); - } - if (iopsReadRateMaxLength != null && iopsReadRateMaxLength > 0) { - newDiskOffering.setIopsReadRateMaxLength(iopsReadRateMaxLength); - } - if (iopsWriteRate != null && iopsWriteRate > 0) { - newDiskOffering.setIopsWriteRate(iopsWriteRate); - } - if (iopsWriteRateMax != null && iopsWriteRateMax > 0) { - newDiskOffering.setIopsWriteRateMax(iopsWriteRateMax); - } - if (iopsWriteRateMaxLength != null && iopsWriteRateMaxLength > 0) { - newDiskOffering.setIopsWriteRateMaxLength(iopsWriteRateMaxLength); - } + setBytesRate(newDiskOffering, bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength); + setIopsRate(newDiskOffering, iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, iopsWriteRateMax, iopsWriteRateMaxLength); + if (cacheMode != null) { newDiskOffering.setCacheMode(DiskOffering.DiskCacheMode.valueOf(cacheMode.toUpperCase())); } @@ -3111,6 +3087,20 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati final List zoneIds = cmd.getZoneIds(); final String tags = cmd.getTags(); + Long bytesReadRate = cmd.getBytesReadRate(); + Long bytesReadRateMax = cmd.getBytesReadRateMax(); + Long bytesReadRateMaxLength = cmd.getBytesReadRateMaxLength(); + Long bytesWriteRate = cmd.getBytesWriteRate(); + Long bytesWriteRateMax = cmd.getBytesWriteRateMax(); + Long bytesWriteRateMaxLength = cmd.getBytesWriteRateMaxLength(); + Long iopsReadRate = cmd.getIopsReadRate(); + Long iopsReadRateMax = cmd.getIopsReadRateMax(); + Long iopsReadRateMaxLength = cmd.getIopsReadRateMaxLength(); + Long iopsWriteRate = cmd.getIopsWriteRate(); + Long iopsWriteRateMax = cmd.getIopsWriteRateMax(); + Long iopsWriteRateMaxLength = cmd.getIopsWriteRateMaxLength(); + String cacheMode = cmd.getCacheMode(); + // Check if diskOffering exists final DiskOffering diskOfferingHandle = _entityMgr.findById(DiskOffering.class, diskOfferingId); if (diskOfferingHandle == null) { @@ -3191,7 +3181,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati throw new InvalidParameterValueException(String.format("Unable to update disk offering: %s by id user: %s because it is not root-admin or domain-admin", diskOfferingHandle.getUuid(), user.getUuid())); } - final boolean updateNeeded = shouldUpdateDiskOffering(name, displayText, sortKey, displayDiskOffering, tags); + boolean updateNeeded = shouldUpdateDiskOffering(name, displayText, sortKey, displayDiskOffering, tags, cacheMode) || + shouldUpdateIopsRateParameters(iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, iopsWriteRateMax, iopsWriteRateMaxLength) || + shouldUpdateBytesRateParameters(bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength); + final boolean detailsUpdateNeeded = !filteredDomainIds.equals(existingDomainIds) || !filteredZoneIds.equals(existingZoneIds); if (!updateNeeded && !detailsUpdateNeeded) { return _diskOfferingDao.findById(diskOfferingId); @@ -3217,6 +3210,20 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati updateDiskOfferingTagsIfIsNotNull(tags, diskOffering); + validateMaxRateEqualsOrGreater(iopsReadRate, iopsReadRateMax, IOPS_READ_RATE); + validateMaxRateEqualsOrGreater(iopsWriteRate, iopsWriteRateMax, IOPS_WRITE_RATE); + validateMaxRateEqualsOrGreater(bytesReadRate, bytesReadRateMax, BYTES_READ_RATE); + validateMaxRateEqualsOrGreater(bytesWriteRate, bytesWriteRateMax, BYTES_WRITE_RATE); + validateMaximumIopsAndBytesLength(iopsReadRateMaxLength, iopsWriteRateMaxLength, bytesReadRateMaxLength, bytesWriteRateMaxLength); + + setBytesRate(diskOffering, bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength); + setIopsRate(diskOffering, iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, iopsWriteRateMax, iopsWriteRateMaxLength); + + if (cacheMode != null) { + validateCacheMode(cacheMode); + diskOffering.setCacheMode(DiskOffering.DiskCacheMode.valueOf(cacheMode.toUpperCase())); + } + if (updateNeeded && !_diskOfferingDao.update(diskOfferingId, diskOffering)) { return null; } @@ -3273,8 +3280,17 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati * Check if it needs to update any parameter when updateDiskoffering is called * Verify if name or displayText are not blank, tags is not null, sortkey and displayDiskOffering is not null */ - protected boolean shouldUpdateDiskOffering(String name, String displayText, Integer sortKey, Boolean displayDiskOffering, String tags) { - return StringUtils.isNotBlank(name) || StringUtils.isNotBlank(displayText) || tags != null || sortKey != null || displayDiskOffering != null; + protected boolean shouldUpdateDiskOffering(String name, String displayText, Integer sortKey, Boolean displayDiskOffering, String tags, String cacheMode) { + return StringUtils.isNotBlank(name) || StringUtils.isNotBlank(displayText) || tags != null || sortKey != null || displayDiskOffering != null || StringUtils.isNotBlank(cacheMode); + } + + protected boolean shouldUpdateBytesRateParameters(Long bytesReadRate, Long bytesReadRateMax, Long bytesReadRateMaxLength, Long bytesWriteRate, Long bytesWriteRateMax, Long bytesWriteRateMaxLength) { + return bytesReadRate != null || bytesReadRateMax != null || bytesReadRateMaxLength != null || bytesWriteRate != null || + bytesWriteRateMax != null || bytesWriteRateMaxLength != null; + } + + protected boolean shouldUpdateIopsRateParameters(Long iopsReadRate, Long iopsReadRateMax, Long iopsReadRateMaxLength, Long iopsWriteRate, Long iopsWriteRateMax, Long iopsWriteRateMaxLength) { + return iopsReadRate != null || iopsReadRateMax != null || iopsReadRateMaxLength != null || iopsWriteRate != null || iopsWriteRateMax != null || iopsWriteRateMaxLength != null; } @Override diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java index 0f95cb074f4..3d18d1a99c0 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java @@ -946,17 +946,47 @@ public class ConfigurationManagerTest { @Test public void shouldUpdateDiskOfferingTests(){ - Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyString())); - Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(Mockito.anyString(), nullable(String.class), nullable(Integer.class), nullable(Boolean.class), nullable(String.class))); - Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), Mockito.anyString(), nullable(Integer.class), nullable(Boolean.class), nullable(String.class))); - Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), nullable(String.class), Mockito.anyInt(), nullable(Boolean.class), nullable(String.class))); - Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), nullable(String.class), nullable(int.class), Mockito.anyBoolean(), nullable(String.class))); - Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), nullable(String.class), nullable(int.class), nullable(Boolean.class), Mockito.anyString())); + Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyString(), Mockito.anyString())); + Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(Mockito.anyString(), nullable(String.class), nullable(Integer.class), nullable(Boolean.class), nullable(String.class), nullable(String.class))); + Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), Mockito.anyString(), nullable(Integer.class), nullable(Boolean.class), nullable(String.class), nullable(String.class))); + Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), nullable(String.class), Mockito.anyInt(), nullable(Boolean.class), nullable(String.class), nullable(String.class))); + Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), nullable(String.class), nullable(int.class), Mockito.anyBoolean(), nullable(String.class), nullable(String.class))); + Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), nullable(String.class), nullable(int.class), nullable(Boolean.class), Mockito.anyString(), Mockito.anyString())); } @Test public void shouldUpdateDiskOfferingTestFalse(){ - Assert.assertFalse(configurationMgr.shouldUpdateDiskOffering(null, null, null, null, null)); + Assert.assertFalse(configurationMgr.shouldUpdateDiskOffering(null, null, null, null, null, null)); + } + + @Test + public void shouldUpdateIopsRateParametersTestFalse() { + Assert.assertFalse(configurationMgr.shouldUpdateIopsRateParameters(null, null, null, null, null, null)); + } + + @Test + public void shouldUpdateIopsRateParametersTests(){ + Assert.assertTrue(configurationMgr.shouldUpdateIopsRateParameters(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong())); + Assert.assertTrue(configurationMgr.shouldUpdateIopsRateParameters(nullable(Long.class), Mockito.anyLong(), nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class))); + Assert.assertTrue(configurationMgr.shouldUpdateIopsRateParameters(nullable(Long.class), nullable(Long.class), Mockito.anyLong(), nullable(Long.class), nullable(Long.class), nullable(Long.class))); + Assert.assertTrue(configurationMgr.shouldUpdateIopsRateParameters(nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.anyLong(), nullable(Long.class), nullable(Long.class))); + Assert.assertTrue(configurationMgr.shouldUpdateIopsRateParameters(nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.anyLong(), nullable(Long.class))); + Assert.assertTrue(configurationMgr.shouldUpdateIopsRateParameters(nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.anyLong())); + } + + @Test + public void shouldUpdateBytesRateParametersTestFalse() { + Assert.assertFalse(configurationMgr.shouldUpdateBytesRateParameters(null, null, null, null, null, null)); + } + + @Test + public void shouldUpdateBytesRateParametersTests(){ + Assert.assertTrue(configurationMgr.shouldUpdateBytesRateParameters(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong())); + Assert.assertTrue(configurationMgr.shouldUpdateBytesRateParameters(nullable(Long.class), Mockito.anyLong(), nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class))); + Assert.assertTrue(configurationMgr.shouldUpdateBytesRateParameters(nullable(Long.class), nullable(Long.class), Mockito.anyLong(), nullable(Long.class), nullable(Long.class), nullable(Long.class))); + Assert.assertTrue(configurationMgr.shouldUpdateBytesRateParameters(nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.anyLong(), nullable(Long.class), nullable(Long.class))); + Assert.assertTrue(configurationMgr.shouldUpdateBytesRateParameters(nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.anyLong(), nullable(Long.class))); + Assert.assertTrue(configurationMgr.shouldUpdateBytesRateParameters(nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.anyLong())); } @Test From 5316810aa3deceb27b01d60e27ffccd6618be51f Mon Sep 17 00:00:00 2001 From: Syed Mushtaq Ahmed Date: Sat, 24 Oct 2020 07:00:57 -0400 Subject: [PATCH 8/9] [CLOUDSTACK-10020] Changes to make marvin work with projects and VPCs (#2206) Co-authored-by: dahn --- tools/marvin/marvin/lib/base.py | 111 ++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 35 deletions(-) diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 10ea666b2de..73c96d761b1 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -451,34 +451,53 @@ class VirtualMachine: @classmethod def access_ssh_over_nat( cls, apiclient, services, virtual_machine, allow_egress=False, - networkid=None): + networkid=None, vpcid=None): """ Program NAT and PF rules to open up ssh access to deployed guest @return: """ - public_ip = PublicIPAddress.create( - apiclient=apiclient, - accountid=virtual_machine.account, - zoneid=virtual_machine.zoneid, - domainid=virtual_machine.domainid, - services=services, - networkid=networkid - ) - FireWallRule.create( - apiclient=apiclient, - ipaddressid=public_ip.ipaddress.id, - protocol='TCP', - cidrlist=['0.0.0.0/0'], - startport=22, - endport=22 - ) - nat_rule = NATRule.create( - apiclient=apiclient, - virtual_machine=virtual_machine, - services=services, - ipaddressid=public_ip.ipaddress.id - ) - if allow_egress: + # VPCs have ACLs managed differently + if vpcid: + public_ip = PublicIPAddress.create( + apiclient=apiclient, + accountid=virtual_machine.account, + zoneid=virtual_machine.zoneid, + domainid=virtual_machine.domainid, + services=services, + vpcid=vpcid + ) + + nat_rule = NATRule.create( + apiclient=apiclient, + virtual_machine=virtual_machine, + services=services, + ipaddressid=public_ip.ipaddress.id, + networkid=networkid) + else: + public_ip = PublicIPAddress.create( + apiclient=apiclient, + accountid=virtual_machine.account, + zoneid=virtual_machine.zoneid, + domainid=virtual_machine.domainid, + services=services, + networkid=networkid, + ) + + FireWallRule.create( + apiclient=apiclient, + ipaddressid=public_ip.ipaddress.id, + protocol='TCP', + cidrlist=['0.0.0.0/0'], + startport=22, + endport=22 + ) + nat_rule = NATRule.create( + apiclient=apiclient, + virtual_machine=virtual_machine, + services=services, + ipaddressid=public_ip.ipaddress.id) + + if allow_egress and not vpcid: try: EgressFireWallRule.create( apiclient=apiclient, @@ -502,7 +521,7 @@ class VirtualMachine: hostid=None, keypair=None, ipaddress=None, mode='default', method='GET', hypervisor=None, customcpunumber=None, customcpuspeed=None, custommemory=None, rootdisksize=None, - rootdiskcontroller=None, macaddress=None, datadisktemplate_diskoffering_list={}): + rootdiskcontroller=None, vpcid=None, macaddress=None, datadisktemplate_diskoffering_list={}): """Create the instance""" cmd = deployVirtualMachine.deployVirtualMachineCmd() @@ -654,7 +673,8 @@ class VirtualMachine: services, virtual_machine, allow_egress=allow_egress, - networkid=cmd.networkids[0] if cmd.networkids else None) + networkid=cmd.networkids[0] if cmd.networkids else None, + vpcid=vpcid) elif mode.lower() == 'basic': if virtual_machine.publicip is not None: # EIP/ELB (netscaler) enabled zone @@ -1042,7 +1062,7 @@ class Volume: @classmethod def create_custom_disk(cls, apiclient, services, account=None, - domainid=None, diskofferingid=None): + domainid=None, diskofferingid=None, projectid=None): """Create Volume from Custom disk offering""" cmd = createVolume.createVolumeCmd() cmd.name = services["diskname"] @@ -1065,19 +1085,22 @@ class Volume: if account: cmd.account = account - else: + elif "account" in services: cmd.account = services["account"] if domainid: cmd.domainid = domainid - else: + elif "domainid" in services: cmd.domainid = services["domainid"] + if projectid: + cmd.projectid = projectid + return Volume(apiclient.createVolume(cmd).__dict__) @classmethod def create_from_snapshot(cls, apiclient, snapshot_id, services, - account=None, domainid=None): + account=None, domainid=None, projectid=None): """Create Volume from snapshot""" cmd = createVolume.createVolumeCmd() cmd.name = "-".join([services["diskname"], random_gen()]) @@ -1091,12 +1114,16 @@ class Volume: cmd.ispublic = False if account: cmd.account = account - else: + elif "account" in services: cmd.account = services["account"] if domainid: cmd.domainid = domainid - else: + elif "domainid" in services: cmd.domainid = services["domainid"] + + if projectid: + cmd.projectid = projectid + return Volume(apiclient.createVolume(cmd).__dict__) @classmethod @@ -1445,8 +1472,8 @@ class Template: return Template(apiclient.createTemplate(cmd).__dict__) @classmethod - def create_from_snapshot(cls, apiclient, snapshot, services, - random_name=True): + def create_from_snapshot(cls, apiclient, snapshot, services, account=None, + domainid=None, projectid=None, random_name=True): """Create Template from snapshot""" # Create template from Snapshot ID cmd = createTemplate.createTemplateCmd() @@ -1485,6 +1512,17 @@ class Template: raise Exception( "Unable to find Ostype is required for creating template") + cmd.snapshotid = snapshot.id + + if account: + cmd.account = account + if domainid: + cmd.domainid = domainid + if projectid: + cmd.projectid = projectid + + return Template(apiclient.createTemplate(cmd).__dict__) + def delete(self, apiclient, zoneid=None): """Delete Template""" @@ -3921,7 +3959,7 @@ class VpnCustomerGateway: @classmethod def create(cls, apiclient, services, name, gateway, cidrlist, - account=None, domainid=None): + account=None, domainid=None, projectid=None): """Create VPN Customer Gateway""" cmd = createVpnCustomerGateway.createVpnCustomerGatewayCmd() cmd.name = name @@ -3945,6 +3983,9 @@ class VpnCustomerGateway: cmd.account = account if domainid: cmd.domainid = domainid + if projectid: + cmd.projectid = projectid + return VpnCustomerGateway( apiclient.createVpnCustomerGateway(cmd).__dict__) From 81c524e9f393157e00474d1f975d3de4d56da2f5 Mon Sep 17 00:00:00 2001 From: davidjumani Date: Sat, 24 Oct 2020 16:38:45 +0530 Subject: [PATCH 9/9] Fixing count for findHostsForMigration (#4375) --- .../cloud/server/ManagementServerImpl.java | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 80b54c09dfa..bd5de221ba8 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -1172,8 +1172,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe final Object resourceState = cmd.getResourceState(); final Object haHosts = cmd.getHaHost(); - final Pair, Integer> result = searchForServers(cmd.getStartIndex(), cmd.getPageSizeVal(), name, type, state, zoneId, pod, cluster, id, keyword, resourceState, haHosts, null, - null); + final Pair, Integer> result = searchForServers(cmd.getStartIndex(), cmd.getPageSizeVal(), name, type, state, zoneId, pod, + cluster, id, keyword, resourceState, haHosts, null, null); return new Pair, Integer>(result.first(), result.second()); } @@ -1275,19 +1275,20 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe final Type hostType = srcHost.getType(); Pair, Integer> allHostsPair = null; List allHosts = null; + List hostsForMigrationWithStorage = null; final Map requiresStorageMotion = new HashMap(); DataCenterDeployment plan = null; if (canMigrateWithStorage) { - allHostsPair = searchForServers(startIndex, pageSize, null, hostType, null, srcHost.getDataCenterId(), null, null, null, keyword, null, null, srcHost.getHypervisorType(), - srcHost.getHypervisorVersion()); + allHostsPair = searchForServers(startIndex, pageSize, null, hostType, null, srcHost.getDataCenterId(), null, null, null, keyword, + null, null, srcHost.getHypervisorType(), srcHost.getHypervisorVersion(), srcHost.getId()); allHosts = allHostsPair.first(); - allHosts.remove(srcHost); + hostsForMigrationWithStorage = new ArrayList<>(allHosts); for (final VolumeVO volume : volumes) { StoragePool storagePool = _poolDao.findById(volume.getPoolId()); Long volClusterId = storagePool.getClusterId(); - for (Iterator iterator = allHosts.iterator(); iterator.hasNext();) { + for (Iterator iterator = hostsForMigrationWithStorage.iterator(); iterator.hasNext();) { final Host host = iterator.next(); if (volClusterId != null) { @@ -1326,10 +1327,9 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe if (s_logger.isDebugEnabled()) { s_logger.debug("Searching for all hosts in cluster " + cluster + " for migrating VM " + vm); } - allHostsPair = searchForServers(startIndex, pageSize, null, hostType, null, null, null, cluster, null, keyword, null, null, null, null); - // Filter out the current host. + allHostsPair = searchForServers(startIndex, pageSize, null, hostType, null, null, null, cluster, null, keyword, null, null, null, + null, srcHost.getId()); allHosts = allHostsPair.first(); - allHosts.remove(srcHost); plan = new DataCenterDeployment(srcHost.getDataCenterId(), srcHost.getPodId(), srcHost.getClusterId(), null, null, null); } @@ -1358,7 +1358,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe for (final HostAllocator allocator : hostAllocators) { if (canMigrateWithStorage) { - suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, allHosts, HostAllocator.RETURN_UPTO_ALL, false); + suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, hostsForMigrationWithStorage, HostAllocator.RETURN_UPTO_ALL, false); } else { suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, HostAllocator.RETURN_UPTO_ALL, false); } @@ -1550,12 +1550,14 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe return suitablePools; } - private Pair, Integer> searchForServers(final Long startIndex, final Long pageSize, final Object name, final Object type, final Object state, final Object zone, final Object pod, - final Object cluster, final Object id, final Object keyword, final Object resourceState, final Object haHosts, final Object hypervisorType, final Object hypervisorVersion) { + private Pair, Integer> searchForServers(final Long startIndex, final Long pageSize, final Object name, final Object type, + final Object state, final Object zone, final Object pod, final Object cluster, final Object id, final Object keyword, + final Object resourceState, final Object haHosts, final Object hypervisorType, final Object hypervisorVersion, final Object... excludes) { final Filter searchFilter = new Filter(HostVO.class, "id", Boolean.TRUE, startIndex, pageSize); final SearchBuilder sb = _hostDao.createSearchBuilder(); sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("idsNotIn", sb.entity().getId(), SearchCriteria.Op.NOTIN); sb.and("name", sb.entity().getName(), SearchCriteria.Op.LIKE); sb.and("type", sb.entity().getType(), SearchCriteria.Op.LIKE); sb.and("status", sb.entity().getStatus(), SearchCriteria.Op.EQ); @@ -1596,6 +1598,10 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe sc.setParameters("id", id); } + if (excludes != null && excludes.length > 0) { + sc.setParameters("idsNotIn", excludes); + } + if (name != null) { sc.setParameters("name", "%" + name + "%"); }