Merge branch 'master' into ovfprops-and-vsphere-adv-together

api/src/main/java/com/cloud/network/Network.java

@@ -43,7 +43,21 @@ import com.cloud.utils.fsm.StateObject;
 public interface Network extends ControlledEntity, StateObject<Network.State>, InternalIdentity, Identity, Serializable, Displayable {
 
     enum GuestType {
-        Shared, Isolated, L2
+        Shared, Isolated, L2;
+
+        public static GuestType fromValue(String type) {
+            if (StringUtils.isBlank(type)) {
+                return null;
+            } else if (type.equalsIgnoreCase("Shared")) {
+                return Shared;
+            } else if (type.equalsIgnoreCase("Isolated")) {
+                return Isolated;
+            } else if (type.equalsIgnoreCase("L2")) {
+                return L2;
+            } else {
+                throw new InvalidParameterValueException("Unexpected Guest type : " + type);
+            }
+        }
     }
 
     enum PVlanType {

api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java

@@ -85,6 +85,44 @@ public class UpdateDiskOfferingCmd extends BaseCmd {
             since = "4.15")
     private String tags;
 
+    @Parameter(name = ApiConstants.BYTES_READ_RATE, type = CommandType.LONG, description = "bytes read rate of the disk offering", since = "4.15")
+    private Long bytesReadRate;
+
+    @Parameter(name = ApiConstants.BYTES_READ_RATE_MAX, type = CommandType.LONG, description = "burst bytes read rate of the disk offering", since = "4.15")
+    private Long bytesReadRateMax;
+
+    @Parameter(name = ApiConstants.BYTES_READ_RATE_MAX_LENGTH, type = CommandType.LONG, description = "length (in seconds) of the burst", since = "4.15")
+    private Long bytesReadRateMaxLength;
+
+    @Parameter(name = ApiConstants.BYTES_WRITE_RATE, type = CommandType.LONG, description = "bytes write rate of the disk offering", since = "4.15")
+    private Long bytesWriteRate;
+
+    @Parameter(name = ApiConstants.BYTES_WRITE_RATE_MAX, type = CommandType.LONG, description = "burst bytes write rate of the disk offering", since = "4.15")
+    private Long bytesWriteRateMax;
+
+    @Parameter(name = ApiConstants.BYTES_WRITE_RATE_MAX_LENGTH,  type = CommandType.LONG, description = "length (in seconds) of the burst", since = "4.15")
+    private Long bytesWriteRateMaxLength;
+
+    @Parameter(name = ApiConstants.IOPS_READ_RATE, type = CommandType.LONG, description = "io requests read rate of the disk offering", since = "4.15")
+    private Long iopsReadRate;
+
+    @Parameter(name = ApiConstants.IOPS_READ_RATE_MAX, type = CommandType.LONG, description = "burst requests read rate of the disk offering", since = "4.15")
+    private Long iopsReadRateMax;
+
+    @Parameter(name = ApiConstants.IOPS_READ_RATE_MAX_LENGTH, type = CommandType.LONG, description = "length (in seconds) of the burst", since = "4.15")
+    private Long iopsReadRateMaxLength;
+
+    @Parameter(name = ApiConstants.IOPS_WRITE_RATE, type = CommandType.LONG, description = "io requests write rate of the disk offering", since = "4.15")
+    private Long iopsWriteRate;
+
+    @Parameter(name = ApiConstants.IOPS_WRITE_RATE_MAX, type = CommandType.LONG, description = "burst io requests write rate of the disk offering", since = "4.15")
+    private Long iopsWriteRateMax;
+
+    @Parameter(name = ApiConstants.IOPS_WRITE_RATE_MAX_LENGTH, type = CommandType.LONG, description = "length (in seconds) of the burst", since = "4.15")
+    private Long iopsWriteRateMaxLength;
+
+    @Parameter(name = ApiConstants.CACHE_MODE, type = CommandType.STRING, description = "the cache mode to use for this disk offering", since = "4.15")
+    private String cacheMode;
 
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
@@ -174,7 +212,59 @@ public class UpdateDiskOfferingCmd extends BaseCmd {
         return tags;
     }
 
-/////////////////////////////////////////////////////
+    public String getCacheMode() {
+        return cacheMode;
+    }
+
+    public Long getBytesReadRate() {
+        return bytesReadRate;
+    }
+
+    public Long getBytesReadRateMax() {
+        return bytesReadRateMax;
+    }
+
+    public Long getBytesReadRateMaxLength() {
+        return bytesReadRateMaxLength;
+    }
+
+    public Long getBytesWriteRate() {
+        return bytesWriteRate;
+    }
+
+    public Long getBytesWriteRateMax() {
+        return bytesWriteRateMax;
+    }
+
+    public Long getBytesWriteRateMaxLength() {
+        return bytesWriteRateMaxLength;
+    }
+
+    public Long getIopsReadRate() {
+        return iopsReadRate;
+    }
+
+    public Long getIopsReadRateMax() {
+        return iopsReadRateMax;
+    }
+
+    public Long getIopsReadRateMaxLength() {
+        return iopsReadRateMaxLength;
+    }
+
+    public Long getIopsWriteRate() {
+        return iopsWriteRate;
+    }
+
+    public Long getIopsWriteRateMax() {
+        return iopsWriteRateMax;
+    }
+
+    public Long getIopsWriteRateMaxLength() {
+        return iopsWriteRateMaxLength;
+    }
+
+    /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
 

api/src/main/java/org/apache/cloudstack/api/command/user/network/ListNetworksCmd.java

@@ -37,6 +37,7 @@ import org.apache.cloudstack.api.response.ZoneResponse;
 
 import com.cloud.network.Network;
 import com.cloud.utils.Pair;
+import com.google.common.base.Strings;
 
 @APICommand(name = "listNetworks", description = "Lists all available networks.", responseObject = NetworkResponse.class, responseView = ResponseView.Restricted, entityType = {Network.class},
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
@@ -53,7 +54,7 @@ public class ListNetworksCmd extends BaseListTaggedResourcesCmd implements UserC
     @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "the zone ID of the network")
     private Long zoneId;
 
-    @Parameter(name = ApiConstants.TYPE, type = CommandType.STRING, description = "the type of the network. Supported values are: isolated and shared")
+    @Parameter(name = ApiConstants.TYPE, type = CommandType.STRING, description = "the type of the network. Supported values are: isolated, l2, shared and all")
     private String guestIpType;
 
     @Parameter(name = ApiConstants.IS_SYSTEM, type = CommandType.BOOLEAN, description = "true if network is system, false otherwise")
@@ -105,7 +106,13 @@ public class ListNetworksCmd extends BaseListTaggedResourcesCmd implements UserC
     }
 
     public String getGuestIpType() {
-        return guestIpType;
+        if (!Strings.isNullOrEmpty(guestIpType)) {
+            if (guestIpType.equalsIgnoreCase("all")) {
+                return null;
+            }
+            return Network.GuestType.fromValue(guestIpType).toString();
+        }
+        return null;
     }
 
     public Boolean getIsSystem() {

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java

@@ -198,6 +198,10 @@ public class StoragePoolVO implements StoragePool {
         return updateTime;
     }
 
+    public void setUpdateTime(Date updateTime) {
+        this.updateTime = updateTime;
+    }
+
     @Override
     public long getDataCenterId() {
         return dataCenterId;

framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java

@@ -475,14 +475,19 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
         return job;
     }
 
-    private String obfuscatePassword(String result, boolean hidePassword) {
+    public String obfuscatePassword(String result, boolean hidePassword) {
         if (hidePassword) {
             String pattern = "\"password\":";
             if (result != null) {
                 if (result.contains(pattern)) {
                     String[] resp = result.split(pattern);
                     String psswd = resp[1].toString().split(",")[0];
-                    result = resp[0] + pattern + psswd.replace(psswd.substring(2, psswd.length() - 1), "*****") + "," + resp[1].split(",", 2)[1];
+                    if (psswd.endsWith("}")) {
+                        psswd = psswd.substring(0, psswd.length() - 1);
+                        result = resp[0] + pattern + psswd.replace(psswd.substring(2, psswd.length() - 1), "*****") + "}," + resp[1].split(",", 2)[1];
+                    } else {
+                        result = resp[0] + pattern + psswd.replace(psswd.substring(2, psswd.length() - 1), "*****") + "," + resp[1].split(",", 2)[1];
+                    }
                 }
             }
         }

framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/AsyncJobManagerTest.java

@@ -16,116 +16,67 @@
 // under the License.
 package org.apache.cloudstack.framework.jobs;
 
-/*
- * This integration test requires real DB setup, it is not meant to run at per-build
- * basis, it can only be opened in developer's run
- *
- *
+import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
 
-@RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(locations = "classpath:/AsyncJobManagerTestContext.xml")
-public class AsyncJobManagerTest extends TestCase {
-    private static final Logger s_logger =
-            Logger.getLogger(AsyncJobManagerTest.class);
+@RunWith (MockitoJUnitRunner.class)
+public class AsyncJobManagerTest {
 
-    @Inject
-    AsyncJobManager _jobMgr;
+    @Spy
+    AsyncJobManagerImpl asyncJobManager;
 
-    @Inject
-    AsyncJobTestDashboard _testDashboard;
+    String input = "\"haprovider\":\"kvmhaprovider\"},\"outofbandmanagement\":{\"powerstate\":\"On\",\"enabled\":true,\"driver\":\"redfish\",\"address\":\"oob-address.com\",\"port\":\"80\",\"username\":\"root\",\"password\":\"password\"},\"resourcestate\":\"PrepareForMaintenance\",\"hahost\":false";
+    String expected = "\"haprovider\":\"kvmhaprovider\"},\"outofbandmanagement\":{\"powerstate\":\"On\",\"enabled\":true,\"driver\":\"redfish\",\"address\":\"oob-address.com\",\"port\":\"80\",\"username\":\"root\",\"password\":\"p*****\"},\"resourcestate\":\"PrepareForMaintenance\",\"hahost\":false";
+    String obfuscatedInput = "\"haprovider\":\"kvmhaprovider\"},\"outofbandmanagement\":{\"powerstate\":\"On\",\"enabled\":true,\"driver\":\"redfish\",\"address\":\"oob-address.com\",\"port\":\"80\",\"username\":\"root\",\"password\":\"p***\"},\"resourcestate\":\"PrepareForMaintenance\",\"hahost\":false";
+    String noPassword = "\"haprovider\":\"kvmhaprovider\"},\"outofbandmanagement\":{\"powerstate\":\"On\",\"enabled\":true,\"driver\":\"redfish\",\"address\":\"oob-address.com\",\"port\":\"80\",\"username\":\"root\"},\"resourcestate\":\"PrepareForMaintenance\",\"hahost\":false";
 
-    @Override
-    @Before
-    public void setUp() throws Exception {
-        try {
-            ComponentContext.initComponentsLifeCycle();
-        } catch (Exception ex) {
-            ex.printStackTrace();
-            s_logger.error(ex.getMessage());
-        }
-    }
+    String inputNoBraces = "\"password\":\"password\"\",\"action\":\"OFF\"";
+    String expectedNoBraces = "\"password\":\"p*****\",\"action\":\"OFF\"";
 
-    @Override
-    @After
-    public void tearDown() throws Exception {
-    }
-
-    public void testWaitBehave() {
-
-        final Object me = this;
-        new Thread(new Runnable() {
-
-            @Override
-            public void run() {
-                s_logger.info("Sleeping...");
-                try {
-                    Thread.sleep(3000);
-                } catch (InterruptedException e) {
-                    s_logger.debug("[ignored] .");
-                }
-
-                s_logger.info("wakeup");
-                synchronized (me) {
-                    me.notifyAll();
-                }
-            }
-
-        }).start();
-
-        s_logger.info("First wait");
-        synchronized (me) {
-            try {
-                wait(5000);
-            } catch (InterruptedException e) {
-                // TODO Auto-generated catch block
-                e.printStackTrace();
-            }
-        }
-        s_logger.info("First wait done");
-
-        s_logger.info("Second wait");
-        synchronized (me) {
-            try {
-                wait(5000);
-            } catch (InterruptedException e) {
-                // TODO Auto-generated catch block
-                e.printStackTrace();
-            }
-        }
-        s_logger.info("Second wait done");
+    @Test
+    public void obfuscatePasswordTest() {
+        String result = asyncJobManager.obfuscatePassword(input, true);
+        Assert.assertEquals(expected, result);
     }
 
     @Test
-    public void test() {
-        final int TOTAL_JOBS_PER_QUEUE = 5;
-        final int TOTAL_QUEUES = 100;
-
-        for (int i = 0; i < TOTAL_QUEUES; i++) {
-            for (int j = 0; j < TOTAL_JOBS_PER_QUEUE; j++) {
-                AsyncJobVO job = new AsyncJobVO();
-                job.setCmd("TestCmd");
-                job.setDispatcher("TestJobDispatcher");
-                job.setCmdInfo("TestCmd info");
-
-                _jobMgr.submitAsyncJob(job, "fakequeue", i);
-
-                s_logger.info("Job submitted. job " + job.getId() + ", queue: " + i);
-            }
-        }
-
-        while (true) {
-            if (_testDashboard.getCompletedJobCount() == TOTAL_JOBS_PER_QUEUE * TOTAL_QUEUES)
-                break;
-
-            try {
-                Thread.sleep(1000);
-            } catch (InterruptedException e) {
-                s_logger.debug("[ignored] .");
-            }
-        }
-
-        s_logger.info("Test done with " + _testDashboard.getCompletedJobCount() + " job executed");
+    public void obfuscatePasswordTestNoBraces() {
+        String result = asyncJobManager.obfuscatePassword(inputNoBraces, true);
+        Assert.assertEquals(expectedNoBraces, result);
     }
-}
 
-*/
+    @Test
+    public void obfuscatePasswordTestHidePasswordFalse() {
+        String result = asyncJobManager.obfuscatePassword(input, false);
+        Assert.assertEquals(input, result);
+    }
+
+    @Test
+    public void obfuscatePasswordTestObfuscatedInput() {
+        String result = asyncJobManager.obfuscatePassword(obfuscatedInput, true);
+        Assert.assertEquals(expected, result);
+    }
+
+    @Test
+    public void obfuscatePasswordTestHidePasswordFalseObfuscatedInput() {
+        String result = asyncJobManager.obfuscatePassword(obfuscatedInput, false);
+        Assert.assertEquals(obfuscatedInput, result);
+    }
+
+    @Test
+    public void obfuscatePasswordTestNoPassword() {
+        String result = asyncJobManager.obfuscatePassword(noPassword, true);
+        Assert.assertEquals(noPassword, result);
+    }
+
+    @Test
+    public void obfuscatePasswordTestHidePasswordNoPassword() {
+        String result = asyncJobManager.obfuscatePassword(noPassword, false);
+        Assert.assertEquals(noPassword, result);
+    }
+
+}

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java

@@ -1187,11 +1187,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
         if (clusterDetailsVO != null && !Strings.isNullOrEmpty(clusterDetailsVO.getValue())) {
             configData = new String(Base64.decodeBase64(clusterDetailsVO.getValue()));
         } else {
-            if (KubernetesCluster.State.Starting.equals(kubernetesCluster.getState())) {
-                throw new CloudRuntimeException(String.format("Setup is in progress for Kubernetes cluster ID: %s, config not available at this moment", kubernetesCluster.getUuid()));
-            } else {
-                throw new CloudRuntimeException((String.format("Config not found for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())));
-            }
+            String exceptionMessage = KubernetesCluster.State.Starting.equals(kubernetesCluster.getState()) ?
+                    String.format("Setup is in progress for Kubernetes cluster : %s, config not available at this moment", kubernetesCluster.getName()) :
+                    String.format("Config not found for Kubernetes cluster : %s", kubernetesCluster.getName());
+            throw new CloudRuntimeException(exceptionMessage);
         }
         response.setConfigData(configData);
         response.setObjectName("clusterconfig");

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java

@@ -579,22 +579,22 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
         try {
             InetAddress address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost());
         } catch (MalformedURLException | UnknownHostException ex) {
-            logTransitStateAndThrow(Level.ERROR, String.format("Kubernetes cluster ID: %s has invalid API endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+            logTransitStateAndThrow(Level.ERROR, String.format("Kubernetes cluster : %s has invalid API endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
         }
         Pair<String, Integer> sshIpPort =  getKubernetesClusterServerIpSshPort(null);
         publicIpAddress = sshIpPort.first();
         sshPort = sshIpPort.second();
         if (Strings.isNullOrEmpty(publicIpAddress)) {
-            logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+            logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s as no public IP found for the cluster" , kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
         }
         if (!KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, CLUSTER_API_PORT, startTimeoutTime, 15000)) {
-            logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+            logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s in usable state", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
         }
         if (!isKubernetesClusterKubeConfigAvailable(startTimeoutTime)) {
-            logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+            logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
         }
         if (!isKubernetesClusterDashboardServiceRunning(false, startTimeoutTime)) {
-            logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+            logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
         }
         stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded);
         if (LOGGER.isInfoEnabled()) {

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -2548,42 +2548,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         offering.setMinIops(minIops);
         offering.setMaxIops(maxIops);
 
-        if (bytesReadRate != null && bytesReadRate > 0) {
-            offering.setBytesReadRate(bytesReadRate);
-        }
-        if (bytesReadRateMax != null && bytesReadRateMax > 0) {
-            offering.setBytesReadRateMax(bytesReadRateMax);
-        }
-        if (bytesReadRateMaxLength != null && bytesReadRateMaxLength > 0) {
-            offering.setBytesReadRateMaxLength(bytesReadRateMaxLength);
-        }
-        if (bytesWriteRate != null && bytesWriteRate > 0) {
-            offering.setBytesWriteRate(bytesWriteRate);
-        }
-        if (bytesWriteRateMax != null && bytesWriteRateMax > 0) {
-            offering.setBytesWriteRateMax(bytesWriteRateMax);
-        }
-        if (bytesWriteRateMaxLength != null && bytesWriteRateMaxLength > 0) {
-            offering.setBytesWriteRateMaxLength(bytesWriteRateMaxLength);
-        }
-        if (iopsReadRate != null && iopsReadRate > 0) {
-            offering.setIopsReadRate(iopsReadRate);
-        }
-        if (iopsReadRateMax != null && iopsReadRateMax > 0) {
-            offering.setIopsReadRateMax(iopsReadRateMax);
-        }
-        if (iopsReadRateMaxLength != null && iopsReadRateMaxLength > 0) {
-            offering.setIopsReadRateMaxLength(iopsReadRateMaxLength);
-        }
-        if (iopsWriteRate != null && iopsWriteRate > 0) {
-            offering.setIopsWriteRate(iopsWriteRate);
-        }
-        if (iopsWriteRateMax != null && iopsWriteRateMax > 0) {
-            offering.setIopsWriteRateMax(iopsWriteRateMax);
-        }
-        if (iopsWriteRateMaxLength != null && iopsWriteRateMaxLength > 0) {
-            offering.setIopsWriteRateMaxLength(iopsWriteRateMaxLength);
-        }
+        setBytesRate(offering, bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength);
+        setIopsRate(offering, iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, iopsWriteRateMax, iopsWriteRateMaxLength);
+
         if(cacheMode != null) {
             offering.setCacheMode(DiskOffering.DiskCacheMode.valueOf(cacheMode.toUpperCase()));
         }
@@ -2653,6 +2620,48 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         }
     }
 
+    private void setIopsRate(DiskOffering offering, Long iopsReadRate, Long iopsReadRateMax, Long iopsReadRateMaxLength, Long iopsWriteRate, Long iopsWriteRateMax, Long iopsWriteRateMaxLength) {
+        if (iopsReadRate != null && iopsReadRate > 0) {
+            offering.setIopsReadRate(iopsReadRate);
+        }
+        if (iopsReadRateMax != null && iopsReadRateMax > 0) {
+            offering.setIopsReadRateMax(iopsReadRateMax);
+        }
+        if (iopsReadRateMaxLength != null && iopsReadRateMaxLength > 0) {
+            offering.setIopsReadRateMaxLength(iopsReadRateMaxLength);
+        }
+        if (iopsWriteRate != null && iopsWriteRate > 0) {
+            offering.setIopsWriteRate(iopsWriteRate);
+        }
+        if (iopsWriteRateMax != null && iopsWriteRateMax > 0) {
+            offering.setIopsWriteRateMax(iopsWriteRateMax);
+        }
+        if (iopsWriteRateMaxLength != null && iopsWriteRateMaxLength > 0) {
+            offering.setIopsWriteRateMaxLength(iopsWriteRateMaxLength);
+        }
+    }
+
+    private void setBytesRate(DiskOffering offering, Long bytesReadRate, Long bytesReadRateMax, Long bytesReadRateMaxLength, Long bytesWriteRate, Long bytesWriteRateMax, Long bytesWriteRateMaxLength) {
+        if (bytesReadRate != null && bytesReadRate > 0) {
+            offering.setBytesReadRate(bytesReadRate);
+        }
+        if (bytesReadRateMax != null && bytesReadRateMax > 0) {
+            offering.setBytesReadRateMax(bytesReadRateMax);
+        }
+        if (bytesReadRateMaxLength != null && bytesReadRateMaxLength > 0) {
+            offering.setBytesReadRateMaxLength(bytesReadRateMaxLength);
+        }
+        if (bytesWriteRate != null && bytesWriteRate > 0) {
+            offering.setBytesWriteRate(bytesWriteRate);
+        }
+        if (bytesWriteRateMax != null && bytesWriteRateMax > 0) {
+            offering.setBytesWriteRateMax(bytesWriteRateMax);
+        }
+        if (bytesWriteRateMaxLength != null && bytesWriteRateMaxLength > 0) {
+            offering.setBytesWriteRateMaxLength(bytesWriteRateMaxLength);
+        }
+    }
+
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_SERVICE_OFFERING_EDIT, eventDescription = "updating service offering")
     public ServiceOffering updateServiceOffering(final UpdateServiceOfferingCmd cmd) {
@@ -2919,42 +2928,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         newDiskOffering.setUseLocalStorage(localStorageRequired);
         newDiskOffering.setDisplayOffering(isDisplayOfferingEnabled);
 
-        if (bytesReadRate != null && bytesReadRate > 0) {
-            newDiskOffering.setBytesReadRate(bytesReadRate);
-        }
-        if (bytesReadRateMax != null && bytesReadRateMax > 0) {
-            newDiskOffering.setBytesReadRateMax(bytesReadRateMax);
-        }
-        if (bytesReadRateMaxLength != null && bytesReadRateMaxLength > 0) {
-            newDiskOffering.setBytesReadRateMaxLength(bytesReadRateMaxLength);
-        }
-        if (bytesWriteRate != null && bytesWriteRate > 0) {
-            newDiskOffering.setBytesWriteRate(bytesWriteRate);
-        }
-        if (bytesWriteRateMax != null && bytesWriteRateMax > 0) {
-            newDiskOffering.setBytesWriteRateMax(bytesWriteRateMax);
-        }
-        if (bytesWriteRateMaxLength != null && bytesWriteRateMaxLength > 0) {
-            newDiskOffering.setBytesWriteRateMaxLength(bytesWriteRateMaxLength);
-        }
-        if (iopsReadRate != null && iopsReadRate > 0) {
-            newDiskOffering.setIopsReadRate(iopsReadRate);
-        }
-        if (iopsReadRateMax != null && iopsReadRateMax > 0) {
-            newDiskOffering.setIopsReadRateMax(iopsReadRateMax);
-        }
-        if (iopsReadRateMaxLength != null && iopsReadRateMaxLength > 0) {
-            newDiskOffering.setIopsReadRateMaxLength(iopsReadRateMaxLength);
-        }
-        if (iopsWriteRate != null && iopsWriteRate > 0) {
-            newDiskOffering.setIopsWriteRate(iopsWriteRate);
-        }
-        if (iopsWriteRateMax != null && iopsWriteRateMax > 0) {
-            newDiskOffering.setIopsWriteRateMax(iopsWriteRateMax);
-        }
-        if (iopsWriteRateMaxLength != null && iopsWriteRateMaxLength > 0) {
-            newDiskOffering.setIopsWriteRateMaxLength(iopsWriteRateMaxLength);
-        }
+        setBytesRate(newDiskOffering, bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength);
+        setIopsRate(newDiskOffering, iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, iopsWriteRateMax, iopsWriteRateMaxLength);
+
         if (cacheMode != null) {
             newDiskOffering.setCacheMode(DiskOffering.DiskCacheMode.valueOf(cacheMode.toUpperCase()));
         }
@@ -3143,6 +3119,20 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         final List<Long> zoneIds = cmd.getZoneIds();
         final String tags = cmd.getTags();
 
+        Long bytesReadRate = cmd.getBytesReadRate();
+        Long bytesReadRateMax = cmd.getBytesReadRateMax();
+        Long bytesReadRateMaxLength = cmd.getBytesReadRateMaxLength();
+        Long bytesWriteRate = cmd.getBytesWriteRate();
+        Long bytesWriteRateMax = cmd.getBytesWriteRateMax();
+        Long bytesWriteRateMaxLength = cmd.getBytesWriteRateMaxLength();
+        Long iopsReadRate = cmd.getIopsReadRate();
+        Long iopsReadRateMax = cmd.getIopsReadRateMax();
+        Long iopsReadRateMaxLength = cmd.getIopsReadRateMaxLength();
+        Long iopsWriteRate = cmd.getIopsWriteRate();
+        Long iopsWriteRateMax = cmd.getIopsWriteRateMax();
+        Long iopsWriteRateMaxLength = cmd.getIopsWriteRateMaxLength();
+        String cacheMode = cmd.getCacheMode();
+
         // Check if diskOffering exists
         final DiskOffering diskOfferingHandle = _entityMgr.findById(DiskOffering.class, diskOfferingId);
         if (diskOfferingHandle == null) {
@@ -3223,7 +3213,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
             throw new InvalidParameterValueException(String.format("Unable to update disk offering: %s by id user: %s because it is not root-admin or domain-admin", diskOfferingHandle.getUuid(), user.getUuid()));
         }
 
-        final boolean updateNeeded = shouldUpdateDiskOffering(name, displayText, sortKey, displayDiskOffering, tags);
+        boolean updateNeeded = shouldUpdateDiskOffering(name, displayText, sortKey, displayDiskOffering, tags, cacheMode) ||
+                shouldUpdateIopsRateParameters(iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, iopsWriteRateMax, iopsWriteRateMaxLength) ||
+                shouldUpdateBytesRateParameters(bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength);
+
         final boolean detailsUpdateNeeded = !filteredDomainIds.equals(existingDomainIds) || !filteredZoneIds.equals(existingZoneIds);
         if (!updateNeeded && !detailsUpdateNeeded) {
             return _diskOfferingDao.findById(diskOfferingId);
@@ -3249,6 +3242,20 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
 
         updateDiskOfferingTagsIfIsNotNull(tags, diskOffering);
 
+        validateMaxRateEqualsOrGreater(iopsReadRate, iopsReadRateMax, IOPS_READ_RATE);
+        validateMaxRateEqualsOrGreater(iopsWriteRate, iopsWriteRateMax, IOPS_WRITE_RATE);
+        validateMaxRateEqualsOrGreater(bytesReadRate, bytesReadRateMax, BYTES_READ_RATE);
+        validateMaxRateEqualsOrGreater(bytesWriteRate, bytesWriteRateMax, BYTES_WRITE_RATE);
+        validateMaximumIopsAndBytesLength(iopsReadRateMaxLength, iopsWriteRateMaxLength, bytesReadRateMaxLength, bytesWriteRateMaxLength);
+
+        setBytesRate(diskOffering, bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength);
+        setIopsRate(diskOffering, iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, iopsWriteRateMax, iopsWriteRateMaxLength);
+
+        if (cacheMode != null) {
+            validateCacheMode(cacheMode);
+            diskOffering.setCacheMode(DiskOffering.DiskCacheMode.valueOf(cacheMode.toUpperCase()));
+        }
+
         if (updateNeeded && !_diskOfferingDao.update(diskOfferingId, diskOffering)) {
             return null;
         }
@@ -3305,8 +3312,17 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
      * Check if it needs to update any parameter when updateDiskoffering is called
      * Verify if name or displayText are not blank, tags is not null, sortkey and displayDiskOffering is not null
      */
-    protected boolean shouldUpdateDiskOffering(String name, String displayText, Integer sortKey, Boolean displayDiskOffering, String tags) {
-        return StringUtils.isNotBlank(name) || StringUtils.isNotBlank(displayText) || tags != null || sortKey != null || displayDiskOffering != null;
+    protected boolean shouldUpdateDiskOffering(String name, String displayText, Integer sortKey, Boolean displayDiskOffering, String tags, String cacheMode) {
+        return StringUtils.isNotBlank(name) || StringUtils.isNotBlank(displayText) || tags != null || sortKey != null || displayDiskOffering != null || StringUtils.isNotBlank(cacheMode);
+    }
+
+    protected boolean shouldUpdateBytesRateParameters(Long bytesReadRate, Long bytesReadRateMax, Long bytesReadRateMaxLength, Long bytesWriteRate, Long bytesWriteRateMax, Long bytesWriteRateMaxLength) {
+        return bytesReadRate != null || bytesReadRateMax != null || bytesReadRateMaxLength != null || bytesWriteRate != null ||
+                bytesWriteRateMax != null || bytesWriteRateMaxLength != null;
+    }
+
+    protected boolean shouldUpdateIopsRateParameters(Long iopsReadRate, Long iopsReadRateMax, Long iopsReadRateMaxLength, Long iopsWriteRate, Long iopsWriteRateMax, Long iopsWriteRateMaxLength) {
+        return iopsReadRate != null || iopsReadRateMax != null || iopsReadRateMaxLength != null || iopsWriteRate != null || iopsWriteRateMax != null || iopsWriteRateMaxLength != null;
     }
 
     @Override

server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManager.java

@@ -44,6 +44,7 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA
     static final String RouterTemplateOvm3CK = "router.template.ovm3";
     static final String SetServiceMonitorCK = "network.router.EnableServiceMonitoring";
     static final String RouterAlertsCheckIntervalCK = "router.alerts.check.interval";
+    static final String VirtualRouterServiceOfferingCK = "router.service.offering";
 
     static final String RouterHealthChecksConfigRefreshIntervalCK = "router.health.checks.config.refresh.interval";
     static final String RouterHealthChecksResultFetchIntervalCK = "router.health.checks.results.fetch.interval";
@@ -74,6 +75,9 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA
     static final ConfigKey<Boolean> ExposeDnsAndBootpServer = new ConfigKey<Boolean>(Boolean.class, "expose.dns.externally", "Advanced", "true",
             "open dns, dhcp and bootp on the public interface", true, ConfigKey.Scope.Zone, null);
 
+    static final ConfigKey<String> VirtualRouterServiceOffering = new ConfigKey<String>(String.class, VirtualRouterServiceOfferingCK, "Advanced", "",
+            "Uuid of the service offering used by virtual routers; if NULL - system offering will be used", true, ConfigKey.Scope.Account, null);
+
     // Health checks
     static final ConfigKey<Boolean> RouterHealthChecksEnabled = new ConfigKey<Boolean>(Boolean.class, "router.health.checks.enabled", "Advanced", "true",
             "If true, router health checks are allowed to be executed and read. If false, all scheduled checks and API calls for on demand checks are disabled.",

server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -3256,6 +3256,7 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
                 UseExternalDnsServers,
                 RouterVersionCheckEnabled,
                 SetServiceMonitor,
+                VirtualRouterServiceOffering,
                 RouterAlertsCheckInterval,
                 RouterHealthChecksEnabled,
                 RouterHealthChecksBasicInterval,

server/src/main/java/com/cloud/server/ManagementServerImpl.java

@@ -1177,8 +1177,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         final Object resourceState = cmd.getResourceState();
         final Object haHosts = cmd.getHaHost();
 
-        final Pair<List<HostVO>, Integer> result = searchForServers(cmd.getStartIndex(), cmd.getPageSizeVal(), name, type, state, zoneId, pod, cluster, id, keyword, resourceState, haHosts, null,
-                null);
+        final Pair<List<HostVO>, Integer> result = searchForServers(cmd.getStartIndex(), cmd.getPageSizeVal(), name, type, state, zoneId, pod,
+            cluster, id, keyword, resourceState, haHosts, null, null);
         return new Pair<List<? extends Host>, Integer>(result.first(), result.second());
     }
 
@@ -1280,19 +1280,20 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         final Type hostType = srcHost.getType();
         Pair<List<HostVO>, Integer> allHostsPair = null;
         List<HostVO> allHosts = null;
+        List<HostVO> hostsForMigrationWithStorage = null;
         final Map<Host, Boolean> requiresStorageMotion = new HashMap<Host, Boolean>();
         DataCenterDeployment plan = null;
         if (canMigrateWithStorage) {
-            allHostsPair = searchForServers(startIndex, pageSize, null, hostType, null, srcHost.getDataCenterId(), null, null, null, keyword, null, null, srcHost.getHypervisorType(),
-                    srcHost.getHypervisorVersion());
+            allHostsPair = searchForServers(startIndex, pageSize, null, hostType, null, srcHost.getDataCenterId(), null, null, null, keyword,
+                null, null, srcHost.getHypervisorType(), srcHost.getHypervisorVersion(), srcHost.getId());
             allHosts = allHostsPair.first();
-            allHosts.remove(srcHost);
+            hostsForMigrationWithStorage = new ArrayList<>(allHosts);
 
             for (final VolumeVO volume : volumes) {
                 StoragePool storagePool = _poolDao.findById(volume.getPoolId());
                 Long volClusterId = storagePool.getClusterId();
 
-                for (Iterator<HostVO> iterator = allHosts.iterator(); iterator.hasNext();) {
+                for (Iterator<HostVO> iterator = hostsForMigrationWithStorage.iterator(); iterator.hasNext();) {
                     final Host host = iterator.next();
 
                     if (volClusterId != null) {
@@ -1331,10 +1332,9 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
             if (s_logger.isDebugEnabled()) {
                 s_logger.debug("Searching for all hosts in cluster " + cluster + " for migrating VM " + vm);
             }
-            allHostsPair = searchForServers(startIndex, pageSize, null, hostType, null, null, null, cluster, null, keyword, null, null, null, null);
-            // Filter out the current host.
+            allHostsPair = searchForServers(startIndex, pageSize, null, hostType, null, null, null, cluster, null, keyword, null, null, null,
+                null, srcHost.getId());
             allHosts = allHostsPair.first();
-            allHosts.remove(srcHost);
             plan = new DataCenterDeployment(srcHost.getDataCenterId(), srcHost.getPodId(), srcHost.getClusterId(), null, null, null);
         }
 
@@ -1363,7 +1363,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
 
         for (final HostAllocator allocator : hostAllocators) {
             if (canMigrateWithStorage) {
-                suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, allHosts, HostAllocator.RETURN_UPTO_ALL, false);
+                suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, hostsForMigrationWithStorage, HostAllocator.RETURN_UPTO_ALL, false);
             } else {
                 suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, HostAllocator.RETURN_UPTO_ALL, false);
             }
@@ -1576,12 +1576,14 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         return suitablePools;
     }
 
-    private Pair<List<HostVO>, Integer> searchForServers(final Long startIndex, final Long pageSize, final Object name, final Object type, final Object state, final Object zone, final Object pod,
-            final Object cluster, final Object id, final Object keyword, final Object resourceState, final Object haHosts, final Object hypervisorType, final Object hypervisorVersion) {
+    private Pair<List<HostVO>, Integer> searchForServers(final Long startIndex, final Long pageSize, final Object name, final Object type,
+        final Object state, final Object zone, final Object pod, final Object cluster, final Object id, final Object keyword,
+        final Object resourceState, final Object haHosts, final Object hypervisorType, final Object hypervisorVersion, final Object... excludes) {
         final Filter searchFilter = new Filter(HostVO.class, "id", Boolean.TRUE, startIndex, pageSize);
 
         final SearchBuilder<HostVO> sb = _hostDao.createSearchBuilder();
         sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ);
+        sb.and("idsNotIn", sb.entity().getId(), SearchCriteria.Op.NOTIN);
         sb.and("name", sb.entity().getName(), SearchCriteria.Op.LIKE);
         sb.and("type", sb.entity().getType(), SearchCriteria.Op.LIKE);
         sb.and("status", sb.entity().getStatus(), SearchCriteria.Op.EQ);
@@ -1622,6 +1624,10 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
             sc.setParameters("id", id);
         }
 
+        if (excludes != null && excludes.length > 0) {
+            sc.setParameters("idsNotIn", excludes);
+        }
+
         if (name != null) {
             sc.setParameters("name", "%" + name + "%");
         }

server/src/main/java/com/cloud/server/StatsCollector.java

@@ -36,6 +36,7 @@ import javax.inject.Inject;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
 import org.apache.cloudstack.framework.config.ConfigKey;
@@ -1027,8 +1028,13 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
                             storagePoolStats.put(pool.getId(), (StorageStats)answer);
 
                             // Seems like we have dynamically updated the pool size since the prev. size and the current do not match
-                            if (_storagePoolStats.get(poolId) != null && _storagePoolStats.get(poolId).getCapacityBytes() != ((StorageStats)answer).getCapacityBytes()) {
+                            if (pool.getCapacityBytes() != ((StorageStats)answer).getCapacityBytes() ||
+                                    pool.getUsedBytes() != ((StorageStats)answer).getByteUsed()) {
                                 pool.setCapacityBytes(((StorageStats)answer).getCapacityBytes());
+                                if (pool.getStorageProviderName().equalsIgnoreCase(DataStoreProvider.DEFAULT_PRIMARY)) {
+                                    pool.setUsedBytes(((StorageStats) answer).getByteUsed());
+                                    pool.setUpdateTime(new Date());
+                                }
                                 _storagePoolDao.update(pool.getId(), pool);
                             }
                         }

server/src/main/java/com/cloud/storage/StorageManagerImpl.java

@@ -159,7 +159,6 @@ import com.cloud.org.Grouping.AllocationState;
 import com.cloud.resource.ResourceState;
 import com.cloud.server.ConfigurationServer;
 import com.cloud.server.ManagementServer;
-import com.cloud.server.StatsCollector;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.Volume.Type;
@@ -1852,31 +1851,21 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
             return true;
         }
 
-        StatsCollector sc = StatsCollector.getInstance();
         double storageUsedThreshold = CapacityManager.StorageCapacityDisableThreshold.valueIn(pool.getDataCenterId());
-        if (sc != null) {
-            long totalSize = pool.getCapacityBytes();
-            StorageStats stats = sc.getStoragePoolStats(pool.getId());
-            if (stats == null) {
-                stats = sc.getStorageStats(pool.getId());
-            }
-            if (stats != null) {
-                double usedPercentage = ((double)stats.getByteUsed() / (double)totalSize);
-                if (s_logger.isDebugEnabled()) {
-                    s_logger.debug("Checking pool " + pool.getId() + " for storage, totalSize: " + toHumanReadableSize(pool.getCapacityBytes()) + ", usedBytes: " + toHumanReadableSize(stats.getByteUsed()) + ", usedPct: " + usedPercentage
-                            + ", disable threshold: " + storageUsedThreshold);
-                }
-                if (usedPercentage >= storageUsedThreshold) {
-                    if (s_logger.isDebugEnabled()) {
-                        s_logger.debug("Insufficient space on pool: " + pool.getId() + " since its usage percentage: " + usedPercentage + " has crossed the pool.storage.capacity.disablethreshold: "
-                                + storageUsedThreshold);
-                    }
-                    return false;
-                }
-            }
-            return true;
+        long totalSize = pool.getCapacityBytes();
+        double usedPercentage = ((double)pool.getUsedBytes() / (double)totalSize);
+        if (s_logger.isDebugEnabled()) {
+            s_logger.debug("Checking pool " + pool.getId() + " for storage, totalSize: " + pool.getCapacityBytes() + ", usedBytes: " + pool.getUsedBytes() +
+                    ", usedPct: " + usedPercentage + ", disable threshold: " + storageUsedThreshold);
         }
-        return false;
+        if (usedPercentage >= storageUsedThreshold) {
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("Insufficient space on pool: " + pool.getId() + " since its usage percentage: " + usedPercentage +
+                        " has crossed the pool.storage.capacity.disablethreshold: " + storageUsedThreshold);
+            }
+            return false;
+        }
+        return true;
     }
 
     @Override

server/src/main/java/org/cloud/network/router/deployment/RouterDeploymentDefinition.java

@@ -54,6 +54,7 @@ import com.cloud.network.dao.PhysicalNetworkServiceProviderDao;
 import com.cloud.network.dao.UserIpv6AddressDao;
 import com.cloud.network.dao.VirtualRouterProviderDao;
 import com.cloud.network.router.NetworkHelper;
+import com.cloud.network.router.VirtualNetworkApplianceManager;
 import com.cloud.network.router.VirtualRouter.Role;
 import com.cloud.network.vpc.Vpc;
 import com.cloud.offering.ServiceOffering;
@@ -389,8 +390,34 @@ public class RouterDeploymentDefinition {
         serviceOfferingId = serviceOffering.getId();
     }
 
+    protected void findAccountServiceOfferingId(long accountId) {
+        String accountRouterOffering = VirtualNetworkApplianceManager.VirtualRouterServiceOffering.valueIn(accountId);
+        String globalRouterOffering = VirtualNetworkApplianceManager.VirtualRouterServiceOffering.value();
+        if (accountRouterOffering != null) {
+            verifyServiceOfferingByUuid(accountRouterOffering);
+        }
+        if (serviceOfferingId == null && globalRouterOffering != accountRouterOffering) {
+            verifyServiceOfferingByUuid(globalRouterOffering);
+        }
+    }
+
+    private void verifyServiceOfferingByUuid(String offeringUuid) {
+        logger.debug("Verifying router service offering with uuid : " + offeringUuid);
+        ServiceOfferingVO serviceOffering = serviceOfferingDao.findByUuid(offeringUuid);
+        if (serviceOffering != null && serviceOffering.isSystemUse()) {
+            boolean isLocalStorage = ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dest.getDataCenter().getId());
+            if (isLocalStorage == serviceOffering.isUseLocalStorage()) {
+                logger.debug(String.format("Service offering %s (uuid: %s) will be used on virtual router", serviceOffering.getName(), serviceOffering.getUuid()));
+                serviceOfferingId = serviceOffering.getId();
+            }
+        }
+    }
+
     protected void findServiceOfferingId() {
         serviceOfferingId = networkOfferingDao.findById(guestNetwork.getNetworkOfferingId()).getServiceOfferingId();
+        if (serviceOfferingId == null) {
+            findAccountServiceOfferingId(guestNetwork.getAccountId());
+        }
         if (serviceOfferingId == null) {
             findDefaultServiceOfferingId();
         }
@@ -482,4 +509,4 @@ public class RouterDeploymentDefinition {
 
         return needReset;
     }
-}
+}

server/src/main/java/org/cloud/network/router/deployment/VpcRouterDeploymentDefinition.java

@@ -159,6 +159,9 @@ public class VpcRouterDeploymentDefinition extends RouterDeploymentDefinition {
     @Override
     protected void findServiceOfferingId() {
         serviceOfferingId = vpcOffDao.findById(vpc.getVpcOfferingId()).getServiceOfferingId();
+        if (serviceOfferingId == null) {
+            findAccountServiceOfferingId(vpc.getAccountId());
+        }
         if (serviceOfferingId == null) {
             findDefaultServiceOfferingId();
         }

server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java

@@ -946,17 +946,47 @@ public class ConfigurationManagerTest {
 
     @Test
     public void shouldUpdateDiskOfferingTests(){
-        Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyString()));
-        Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(Mockito.anyString(), nullable(String.class), nullable(Integer.class), nullable(Boolean.class), nullable(String.class)));
-        Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), Mockito.anyString(), nullable(Integer.class), nullable(Boolean.class), nullable(String.class)));
-        Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), nullable(String.class), Mockito.anyInt(), nullable(Boolean.class), nullable(String.class)));
-        Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), nullable(String.class), nullable(int.class), Mockito.anyBoolean(), nullable(String.class)));
-        Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), nullable(String.class), nullable(int.class), nullable(Boolean.class), Mockito.anyString()));
+        Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyString(), Mockito.anyString()));
+        Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(Mockito.anyString(), nullable(String.class), nullable(Integer.class), nullable(Boolean.class), nullable(String.class), nullable(String.class)));
+        Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), Mockito.anyString(), nullable(Integer.class), nullable(Boolean.class), nullable(String.class), nullable(String.class)));
+        Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), nullable(String.class), Mockito.anyInt(), nullable(Boolean.class), nullable(String.class), nullable(String.class)));
+        Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), nullable(String.class), nullable(int.class), Mockito.anyBoolean(), nullable(String.class), nullable(String.class)));
+        Assert.assertTrue(configurationMgr.shouldUpdateDiskOffering(nullable(String.class), nullable(String.class), nullable(int.class), nullable(Boolean.class), Mockito.anyString(), Mockito.anyString()));
     }
 
     @Test
     public void shouldUpdateDiskOfferingTestFalse(){
-        Assert.assertFalse(configurationMgr.shouldUpdateDiskOffering(null, null, null, null, null));
+        Assert.assertFalse(configurationMgr.shouldUpdateDiskOffering(null, null, null, null, null, null));
+    }
+
+    @Test
+    public void shouldUpdateIopsRateParametersTestFalse() {
+        Assert.assertFalse(configurationMgr.shouldUpdateIopsRateParameters(null, null, null, null, null, null));
+    }
+
+    @Test
+    public void shouldUpdateIopsRateParametersTests(){
+        Assert.assertTrue(configurationMgr.shouldUpdateIopsRateParameters(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()));
+        Assert.assertTrue(configurationMgr.shouldUpdateIopsRateParameters(nullable(Long.class), Mockito.anyLong(), nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class)));
+        Assert.assertTrue(configurationMgr.shouldUpdateIopsRateParameters(nullable(Long.class), nullable(Long.class), Mockito.anyLong(), nullable(Long.class), nullable(Long.class), nullable(Long.class)));
+        Assert.assertTrue(configurationMgr.shouldUpdateIopsRateParameters(nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.anyLong(), nullable(Long.class), nullable(Long.class)));
+        Assert.assertTrue(configurationMgr.shouldUpdateIopsRateParameters(nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.anyLong(), nullable(Long.class)));
+        Assert.assertTrue(configurationMgr.shouldUpdateIopsRateParameters(nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.anyLong()));
+    }
+
+    @Test
+    public void shouldUpdateBytesRateParametersTestFalse() {
+        Assert.assertFalse(configurationMgr.shouldUpdateBytesRateParameters(null, null, null, null, null, null));
+    }
+
+    @Test
+    public void shouldUpdateBytesRateParametersTests(){
+        Assert.assertTrue(configurationMgr.shouldUpdateBytesRateParameters(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()));
+        Assert.assertTrue(configurationMgr.shouldUpdateBytesRateParameters(nullable(Long.class), Mockito.anyLong(), nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class)));
+        Assert.assertTrue(configurationMgr.shouldUpdateBytesRateParameters(nullable(Long.class), nullable(Long.class), Mockito.anyLong(), nullable(Long.class), nullable(Long.class), nullable(Long.class)));
+        Assert.assertTrue(configurationMgr.shouldUpdateBytesRateParameters(nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.anyLong(), nullable(Long.class), nullable(Long.class)));
+        Assert.assertTrue(configurationMgr.shouldUpdateBytesRateParameters(nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.anyLong(), nullable(Long.class)));
+        Assert.assertTrue(configurationMgr.shouldUpdateBytesRateParameters(nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.anyLong()));
     }
 
     @Test

systemvm/debian/opt/cloud/bin/cs/CsApp.py

@@ -17,7 +17,6 @@
 # under the License.
 import os
 from CsFile import CsFile
-from CsProcess import CsProcess
 import CsHelper
 
 

systemvm/debian/opt/cloud/bin/cs/CsProcess.py

@@ -40,9 +40,9 @@ class CsProcess(object):
 
     def find_pid(self):
         self.pid = []
+        items = len(self.search)
         for i in CsHelper.execute("ps aux"):
-            items = len(self.search)
-            proc = re.split(r"\s+", i)[items*-1:]
+            proc = re.split(r"\s+", i)[10:]
             matches = len([m for m in proc if m in self.search])
             if matches == items:
                 self.pid.append(re.split(r"\s+", i)[1])

tools/marvin/marvin/lib/base.py

@@ -451,34 +451,53 @@ class VirtualMachine:
     @classmethod
     def access_ssh_over_nat(
             cls, apiclient, services, virtual_machine, allow_egress=False,
-            networkid=None):
+            networkid=None, vpcid=None):
         """
         Program NAT and PF rules to open up ssh access to deployed guest
         @return:
         """
-        public_ip = PublicIPAddress.create(
-            apiclient=apiclient,
-            accountid=virtual_machine.account,
-            zoneid=virtual_machine.zoneid,
-            domainid=virtual_machine.domainid,
-            services=services,
-            networkid=networkid
-        )
-        FireWallRule.create(
-            apiclient=apiclient,
-            ipaddressid=public_ip.ipaddress.id,
-            protocol='TCP',
-            cidrlist=['0.0.0.0/0'],
-            startport=22,
-            endport=22
-        )
-        nat_rule = NATRule.create(
-            apiclient=apiclient,
-            virtual_machine=virtual_machine,
-            services=services,
-            ipaddressid=public_ip.ipaddress.id
-        )
-        if allow_egress:
+        # VPCs have ACLs managed differently
+        if vpcid:
+            public_ip = PublicIPAddress.create(
+                apiclient=apiclient,
+                accountid=virtual_machine.account,
+                zoneid=virtual_machine.zoneid,
+                domainid=virtual_machine.domainid,
+                services=services,
+                vpcid=vpcid
+            )
+
+            nat_rule = NATRule.create(
+                apiclient=apiclient,
+                virtual_machine=virtual_machine,
+                services=services,
+                ipaddressid=public_ip.ipaddress.id,
+                networkid=networkid)
+        else:
+            public_ip = PublicIPAddress.create(
+                apiclient=apiclient,
+                accountid=virtual_machine.account,
+                zoneid=virtual_machine.zoneid,
+                domainid=virtual_machine.domainid,
+                services=services,
+                networkid=networkid,
+            )
+
+            FireWallRule.create(
+                apiclient=apiclient,
+                ipaddressid=public_ip.ipaddress.id,
+                protocol='TCP',
+                cidrlist=['0.0.0.0/0'],
+                startport=22,
+                endport=22
+            )
+            nat_rule = NATRule.create(
+                apiclient=apiclient,
+                virtual_machine=virtual_machine,
+                services=services,
+                ipaddressid=public_ip.ipaddress.id)
+
+        if allow_egress and not vpcid:
             try:
                 EgressFireWallRule.create(
                     apiclient=apiclient,
@@ -502,7 +521,7 @@ class VirtualMachine:
                hostid=None, keypair=None, ipaddress=None, mode='default',
                method='GET', hypervisor=None, customcpunumber=None,
                customcpuspeed=None, custommemory=None, rootdisksize=None,
-               rootdiskcontroller=None, macaddress=None, datadisktemplate_diskoffering_list={},
+               rootdiskcontroller=None, vpcid=None, macaddress=None, datadisktemplate_diskoffering_list={},
                properties=None, nicnetworklist=None):
         """Create the instance"""
 
@@ -661,7 +680,8 @@ class VirtualMachine:
                         services,
                         virtual_machine,
                         allow_egress=allow_egress,
-                        networkid=cmd.networkids[0] if cmd.networkids else None)
+                        networkid=cmd.networkids[0] if cmd.networkids else None,
+                        vpcid=vpcid)
                 elif mode.lower() == 'basic':
                     if virtual_machine.publicip is not None:
                         # EIP/ELB (netscaler) enabled zone
@@ -1049,7 +1069,7 @@ class Volume:
 
     @classmethod
     def create_custom_disk(cls, apiclient, services, account=None,
-                           domainid=None, diskofferingid=None):
+                           domainid=None, diskofferingid=None, projectid=None):
         """Create Volume from Custom disk offering"""
         cmd = createVolume.createVolumeCmd()
         cmd.name = services["diskname"]
@@ -1072,19 +1092,22 @@ class Volume:
 
         if account:
             cmd.account = account
-        else:
+        elif "account" in services:
             cmd.account = services["account"]
 
         if domainid:
             cmd.domainid = domainid
-        else:
+        elif "domainid" in services:
             cmd.domainid = services["domainid"]
 
+        if projectid:
+            cmd.projectid = projectid
+
         return Volume(apiclient.createVolume(cmd).__dict__)
 
     @classmethod
     def create_from_snapshot(cls, apiclient, snapshot_id, services,
-                             account=None, domainid=None):
+                             account=None, domainid=None, projectid=None):
         """Create Volume from snapshot"""
         cmd = createVolume.createVolumeCmd()
         cmd.name = "-".join([services["diskname"], random_gen()])
@@ -1098,12 +1121,16 @@ class Volume:
             cmd.ispublic = False
         if account:
             cmd.account = account
-        else:
+        elif "account" in services:
             cmd.account = services["account"]
         if domainid:
             cmd.domainid = domainid
-        else:
+        elif "domainid" in services:
             cmd.domainid = services["domainid"]
+
+        if projectid:
+            cmd.projectid = projectid
+
         return Volume(apiclient.createVolume(cmd).__dict__)
 
     @classmethod
@@ -1454,8 +1481,8 @@ class Template:
         return Template(apiclient.createTemplate(cmd).__dict__)
 
     @classmethod
-    def create_from_snapshot(cls, apiclient, snapshot, services,
-                             random_name=True):
+    def create_from_snapshot(cls, apiclient, snapshot, services, account=None,
+                             domainid=None, projectid=None, random_name=True):
         """Create Template from snapshot"""
         # Create template from Snapshot ID
         cmd = createTemplate.createTemplateCmd()
@@ -1494,6 +1521,17 @@ class Template:
             raise Exception(
                 "Unable to find Ostype is required for creating template")
 
+        cmd.snapshotid = snapshot.id
+
+        if account:
+            cmd.account = account
+        if domainid:
+            cmd.domainid = domainid
+        if projectid:
+            cmd.projectid = projectid
+
+        return Template(apiclient.createTemplate(cmd).__dict__)
+
     def delete(self, apiclient, zoneid=None):
         """Delete Template"""
 
@@ -3930,7 +3968,7 @@ class VpnCustomerGateway:
 
     @classmethod
     def create(cls, apiclient, services, name, gateway, cidrlist,
-               account=None, domainid=None):
+               account=None, domainid=None, projectid=None):
         """Create VPN Customer Gateway"""
         cmd = createVpnCustomerGateway.createVpnCustomerGatewayCmd()
         cmd.name = name
@@ -3954,6 +3992,9 @@ class VpnCustomerGateway:
             cmd.account = account
         if domainid:
             cmd.domainid = domainid
+        if projectid:
+            cmd.projectid = projectid
+
         return VpnCustomerGateway(
             apiclient.createVpnCustomerGateway(cmd).__dict__)