Merge release branch 4.8 to master

* 4.8:
  CLOUDSTACK-6928: fix issue disk I/O throttling not applied
  CLOUDSTACK-6975: Prevent dnsmasq from starting on backup redundant RvR.

engine/components-api/src/com/cloud/storage/StorageManager.java

@@ -27,6 +27,8 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.StoragePoolInfo;
+import com.cloud.agent.api.to.DataTO;
+import com.cloud.agent.api.to.DiskTO;
 import com.cloud.agent.manager.Commands;
 import com.cloud.capacity.CapacityVO;
 import com.cloud.exception.ConnectionException;
@@ -34,9 +36,11 @@ import com.cloud.exception.StorageConflictException;
 import com.cloud.exception.StorageUnavailableException;
 import com.cloud.host.Host;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import com.cloud.service.ServiceOfferingVO;
+import com.cloud.offering.DiskOffering;
+import com.cloud.offering.ServiceOffering;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.utils.Pair;
+import com.cloud.vm.DiskProfile;
 import com.cloud.vm.VMInstanceVO;
 
 public interface StorageManager extends StorageService {
@@ -140,13 +144,18 @@ public interface StorageManager extends StorageService {
 
     BigDecimal getStorageOverProvisioningFactor(Long dcId);
 
-    Long getDiskBytesReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering);
+    Long getDiskBytesReadRate(ServiceOffering offering, DiskOffering diskOffering);
 
-    Long getDiskBytesWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering);
+    Long getDiskBytesWriteRate(ServiceOffering offering, DiskOffering diskOffering);
 
-    Long getDiskIopsReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering);
+    Long getDiskIopsReadRate(ServiceOffering offering, DiskOffering diskOffering);
 
-    Long getDiskIopsWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering);
+    Long getDiskIopsWriteRate(ServiceOffering offering, DiskOffering diskOffering);
 
     void cleanupDownloadUrls();
+
+    void setDiskProfileThrottling(DiskProfile dskCh, ServiceOffering offering, DiskOffering diskOffering);
+
+    DiskTO getDiskWithThrottling(DataTO volTO, Volume.Type volumeType, long deviceId, String path, long offeringId, long diskOfferingId);
+
 }

engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java

@@ -94,6 +94,7 @@ import com.cloud.storage.ScopeType;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.Volume;
@@ -175,6 +176,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
     protected AsyncJobManager _jobMgr;
     @Inject
     ClusterManager clusterManager;
+    @Inject
+    StorageManager storageMgr;
 
     private final StateMachine2<Volume.State, Volume.Event, Volume> _volStateMachine;
     protected List<StoragePoolAllocator> _storagePoolAllocators;
@@ -207,10 +210,11 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
 
         // Find a destination storage pool with the specified criteria
         DiskOffering diskOffering = _entityMgr.findById(DiskOffering.class, volume.getDiskOfferingId());
-        ;
         DiskProfile dskCh = new DiskProfile(volume.getId(), volume.getVolumeType(), volume.getName(), diskOffering.getId(), diskOffering.getDiskSize(),
                 diskOffering.getTagsArray(), diskOffering.getUseLocalStorage(), diskOffering.isRecreatable(), null);
         dskCh.setHyperType(dataDiskHyperType);
+        storageMgr.setDiskProfileThrottling(dskCh, null, diskOffering);
+
         DataCenter destPoolDataCenter = _entityMgr.findById(DataCenter.class, destPoolDcId);
         Pod destPoolPod = _entityMgr.findById(Pod.class, destPoolPodId);
 
@@ -458,6 +462,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
         final HashSet<StoragePool> avoidPools = new HashSet<StoragePool>(avoids);
         DiskProfile dskCh = createDiskCharacteristics(volume, template, dc, diskOffering);
         dskCh.setHyperType(vm.getHypervisorType());
+        storageMgr.setDiskProfileThrottling(dskCh, null, diskOffering);
+
         // Find a suitable storage to create volume on
         StoragePool destPool = findStoragePool(dskCh, dc, pod, clusterId, null, vm, avoidPools);
         DataStore destStore = dataStoreMgr.getDataStore(destPool.getId(), DataStoreRole.Primary);
@@ -490,8 +496,10 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
         DiskProfile dskCh = null;
         if (volume.getVolumeType() == Type.ROOT && Storage.ImageFormat.ISO != template.getFormat()) {
             dskCh = createDiskCharacteristics(volume, template, dc, offering);
+            storageMgr.setDiskProfileThrottling(dskCh, offering, diskOffering);
         } else {
             dskCh = createDiskCharacteristics(volume, template, dc, diskOffering);
+            storageMgr.setDiskProfileThrottling(dskCh, null, diskOffering);
         }
 
         if (diskOffering != null && diskOffering.isCustomized()) {
@@ -1054,9 +1062,10 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
         }
 
         for (VolumeVO vol : vols) {
-            DataTO volTO = volFactory.getVolume(vol.getId()).getTO();
-            DiskTO disk = new DiskTO(volTO, vol.getDeviceId(), vol.getPath(), vol.getVolumeType());
             VolumeInfo volumeInfo = volFactory.getVolume(vol.getId());
+            DataTO volTO = volumeInfo.getTO();
+            DiskTO disk = storageMgr.getDiskWithThrottling(volTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(),
+                    vm.getServiceOfferingId(), vol.getDiskOfferingId());
             DataStore dataStore = dataStoreMgr.getDataStore(vol.getPoolId(), DataStoreRole.Primary);
 
             disk.setDetails(getDetails(volumeInfo, dataStore));
@@ -1335,9 +1344,10 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
                 pool = (StoragePool)dataStoreMgr.getDataStore(result.second().getId(), DataStoreRole.Primary);
                 vol = result.first();
             }
-            DataTO volumeTO = volFactory.getVolume(vol.getId()).getTO();
-            DiskTO disk = new DiskTO(volumeTO, vol.getDeviceId(), vol.getPath(), vol.getVolumeType());
             VolumeInfo volumeInfo = volFactory.getVolume(vol.getId());
+            DataTO volTO = volumeInfo.getTO();
+            DiskTO disk = storageMgr.getDiskWithThrottling(volTO, vol.getVolumeType(), vol.getDeviceId(), vol.getPath(),
+                    vm.getServiceOfferingId(), vol.getDiskOfferingId());
             DataStore dataStore = dataStoreMgr.getDataStore(vol.getPoolId(), DataStoreRole.Primary);
 
             disk.setDetails(getDetails(volumeInfo, dataStore));

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -953,8 +953,8 @@ public class KVMStorageProcessor implements StorageProcessor {
         return null;
     }
 
-    protected synchronized String attachOrDetachDisk(final Connect conn, final boolean attach, final String vmName, final KVMPhysicalDisk attachingDisk, final int devId, final String serial) throws LibvirtException,
+    protected synchronized String attachOrDetachDisk(final Connect conn, final boolean attach, final String vmName, final KVMPhysicalDisk attachingDisk, final int devId, final String serial,
-    InternalErrorException {
+            final Long bytesReadRate, final Long bytesWriteRate, final Long iopsReadRate, final Long iopsWriteRate) throws LibvirtException, InternalErrorException {
         List<DiskDef> disks = null;
         Domain dm = null;
         DiskDef diskdef = null;
@@ -1015,6 +1015,19 @@ public class KVMStorageProcessor implements StorageProcessor {
                 } else if (attachingDisk.getFormat() == PhysicalDiskFormat.RAW) {
                     diskdef.defBlockBasedDisk(attachingDisk.getPath(), devId, DiskDef.DiskBus.VIRTIO);
                 }
+
+                if ((bytesReadRate != null) && (bytesReadRate > 0)) {
+                    diskdef.setBytesReadRate(bytesReadRate);
+                }
+                if ((bytesWriteRate != null) && (bytesWriteRate > 0)) {
+                    diskdef.setBytesWriteRate(bytesWriteRate);
+                }
+                if ((iopsReadRate != null) && (iopsReadRate > 0)) {
+                    diskdef.setIopsReadRate(iopsReadRate);
+                }
+                if ((iopsWriteRate != null) && (iopsWriteRate > 0)) {
+                    diskdef.setIopsWriteRate(iopsWriteRate);
+                }
             }
 
             final String xml = diskdef.toString();
@@ -1040,7 +1053,7 @@ public class KVMStorageProcessor implements StorageProcessor {
 
             final KVMPhysicalDisk phyDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath());
 
-            attachOrDetachDisk(conn, true, vmName, phyDisk, disk.getDiskSeq().intValue(), serial);
+            attachOrDetachDisk(conn, true, vmName, phyDisk, disk.getDiskSeq().intValue(), serial, vol.getBytesReadRate(), vol.getBytesWriteRate(), vol.getIopsReadRate(), vol.getIopsWriteRate());
 
             return new AttachAnswer(disk);
         } catch (final LibvirtException e) {
@@ -1065,7 +1078,7 @@ public class KVMStorageProcessor implements StorageProcessor {
 
             final KVMPhysicalDisk phyDisk = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath());
 
-            attachOrDetachDisk(conn, false, vmName, phyDisk, disk.getDiskSeq().intValue(), serial);
+            attachOrDetachDisk(conn, false, vmName, phyDisk, disk.getDiskSeq().intValue(), serial, vol.getBytesReadRate(), vol.getBytesWriteRate(), vol.getIopsReadRate(), vol.getIopsWriteRate());
 
             storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath());
 

server/src/com/cloud/storage/StorageManagerImpl.java

@@ -96,11 +96,14 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
 import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.StoragePoolInfo;
+import com.cloud.agent.api.to.DataTO;
+import com.cloud.agent.api.to.DiskTO;
 import com.cloud.agent.manager.Commands;
 import com.cloud.api.ApiDBUtils;
 import com.cloud.api.query.dao.TemplateJoinDao;
@@ -139,13 +142,14 @@ import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.hypervisor.HypervisorGuruManager;
+import com.cloud.offering.DiskOffering;
+import com.cloud.offering.ServiceOffering;
 import com.cloud.org.Grouping;
 import com.cloud.org.Grouping.AllocationState;
 import com.cloud.resource.ResourceState;
 import com.cloud.server.ConfigurationServer;
 import com.cloud.server.ManagementServer;
 import com.cloud.server.StatsCollector;
-import com.cloud.service.ServiceOfferingVO;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.Volume.Type;
@@ -174,6 +178,7 @@ import com.cloud.utils.component.ComponentContext;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
 import com.cloud.utils.db.DB;
+import com.cloud.utils.db.EntityManager;
 import com.cloud.utils.db.GenericSearchBuilder;
 import com.cloud.utils.db.GlobalLock;
 import com.cloud.utils.db.JoinBuilder;
@@ -186,6 +191,7 @@ import com.cloud.utils.db.TransactionCallbackNoReturn;
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.db.TransactionStatus;
 import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.DiskProfile;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine.State;
 import com.cloud.vm.dao.VMInstanceDao;
@@ -281,6 +287,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
     private DiskOfferingDao _diskOfferingDao;
     @Inject
     ResourceLimitService _resourceLimitMgr;
+    @Inject
+    EntityManager _entityMgr;
 
     protected List<StoragePoolDiscoverer> _discoverers;
 
@@ -2280,7 +2288,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
     // get bytesReadRate from service_offering, disk_offering and vm.disk.throttling.bytes_read_rate
     @Override
-    public Long getDiskBytesReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) {
+    public Long getDiskBytesReadRate(final ServiceOffering offering, final DiskOffering diskOffering) {
         if ((offering != null) && (offering.getBytesReadRate() != null) && (offering.getBytesReadRate() > 0)) {
             return offering.getBytesReadRate();
         } else if ((diskOffering != null) && (diskOffering.getBytesReadRate() != null) && (diskOffering.getBytesReadRate() > 0)) {
@@ -2296,7 +2304,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
     // get bytesWriteRate from service_offering, disk_offering and vm.disk.throttling.bytes_write_rate
     @Override
-    public Long getDiskBytesWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) {
+    public Long getDiskBytesWriteRate(final ServiceOffering offering, final DiskOffering diskOffering) {
         if ((offering != null) && (offering.getBytesWriteRate() != null) && (offering.getBytesWriteRate() > 0)) {
             return offering.getBytesWriteRate();
         } else if ((diskOffering != null) && (diskOffering.getBytesWriteRate() != null) && (diskOffering.getBytesWriteRate() > 0)) {
@@ -2312,7 +2320,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
     // get iopsReadRate from service_offering, disk_offering and vm.disk.throttling.iops_read_rate
     @Override
-    public Long getDiskIopsReadRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) {
+    public Long getDiskIopsReadRate(final ServiceOffering offering, final DiskOffering diskOffering) {
         if ((offering != null) && (offering.getIopsReadRate() != null) && (offering.getIopsReadRate() > 0)) {
             return offering.getIopsReadRate();
         } else if ((diskOffering != null) && (diskOffering.getIopsReadRate() != null) && (diskOffering.getIopsReadRate() > 0)) {
@@ -2328,7 +2336,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
     // get iopsWriteRate from service_offering, disk_offering and vm.disk.throttling.iops_write_rate
     @Override
-    public Long getDiskIopsWriteRate(ServiceOfferingVO offering, DiskOfferingVO diskOffering) {
+    public Long getDiskIopsWriteRate(final ServiceOffering offering, final DiskOffering diskOffering) {
         if ((offering != null) && (offering.getIopsWriteRate() != null) && (offering.getIopsWriteRate() > 0)) {
             return offering.getIopsWriteRate();
         } else if ((diskOffering != null) && (diskOffering.getIopsWriteRate() != null) && (diskOffering.getIopsWriteRate() > 0)) {
@@ -2351,4 +2359,39 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {StorageCleanupInterval, StorageCleanupDelay, StorageCleanupEnabled};
     }
+
+    @Override
+    public void setDiskProfileThrottling(DiskProfile dskCh, final ServiceOffering offering, final DiskOffering diskOffering) {
+        dskCh.setBytesReadRate(getDiskBytesReadRate(offering, diskOffering));
+        dskCh.setBytesWriteRate(getDiskBytesWriteRate(offering, diskOffering));
+        dskCh.setIopsReadRate(getDiskIopsReadRate(offering, diskOffering));
+        dskCh.setIopsWriteRate(getDiskIopsWriteRate(offering, diskOffering));
+    }
+
+    @Override
+    public DiskTO getDiskWithThrottling(final DataTO volTO, final Volume.Type volumeType, final long deviceId, final String path, final long offeringId, final long diskOfferingId) {
+        DiskTO disk = null;
+        if (volTO != null && volTO instanceof VolumeObjectTO) {
+            VolumeObjectTO volumeTO = (VolumeObjectTO) volTO;
+            ServiceOffering offering = _entityMgr.findById(ServiceOffering.class, offeringId);
+            DiskOffering diskOffering = _entityMgr.findById(DiskOffering.class, diskOfferingId);
+            if (volumeType == Volume.Type.ROOT) {
+                setVolumeObjectTOThrottling(volumeTO, offering, diskOffering);
+            } else {
+                setVolumeObjectTOThrottling(volumeTO, null, diskOffering);
+            }
+            disk = new DiskTO(volumeTO, deviceId, path, volumeType);
+        } else {
+            disk = new DiskTO(volTO, deviceId, path, volumeType);
+        }
+        return disk;
+    }
+
+    private void setVolumeObjectTOThrottling(VolumeObjectTO volumeTO, final ServiceOffering offering, final DiskOffering diskOffering) {
+        volumeTO.setBytesReadRate(getDiskBytesReadRate(offering, diskOffering));
+        volumeTO.setBytesWriteRate(getDiskBytesWriteRate(offering, diskOffering));
+        volumeTO.setIopsReadRate(getDiskIopsReadRate(offering, diskOffering));
+        volumeTO.setIopsWriteRate(getDiskIopsWriteRate(offering, diskOffering));
+    }
+
 }

server/src/com/cloud/storage/VolumeApiServiceImpl.java

@@ -251,6 +251,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
     @Inject
     UserVmManager _userVmMgr;
     protected Gson _gson;
+    @Inject
+    StorageManager storageMgr;
 
     private List<StoragePoolAllocator> _storagePoolAllocators;
 
@@ -2469,7 +2471,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
 
             deviceId = getDeviceId(vm.getId(), deviceId);
 
-            DiskTO disk = new DiskTO(volTO, deviceId, volumeToAttach.getPath(), volumeToAttach.getVolumeType());
+            DiskTO disk = storageMgr.getDiskWithThrottling(volTO, volumeToAttach.getVolumeType(), deviceId, volumeToAttach.getPath(),
+                    vm.getServiceOfferingId(), volumeToAttach.getDiskOfferingId());
 
             AttachCommand cmd = new AttachCommand(disk, vm.getInstanceName());
 

systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py

@@ -54,7 +54,8 @@ class CsDhcp(CsDataBag):
         self.cloud.commit()
 
         # We restart DNSMASQ every time the configure.py is called in order to avoid lease problems.
-        CsHelper.service("dnsmasq", "restart")
+        if not self.cl.is_redundant() or self.cl.is_master():
+            CsHelper.service("dnsmasq", "restart")
 
     def configure_server(self):
         # self.conf.addeq("dhcp-hostsfile=%s" % DHCP_HOSTS)