apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

server: Submitting multiple dynamic VM Scaling API commands for the same instance can result in two usage events in the same second causing a compound key violation in usage service (#3991)

Browse Source 
Root cause:
Even though dynamic scaling job is handled in vmworkjob queue which ensures serilizing multiple jobs but the database updating and generating usage events are out of the job queue.

Solution:
Moved all updations into the job queue

Firstly I have tested all the scenarios to check if nothing is broken:
Scaling on a running VM with normal compute offering
Scaling on a stopped VM with normal compute offering
Scaling on a running VM with custom compute offering
Scaling on stopped VM with custom compute offering
Scaling on stopped/running VM between custom compute offering and normal compute offering and combinations among these. Checked if the custom parameters have been populated or deleted accordingly based on the offering to which the VM is scaled
Since this is a corner scenario I could not test the exact point where two usage events are recorded at the same time for two different API calls on same VM.
This commit is contained in:
harikrishna-patnala 2020-06-16 11:41:14 +05:30 committed by GitHub
parent 6a683dcf77
commit 5054766d9f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 88 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java
View File

@ -152,7 +152,7 @@ public interface VirtualMachineManager extends Manager {
* @param serviceOfferingId
* @return
*/
boolean upgradeVmDb(long vmId, long serviceOfferingId);
boolean upgradeVmDb(long vmId, ServiceOffering newServiceOffering, ServiceOffering currentServiceOffering);
/**
* @param vm
@ -201,7 +201,7 @@ public interface VirtualMachineManager extends Manager {
boolean replugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) throws ConcurrentOperationException,
ResourceUnavailableException, InsufficientCapacityException;
VirtualMachine reConfigureVm(String vmUuid, ServiceOffering newServiceOffering, boolean sameHost) throws ResourceUnavailableException, ConcurrentOperationException,
VirtualMachine reConfigureVm(String vmUuid, ServiceOffering oldServiceOffering, ServiceOffering newServiceOffering, Map<String, String> customParameters, boolean sameHost) throws ResourceUnavailableException, ConcurrentOperationException,
InsufficientServerCapacityException;
void findHostAndMigrate(String vmUuid, Long newSvcOfferingId, DeploymentPlanner.ExcludeList excludeHostList) throws InsufficientCapacityException,

83
engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
View File

@ -41,6 +41,7 @@ import javax.naming.ConfigurationException;
import com.cloud.agent.api.PrepareForMigrationAnswer;
import com.cloud.agent.api.to.DpdkTO;
import com.cloud.event.UsageEventVO;
import com.cloud.offering.NetworkOffering;
import com.cloud.offerings.dao.NetworkOfferingDetailsDao;
import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
@ -232,6 +233,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
private static final String VM_SYNC_ALERT_SUBJECT = "VM state sync alert";
@Inject
private UserVmManager _userVmMgr;
@Inject
private DataStoreManager dataStoreMgr;
@Inject
@ -3406,13 +3409,20 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
@Override
public boolean upgradeVmDb(final long vmId, final long serviceOfferingId) {
final VMInstanceVO vmForUpdate = _vmDao.createForUpdate();
vmForUpdate.setServiceOfferingId(serviceOfferingId);
final ServiceOffering newSvcOff = _entityMgr.findById(ServiceOffering.class, serviceOfferingId);
public boolean upgradeVmDb(final long vmId, final ServiceOffering newServiceOffering, ServiceOffering currentServiceOffering) {
final VMInstanceVO vmForUpdate = _vmDao.findById(vmId);
vmForUpdate.setServiceOfferingId(newServiceOffering.getId());
final ServiceOffering newSvcOff = _entityMgr.findById(ServiceOffering.class, newServiceOffering.getId());
vmForUpdate.setHaEnabled(newSvcOff.isOfferHA());
vmForUpdate.setLimitCpuUse(newSvcOff.getLimitCpuUse());
vmForUpdate.setServiceOfferingId(newSvcOff.getId());
if (newServiceOffering.isDynamic()) {
saveCustomOfferingDetails(vmId, newServiceOffering);
}
if (currentServiceOffering.isDynamic() && !newServiceOffering.isDynamic()) {
removeCustomOfferingDetails(vmId);
}
return _vmDao.update(vmId, vmForUpdate);
}
@ -4087,8 +4097,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
@Override
public VMInstanceVO reConfigureVm(final String vmUuid, final ServiceOffering oldServiceOffering,
final boolean reconfiguringOnExistingHost)
public VMInstanceVO reConfigureVm(final String vmUuid, final ServiceOffering oldServiceOffering, final ServiceOffering newServiceOffering,
Map<String, String> customParameters, final boolean reconfiguringOnExistingHost)
throws ResourceUnavailableException, InsufficientServerCapacityException, ConcurrentOperationException {
final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext();
@ -4098,14 +4108,14 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
final VirtualMachine vm = _vmDao.findByUuid(vmUuid);
placeHolder = createPlaceHolderWork(vm.getId());
try {
return orchestrateReConfigureVm(vmUuid, oldServiceOffering, reconfiguringOnExistingHost);
return orchestrateReConfigureVm(vmUuid, oldServiceOffering, newServiceOffering, reconfiguringOnExistingHost);
} finally {
if (placeHolder != null) {
_workJobDao.expunge(placeHolder.getId());
}
}
} else {
final Outcome<VirtualMachine> outcome = reconfigureVmThroughJobQueue(vmUuid, oldServiceOffering, reconfiguringOnExistingHost);
final Outcome<VirtualMachine> outcome = reconfigureVmThroughJobQueue(vmUuid, oldServiceOffering, newServiceOffering, customParameters, reconfiguringOnExistingHost);
VirtualMachine vm = null;
try {
@ -4134,14 +4144,12 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
}
private VMInstanceVO orchestrateReConfigureVm(final String vmUuid, final ServiceOffering oldServiceOffering, final boolean reconfiguringOnExistingHost) throws ResourceUnavailableException,
ConcurrentOperationException {
private VMInstanceVO orchestrateReConfigureVm(final String vmUuid, final ServiceOffering oldServiceOffering, final ServiceOffering newServiceOffering,
final boolean reconfiguringOnExistingHost) throws ResourceUnavailableException, ConcurrentOperationException {
final VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
upgradeVmDb(vm.getId(), newServiceOffering, oldServiceOffering);
final long newServiceofferingId = vm.getServiceOfferingId();
final ServiceOffering newServiceOffering = _offeringDao.findById(vm.getId(), newServiceofferingId);
final HostVO hostVo = _hostDao.findById(vm.getHostId());
final Float memoryOvercommitRatio = CapacityManager.MemOverprovisioningFactor.valueIn(hostVo.getClusterId());
final Float cpuOvercommitRatio = CapacityManager.CpuOverprovisioningFactor.valueIn(hostVo.getClusterId());
final long minMemory = (long)(newServiceOffering.getRamSize() / memoryOvercommitRatio);
@ -4168,7 +4176,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
if (reconfiguringOnExistingHost) {
vm.setServiceOfferingId(oldServiceOffering.getId());
_capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); //release the old capacity
vm.setServiceOfferingId(newServiceofferingId);
vm.setServiceOfferingId(newServiceOffering.getId());
_capacityMgr.allocateVmCapacity(vm, false); // lock the new capacity
}
@ -4177,7 +4185,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
s_logger.error("Unable to scale vm due to " + (reconfigureAnswer == null ? "" : reconfigureAnswer.getDetails()));
throw new CloudRuntimeException("Unable to scale vm due to " + (reconfigureAnswer == null ? "" : reconfigureAnswer.getDetails()));
}
if (vm.getType().equals(VirtualMachine.Type.User)) {
_userVmMgr.generateUsageEvent(vm, vm.isDisplayVm(), EventTypes.EVENT_VM_DYNAMIC_SCALE);
}
success = true;
} catch (final OperationTimedoutException e) {
throw new AgentUnavailableException("Operation timed out on reconfiguring " + vm, dstHostId);
@ -4186,7 +4196,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
} finally {
if (!success) {
_capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); // release the new capacity
vm.setServiceOfferingId(oldServiceOffering.getId());
upgradeVmDb(vm.getId(), oldServiceOffering, newServiceOffering); // rollback
_capacityMgr.allocateVmCapacity(vm, false); // allocate the old capacity
}
}
@ -4195,6 +4205,33 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
private void removeCustomOfferingDetails(long vmId) {
Map<String, String> details = userVmDetailsDao.listDetailsKeyPairs(vmId);
details.remove(UsageEventVO.DynamicParameters.cpuNumber.name());
details.remove(UsageEventVO.DynamicParameters.cpuSpeed.name());
details.remove(UsageEventVO.DynamicParameters.memory.name());
List<UserVmDetailVO> detailList = new ArrayList<UserVmDetailVO>();
for(Map.Entry<String, String> entry: details.entrySet()) {
UserVmDetailVO detailVO = new UserVmDetailVO(vmId, entry.getKey(), entry.getValue(), true);
detailList.add(detailVO);
}
userVmDetailsDao.saveDetails(detailList);
}
private void saveCustomOfferingDetails(long vmId, ServiceOffering serviceOffering) {
//save the custom values to the database.
Map<String, String> details = userVmDetailsDao.listDetailsKeyPairs(vmId);
details.put(UsageEventVO.DynamicParameters.cpuNumber.name(), serviceOffering.getCpu().toString());
details.put(UsageEventVO.DynamicParameters.cpuSpeed.name(), serviceOffering.getSpeed().toString());
details.put(UsageEventVO.DynamicParameters.memory.name(), serviceOffering.getRamSize().toString());
List<UserVmDetailVO> detailList = new ArrayList<UserVmDetailVO>();
for (Map.Entry<String, String> entry: details.entrySet()) {
UserVmDetailVO detailVO = new UserVmDetailVO(vmId, entry.getKey(), entry.getValue(), true);
detailList.add(detailVO);
}
userVmDetailsDao.saveDetails(detailList);
}
@Override
public String getConfigComponentName() {
return VirtualMachineManager.class.getSimpleName();
@ -5090,7 +5127,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
}
public Outcome<VirtualMachine> reconfigureVmThroughJobQueue(
final String vmUuid, final ServiceOffering newServiceOffering, final boolean reconfiguringOnExistingHost) {
final String vmUuid, final ServiceOffering oldServiceOffering, final ServiceOffering newServiceOffering, Map<String, String> customParameters, final boolean reconfiguringOnExistingHost) {
final CallContext context = CallContext.current();
final User user = context.getCallingUser();
@ -5121,7 +5158,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
// save work context info (there are some duplications)
final VmWorkReconfigure workInfo = new VmWorkReconfigure(user.getId(), account.getId(), vm.getId(),
VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, newServiceOffering.getId(), reconfiguringOnExistingHost);
VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, oldServiceOffering.getId(), newServiceOffering.getId(), customParameters, reconfiguringOnExistingHost);
workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo));
_jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId());
@ -5280,10 +5317,14 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
s_logger.info("Unable to find vm " + work.getVmId());
}
assert vm != null;
ServiceOfferingVO oldServiceOffering = _offeringDao.findById(work.getOldServiceOfferingId());
ServiceOfferingVO newServiceOffering = _offeringDao.findById(work.getNewServiceOfferingId());
if (newServiceOffering.isDynamic()) {
// update the service offering object with the custom parameters like cpu, memory
newServiceOffering = _offeringDao.getcomputeOffering(newServiceOffering, work.getCustomParameters());
}
final ServiceOffering newServiceOffering = _offeringDao.findById(vm.getId(), work.getNewServiceOfferingId());
reConfigureVm(vm.getUuid(), newServiceOffering,
reConfigureVm(vm.getUuid(), oldServiceOffering, newServiceOffering, work.getCustomParameters(),
work.isSameHost());
return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, null);
}

19
engine/orchestration/src/main/java/com/cloud/vm/VmWorkReconfigure.java
View File

@ -17,25 +17,40 @@
package com.cloud.vm;
import java.util.Map;
public class VmWorkReconfigure extends VmWork {
private static final long serialVersionUID = -4517030323758086615L;
Long oldServiceOfferingId;
Long newServiceOfferingId;
Map<String, String> customParameters;
boolean sameHost;
public VmWorkReconfigure(long userId, long accountId, long vmId, String handlerName,
Long newServiceOfferingId, boolean sameHost) {
public VmWorkReconfigure(long userId, long accountId, long vmId, String handlerName, Long oldServiceOfferingId,
Long newServiceOfferingId, Map<String, String> customParameters, boolean sameHost) {
super(userId, accountId, vmId, handlerName);
this.oldServiceOfferingId = oldServiceOfferingId;
this.newServiceOfferingId = newServiceOfferingId;
this.customParameters = customParameters;
this.sameHost = sameHost;
}
public Long getOldServiceOfferingId() {
return oldServiceOfferingId;
}
public Long getNewServiceOfferingId() {
return newServiceOfferingId;
}
public Map<String, String> getCustomParameters() {
return customParameters;
}
public boolean isSameHost() {
return sameHost;
}

10
server/src/main/java/com/cloud/server/ManagementServerImpl.java
View File

@ -4086,15 +4086,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
}
_itMgr.checkIfCanUpgrade(systemVm, newServiceOffering);
final boolean result = _itMgr.upgradeVmDb(systemVmId, serviceOfferingId);
if (newServiceOffering.isDynamic()) {
//save the custom values to the database.
_userVmMgr.saveCustomOfferingDetails(systemVmId, newServiceOffering);
}
if (currentServiceOffering.isDynamic() && !newServiceOffering.isDynamic()) {
_userVmMgr.removeCustomOfferingDetails(systemVmId);
}
final boolean result = _itMgr.upgradeVmDb(systemVmId, newServiceOffering, currentServiceOffering);
if (result) {
return _vmInstanceDao.findById(systemVmId);

5
server/src/main/java/com/cloud/vm/UserVmManager.java
View File

@ -32,7 +32,6 @@ import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.ManagementServerException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.exception.VirtualMachineMigrationException;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.user.Account;
@ -117,10 +116,6 @@ public interface UserVmManager extends UserVmService {
//find a common place for all the scaling and upgrading code of both user and systemvms.
void validateCustomParameters(ServiceOfferingVO serviceOffering, Map<String, String> customParameters);
public void saveCustomOfferingDetails(long vmId, ServiceOffering serviceOffering);
public void removeCustomOfferingDetails(long vmId);
void generateUsageEvent(VirtualMachine vm, boolean isDisplay, String eventType);
void persistDeviceBusInfo(UserVmVO paramUserVmVO, String paramString);

67
server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
View File

@ -1007,14 +1007,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
// Check that the specified service offering ID is valid
_itMgr.checkIfCanUpgrade(vmInstance, newServiceOffering);
_itMgr.upgradeVmDb(vmId, svcOffId);
if (newServiceOffering.isDynamic()) {
//save the custom values to the database.
saveCustomOfferingDetails(vmId, newServiceOffering);
}
if (currentServiceOffering.isDynamic() && !newServiceOffering.isDynamic()) {
removeCustomOfferingDetails(vmId);
}
_itMgr.upgradeVmDb(vmId, newServiceOffering, currentServiceOffering);
// Increment or decrement CPU and Memory count accordingly.
if (newCpu > currentCpu) {
@ -1132,14 +1125,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
ServiceOffering newSvcOffering = _offeringDao.findById(svcOffId);
_accountMgr.checkAccess(owner, newSvcOffering, _dcDao.findById(vmInstance.getDataCenterId()));
_itMgr.upgradeVmDb(vmId, svcOffId);
if (newServiceOffering.isDynamic()) {
//save the custom values to the database.
saveCustomOfferingDetails(vmId, newServiceOffering);
}
if (currentServiceOffering.isDynamic() && !newServiceOffering.isDynamic()) {
removeCustomOfferingDetails(vmId);
}
_itMgr.upgradeVmDb(vmId, newServiceOffering, currentServiceOffering);
// Increment or decrement CPU and Memory count accordingly.
if (newCpu > currentCpu) {
@ -1659,10 +1645,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
// Generate usage event for VM upgrade
generateUsageEvent(vmInstance, vmInstance.isDisplayVm(), EventTypes.EVENT_VM_UPGRADE);
}
if (vmInstance.getState().equals(State.Running)) {
// Generate usage event for Dynamic scaling of VM
generateUsageEvent( vmInstance, vmInstance.isDisplayVm(), EventTypes.EVENT_VM_DYNAMIC_SCALE);
}
return vmInstance;
} else {
throw new CloudRuntimeException("Failed to scale the VM");
@ -1835,17 +1817,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
}
// #3 scale the vm now
_itMgr.upgradeVmDb(vmId, newServiceOfferingId);
if (newServiceOffering.isDynamic()) {
//save the custom values to the database.
saveCustomOfferingDetails(vmId, newServiceOffering);
}
vmInstance = _vmInstanceDao.findById(vmId);
_itMgr.reConfigureVm(vmInstance.getUuid(), currentServiceOffering, existingHostHasCapacity);
_itMgr.reConfigureVm(vmInstance.getUuid(), currentServiceOffering, newServiceOffering, customParameters, existingHostHasCapacity);
success = true;
if (currentServiceOffering.isDynamic() && !newServiceOffering.isDynamic()) {
removeCustomOfferingDetails(vmId);
}
return success;
} catch (InsufficientCapacityException e) {
s_logger.warn("Received exception while scaling ", e);
@ -1857,16 +1831,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
s_logger.warn("Received exception while scaling ", e);
} finally {
if (!success) {
_itMgr.upgradeVmDb(vmId, currentServiceOffering.getId()); // rollback
// Decrement CPU and Memory count accordingly.
if (newCpu > currentCpu) {
_resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu));
}
//restoring old service offering will take care of removing new SO.
if(currentServiceOffering.isDynamic()){
saveCustomOfferingDetails(vmId, currentServiceOffering);
}
if (memoryDiff > 0) {
_resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long(memoryDiff));
@ -1878,35 +1846,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
return success;
}
@Override
public void saveCustomOfferingDetails(long vmId, ServiceOffering serviceOffering) {
//save the custom values to the database.
Map<String, String> details = userVmDetailsDao.listDetailsKeyPairs(vmId);
details.put(UsageEventVO.DynamicParameters.cpuNumber.name(), serviceOffering.getCpu().toString());
details.put(UsageEventVO.DynamicParameters.cpuSpeed.name(), serviceOffering.getSpeed().toString());
details.put(UsageEventVO.DynamicParameters.memory.name(), serviceOffering.getRamSize().toString());
List<UserVmDetailVO> detailList = new ArrayList<UserVmDetailVO>();
for (Map.Entry<String, String> entry: details.entrySet()) {
UserVmDetailVO detailVO = new UserVmDetailVO(vmId, entry.getKey(), entry.getValue(), true);
detailList.add(detailVO);
}
userVmDetailsDao.saveDetails(detailList);
}
@Override
public void removeCustomOfferingDetails(long vmId) {
Map<String, String> details = userVmDetailsDao.listDetailsKeyPairs(vmId);
details.remove(UsageEventVO.DynamicParameters.cpuNumber.name());
details.remove(UsageEventVO.DynamicParameters.cpuSpeed.name());
details.remove(UsageEventVO.DynamicParameters.memory.name());
List<UserVmDetailVO> detailList = new ArrayList<UserVmDetailVO>();
for(Map.Entry<String, String> entry: details.entrySet()) {
UserVmDetailVO detailVO = new UserVmDetailVO(vmId, entry.getKey(), entry.getValue(), true);
detailList.add(detailVO);
}
userVmDetailsDao.saveDetails(detailList);
}
@Override
public HashMap<Long, VmStatsEntry> getVirtualMachineStatistics(long hostId, String hostName, List<Long> vmIds) throws CloudRuntimeException {
HashMap<Long, VmStatsEntry> vmStatsById = new HashMap<Long, VmStatsEntry>();

6
server/src/test/java/com/cloud/vm/UserVmManagerTest.java
View File

@ -572,7 +572,7 @@ public class UserVmManagerTest {
doReturn(VirtualMachine.State.Stopped).when(_vmInstance).getState();
when(_vmDao.findById(anyLong())).thenReturn(null);
doReturn(true).when(_itMgr).upgradeVmDb(anyLong(), anyLong());
doReturn(true).when(_itMgr).upgradeVmDb(anyLong(), so1, so2);
//when(_vmDao.findById(anyLong())).thenReturn(_vmMock);
@ -619,9 +619,9 @@ public class UserVmManagerTest {
//when(ApiDBUtils.getCpuOverprovisioningFactor()).thenReturn(3f);
when(_capacityMgr.checkIfHostHasCapacity(anyLong(), anyInt(), anyLong(), anyBoolean(), anyFloat(), anyFloat(), anyBoolean())).thenReturn(false);
when(_itMgr.reConfigureVm(_vmInstance.getUuid(), so1, false)).thenReturn(_vmInstance);
when(_itMgr.reConfigureVm(_vmInstance.getUuid(), so2, so1, new HashMap<String, String>(), false)).thenReturn(_vmInstance);
doReturn(true).when(_itMgr).upgradeVmDb(anyLong(), anyLong());
doReturn(true).when(_itMgr).upgradeVmDb(anyLong(), so1, so2);
when(_vmDao.findById(anyLong())).thenReturn(_vmMock);