Track volume usage data at a vm granularity as well (#11531)

Co-authored-by: Vishesh <8760112+vishesh92@users.noreply.github.com>
This commit is contained in:
Abhisar Sinha 2025-11-12 14:02:01 +05:30 committed by GitHub
parent 15439ede7d
commit 671d8ad704
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 243 additions and 156 deletions

View File

@ -94,6 +94,14 @@ public class UsageEventUtils {
}
public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId,
Long size, String entityType, String entityUUID, Long vmId, boolean displayResource) {
if (displayResource) {
saveUsageEvent(usageType, accountId, zoneId, resourceId, offeringId, templateId, size, vmId, resourceName);
}
publishUsageEvent(usageType, accountId, zoneId, entityType, entityUUID);
}
public static void publishUsageEvent(String usageType, long accountId, long zoneId, long resourceId, String resourceName, Long offeringId, Long templateId,
Long size, Long virtualSize, String entityType, String entityUUID, Map<String, String> details) {
saveUsageEvent(usageType, accountId, zoneId, resourceId, resourceName, offeringId, templateId, size, virtualSize, details);
@ -202,6 +210,10 @@ public class UsageEventUtils {
s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, vmId, securityGroupId));
}
public static void saveUsageEvent(String usageType, long accountId, long zoneId, long resourceId, Long offeringId, Long templateId, Long size, Long vmId, String resourceName) {
s_usageEventDao.persist(new UsageEventVO(usageType, accountId, zoneId, resourceId, offeringId, templateId, size, vmId, resourceName));
}
private static void publishUsageEvent(String usageEventType, Long accountId, Long zoneId, String resourceType, String resourceUUID) {
String configKey = "publish.usage.events";
String value = s_configDao.getValue(configKey);

View File

@ -903,7 +903,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
// Save usage event and update resource count for user vm volumes
if (vm.getType() == VirtualMachine.Type.User) {
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offering.getId(), null, size,
Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume());
Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume());
_resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering);
}
DiskProfile diskProfile = toDiskProfile(vol, offering);
@ -981,7 +981,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
}
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offeringId, vol.getTemplateId(), size,
Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume());
Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume());
_resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering);
}

View File

@ -75,6 +75,9 @@ public class UsageEventVO implements UsageEvent {
@Column(name = "virtual_size")
private Long virtualSize;
@Column(name = "vm_id")
private Long vmId;
public UsageEventVO() {
}
@ -143,6 +146,18 @@ public class UsageEventVO implements UsageEvent {
this.offeringId = securityGroupId;
}
public UsageEventVO(String usageType, long accountId, long zoneId, long resourceId, Long offeringId, Long templateId, Long size, Long vmId, String resourceName) {
this.type = usageType;
this.accountId = accountId;
this.zoneId = zoneId;
this.resourceId = resourceId;
this.offeringId = offeringId;
this.templateId = templateId;
this.size = size;
this.vmId = vmId;
this.resourceName = resourceName;
}
@Override
public long getId() {
return id;
@ -248,4 +263,11 @@ public class UsageEventVO implements UsageEvent {
this.virtualSize = virtualSize;
}
public Long getVmId() {
return vmId;
}
public void setVmId(Long vmId) {
this.vmId = vmId;
}
}

View File

@ -45,11 +45,11 @@ public class UsageEventDaoImpl extends GenericDaoBase<UsageEventVO, Long> implem
private final SearchBuilder<UsageEventVO> latestEventsSearch;
private final SearchBuilder<UsageEventVO> IpeventsSearch;
private static final String COPY_EVENTS =
"INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) "
+ "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id > ? and vmevt.id <= ? ";
"INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id) "
+ "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id FROM cloud.usage_event vmevt WHERE vmevt.id > ? and vmevt.id <= ? ";
private static final String COPY_ALL_EVENTS =
"INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size) "
+ "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size FROM cloud.usage_event vmevt WHERE vmevt.id <= ?";
"INSERT INTO cloud_usage.usage_event (id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id) "
+ "SELECT id, type, account_id, created, zone_id, resource_id, resource_name, offering_id, template_id, size, resource_type, virtual_size, vm_id FROM cloud.usage_event vmevt WHERE vmevt.id <= ?";
private static final String COPY_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) "
+ "SELECT id, usage_event_id, name, value FROM cloud.usage_event_details vmevtDetails WHERE vmevtDetails.usage_event_id > ? and vmevtDetails.usage_event_id <= ? ";
private static final String COPY_ALL_EVENT_DETAILS = "INSERT INTO cloud_usage.usage_event_details (id, usage_event_id, name, value) "

View File

@ -59,6 +59,9 @@ public class UsageVolumeVO implements InternalIdentity {
@Column(name = "size")
private long size;
@Column(name = "vm_id")
private Long vmId;
@Column(name = "created")
@Temporal(value = TemporalType.TIMESTAMP)
private Date created = null;
@ -70,13 +73,14 @@ public class UsageVolumeVO implements InternalIdentity {
protected UsageVolumeVO() {
}
public UsageVolumeVO(long id, long zoneId, long accountId, long domainId, Long diskOfferingId, Long templateId, long size, Date created, Date deleted) {
public UsageVolumeVO(long id, long zoneId, long accountId, long domainId, Long diskOfferingId, Long templateId, Long vmId, long size, Date created, Date deleted) {
this.volumeId = id;
this.zoneId = zoneId;
this.accountId = accountId;
this.domainId = domainId;
this.diskOfferingId = diskOfferingId;
this.templateId = templateId;
this.vmId = vmId;
this.size = size;
this.created = created;
this.deleted = deleted;
@ -126,4 +130,12 @@ public class UsageVolumeVO implements InternalIdentity {
public long getVolumeId() {
return volumeId;
}
public Long getVmId() {
return vmId;
}
public void setVmId(Long vmId) {
this.vmId = vmId;
}
}

View File

@ -57,6 +57,7 @@ public class UsageStorageDaoImpl extends GenericDaoBase<UsageStorageVO, Long> im
IdSearch.and("accountId", IdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
IdSearch.and("id", IdSearch.entity().getEntityId(), SearchCriteria.Op.EQ);
IdSearch.and("type", IdSearch.entity().getStorageType(), SearchCriteria.Op.EQ);
IdSearch.and("deleted", IdSearch.entity().getDeleted(), SearchCriteria.Op.NULL);
IdSearch.done();
IdZoneSearch = createSearchBuilder();
@ -74,6 +75,7 @@ public class UsageStorageDaoImpl extends GenericDaoBase<UsageStorageVO, Long> im
sc.setParameters("accountId", accountId);
sc.setParameters("id", id);
sc.setParameters("type", type);
sc.setParameters("deleted", null);
return listBy(sc, null);
}

View File

@ -23,9 +23,7 @@ import com.cloud.usage.UsageVolumeVO;
import com.cloud.utils.db.GenericDao;
public interface UsageVolumeDao extends GenericDao<UsageVolumeVO, Long> {
public void removeBy(long userId, long id);
public void update(UsageVolumeVO usage);
public List<UsageVolumeVO> getUsageRecords(Long accountId, Long domainId, Date startDate, Date endDate, boolean limit, int page);
List<UsageVolumeVO> listByVolumeId(long volumeId, long accountId);
}

View File

@ -18,81 +18,46 @@ package com.cloud.usage.dao;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
import com.cloud.exception.CloudException;
import javax.annotation.PostConstruct;
import org.springframework.stereotype.Component;
import com.cloud.usage.UsageVolumeVO;
import com.cloud.utils.DateUtil;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.TransactionLegacy;
@Component
public class UsageVolumeDaoImpl extends GenericDaoBase<UsageVolumeVO, Long> implements UsageVolumeDao {
protected static final String REMOVE_BY_USERID_VOLID = "DELETE FROM usage_volume WHERE account_id = ? AND volume_id = ?";
protected static final String UPDATE_DELETED = "UPDATE usage_volume SET deleted = ? WHERE account_id = ? AND volume_id = ? and deleted IS NULL";
protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted "
protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted "
+ "FROM usage_volume " + "WHERE account_id = ? AND ((deleted IS NULL) OR (created BETWEEN ? AND ?) OR "
+ " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?)))";
protected static final String GET_USAGE_RECORDS_BY_DOMAIN = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted "
protected static final String GET_USAGE_RECORDS_BY_DOMAIN = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted "
+ "FROM usage_volume " + "WHERE domain_id = ? AND ((deleted IS NULL) OR (created BETWEEN ? AND ?) OR "
+ " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?)))";
protected static final String GET_ALL_USAGE_RECORDS = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, size, created, deleted "
protected static final String GET_ALL_USAGE_RECORDS = "SELECT volume_id, zone_id, account_id, domain_id, disk_offering_id, template_id, vm_id, size, created, deleted "
+ "FROM usage_volume " + "WHERE (deleted IS NULL) OR (created BETWEEN ? AND ?) OR " + " (deleted BETWEEN ? AND ?) OR ((created <= ?) AND (deleted >= ?))";
private SearchBuilder<UsageVolumeVO> volumeSearch;
public UsageVolumeDaoImpl() {
}
@Override
public void removeBy(long accountId, long volId) {
TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
try {
txn.start();
try(PreparedStatement pstmt = txn.prepareStatement(REMOVE_BY_USERID_VOLID);) {
if (pstmt != null) {
pstmt.setLong(1, accountId);
pstmt.setLong(2, volId);
pstmt.executeUpdate();
}
}catch (SQLException e) {
throw new CloudException("Error removing usageVolumeVO:"+e.getMessage(), e);
}
txn.commit();
} catch (Exception e) {
txn.rollback();
logger.warn("Error removing usageVolumeVO:"+e.getMessage(), e);
} finally {
txn.close();
}
}
@Override
public void update(UsageVolumeVO usage) {
TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
PreparedStatement pstmt = null;
try {
txn.start();
if (usage.getDeleted() != null) {
pstmt = txn.prepareAutoCloseStatement(UPDATE_DELETED);
pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), usage.getDeleted()));
pstmt.setLong(2, usage.getAccountId());
pstmt.setLong(3, usage.getVolumeId());
pstmt.executeUpdate();
}
txn.commit();
} catch (Exception e) {
txn.rollback();
logger.warn("Error updating UsageVolumeVO", e);
} finally {
txn.close();
}
@PostConstruct
protected void init() {
volumeSearch = createSearchBuilder();
volumeSearch.and("accountId", volumeSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
volumeSearch.and("volumeId", volumeSearch.entity().getVolumeId(), SearchCriteria.Op.EQ);
volumeSearch.and("deleted", volumeSearch.entity().getDeleted(), SearchCriteria.Op.NULL);
volumeSearch.done();
}
@Override
@ -150,11 +115,15 @@ public class UsageVolumeDaoImpl extends GenericDaoBase<UsageVolumeVO, Long> impl
if (tId == 0) {
tId = null;
}
long size = Long.valueOf(rs.getLong(7));
Long vmId = Long.valueOf(rs.getLong(7));
if (vmId == 0) {
vmId = null;
}
long size = Long.valueOf(rs.getLong(8));
Date createdDate = null;
Date deletedDate = null;
String createdTS = rs.getString(8);
String deletedTS = rs.getString(9);
String createdTS = rs.getString(9);
String deletedTS = rs.getString(10);
if (createdTS != null) {
createdDate = DateUtil.parseDateString(s_gmtTimeZone, createdTS);
@ -163,7 +132,7 @@ public class UsageVolumeDaoImpl extends GenericDaoBase<UsageVolumeVO, Long> impl
deletedDate = DateUtil.parseDateString(s_gmtTimeZone, deletedTS);
}
usageRecords.add(new UsageVolumeVO(vId, zoneId, acctId, dId, doId, tId, size, createdDate, deletedDate));
usageRecords.add(new UsageVolumeVO(vId, zoneId, acctId, dId, doId, tId, vmId, size, createdDate, deletedDate));
}
} catch (Exception e) {
txn.rollback();
@ -174,4 +143,13 @@ public class UsageVolumeDaoImpl extends GenericDaoBase<UsageVolumeVO, Long> impl
return usageRecords;
}
@Override
public List<UsageVolumeVO> listByVolumeId(long volumeId, long accountId) {
SearchCriteria<UsageVolumeVO> sc = volumeSearch.create();
sc.setParameters("accountId", accountId);
sc.setParameters("volumeId", volumeId);
sc.setParameters("deleted", null);
return listBy(sc);
}
}

View File

@ -41,6 +41,13 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.ldap_configuration', 'uuid', 'VARCHA
-- Populate uuid for existing rows where uuid is NULL or empty
UPDATE `cloud`.`ldap_configuration` SET uuid = UUID() WHERE uuid IS NULL OR uuid = '';
-- Add vm_id column to usage_event table for volume usage events
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.usage_event','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with volume usage events"');
CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_event','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with volume usage events"');
-- Add vm_id column to cloud_usage.usage_volume table
CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_volume','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with the volume usage"');
-- Add the column cross_zone_instance_creation to cloud.backup_repository. if enabled it means that new Instance can be created on all Zones from Backups on this Repository.
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_repository', 'cross_zone_instance_creation', 'TINYINT(1) DEFAULT NULL COMMENT ''Backup Repository can be used for disaster recovery on another zone''');

View File

@ -4298,6 +4298,9 @@ public class ApiResponseHelper implements ResponseGenerator {
if (volume != null) {
builder.append("for ").append(volume.getName()).append(" (").append(volume.getUuid()).append(")");
}
if (vmInstance != null) {
builder.append(" attached to VM ").append(vmInstance.getHostName()).append(" (").append(vmInstance.getUuid()).append(")");
}
if (diskOff != null) {
builder.append(" with disk offering ").append(diskOff.getName()).append(" (").append(diskOff.getUuid()).append(")");
}

View File

@ -21,6 +21,8 @@ import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.NicTO;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.configuration.ConfigurationManagerImpl;
import com.cloud.event.EventTypes;
import com.cloud.event.UsageEventUtils;
import com.cloud.host.HostVO;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao;
@ -370,6 +372,8 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
_volumeDao.update(volume.getId(), volume);
_volumeDao.attachVolume(volume.getId(), vm.getId(), getNextAvailableDeviceId(vmVolumes));
}
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(),
volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), vm.getId(), volume.isDisplay());
}
} catch (Exception e) {
throw new RuntimeException("Could not restore VM " + vm.getName() + " due to : " + e.getMessage());
@ -387,6 +391,8 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
_volumeDao.attachVolume(restoredVolume.getId(), vm.getId(), getNextAvailableDeviceId(vmVolumes));
restoredVolume.setState(Volume.State.Ready);
_volumeDao.update(restoredVolume.getId(), restoredVolume);
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, restoredVolume.getAccountId(), restoredVolume.getDataCenterId(), restoredVolume.getId(), restoredVolume.getName(),
restoredVolume.getDiskOfferingId(), restoredVolume.getTemplateId(), restoredVolume.getSize(), Volume.class.getName(), restoredVolume.getUuid(), vm.getId(), restoredVolume.isDisplay());
return true;
} catch (Exception e) {
restoredVolume.setDisplay(false);

View File

@ -1004,7 +1004,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
if (snapshotId == null && displayVolume) {
// for volume created from snapshot, create usage event after volume creation
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), diskOfferingId, null, size,
Volume.class.getName(), volume.getUuid(), displayVolume);
Volume.class.getName(), volume.getUuid(), volume.getInstanceId(), displayVolume);
}
if (volume != null && details != null) {
@ -1106,7 +1106,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
createdVolume = _volumeMgr.createVolumeFromSnapshot(volume, snapshot, vm);
VolumeVO volumeVo = _volsDao.findById(createdVolume.getId());
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, createdVolume.getAccountId(), createdVolume.getDataCenterId(), createdVolume.getId(), createdVolume.getName(),
createdVolume.getDiskOfferingId(), null, createdVolume.getSize(), Volume.class.getName(), createdVolume.getUuid(), volumeVo.isDisplayVolume());
createdVolume.getDiskOfferingId(), null, createdVolume.getSize(), Volume.class.getName(), createdVolume.getUuid(), volume.getInstanceId(), volumeVo.isDisplayVolume());
return volumeVo;
}
@ -1903,7 +1903,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
}
UsageEventUtils
.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), offeringId,
volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.isDisplay());
volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.getInstanceId(), volume.isDisplay());
logger.debug("Volume [{}] has been successfully recovered, thus a new usage event {} has been published.", volume, EventTypes.EVENT_VOLUME_CREATE);
}
@ -2998,7 +2998,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
if (displayVolume) {
// flag turned 1 equivalent to freshly created volume
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), volume.getDiskOfferingId(),
volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid());
volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.getInstanceId(), displayVolume);
} else {
// flag turned 0 equivalent to deleting a volume
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), Volume.class.getName(),
@ -3259,6 +3259,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
handleTargetsForVMware(hostId, volumePool.getHostAddress(), volumePool.getPort(), volume.get_iScsiName());
}
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DETACH, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(),
volume.getDiskOfferingId(), null, volume.getSize(), Volume.class.getName(), volume.getUuid(), null, volume.isDisplay());
return _volsDao.findById(volumeId);
} else {
@ -4339,7 +4341,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
diskOfferingVO);
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(),
volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(),
volume.getUuid(), volume.isDisplayVolume());
volume.getUuid(), volume.getInstanceId(), volume.isDisplayVolume());
volService.moveVolumeOnSecondaryStorageToAnotherAccount(volume, oldAccount, newAccount);
}
@ -4863,6 +4865,9 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
if (attached) {
ev = Volume.Event.OperationSucceeded;
logger.debug("Volume: {} successfully attached to VM: {}", volInfo.getVolume(), volInfo.getAttachedVM());
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, volumeToAttach.getAccountId(), volumeToAttach.getDataCenterId(), volumeToAttach.getId(), volumeToAttach.getName(),
volumeToAttach.getDiskOfferingId(), volumeToAttach.getTemplateId(), volumeToAttach.getSize(), Volume.class.getName(), volumeToAttach.getUuid(), vm.getId(), volumeToAttach.isDisplay());
provideVMInfo(dataStore, vm.getId(), volInfo.getId());
} else {
logger.debug("Volume: {} failed to attach to VM: {}", volInfo.getVolume(), volInfo.getAttachedVM());

View File

@ -81,7 +81,7 @@ public class VolumeStateListener implements StateListener<State, Event, Volume>
// For the Resize Volume Event, this publishes an event with an incorrect disk offering ID, so do nothing for now
} else {
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), vol.getDiskOfferingId(), null, vol.getSize(),
Volume.class.getName(), vol.getUuid(), vol.isDisplayVolume());
Volume.class.getName(), vol.getUuid(), instanceId, vol.isDisplayVolume());
}
} else if (transition.getToState() == State.Destroy && vol.getVolumeType() != Volume.Type.ROOT) { //Do not Publish Usage Event for ROOT Disk as it would have been published already while destroying a VM
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_DELETE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(),

View File

@ -2408,6 +2408,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
if (Volume.State.Destroy.equals(volume.getState())) {
_volumeService.recoverVolume(volume.getId());
_volsDao.attachVolume(volume.getId(), vmId, ROOT_DEVICE_ID);
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(),
volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), vmId, volume.isDisplay());
} else {
_volumeService.publishVolumeCreationUsageEvent(volume);
}
@ -8156,7 +8158,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
logger.trace("Generating a create volume event for volume [{}].", volume);
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(),
volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.isDisplayVolume());
volume.getDiskOfferingId(), volume.getTemplateId(), volume.getSize(), Volume.class.getName(), volume.getUuid(), volume.getInstanceId(), volume.isDisplayVolume());
}
}
@ -8959,6 +8961,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
handleManagedStorage(vm, root);
_volsDao.attachVolume(newVol.getId(), vmId, newVol.getDeviceId());
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_ATTACH, newVol.getAccountId(), newVol.getDataCenterId(), newVol.getId(), newVol.getName(),
newVol.getDiskOfferingId(), newVol.getTemplateId(), newVol.getSize(), Volume.class.getName(), newVol.getUuid(), vmId, newVol.isDisplay());
// Detach, destroy and create the usage event for the old root volume.
_volsDao.detachVolume(root.getId());

View File

@ -1545,7 +1545,7 @@ public class VolumeApiServiceImplTest {
volumeApiServiceImpl.publishVolumeCreationUsageEvent(volumeVoMock);
usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(),
null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplay()));
null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.getInstanceId(), volumeVoMock.isDisplay()));
}
}
@ -1558,7 +1558,7 @@ public class VolumeApiServiceImplTest {
volumeApiServiceImpl.publishVolumeCreationUsageEvent(volumeVoMock);
usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(),
null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplay()));
null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.getInstanceId(), volumeVoMock.isDisplay()));
}
}
@ -1573,7 +1573,7 @@ public class VolumeApiServiceImplTest {
volumeApiServiceImpl.publishVolumeCreationUsageEvent(volumeVoMock);
usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(),
null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplay()));
null, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.getInstanceId(), volumeVoMock.isDisplay()));
}
}
@ -1589,7 +1589,7 @@ public class VolumeApiServiceImplTest {
volumeApiServiceImpl.publishVolumeCreationUsageEvent(volumeVoMock);
usageEventUtilsMocked.verify(() -> UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volumeVoMock.getAccountId(), volumeVoMock.getDataCenterId(), volumeVoMock.getId(), volumeVoMock.getName(),
offeringMockId, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.isDisplay()));
offeringMockId, volumeVoMock.getTemplateId(), volumeVoMock.getSize(), Volume.class.getName(), volumeVoMock.getUuid(), volumeVoMock.getInstanceId(), volumeVoMock.isDisplay()));
}
}

View File

@ -1120,10 +1120,12 @@ public class UserVmManagerImplTest {
public void recoverRootVolumeTestDestroyState() {
Mockito.doReturn(Volume.State.Destroy).when(volumeVOMock).getState();
userVmManagerImpl.recoverRootVolume(volumeVOMock, vmId);
try (MockedStatic<UsageEventUtils> ignored = Mockito.mockStatic(UsageEventUtils.class)) {
userVmManagerImpl.recoverRootVolume(volumeVOMock, vmId);
Mockito.verify(volumeApiService).recoverVolume(volumeVOMock.getId());
Mockito.verify(volumeDaoMock).attachVolume(volumeVOMock.getId(), vmId, UserVmManagerImpl.ROOT_DEVICE_ID);
Mockito.verify(volumeApiService).recoverVolume(volumeVOMock.getId());
Mockito.verify(volumeDaoMock).attachVolume(volumeVOMock.getId(), vmId, UserVmManagerImpl.ROOT_DEVICE_ID);
}
}
@Test(expected = InvalidParameterValueException.class)

View File

@ -1008,7 +1008,12 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
private boolean isVolumeEvent(String eventType) {
return eventType != null &&
(eventType.equals(EventTypes.EVENT_VOLUME_CREATE) || eventType.equals(EventTypes.EVENT_VOLUME_DELETE) || eventType.equals(EventTypes.EVENT_VOLUME_RESIZE) || eventType.equals(EventTypes.EVENT_VOLUME_UPLOAD));
(eventType.equals(EventTypes.EVENT_VOLUME_CREATE) ||
eventType.equals(EventTypes.EVENT_VOLUME_DELETE) ||
eventType.equals(EventTypes.EVENT_VOLUME_RESIZE) ||
eventType.equals(EventTypes.EVENT_VOLUME_UPLOAD) ||
eventType.equals(EventTypes.EVENT_VOLUME_ATTACH) ||
eventType.equals(EventTypes.EVENT_VOLUME_DETACH));
}
private boolean isTemplateEvent(String eventType) {
@ -1424,92 +1429,112 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
}
}
private void deleteExistingSecondaryStorageUsageForVolume(long volId, long accountId, Date deletedDate) {
List<UsageStorageVO> storageVOs = _usageStorageDao.listById(accountId, volId, StorageTypes.VOLUME);
for (UsageStorageVO storageVO : storageVOs) {
logger.debug("Setting the volume with id: {} to 'deleted' in the usage_storage table for account: {}.", volId, accountId);
storageVO.setDeleted(deletedDate);
_usageStorageDao.update(storageVO);
}
}
private void deleteExistingInstanceVolumeUsage(long volId, long accountId, Date deletedDate) {
List<UsageVolumeVO> volumesVOs = _usageVolumeDao.listByVolumeId(volId, accountId);
for (UsageVolumeVO volumesVO : volumesVOs) {
if (volumesVO.getVmId() != null) {
logger.debug("Setting the volume with id: {} for instance id: {} to 'deleted' in the usage_volume table for account {}.",
volumesVO.getVolumeId(), volumesVO.getVmId(), accountId);
volumesVO.setDeleted(deletedDate);
_usageVolumeDao.update(volumesVO.getId(), volumesVO);
}
}
}
private void deleteExistingVolumeUsage(long volId, long accountId, Date deletedDate) {
List<UsageVolumeVO> volumesVOs = _usageVolumeDao.listByVolumeId(volId, accountId);
for (UsageVolumeVO volumesVO : volumesVOs) {
logger.debug("Setting the volume with id: {} to 'deleted' in the usage_volume table for account: {}.", volId, accountId);
volumesVO.setDeleted(deletedDate);
_usageVolumeDao.update(volumesVO.getId(), volumesVO);
}
}
private void createVolumeHelperEvent(UsageEventVO event) {
long volId = event.getResourceId();
Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId());
List<UsageVolumeVO> volumesVOs;
UsageVolumeVO volumeVO;
if (EventTypes.EVENT_VOLUME_CREATE.equals(event.getType())) {
//For volumes which are 'attached' successfully, set the 'deleted' column in the usage_storage table,
switch (event.getType()) {
case EventTypes.EVENT_VOLUME_CREATE:
//For volumes which are 'attached' successfully from uploaded state, set the 'deleted' column in the usage_storage table,
//so that the secondary storage should stop accounting and only primary will be accounted.
SearchCriteria<UsageStorageVO> sc = _usageStorageDao.createSearchCriteria();
sc.addAnd("entityId", SearchCriteria.Op.EQ, volId);
sc.addAnd("storageType", SearchCriteria.Op.EQ, StorageTypes.VOLUME);
List<UsageStorageVO> volumesVOs = _usageStorageDao.search(sc, null);
if (volumesVOs != null) {
if (volumesVOs.size() == 1) {
logger.debug("Setting the volume with id: " + volId + " to 'deleted' in the usage_storage table.");
volumesVOs.get(0).setDeleted(event.getCreateDate());
_usageStorageDao.update(volumesVOs.get(0));
}
}
}
if (EventTypes.EVENT_VOLUME_CREATE.equals(event.getType()) || EventTypes.EVENT_VOLUME_RESIZE.equals(event.getType())) {
SearchCriteria<UsageVolumeVO> sc = _usageVolumeDao.createSearchCriteria();
sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId());
sc.addAnd("volumeId", SearchCriteria.Op.EQ, volId);
sc.addAnd("deleted", SearchCriteria.Op.NULL);
List<UsageVolumeVO> volumesVOs = _usageVolumeDao.search(sc, null);
deleteExistingSecondaryStorageUsageForVolume(volId, event.getAccountId(), event.getCreateDate());
volumesVOs = _usageVolumeDao.listByVolumeId(volId, event.getAccountId());
if (volumesVOs.size() > 0) {
//This is a safeguard to avoid double counting of volumes.
logger.error("Found duplicate usage entry for volume: " + volId + " assigned to account: " + event.getAccountId() + "; marking as deleted...");
deleteExistingVolumeUsage(volId, event.getAccountId(), event.getCreateDate());
}
//an entry exists if it is a resize volume event. marking the existing deleted and creating a new one in the case of resize.
for (UsageVolumeVO volumesVO : volumesVOs) {
if (logger.isDebugEnabled()) {
logger.debug("deleting volume: " + volumesVO.getId() + " from account: " + volumesVO.getAccountId());
}
volumesVO.setDeleted(event.getCreateDate());
_usageVolumeDao.update(volumesVO);
}
if (logger.isDebugEnabled()) {
logger.debug("create volume with id : " + volId + " for account: " + event.getAccountId());
}
Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId());
UsageVolumeVO volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), event.getSize(), event.getCreateDate(), null);
logger.debug("Creating a new entry in usage_volume for volume with id: {} for account: {}", volId, event.getAccountId());
volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), null, event.getSize(), event.getCreateDate(), null);
_usageVolumeDao.persist(volumeVO);
} else if (EventTypes.EVENT_VOLUME_DELETE.equals(event.getType())) {
SearchCriteria<UsageVolumeVO> sc = _usageVolumeDao.createSearchCriteria();
sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId());
sc.addAnd("volumeId", SearchCriteria.Op.EQ, volId);
sc.addAnd("deleted", SearchCriteria.Op.NULL);
List<UsageVolumeVO> volumesVOs = _usageVolumeDao.search(sc, null);
if (volumesVOs.size() > 1) {
logger.warn("More that one usage entry for volume: " + volId + " assigned to account: " + event.getAccountId() + "; marking them all as deleted...");
if (event.getVmId() != null) {
volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), event.getVmId(), event.getSize(), event.getCreateDate(), null);
_usageVolumeDao.persist(volumeVO);
}
break;
case EventTypes.EVENT_VOLUME_RESIZE:
volumesVOs = _usageVolumeDao.listByVolumeId(volId, event.getAccountId());
for (UsageVolumeVO volumesVO : volumesVOs) {
if (logger.isDebugEnabled()) {
logger.debug("deleting volume: " + volumesVO.getId() + " from account: " + volumesVO.getAccountId());
}
volumesVO.setDeleted(event.getCreateDate()); // there really shouldn't be more than one
_usageVolumeDao.update(volumesVO);
}
} else if (EventTypes.EVENT_VOLUME_UPLOAD.equals(event.getType())) {
//For Upload event add an entry to the usage_storage table.
SearchCriteria<UsageStorageVO> sc = _usageStorageDao.createSearchCriteria();
sc.addAnd("accountId", SearchCriteria.Op.EQ, event.getAccountId());
sc.addAnd("entityId", SearchCriteria.Op.EQ, volId);
sc.addAnd("storageType", SearchCriteria.Op.EQ, StorageTypes.VOLUME);
sc.addAnd("deleted", SearchCriteria.Op.NULL);
List<UsageStorageVO> volumesVOs = _usageStorageDao.search(sc, null);
if (volumesVOs.size() > 0) {
//This is a safeguard to avoid double counting of volumes.
logger.error("Found duplicate usage entry for volume: " + volId + " assigned to account: " + event.getAccountId() + "; marking as deleted...");
}
for (UsageStorageVO volumesVO : volumesVOs) {
if (logger.isDebugEnabled()) {
logger.debug("deleting volume: " + volumesVO.getId() + " from account: " + volumesVO.getAccountId());
String delete_msg = String.format("Setting the volume with id: %s to 'deleted' in the usage_volume table for account: %s.", volId, event.getAccountId());
String create_msg = String.format("Creating a new entry in usage_volume for volume with id: %s after resize for account: %s", volId, event.getAccountId());
Long vmId = volumesVO.getVmId();
if (vmId != null) {
delete_msg = String.format("Setting the volume with id: %s for instance id: %s to 'deleted' in the usage_volume table for account: %s.",
volId, vmId, event.getAccountId());
create_msg = String.format("Creating a new entry in usage_volume for volume with id: %s and instance id: %s after resize for account: %s",
volId, vmId, event.getAccountId());
}
logger.debug(delete_msg);
volumesVO.setDeleted(event.getCreateDate());
_usageStorageDao.update(volumesVO);
}
_usageVolumeDao.update(volumesVO.getId(), volumesVO);
if (logger.isDebugEnabled()) {
logger.debug("create volume with id : " + volId + " for account: " + event.getAccountId());
logger.debug(create_msg);
volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), vmId, event.getSize(), event.getCreateDate(), null);
_usageVolumeDao.persist(volumeVO);
}
Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId());
UsageStorageVO volumeVO = new UsageStorageVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), StorageTypes.VOLUME, event.getTemplateId(), event.getSize(), event.getCreateDate(), null);
_usageStorageDao.persist(volumeVO);
break;
case EventTypes.EVENT_VOLUME_DELETE:
deleteExistingVolumeUsage(volId, event.getAccountId(), event.getCreateDate());
break;
case EventTypes.EVENT_VOLUME_ATTACH:
deleteExistingInstanceVolumeUsage(event.getResourceId(), event.getAccountId(), event.getCreateDate());
logger.debug("Creating a new entry in usage_volume for volume with id: {}, and instance id: {} for account: {}",
volId, event.getVmId(), event.getAccountId());
volumeVO = new UsageVolumeVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), event.getOfferingId(), event.getTemplateId(), event.getVmId(), event.getSize(), event.getCreateDate(), null);
_usageVolumeDao.persist(volumeVO);
break;
case EventTypes.EVENT_VOLUME_DETACH:
deleteExistingInstanceVolumeUsage(event.getResourceId(), event.getAccountId(), event.getCreateDate());
break;
case EventTypes.EVENT_VOLUME_UPLOAD:
deleteExistingSecondaryStorageUsageForVolume(volId, event.getAccountId(), event.getCreateDate());
logger.debug("Creating a new entry in usage_storage for volume with id : {} for account: {}", volId, event.getAccountId());
UsageStorageVO storageVO = new UsageStorageVO(volId, event.getZoneId(), event.getAccountId(), acct.getDomainId(), StorageTypes.VOLUME, event.getTemplateId(), event.getSize(), event.getCreateDate(), null);
_usageStorageDao.persist(storageVO);
break;
}
}

View File

@ -73,12 +73,13 @@ public class VolumeUsageParser extends UsageParser {
for (UsageVolumeVO usageVol : usageUsageVols) {
long volId = usageVol.getVolumeId();
Long doId = usageVol.getDiskOfferingId();
Long vmId = usageVol.getVmId();
long zoneId = usageVol.getZoneId();
Long templateId = usageVol.getTemplateId();
long size = usageVol.getSize();
String key = volId + "-" + doId + "-" + size;
String key = volId + "-" + doId + "-" + vmId + "-" + size;
diskOfferingMap.put(key, new VolInfo(volId, zoneId, doId, templateId, size));
diskOfferingMap.put(key, new VolInfo(volId, zoneId, doId, templateId, size, vmId));
Date volCreateDate = usageVol.getCreated();
Date volDeleteDate = usageVol.getDeleted();
@ -110,7 +111,7 @@ public class VolumeUsageParser extends UsageParser {
if (useTime > 0L) {
VolInfo info = diskOfferingMap.get(volIdKey);
createUsageRecord(UsageTypes.VOLUME, useTime, startDate, endDate, account, info.getVolumeId(), info.getZoneId(), info.getDiskOfferingId(),
info.getTemplateId(), info.getSize());
info.getTemplateId(), info.getVmId(), info.getSize());
}
}
@ -130,7 +131,7 @@ public class VolumeUsageParser extends UsageParser {
}
private void createUsageRecord(int type, long runningTime, Date startDate, Date endDate, AccountVO account, long volId, long zoneId, Long doId,
Long templateId, long size) {
Long templateId, Long vmId, long size) {
// Our smallest increment is hourly for now
logger.debug("Total running time {} ms", runningTime);
@ -152,7 +153,11 @@ public class VolumeUsageParser extends UsageParser {
usageDesc += " (DiskOffering: " + doId + ")";
}
UsageVO usageRecord = new UsageVO(zoneId, account.getId(), account.getDomainId(), usageDesc, usageDisplay + " Hrs", type, new Double(usage), null, null, doId, templateId, volId,
if (vmId != null) {
usageDesc += " (VM: " + vmId + ")";
}
UsageVO usageRecord = new UsageVO(zoneId, account.getId(), account.getDomainId(), usageDesc, usageDisplay + " Hrs", type, new Double(usage), vmId, null, doId, templateId, volId,
size, startDate, endDate);
usageDao.persist(usageRecord);
}
@ -163,13 +168,15 @@ public class VolumeUsageParser extends UsageParser {
private Long diskOfferingId;
private Long templateId;
private long size;
private Long vmId;
public VolInfo(long volId, long zoneId, Long diskOfferingId, Long templateId, long size) {
public VolInfo(long volId, long zoneId, Long diskOfferingId, Long templateId, long size, Long vmId) {
this.volId = volId;
this.zoneId = zoneId;
this.diskOfferingId = diskOfferingId;
this.templateId = templateId;
this.size = size;
this.vmId = vmId;
}
public long getZoneId() {
@ -191,5 +198,9 @@ public class VolumeUsageParser extends UsageParser {
public long getSize() {
return size;
}
public Long getVmId() {
return vmId;
}
}
}