apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add configuration to limit the number of rows deleted from vm_stats (#8740)

Browse Source 
Co-authored-by: João Jandre <joao@scclouds.com.br>
This commit is contained in:
João Jandre 2024-06-20 09:26:36 -03:00 committed by GitHub
parent 7dce3d87d4
commit ae3fa5d0de
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 57 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDao.java
View File

@ -75,8 +75,10 @@ public interface VmStatsDao extends GenericDao<VmStatsVO, Long> {
/**
* Removes (expunges) all VM stats with {@code timestamp} less than
* a given Date.
* @param limit the maximum date to keep stored. Records that exceed this limit will be removed.
* @param limitDate the maximum date to keep stored. Records that exceed this limit will be removed.
* @param limitPerQuery the maximum amount of rows to be removed in a single query. We loop if there are still rows to be removed after a given query.
* If 0 or negative, no limit is used.
*/
void removeAllByTimestampLessThan(Date limit);
void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery);
}

21
engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java
View File

@ -21,6 +21,7 @@ import java.util.List;
import javax.annotation.PostConstruct;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.utils.db.Filter;
@ -33,6 +34,8 @@ import com.cloud.vm.VmStatsVO;
@Component
public class VmStatsDaoImpl extends GenericDaoBase<VmStatsVO, Long> implements VmStatsDao {
protected Logger logger = Logger.getLogger(getClass());
protected SearchBuilder<VmStatsVO> vmIdSearch;
protected SearchBuilder<VmStatsVO> vmIdTimestampGreaterThanEqualSearch;
protected SearchBuilder<VmStatsVO> vmIdTimestampLessThanEqualSearch;
@ -113,10 +116,22 @@ public class VmStatsDaoImpl extends GenericDaoBase<VmStatsVO, Long> implements V
}
@Override
public void removeAllByTimestampLessThan(Date limit) {
public void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery) {
SearchCriteria<VmStatsVO> sc = timestampSearch.create();
sc.setParameters("timestamp", limit);
expunge(sc);
sc.setParameters("timestamp", limitDate);
logger.debug(String.format("Starting to remove all vm_stats rows older than [%s].", limitDate));
long totalRemoved = 0;
long removed;
do {
removed = expunge(sc, limitPerQuery);
totalRemoved += removed;
logger.trace(String.format("Removed [%s] vm_stats rows on the last update and a sum of [%s] vm_stats rows older than [%s] until now.", removed, totalRemoved, limitDate));
} while (limitPerQuery > 0 && removed >= limitPerQuery);
logger.info(String.format("Removed a total of [%s] vm_stats rows older than [%s].", totalRemoved, limitDate));
}
}

8
framework/db/src/main/java/com/cloud/utils/db/GenericDao.java
View File

@ -229,6 +229,14 @@ public interface GenericDao<T, ID extends Serializable> {
*/
int expunge(final SearchCriteria<T> sc);
/**
* Delete the entity beans specified by the search criteria with a given limit
* @param sc Search criteria
* @param limit Maximum number of rows that will be affected
* @return Number of rows deleted
*/
int expunge(SearchCriteria<T> sc, long limit);
/**
* expunge the removed rows.
*/

12
framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java
View File

@ -1226,9 +1226,14 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
}
}
// FIXME: Does not work for joins.
@Override
public int expunge(final SearchCriteria<T> sc) {
return expunge(sc, -1);
}
// FIXME: Does not work for joins.
@Override
public int expunge(final SearchCriteria<T> sc, long limit) {
if (sc == null) {
throw new CloudRuntimeException("Call to throw new expunge with null search Criteria");
}
@ -1241,6 +1246,11 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
str.append(sc.getWhereClause());
}
if (limit > 0) {
str.append(" LIMIT ");
str.append(limit);
}
final String sql = str.toString();
final TransactionLegacy txn = TransactionLegacy.currentTxn();

12
server/src/main/java/com/cloud/server/StatsCollector.java
View File

@ -114,9 +114,6 @@ import com.cloud.org.Cluster;
import com.cloud.resource.ResourceManager;
import com.cloud.resource.ResourceState;
import com.cloud.serializer.GsonHelper;
import com.cloud.server.StatsCollector.AbstractStatsCollector;
import com.cloud.server.StatsCollector.AutoScaleMonitor;
import com.cloud.server.StatsCollector.StorageCollector;
import com.cloud.storage.ImageStoreDetailsUtil;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage;
@ -287,6 +284,11 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
protected static ConfigKey<Boolean> vmStatsCollectUserVMOnly = new ConfigKey<>("Advanced", Boolean.class, "vm.stats.user.vm.only", "false",
"When set to 'false' stats for system VMs will be collected otherwise stats collection will be done only for user VMs", true);
protected static ConfigKey<Long> vmStatsRemoveBatchSize = new ConfigKey<>("Advanced", Long.class, "vm.stats.remove.batch.size", "0", "Indicates the" +
" limit applied to delete vm_stats entries while running the clean-up task. With this, ACS will run the delete query, applying the limit, as many times as necessary" +
" to delete all entries older than the value defined in vm.stats.max.retention.time. This is advised when retaining several days of records, which can lead to slowness" +
" on the delete query. Zero (0) means that no limit will be applied, therefore, the query will run once and without limit, keeping the default behavior.", true);
protected static ConfigKey<Boolean> vmDiskStatsRetentionEnabled = new ConfigKey<>("Advanced", Boolean.class, "vm.disk.stats.retention.enabled", "false",
"When set to 'true' stats for VM disks will be stored in the database otherwise disk stats will not be stored", true);
@ -1965,7 +1967,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
LOGGER.trace("Removing older VM stats records.");
Date now = new Date();
Date limit = DateUtils.addMinutes(now, -maxRetentionTime);
vmStatsDao.removeAllByTimestampLessThan(limit);
vmStatsDao.removeAllByTimestampLessThan(limit, vmStatsRemoveBatchSize.value());
}
/**
@ -2139,7 +2141,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] {vmDiskStatsInterval, vmDiskStatsIntervalMin, vmNetworkStatsInterval, vmNetworkStatsIntervalMin, StatsTimeout, statsOutputUri,
return new ConfigKey<?>[] {vmDiskStatsInterval, vmDiskStatsIntervalMin, vmNetworkStatsInterval, vmNetworkStatsIntervalMin, StatsTimeout, statsOutputUri, vmStatsRemoveBatchSize,
vmStatsIncrementMetrics, vmStatsMaxRetentionTime, vmStatsCollectUserVMOnly, vmDiskStatsRetentionEnabled, vmDiskStatsMaxRetentionTime,
MANAGEMENT_SERVER_STATUS_COLLECTION_INTERVAL,
DATABASE_SERVER_STATUS_COLLECTION_INTERVAL,

8
server/src/test/java/com/cloud/server/StatsCollectorTest.java
View File

@ -28,7 +28,7 @@ import com.cloud.user.VmDiskStatisticsVO;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VmStats;
import com.cloud.vm.VmStatsVO;
import com.cloud.vm.dao.VmStatsDao;
import com.cloud.vm.dao.VmStatsDaoImpl;
import com.google.gson.Gson;
import com.tngtech.java.junit.dataprovider.DataProvider;
import com.tngtech.java.junit.dataprovider.DataProviderRunner;
@ -81,7 +81,7 @@ public class StatsCollectorTest {
private static final String DEFAULT_DATABASE_NAME = "cloudstack";
@Mock
VmStatsDao vmStatsDaoMock = Mockito.mock(VmStatsDao.class);
VmStatsDaoImpl vmStatsDaoMock;
@Mock
VmStatsEntry statsForCurrentIterationMock;
@ -304,7 +304,7 @@ public class StatsCollectorTest {
statsCollector.cleanUpVirtualMachineStats();
Mockito.verify(vmStatsDaoMock, Mockito.never()).removeAllByTimestampLessThan(Mockito.any());
Mockito.verify(vmStatsDaoMock, Mockito.never()).removeAllByTimestampLessThan(Mockito.any(), Mockito.anyLong());
}
@Test
@ -313,7 +313,7 @@ public class StatsCollectorTest {
statsCollector.cleanUpVirtualMachineStats();
Mockito.verify(vmStatsDaoMock).removeAllByTimestampLessThan(Mockito.any());
Mockito.verify(vmStatsDaoMock).removeAllByTimestampLessThan(Mockito.any(), Mockito.anyLong());
}
@Test

5
server/src/test/java/com/cloud/user/MockUsageEventDao.java
View File

@ -210,6 +210,11 @@ public class MockUsageEventDao implements UsageEventDao{
return 0;
}
@Override
public int expunge(SearchCriteria<UsageEventVO> sc, long limit) {
return 0;
}
@Override
public void expunge() {