diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java index f22a906054d..d4038d4ceeb 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java @@ -37,4 +37,6 @@ public interface UsageJobDao extends GenericDao { UsageJobVO isOwner(String hostname, int pid); void updateJobSuccess(Long jobId, long startMillis, long endMillis, long execTime, boolean success); + + void removeLastOpenJobsOwned(String hostname, int pid); } diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java index 6d460aadd09..44a7d1a8b72 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java @@ -22,6 +22,7 @@ import java.util.Date; import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.usage.UsageJobVO; @@ -114,7 +115,7 @@ public class UsageJobDaoImpl extends GenericDaoBase implements public UsageJobVO isOwner(String hostname, int pid) { TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); try { - if ((hostname == null) || (pid <= 0)) { + if (hostname == null || pid <= 0) { return null; } @@ -174,7 +175,7 @@ public class UsageJobDaoImpl extends GenericDaoBase implements SearchCriteria sc = createSearchCriteria(); sc.addAnd("endMillis", SearchCriteria.Op.EQ, Long.valueOf(0)); sc.addAnd("jobType", SearchCriteria.Op.EQ, Integer.valueOf(UsageJobVO.JOB_TYPE_SINGLE)); - sc.addAnd("scheduled", SearchCriteria.Op.EQ, Integer.valueOf(0)); + sc.addAnd("scheduled", SearchCriteria.Op.EQ, Integer.valueOf(UsageJobVO.JOB_NOT_SCHEDULED)); List jobs = search(sc, filter); if ((jobs == null) || jobs.isEmpty()) { @@ -194,4 +195,36 @@ public class UsageJobDaoImpl extends GenericDaoBase implements } return jobs.get(0).getHeartbeat(); } + + private List getLastOpenJobsOwned(String hostname, int pid) { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("endMillis", SearchCriteria.Op.EQ, Long.valueOf(0)); + sc.addAnd("host", SearchCriteria.Op.EQ, hostname); + if (pid > 0) { + sc.addAnd("pid", SearchCriteria.Op.EQ, Integer.valueOf(pid)); + } + return listBy(sc); + } + + @Override + public void removeLastOpenJobsOwned(String hostname, int pid) { + if (hostname == null) { + return; + } + + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + try { + List jobs = getLastOpenJobsOwned(hostname, pid); + if (CollectionUtils.isNotEmpty(jobs)) { + logger.info("Found {} opens job, to remove", jobs.size()); + for (UsageJobVO job : jobs) { + logger.debug("Removing job - id: {}, pid: {}, job type: {}, scheduled: {}, heartbeat: {}", + job.getId(), job.getPid(), job.getJobType(), job.getScheduled(), job.getHeartbeat()); + remove(job.getId()); + } + } + } finally { + txn.close(); + } + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index efa8024a34b..46e6068a837 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -653,24 +653,10 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { if (!ScaleIOUtil.startSDCService()) { return new Ternary<>(false, null, "Couldn't start SDC service on host"); } - } - - if (MapUtils.isNotEmpty(details) && details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) { - // Assuming SDC service is started, add mdms - String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS); - String[] mdmAddresses = mdms.split(","); - if (mdmAddresses.length > 0) { - if (ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { - return new Ternary<>(true, getSDCDetails(details), "MDM added, no need to prepare the SDC client"); - } - - ScaleIOUtil.addMdms(mdmAddresses); - if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { - return new Ternary<>(false, null, "Failed to add MDMs"); - } else { - logger.debug(String.format("MDMs %s added to storage pool %s", mdms, uuid)); - applyMdmsChangeWaitTime(details); - } + } else { + logger.debug("SDC service is active on host, re-starting it"); + if (!ScaleIOUtil.restartSDCService()) { + return new Ternary<>(false, null, "Couldn't restart SDC service on host"); } } @@ -798,12 +784,12 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { if (sdcId != null) { sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId); return sdcDetails; - } - - String sdcGuId = ScaleIOUtil.getSdcGuid(); - if (sdcGuId != null) { - sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId); - return sdcDetails; + } else { + String sdcGuId = ScaleIOUtil.getSdcGuid(); + if (sdcGuId != null) { + sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId); + return sdcDetails; + } } try { diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java index eddaa8f6499..dba31f84dbd 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java @@ -92,6 +92,34 @@ public class ScaleIOStorageAdaptorTest { Assert.assertEquals("SDC service not enabled on host", result.third()); } + @Test + public void testPrepareStorageClient_SDCServiceNotRestarted() { + when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3); + when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0); + when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-active scini"))).thenReturn(0); + when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl restart scini"))).thenReturn(1); + + Ternary, String> result = scaleIOStorageAdaptor.prepareStorageClient(poolUuid, new HashMap<>()); + + Assert.assertFalse(result.first()); + Assert.assertNull(result.second()); + Assert.assertEquals("Couldn't restart SDC service on host", result.third()); + } + + @Test + public void testPrepareStorageClient_SDCServiceRestarted() { + when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3); + when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0); + when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-active scini"))).thenReturn(0); + when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl restart scini"))).thenReturn(0); + + Ternary, String> result = scaleIOStorageAdaptor.prepareStorageClient(poolUuid, new HashMap<>()); + + Assert.assertFalse(result.first()); + Assert.assertNull(result.second()); + Assert.assertEquals("Couldn't get the SDC details on the host", result.third()); + } + @Test public void testPrepareStorageClient_SDCServiceNotStarted() { when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3); diff --git a/pom.xml b/pom.xml index 81e297608bf..663e2cdf0fa 100644 --- a/pom.xml +++ b/pom.xml @@ -50,7 +50,7 @@ UTF-8 UTF-8 https://download.cloudstack.org/systemvm - 4.20.1.0 + 4.20.2.0 apache https://sonarcloud.io diff --git a/tools/appliance/systemvmtemplate/scripts/finalize.sh b/tools/appliance/systemvmtemplate/scripts/finalize.sh index e5d15ecb61c..507d4a4133a 100644 --- a/tools/appliance/systemvmtemplate/scripts/finalize.sh +++ b/tools/appliance/systemvmtemplate/scripts/finalize.sh @@ -68,6 +68,7 @@ function zero_disk() { } function finalize() { + depmod -a configure_misc configure_rundisk_size configure_sudoers diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index dd58760d275..2e97e238f09 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -312,6 +312,9 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna logger.info("Starting Usage Manager"); } + _usageJobDao.removeLastOpenJobsOwned(_hostname, 0); + Runtime.getRuntime().addShutdownHook(new AbandonJob()); + // use the configured exec time and aggregation duration for scheduling the job _scheduledFuture = _executor.scheduleAtFixedRate(this, _jobExecTime.getTimeInMillis() - System.currentTimeMillis(), _aggregationDuration * 60 * 1000, TimeUnit.MILLISECONDS); @@ -324,7 +327,6 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna _sanity = _sanityExecutor.scheduleAtFixedRate(new SanityCheck(), 1, _sanityCheckInterval, TimeUnit.DAYS); } - Runtime.getRuntime().addShutdownHook(new AbandonJob()); TransactionLegacy usageTxn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); try { if (_heartbeatLock.lock(3)) { // 3 second timeout @@ -2148,19 +2150,17 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna // the aggregation range away from executing the next job long now = System.currentTimeMillis(); long timeToJob = _jobExecTime.getTimeInMillis() - now; - long timeSinceJob = 0; + long timeSinceLastSuccessJob = 0; long aggregationDurationMillis = _aggregationDuration * 60L * 1000L; long lastSuccess = _usageJobDao.getLastJobSuccessDateMillis(); if (lastSuccess > 0) { - timeSinceJob = now - lastSuccess; + timeSinceLastSuccessJob = now - lastSuccess; } - if ((timeSinceJob > 0) && (timeSinceJob > (aggregationDurationMillis - 100))) { + if ((timeSinceLastSuccessJob > 0) && (timeSinceLastSuccessJob > (aggregationDurationMillis - 100))) { if (timeToJob > (aggregationDurationMillis / 2)) { - if (logger.isDebugEnabled()) { - logger.debug("it's been " + timeSinceJob + " ms since last usage job and " + timeToJob + - " ms until next job, scheduling an immediate job to catch up (aggregation duration is " + _aggregationDuration + " minutes)"); - } + logger.debug("it's been {} ms since last usage job and {} ms until next job, scheduling an immediate job to catch up (aggregation duration is {} minutes)" + , timeSinceLastSuccessJob, timeToJob, _aggregationDuration); scheduleParse(); } } @@ -2245,17 +2245,12 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna } } } + private class AbandonJob extends Thread { @Override public void run() { - logger.info("exitting Usage Manager"); - deleteOpenjob(); - } - private void deleteOpenjob() { - UsageJobVO job = _usageJobDao.isOwner(_hostname, _pid); - if (job != null) { - _usageJobDao.remove(job.getId()); - } + logger.info("exiting Usage Manager"); + _usageJobDao.removeLastOpenJobsOwned(_hostname, _pid); } } }