Merge branch '4.20'

agent/src/main/java/com/cloud/agent/Agent.java

@@ -613,7 +613,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
     }
 
     protected String getAgentArch() {
-        String arch = Script.runSimpleBashScript(Script.getExecutableAbsolutePath("arch"), 1000);
+        String arch = Script.runSimpleBashScript(Script.getExecutableAbsolutePath("arch"), 2000);
         logger.debug("Arch for agent: {} found: {}", _name, arch);
         return arch;
     }

api/src/main/java/com/cloud/server/ManagementService.java

@@ -20,7 +20,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
-import com.cloud.user.UserData;
 import org.apache.cloudstack.api.command.admin.cluster.ListClustersCmd;
 import org.apache.cloudstack.api.command.admin.config.ListCfgGroupsByCmd;
 import org.apache.cloudstack.api.command.admin.config.ListCfgsByCmd;
@@ -72,6 +71,7 @@ import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd;
 import org.apache.cloudstack.api.command.user.vmgroup.UpdateVMGroupCmd;
 import org.apache.cloudstack.config.Configuration;
 import org.apache.cloudstack.config.ConfigurationGroup;
+import org.apache.cloudstack.framework.config.ConfigKey;
 
 import com.cloud.alert.Alert;
 import com.cloud.capacity.Capacity;
@@ -91,6 +91,7 @@ import com.cloud.storage.GuestOSHypervisor;
 import com.cloud.storage.GuestOsCategory;
 import com.cloud.storage.StoragePool;
 import com.cloud.user.SSHKeyPair;
+import com.cloud.user.UserData;
 import com.cloud.utils.Pair;
 import com.cloud.utils.Ternary;
 import com.cloud.vm.InstanceGroup;
@@ -104,6 +105,14 @@ import com.cloud.vm.VirtualMachine.Type;
 public interface ManagementService {
     static final String Name = "management-server";
 
+    ConfigKey<Boolean> JsInterpretationEnabled = new ConfigKey<>("Hidden"
+            , Boolean.class
+            , "js.interpretation.enabled"
+            , "false"
+            , "Enable/Disable all JavaScript interpretation related functionalities to create or update Javascript rules."
+            , false
+            , ConfigKey.Scope.Global);
+
     /**
      * returns the a map of the names/values in the configuration table
      *
@@ -509,4 +518,6 @@ public interface ManagementService {
 
     boolean removeManagementServer(RemoveManagementServerCmd cmd);
 
+    void checkJsInterpretationAllowedIfNeededForParameterValue(String paramName, boolean paramValue);
+
 }

api/src/main/java/com/cloud/server/ResourceManagerUtil.java

@@ -18,6 +18,7 @@ package com.cloud.server;
 
 public interface ResourceManagerUtil {
     long getResourceId(String resourceId, ResourceTag.ResourceObjectType resourceType);
+    long getResourceId(String resourceId, ResourceTag.ResourceObjectType resourceType, boolean checkAccess);
     String getUuid(String resourceId, ResourceTag.ResourceObjectType resourceType);
     ResourceTag.ResourceObjectType getResourceType(String resourceTypeStr);
     void checkResourceAccessible(Long accountId, Long domainId, String exceptionMessage);

engine/components-api/src/main/java/com/cloud/template/TemplateManager.java

@@ -57,6 +57,13 @@ public interface TemplateManager {
             + "will validate if the provided URL is resolvable during the register of templates/ISOs before persisting them in the database.",
             true);
 
+    ConfigKey<Boolean> TemplateDeleteFromPrimaryStorage = new ConfigKey<Boolean>("Advanced",
+            Boolean.class,
+            "template.delete.from.primary.storage", "true",
+            "Template when deleted will be instantly deleted from the Primary Storage",
+            true,
+            ConfigKey.Scope.Global);
+
     static final String VMWARE_TOOLS_ISO = "vmware-tools.iso";
     static final String XS_TOOLS_ISO = "xs-tools.iso";
 
@@ -104,6 +111,8 @@ public interface TemplateManager {
      */
     List<VMTemplateStoragePoolVO> getUnusedTemplatesInPool(StoragePoolVO pool);
 
+    void evictTemplateFromStoragePoolsForZones(Long templateId, List<Long> zoneId);
+
     /**
      * Deletes a template in the specified storage pool.
      *

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -4799,6 +4799,18 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             }
         });
 
+        if (selectedIp != null && GuestType.Shared.equals(network.getGuestType())) {
+            IPAddressVO ipAddressVO = _ipAddressDao.findByIpAndSourceNetworkId(network.getId(), selectedIp);
+            if (ipAddressVO != null && IpAddress.State.Free.equals(ipAddressVO.getState())) {
+                ipAddressVO.setState(IPAddressVO.State.Allocated);
+                ipAddressVO.setAllocatedTime(new Date());
+                Account account = _accountDao.findById(vm.getAccountId());
+                ipAddressVO.setAllocatedInDomainId(account.getDomainId());
+                ipAddressVO.setAllocatedToAccountId(account.getId());
+                _ipAddressDao.update(ipAddressVO.getId(), ipAddressVO);
+            }
+        }
+
         final Integer networkRate = _networkModel.getNetworkRate(network.getId(), vm.getId());
         final NicProfile vmNic = new NicProfile(vo, network, vo.getBroadcastUri(), vo.getIsolationUri(), networkRate, _networkModel.isSecurityGroupSupportedInNetwork(network),
                 _networkModel.getNetworkTag(vm.getHypervisorType(), network));
@@ -4810,15 +4822,15 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         if (network.getGuestType() == GuestType.L2) {
             return null;
         }
-        return dataCenter.getNetworkType() == NetworkType.Basic ?
+        return GuestType.Shared.equals(network.getGuestType()) ?
-                getSelectedIpForNicImportOnBasicZone(ipAddresses.getIp4Address(), network, dataCenter):
+                getSelectedIpForNicImportOnSharedNetwork(ipAddresses.getIp4Address(), network, dataCenter):
                 _ipAddrMgr.acquireGuestIpAddress(network, ipAddresses.getIp4Address());
     }
 
-    protected String getSelectedIpForNicImportOnBasicZone(String requestedIp, Network network, DataCenter dataCenter) {
+    protected String getSelectedIpForNicImportOnSharedNetwork(String requestedIp, Network network, DataCenter dataCenter) {
         IPAddressVO ipAddressVO = StringUtils.isBlank(requestedIp) ?
                 _ipAddressDao.findBySourceNetworkIdAndDatacenterIdAndState(network.getId(), dataCenter.getId(), IpAddress.State.Free):
-                _ipAddressDao.findByIp(requestedIp);
+                _ipAddressDao.findByIpAndSourceNetworkId(network.getId(), requestedIp);
         if (ipAddressVO == null || ipAddressVO.getState() != IpAddress.State.Free) {
             String msg = String.format("Cannot find a free IP to assign to VM NIC on network %s", network.getName());
             logger.error(msg);

engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java

@@ -822,7 +822,7 @@ public class NetworkOrchestratorTest extends TestCase {
         Mockito.when(network.getId()).thenReturn(networkId);
         Mockito.when(dataCenter.getId()).thenReturn(dataCenterId);
         Mockito.when(ipAddresses.getIp4Address()).thenReturn(requestedIp);
-        Mockito.when(testOrchestrator._ipAddressDao.findByIp(requestedIp)).thenReturn(ipAddressVO);
+        Mockito.when(testOrchestrator._ipAddressDao.findByIpAndSourceNetworkId(networkId, requestedIp)).thenReturn(ipAddressVO);
         String ipAddress = testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses);
         Assert.assertEquals(requestedIp, ipAddress);
     }

engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java

@@ -35,6 +35,8 @@ public interface VMTemplatePoolDao extends GenericDao<VMTemplateStoragePoolVO, L
 
     List<VMTemplateStoragePoolVO> listByPoolIdAndState(long poolId, ObjectInDataStoreStateMachine.State state);
 
+    List<VMTemplateStoragePoolVO> listByPoolIdsAndTemplate(List<Long> poolIds, Long templateId);
+
     List<VMTemplateStoragePoolVO> listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState);
 
     List<VMTemplateStoragePoolVO> listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState, long poolId);

engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java

@@ -150,6 +150,16 @@ public class VMTemplatePoolDaoImpl extends GenericDaoBase<VMTemplateStoragePoolV
         return findOneIncludingRemovedBy(sc);
     }
 
+    @Override
+    public List<VMTemplateStoragePoolVO> listByPoolIdsAndTemplate(List<Long> poolIds, Long templateId) {
+        SearchCriteria<VMTemplateStoragePoolVO> sc = PoolTemplateSearch.create();
+        if (CollectionUtils.isNotEmpty(poolIds)) {
+            sc.setParameters("pool_id", poolIds.toArray());
+        }
+        sc.setParameters("template_id", templateId);
+        return listBy(sc);
+    }
+
     @Override
     public List<VMTemplateStoragePoolVO> listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState) {
         SearchCriteria<VMTemplateStoragePoolVO> sc = TemplateStatusSearch.create();

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java

@@ -171,4 +171,5 @@ public interface PrimaryDataStoreDao extends GenericDao<StoragePoolVO, Long> {
 
     List<StoragePoolVO> findPoolsByStorageTypeAndZone(Storage.StoragePoolType storageType, Long zoneId);
 
+    List<StoragePoolVO> listByDataCenterIds(List<Long> dataCenterIds);
 }

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java

@@ -65,6 +65,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
     private final GenericSearchBuilder<StoragePoolVO, Long> StatusCountSearch;
     private final SearchBuilder<StoragePoolVO> ClustersSearch;
     private final SearchBuilder<StoragePoolVO> IdsSearch;
+    private final SearchBuilder<StoragePoolVO> DcsSearch;
 
     @Inject
     private StoragePoolDetailsDao _detailsDao;
@@ -167,6 +168,9 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
         IdsSearch.and("ids", IdsSearch.entity().getId(), SearchCriteria.Op.IN);
         IdsSearch.done();
 
+        DcsSearch = createSearchBuilder();
+        DcsSearch.and("dataCenterId", DcsSearch.entity().getDataCenterId(), SearchCriteria.Op.IN);
+        DcsSearch.done();
     }
 
     @Override
@@ -927,6 +931,16 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
         return listBy(sc);
     }
 
+    @Override
+    public List<StoragePoolVO> listByDataCenterIds(List<Long> dataCenterIds) {
+        if (CollectionUtils.isEmpty(dataCenterIds)) {
+            return Collections.emptyList();
+        }
+        SearchCriteria<StoragePoolVO> sc = DcsSearch.create();
+        sc.setParameters("dataCenterId", dataCenterIds.toArray());
+        return listBy(sc);
+    }
+
     private SearchCriteria<StoragePoolVO> createStoragePoolSearchCriteria(Long storagePoolId, String storagePoolName,
                                                                           Long zoneId, String path, Long podId, Long clusterId, Long hostId, String address, ScopeType scopeType,
                                                                           StoragePoolStatus status, String keyword, String storageAccessGroup) {

framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java

@@ -32,7 +32,6 @@ import java.util.stream.Collectors;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import com.cloud.user.Account;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.quota.activationrule.presetvariables.Configuration;
 import org.apache.cloudstack.quota.activationrule.presetvariables.GenericPresetVariable;
@@ -63,6 +62,7 @@ import org.springframework.stereotype.Component;
 
 import com.cloud.usage.UsageVO;
 import com.cloud.usage.dao.UsageDao;
+import com.cloud.user.Account;
 import com.cloud.user.AccountVO;
 import com.cloud.user.dao.AccountDao;
 import com.cloud.utils.DateUtil;
@@ -473,7 +473,7 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager {
             jsInterpreter.injectVariable("configuration", configuration.toString());
         }
 
-        jsInterpreter.injectStringVariable("resourceType", presetVariables.getResourceType());
+        jsInterpreter.injectVariable("resourceType", presetVariables.getResourceType());
         jsInterpreter.injectVariable("value", presetVariables.getValue().toString());
         jsInterpreter.injectVariable("zone", presetVariables.getZone().toString());
     }

framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java

@@ -270,8 +270,7 @@ public class QuotaManagerImplTest {
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("account"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("domain"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock, Mockito.never()).injectVariable(Mockito.eq("project"), Mockito.anyString());
-        Mockito.verify(jsInterpreterMock, Mockito.never()).injectVariable(Mockito.eq("configuration"), Mockito.anyString());
+        Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("resourceType"), Mockito.anyString());
-        Mockito.verify(jsInterpreterMock).injectStringVariable(Mockito.eq("resourceType"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("value"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("zone"), Mockito.anyString());
     }
@@ -292,7 +291,7 @@ public class QuotaManagerImplTest {
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("account"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("domain"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("project"), Mockito.anyString());
-        Mockito.verify(jsInterpreterMock).injectStringVariable(Mockito.eq("resourceType"), Mockito.anyString());
+        Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("resourceType"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("value"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("zone"), Mockito.anyString());
     }

plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java

@@ -80,8 +80,8 @@ import org.apache.cloudstack.quota.dao.QuotaCreditsDao;
 import org.apache.cloudstack.quota.dao.QuotaEmailConfigurationDao;
 import org.apache.cloudstack.quota.dao.QuotaEmailTemplatesDao;
 import org.apache.cloudstack.quota.dao.QuotaTariffDao;
-import org.apache.cloudstack.quota.vo.QuotaAccountVO;
 import org.apache.cloudstack.quota.dao.QuotaUsageDao;
+import org.apache.cloudstack.quota.vo.QuotaAccountVO;
 import org.apache.cloudstack.quota.vo.QuotaBalanceVO;
 import org.apache.cloudstack.quota.vo.QuotaCreditsVO;
 import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO;
@@ -90,15 +90,17 @@ import org.apache.cloudstack.quota.vo.QuotaTariffVO;
 import org.apache.cloudstack.quota.vo.QuotaUsageVO;
 import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter;
 import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
+import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.reflect.FieldUtils;
-import org.apache.commons.lang3.ObjectUtils;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.springframework.stereotype.Component;
 
 import com.cloud.domain.DomainVO;
 import com.cloud.domain.dao.DomainDao;
+import com.cloud.event.ActionEvent;
+import com.cloud.event.EventTypes;
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.user.Account;
 import com.cloud.user.AccountManager;
@@ -107,8 +109,6 @@ import com.cloud.user.dao.AccountDao;
 import com.cloud.user.dao.UserDao;
 import com.cloud.utils.Pair;
 import com.cloud.utils.db.Filter;
-import com.cloud.event.ActionEvent;
-import com.cloud.event.EventTypes;
 
 @Component
 public class QuotaResponseBuilderImpl implements QuotaResponseBuilder {
@@ -150,6 +150,11 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder {
 
     private final Class<?>[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class};
 
+    protected void checkActivationRulesAllowed(String activationRule) {
+        if (!_quotaService.isJsInterpretationEnabled() && StringUtils.isNotEmpty(activationRule)) {
+            throw new PermissionDeniedException("Quota Tariff Activation Rule cannot be set, as Javascript interpretation is disabled in the configuration.");
+        }
+    }
 
     @Override
     public QuotaTariffResponse createQuotaTariffResponse(QuotaTariffVO tariff, boolean returnActivationRule) {
@@ -452,6 +457,8 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder {
             throw new InvalidParameterValueException(String.format("There is no quota tariffs with name [%s].", name));
         }
 
+        checkActivationRulesAllowed(activationRule);
+
         Date currentQuotaTariffStartDate = currentQuotaTariff.getEffectiveOn();
 
         currentQuotaTariff.setRemoved(now);
@@ -706,6 +713,8 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder {
             throw new InvalidParameterValueException(String.format("A quota tariff with name [%s] already exist.", name));
         }
 
+        checkActivationRulesAllowed(activationRule);
+
         if (startDate.compareTo(now) < 0) {
             throw new InvalidParameterValueException(String.format("The value passed as Quota tariff's start date is in the past: [%s]. " +
                     "Please, inform a date in the future or do not pass the parameter to use the current date and time.", startDate));

plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java

@@ -16,15 +16,15 @@
 //under the License.
 package org.apache.cloudstack.quota;
 
-import com.cloud.user.AccountVO;
+import java.math.BigDecimal;
-import com.cloud.utils.component.PluggableService;
+import java.util.Date;
+import java.util.List;
 
 import org.apache.cloudstack.quota.vo.QuotaBalanceVO;
 import org.apache.cloudstack.quota.vo.QuotaUsageVO;
 
-import java.math.BigDecimal;
+import com.cloud.user.AccountVO;
-import java.util.Date;
+import com.cloud.utils.component.PluggableService;
-import java.util.List;
 
 public interface QuotaService extends PluggableService {
 
@@ -40,4 +40,6 @@ public interface QuotaService extends PluggableService {
 
     boolean saveQuotaAccount(AccountVO account, BigDecimal aggrUsage, Date endDate);
 
+    boolean isJsInterpretationEnabled();
+
 }

plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java

@@ -62,6 +62,7 @@ import com.cloud.configuration.Config;
 import com.cloud.domain.dao.DomainDao;
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.PermissionDeniedException;
+import com.cloud.server.ManagementService;
 import com.cloud.user.Account;
 import com.cloud.user.AccountVO;
 import com.cloud.user.dao.AccountDao;
@@ -88,6 +89,8 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi
 
     private TimeZone _usageTimezone;
 
+    private boolean jsInterpretationEnabled = false;
+
     public QuotaServiceImpl() {
         super();
     }
@@ -99,6 +102,8 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi
         String timeZoneStr = ObjectUtils.defaultIfNull(_configDao.getValue(Config.UsageAggregationTimezone.toString()), "GMT");
         _usageTimezone = TimeZone.getTimeZone(timeZoneStr);
 
+        jsInterpretationEnabled = ManagementService.JsInterpretationEnabled.value();
+
         return true;
     }
 
@@ -288,4 +293,8 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi
         }
     }
 
+    @Override
+    public boolean isJsInterpretationEnabled() {
+        return jsInterpretationEnabled;
+    }
 }

plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java

@@ -22,6 +22,7 @@ import static com.cloud.utils.NumbersUtil.toReadableSize;
 import java.lang.reflect.InvocationTargetException;
 import java.text.DecimalFormat;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -32,8 +33,6 @@ import java.util.stream.Collectors;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import com.cloud.dc.ClusterVO;
-import com.cloud.utils.Ternary;
 import org.apache.cloudstack.api.ApiErrorCode;
 import org.apache.cloudstack.api.ListClustersMetricsCmd;
 import org.apache.cloudstack.api.ListDbMetricsCmd;
@@ -102,6 +101,7 @@ import com.cloud.capacity.CapacityManager;
 import com.cloud.capacity.dao.CapacityDao;
 import com.cloud.capacity.dao.CapacityDaoImpl;
 import com.cloud.cluster.dao.ManagementServerHostDao;
+import com.cloud.dc.ClusterVO;
 import com.cloud.dc.DataCenter;
 import com.cloud.dc.dao.ClusterDao;
 import com.cloud.dc.dao.DataCenterDao;
@@ -114,6 +114,7 @@ import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.network.router.VirtualRouter;
 import com.cloud.org.Cluster;
+import com.cloud.projects.Project;
 import com.cloud.server.DbStatsCollection;
 import com.cloud.server.ManagementServerHostStats;
 import com.cloud.server.StatsCollector;
@@ -126,6 +127,7 @@ import com.cloud.usage.dao.UsageJobDao;
 import com.cloud.user.Account;
 import com.cloud.user.AccountManager;
 import com.cloud.utils.Pair;
+import com.cloud.utils.Ternary;
 import com.cloud.utils.db.DbProperties;
 import com.cloud.utils.db.DbUtil;
 import com.cloud.utils.db.Filter;
@@ -188,6 +190,10 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
 
     private static Gson gson = new Gson();
 
+    private final List<Account.Type> AccountTypesWithRecursiveUsageAccess = Arrays.asList(
+            Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN, Account.Type.READ_ONLY_ADMIN
+    );
+
     protected MetricsServiceImpl() {
         super();
     }
@@ -249,17 +255,30 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
      * @return the list of VMs.
      */
     protected Pair<List<UserVmVO>, Integer> searchForUserVmsInternal(ListVMsUsageHistoryCmd cmd) {
+        final Long id = cmd.getId();
+        Account caller = CallContext.current().getCallingAccount();
+        List<Long> permittedAccounts = new ArrayList<>();
+        boolean recursive = AccountTypesWithRecursiveUsageAccess.contains(caller.getType());
+        Ternary<Long, Boolean, Project.ListProjectResourcesCriteria> domainIdRecursiveListProject = new Ternary<>(null, recursive, null);
+        accountMgr.buildACLSearchParameters(caller, id, null, null, permittedAccounts, domainIdRecursiveListProject, true, false);
+        Long domainId = domainIdRecursiveListProject.first();
+        Boolean isRecursive = domainIdRecursiveListProject.second();
+        Project.ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third();
+
         Filter searchFilter = new Filter(UserVmVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal());
         List<Long> ids = getIdsListFromCmd(cmd.getId(), cmd.getIds());
         String name = cmd.getName();
         String keyword = cmd.getKeyword();
 
         SearchBuilder<UserVmVO> sb =  userVmDao.createSearchBuilder();
+        sb.select(null, SearchCriteria.Func.DISTINCT, sb.entity().getId()); // select distinct
+        accountMgr.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
         sb.and("idIN", sb.entity().getId(), SearchCriteria.Op.IN);
         sb.and("displayName", sb.entity().getDisplayName(), SearchCriteria.Op.LIKE);
         sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ);
 
         SearchCriteria<UserVmVO> sc = sb.create();
+        accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
         if (CollectionUtils.isNotEmpty(ids)) {
             sc.setParameters("idIN", ids.toArray());
         }
@@ -273,7 +292,14 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
             sc.addAnd("displayName", SearchCriteria.Op.SC, ssc);
         }
 
-        return userVmDao.searchAndCount(sc, searchFilter);
+        Pair<List<UserVmVO>, Integer> uniqueVmPair = userVmDao.searchAndCount(sc, searchFilter);
+        Integer count = uniqueVmPair.second();
+        if (count == 0) {
+            return new Pair<>(new ArrayList<>(), count);
+        }
+        List<Long> vmIds = uniqueVmPair.first().stream().map(UserVmVO::getId).collect(Collectors.toList());
+        List<UserVmVO> vms = userVmDao.listByIds(vmIds);
+        return new Pair<>(vms, count);
     }
 
     /**
@@ -335,17 +361,49 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
      * @return the list of VMs.
      */
     protected Pair<List<VolumeVO>, Integer> searchForVolumesInternal(ListVolumesUsageHistoryCmd cmd) {
+        final Long id = cmd.getId();
+        Account caller = CallContext.current().getCallingAccount();
+        List<Long> permittedAccounts = new ArrayList<>();
+        boolean recursive = AccountTypesWithRecursiveUsageAccess.contains(caller.getType());
+        Ternary<Long, Boolean, Project.ListProjectResourcesCriteria> domainIdRecursiveListProject = new Ternary<>(null, recursive, null);
+        accountMgr.buildACLSearchParameters(caller, id, null, null, permittedAccounts, domainIdRecursiveListProject, true, false);
+        Long domainId = domainIdRecursiveListProject.first();
+        Boolean isRecursive = domainIdRecursiveListProject.second();
+        Project.ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third();
+
         Filter searchFilter = new Filter(VolumeVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal());
         List<Long> ids = getIdsListFromCmd(cmd.getId(), cmd.getIds());
         String name = cmd.getName();
         String keyword = cmd.getKeyword();
 
         SearchBuilder<VolumeVO> sb =  volumeDao.createSearchBuilder();
+        sb.select(null, SearchCriteria.Func.DISTINCT, sb.entity().getId()); // select distinct
+        accountMgr.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
         sb.and("idIN", sb.entity().getId(), SearchCriteria.Op.IN);
         sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ);
         sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ);
 
+        boolean shouldListSystemVmVolumes = accountMgr.isRootAdmin(CallContext.current().getCallingAccountId());
+        List<Long> vmIds = new ArrayList<>();
+        if (!shouldListSystemVmVolumes) {
+            SearchBuilder<UserVmVO> vmSearch = userVmDao.createSearchBuilder();
+            vmSearch.select(null, SearchCriteria.Func.DISTINCT, vmSearch.entity().getId());
+            accountMgr.buildACLSearchBuilder(vmSearch, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
+            SearchCriteria<UserVmVO> vmSc = vmSearch.create();
+            accountMgr.buildACLSearchCriteria(vmSc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
+            List<UserVmVO> vms = userVmDao.search(vmSc, null);
+            vmIds = vms.stream().map(UserVmVO::getId).collect(Collectors.toList());
+            if (vmIds.isEmpty()) {
+                sb.and("instanceIdNull", sb.entity().getInstanceId(), SearchCriteria.Op.NULL);
+            } else {
+                sb.and().op("instanceIdNull", sb.entity().getInstanceId(), SearchCriteria.Op.NULL);
+                sb.or("instanceIds", sb.entity().getInstanceId(), SearchCriteria.Op.IN);
+                sb.cp();
+            }
+        }
+
         SearchCriteria<VolumeVO> sc = sb.create();
+        accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
         if (CollectionUtils.isNotEmpty(ids)) {
             sc.setParameters("idIN", ids.toArray());
         }
@@ -358,8 +416,18 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
             ssc.addOr("state", SearchCriteria.Op.EQ, keyword);
             sc.addAnd("name", SearchCriteria.Op.SC, ssc);
         }
+        if (!shouldListSystemVmVolumes && CollectionUtils.isNotEmpty(vmIds)) {
+            sc.setParameters("instanceIds", vmIds.toArray());
+        }
 
-        return volumeDao.searchAndCount(sc, searchFilter);
+        Pair<List<VolumeVO>, Integer> uniqueVolumePair = volumeDao.searchAndCount(sc, searchFilter);
+        Integer count = uniqueVolumePair.second();
+        if (count == 0) {
+            return new Pair<>(new ArrayList<>(), count);
+        }
+        List<Long> volumeIds = uniqueVolumePair.first().stream().map(VolumeVO::getId).collect(Collectors.toList());
+        List<VolumeVO> volumes = volumeDao.listByIds(volumeIds);
+        return new Pair<>(volumes, count);
     }
 
     /**

plugins/metrics/src/test/java/org/apache/cloudstack/metrics/MetricsServiceImplTest.java

@@ -19,13 +19,16 @@ package org.apache.cloudstack.metrics;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import org.apache.cloudstack.api.ListVMsUsageHistoryCmd;
+import org.apache.cloudstack.api.ListVolumesUsageHistoryCmd;
 import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.response.VmMetricsStatsResponse;
 import org.apache.commons.lang3.time.DateUtils;
 import org.junit.Assert;
@@ -35,12 +38,18 @@ import org.mockito.ArgumentCaptor;
 import org.mockito.Captor;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
+import org.mockito.MockedStatic;
 import org.mockito.Mockito;
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
 import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.user.Account;
+import com.cloud.user.AccountManager;
 import com.cloud.utils.Pair;
+import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.vm.UserVmVO;
@@ -75,6 +84,12 @@ public class MetricsServiceImplTest {
     @Mock
     VmStatsDao vmStatsDaoMock;
 
+    @Mock
+    AccountManager accountManager;
+
+    @Mock
+    VolumeDao volumeDao;
+
     @Captor
     ArgumentCaptor<String> stringCaptor1, stringCaptor2;
 
@@ -95,12 +110,26 @@ public class MetricsServiceImplTest {
 
     @Mock
     VmStatsVO vmStatsVOMock;
+    @Mock
+    Account mockAccount;
+    @Mock
+    ListVolumesUsageHistoryCmd listVolumesUsageHistoryCmdMock;
+    @Mock
+    VolumeVO volumeVOMock;
+    @Mock
+    SearchBuilder<VolumeVO> volumeSearchBuilderMock;
+    @Mock
+    SearchCriteria<VolumeVO> volumeSearchCriteriaMock;
+    @Mock
+    Filter filterMock;
+
 
     private void prepareSearchCriteriaWhenUseSetParameters() {
         Mockito.doNothing().when(scMock).setParameters(Mockito.anyString(), Mockito.any());
     }
 
     private void preparesearchForUserVmsInternalTest() {
+        Mockito.when(mockAccount.getType()).thenReturn(Account.Type.NORMAL);
         expectedVmListAndCounter = new Pair<>(Arrays.asList(userVmVOMock), 1);
 
         Mockito.doReturn(1L).when(listVMsUsageHistoryCmdMock).getStartIndex();
@@ -111,8 +140,10 @@ public class MetricsServiceImplTest {
         Mockito.doReturn(userVmVOMock).when(sbMock).entity();
         Mockito.doReturn(scMock).when(sbMock).create();
 
-        Mockito.doReturn(new Pair<List<UserVmVO>, Integer>(Arrays.asList(userVmVOMock), 1))
+        Mockito.doReturn(expectedVmListAndCounter)
         .when(userVmDaoMock).searchAndCount(Mockito.any(), Mockito.any());
+        Mockito.doReturn(expectedVmListAndCounter.first())
+                .when(userVmDaoMock).listByIds(Mockito.anyList());
     }
 
     @Test
@@ -124,12 +155,17 @@ public class MetricsServiceImplTest {
         Mockito.doReturn(null).when(listVMsUsageHistoryCmdMock).getName();
         Mockito.doReturn(null).when(listVMsUsageHistoryCmdMock).getKeyword();
 
-        Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
+        try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
+            CallContext callContextMock = Mockito.mock(CallContext.class);
+            callContextMocked.when(CallContext::current).thenReturn(callContextMock);
+            Mockito.when(callContextMock.getCallingAccount()).thenReturn(mockAccount);
+            Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
 
-        Mockito.verify(scMock).setParameters(stringCaptor1.capture(), objectArrayCaptor.capture());
+            Mockito.verify(scMock).setParameters(stringCaptor1.capture(), objectArrayCaptor.capture());
-        Assert.assertEquals("idIN", stringCaptor1.getValue());
+            Assert.assertEquals("idIN", stringCaptor1.getValue());
-        Assert.assertEquals(fakeVmId1, objectArrayCaptor.getAllValues().get(0)[0]);
+            Assert.assertEquals(fakeVmId1, objectArrayCaptor.getAllValues().get(0)[0]);
-        Assert.assertEquals(expectedVmListAndCounter, result);
+            Assert.assertEquals(expectedVmListAndCounter, result);
+        }
     }
 
     @Test
@@ -141,13 +177,17 @@ public class MetricsServiceImplTest {
         Mockito.doReturn(expected).when(listVMsUsageHistoryCmdMock).getIds();
         Mockito.doReturn(null).when(listVMsUsageHistoryCmdMock).getName();
         Mockito.doReturn(null).when(listVMsUsageHistoryCmdMock).getKeyword();
+        try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
+            CallContext callContextMock = Mockito.mock(CallContext.class);
+            callContextMocked.when(CallContext::current).thenReturn(callContextMock);
+            Mockito.when(callContextMock.getCallingAccount()).thenReturn(mockAccount);
+            Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
 
-        Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
+            Mockito.verify(scMock).setParameters(stringCaptor1.capture(), objectArrayCaptor.capture());
-
+            Assert.assertEquals("idIN", stringCaptor1.getValue());
-        Mockito.verify(scMock).setParameters(stringCaptor1.capture(), objectArrayCaptor.capture());
+            Assert.assertArrayEquals(expected.toArray(), objectArrayCaptor.getAllValues().get(0));
-        Assert.assertEquals("idIN", stringCaptor1.getValue());
+            Assert.assertEquals(expectedVmListAndCounter, result);
-        Assert.assertArrayEquals(expected.toArray(), objectArrayCaptor.getAllValues().get(0));
+        }
-        Assert.assertEquals(expectedVmListAndCounter, result);
     }
 
     @Test
@@ -159,12 +199,17 @@ public class MetricsServiceImplTest {
         Mockito.doReturn("fakeName").when(listVMsUsageHistoryCmdMock).getName();
         Mockito.doReturn(null).when(listVMsUsageHistoryCmdMock).getKeyword();
 
-        Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
+        try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
+            CallContext callContextMock = Mockito.mock(CallContext.class);
+            callContextMocked.when(CallContext::current).thenReturn(callContextMock);
+            Mockito.when(callContextMock.getCallingAccount()).thenReturn(mockAccount);
+            Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
 
-        Mockito.verify(scMock).setParameters(stringCaptor1.capture(), objectArrayCaptor.capture());
+            Mockito.verify(scMock).setParameters(stringCaptor1.capture(), objectArrayCaptor.capture());
-        Assert.assertEquals("displayName", stringCaptor1.getValue());
+            Assert.assertEquals("displayName", stringCaptor1.getValue());
-        Assert.assertEquals("%fakeName%", objectArrayCaptor.getValue()[0]);
+            Assert.assertEquals("%fakeName%", objectArrayCaptor.getValue()[0]);
-        Assert.assertEquals(expectedVmListAndCounter, result);
+            Assert.assertEquals(expectedVmListAndCounter, result);
+        }
     }
 
     @Test
@@ -177,16 +222,21 @@ public class MetricsServiceImplTest {
         Mockito.doReturn(null).when(listVMsUsageHistoryCmdMock).getName();
         Mockito.doReturn("fakeKeyword").when(listVMsUsageHistoryCmdMock).getKeyword();
 
-        Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
+        try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
+            CallContext callContextMock = Mockito.mock(CallContext.class);
+            callContextMocked.when(CallContext::current).thenReturn(callContextMock);
+            Mockito.when(callContextMock.getCallingAccount()).thenReturn(mockAccount);
+            Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
 
-        Mockito.verify(scMock, Mockito.times(2)).addOr(stringCaptor1.capture(), opCaptor.capture(), objectArrayCaptor.capture());
+            Mockito.verify(scMock, Mockito.times(2)).addOr(stringCaptor1.capture(), opCaptor.capture(), objectArrayCaptor.capture());
-        List<String> conditions = stringCaptor1.getAllValues();
+            List<String> conditions = stringCaptor1.getAllValues();
-        List<Object[]> params = objectArrayCaptor.getAllValues();
+            List<Object[]> params = objectArrayCaptor.getAllValues();
-        Assert.assertEquals("displayName", conditions.get(0));
+            Assert.assertEquals("displayName", conditions.get(0));
-        Assert.assertEquals("state", conditions.get(1));
+            Assert.assertEquals("state", conditions.get(1));
-        Assert.assertEquals("%fakeKeyword%", params.get(0)[0]);
+            Assert.assertEquals("%fakeKeyword%", params.get(0)[0]);
-        Assert.assertEquals("fakeKeyword", params.get(1)[0]);
+            Assert.assertEquals("fakeKeyword", params.get(1)[0]);
-        Assert.assertEquals(expectedVmListAndCounter, result);
+            Assert.assertEquals(expectedVmListAndCounter, result);
+        }
     }
 
     @Test
@@ -317,4 +367,57 @@ public class MetricsServiceImplTest {
 
         spy.createStatsResponse(Arrays.asList(vmStatsVOMock));
     }
+
+    @Test
+    public void searchForVolumesInternalWithValidParameters() {
+        Mockito.doReturn(null).when(listVolumesUsageHistoryCmdMock).getId();
+        Mockito.doReturn(Arrays.asList(1L, 2L)).when(listVolumesUsageHistoryCmdMock).getIds();
+        Mockito.doReturn("volumeName").when(listVolumesUsageHistoryCmdMock).getName();
+        Mockito.doReturn("keyword").when(listVolumesUsageHistoryCmdMock).getKeyword();
+        Mockito.doReturn(volumeSearchBuilderMock).when(volumeDao).createSearchBuilder();
+        Mockito.doReturn(volumeVOMock).when(volumeSearchBuilderMock).entity();
+        SearchBuilder vmSearchBuilderMock = Mockito.mock(SearchBuilder.class);
+        Mockito.doReturn(vmSearchBuilderMock).when(userVmDaoMock).createSearchBuilder();
+        Mockito.doReturn(userVmVOMock).when(vmSearchBuilderMock).entity();
+        Mockito.doReturn(volumeSearchCriteriaMock).when(volumeSearchBuilderMock).create();
+        Mockito.doReturn(volumeSearchCriteriaMock).when(volumeDao).createSearchCriteria();
+        Mockito.doReturn(new Pair<>(Arrays.asList(volumeVOMock), 1)).when(volumeDao).searchAndCount(Mockito.any(), Mockito.any());
+        Mockito.doReturn(Arrays.asList(volumeVOMock)).when(volumeDao).listByIds(Mockito.anyList());
+
+
+        try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
+            CallContext callContextMock = Mockito.mock(CallContext.class);
+            callContextMocked.when(CallContext::current).thenReturn(callContextMock);
+            Mockito.when(callContextMock.getCallingAccount()).thenReturn(mockAccount);
+            Pair<List<VolumeVO>, Integer> result = spy.searchForVolumesInternal(listVolumesUsageHistoryCmdMock);
+
+            Assert.assertNotNull(result);
+            Assert.assertEquals(1, result.second().intValue());
+            Assert.assertEquals(volumeVOMock, result.first().get(0));
+        }
+    }
+
+    @Test
+    public void searchForVolumesInternalWithValidParametersNoItem() {
+        Mockito.doReturn(1L).when(listVolumesUsageHistoryCmdMock).getId();
+        Mockito.doReturn(volumeSearchBuilderMock).when(volumeDao).createSearchBuilder();
+        Mockito.doReturn(volumeVOMock).when(volumeSearchBuilderMock).entity();
+        Mockito.doReturn(volumeSearchCriteriaMock).when(volumeSearchBuilderMock).create();
+        Mockito.doReturn(new Pair<>(Collections.emptyList(), 0)).when(volumeDao).searchAndCount(Mockito.any(), Mockito.any());
+
+
+        try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
+            CallContext callContextMock = Mockito.mock(CallContext.class);
+            callContextMocked.when(CallContext::current).thenReturn(callContextMock);
+            Mockito.when(callContextMock.getCallingAccount()).thenReturn(mockAccount);
+            Mockito.when(callContextMock.getCallingAccountId()).thenReturn(1L);
+            Mockito.when(accountManager.isRootAdmin(1L)).thenReturn(true);
+            Mockito.when(mockAccount.getType()).thenReturn(Account.Type.ADMIN);
+            Pair<List<VolumeVO>, Integer> result = spy.searchForVolumesInternal(listVolumesUsageHistoryCmdMock);
+
+            Assert.assertNotNull(result);
+            Assert.assertEquals(0, result.second().intValue());
+        }
+    }
+
 }

pom.xml

@@ -156,7 +156,7 @@
         <cs.jakarta.xml.bind.version>2.3.3</cs.jakarta.xml.bind.version>
         <cs.jaxws.version>2.3.7</cs.jaxws.version>
         <cs.jersey-client.version>2.26</cs.jersey-client.version>
-        <cs.jetty.version>9.4.51.v20230217</cs.jetty.version>
+        <cs.jetty.version>9.4.58.v20250814</cs.jetty.version>
         <cs.jetty-maven-plugin.version>9.4.27.v20200227</cs.jetty-maven-plugin.version>
         <cs.jna.version>5.5.0</cs.jna.version>
         <cs.joda-time.version>2.12.5</cs.joda-time.version>

python/lib/cloudutils/configFileOps.py

@@ -68,14 +68,14 @@ class configFileOps:
                 for entry in self.entries:
                     if entry.op == "add":
                         if entry.separator == "=":
-                            matchString = "^\ *" + entry.name + ".*"
+                            matchString = r"^\ *" + entry.name + ".*"
                         elif entry.separator == " ":
-                            matchString = "^\ *" + entry.name + "\ *" + entry.value
+                            matchString = r"^\ *" + entry.name + r"\ *" + entry.value
                     else:
                         if entry.separator == "=":
-                            matchString = "^\ *" + entry.name + "\ *=\ *" + entry.value
+                            matchString = r"^\ *" + entry.name + r"\ *=\ *" + entry.value
                         else:
-                            matchString = "^\ *" + entry.name + "\ *" + entry.value
+                            matchString = r"^\ *" + entry.name + r"\ *" + entry.value
 
                     match = re.match(matchString, line)
                     if match is not None:

python/lib/cloudutils/networkConfig.py

@@ -45,8 +45,11 @@ class networkConfig:
         if not cmd.isSuccess():
             logging.debug("Failed to get default route")
             raise CloudRuntimeException("Failed to get default route")
-
+        output = cmd.getStdout().strip()
-        result = cmd.getStdout().split(" ")
+        result = output.split(" ")
+        if len(result) < 2:
+            logging.debug("Output for the default route incomplete: %s. It should have been '<GATEWAY> <DEVICE>'" % output)
+            raise CloudRuntimeException("Output for the default route incomplete")
         gateway = result[0]
         dev = result[1]
 
@@ -150,10 +153,10 @@ class networkConfig:
             if line.find("HWaddr") != -1:
                 macAddr = line.split("HWaddr ")[1].strip(" ")
             elif line.find("inet ") != -1:
-                m = re.search("addr:(.*)\ *Bcast:(.*)\ *Mask:(.*)", line)
+                m = re.search(r"addr:([^\s]+)\s*Bcast:([^\s]+)\s*Mask:([^\s]+)", line)
                 if m is not None:
-                    ipAddr = m.group(1).rstrip(" ")
+                    ipAddr = m.group(1).strip()
-                    netmask = m.group(3).rstrip(" ")
+                    netmask = m.group(3).strip()
 
         if networkConfig.isBridgePort(dev):
             type = "brport"

python/lib/cloudutils/utilities.py

@@ -63,7 +63,7 @@ class bash:
         return self.stdout.decode('utf-8').strip('\n')
 
     def getLines(self):
-        return self.stdout.decode('utf-8').strip('\n')
+        return self.stdout.decode('utf-8').strip('\n').split('\n')
 
     def getStderr(self):
         return self.stderr.decode('utf-8').strip('\n')

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -5653,7 +5653,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
 
         //Validation - 1.3
         if (resourceIdStr != null) {
-            resourceId = resourceManagerUtil.getResourceId(resourceIdStr, resourceType);
+            resourceId = resourceManagerUtil.getResourceId(resourceIdStr, resourceType, true);
         }
 
         List<? extends ResourceDetail> detailList = new ArrayList<>();

server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java

@@ -488,6 +488,8 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
             throw new InvalidParameterValueException("Cannot create Network ACL Item. ACL Id or network Id is required");
         }
         Network network = networkModel.getNetwork(createNetworkACLCmd.getNetworkId());
+        Account caller = CallContext.current().getCallingAccount();
+        _accountMgr.checkAccess(caller, null, true, network);
         if (network.getVpcId() == null) {
             throw new InvalidParameterValueException("Network: " + network.getUuid() + " does not belong to VPC");
         }
@@ -749,6 +751,7 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
 
         if (networkId != null) {
             final Network network = _networkDao.findById(networkId);
+            _accountMgr.checkAccess(caller, null, true, network);
             aclId = network.getNetworkACLId();
             if (aclId == null) {
                 // No aclId associated with the network.

server/src/main/java/com/cloud/resource/ResourceManagerImpl.java

@@ -179,6 +179,7 @@ import com.cloud.org.Cluster;
 import com.cloud.org.Grouping;
 import com.cloud.org.Managed;
 import com.cloud.serializer.GsonHelper;
+import com.cloud.server.ManagementService;
 import com.cloud.service.ServiceOfferingVO;
 import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.service.dao.ServiceOfferingDetailsDao;
@@ -307,6 +308,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
     private UserVmManager userVmManager;
     @Inject
     private GpuService gpuService;
+    @Inject
+    ManagementService managementService;
 
     private List<? extends Discoverer> _discoverers;
 
@@ -2818,8 +2821,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
 
     @Override
     public Host updateHost(final UpdateHostCmd cmd) throws NoTransitionException {
-        return updateHost(cmd.getId(), cmd.getName(), cmd.getOsCategoryId(), cmd.getAllocationState(), cmd.getUrl(),
+        managementService.checkJsInterpretationAllowedIfNeededForParameterValue(ApiConstants.IS_TAG_A_RULE,
-                cmd.getHostTags(), cmd.getIsTagARule(), cmd.getAnnotation(), false,
+                Boolean.TRUE.equals(cmd.getIsTagARule()));
+
+        return updateHost(cmd.getId(), cmd.getName(), cmd.getOsCategoryId(),
+                cmd.getAllocationState(), cmd.getUrl(), cmd.getHostTags(), cmd.getIsTagARule(), cmd.getAnnotation(), false,
                 cmd.getExternalDetails(), cmd.isCleanupExternalDetails());
     }
 

server/src/main/java/com/cloud/server/ManagementServerImpl.java

@@ -1067,6 +1067,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
 
     protected List<DeploymentPlanner> _planners;
 
+    private boolean jsInterpretationEnabled = false;
+
     private final List<HypervisorType> supportedHypervisors = new ArrayList<>();
 
     public List<DeploymentPlanner> getPlanners() {
@@ -1147,6 +1149,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         supportedHypervisors.add(HypervisorType.KVM);
         supportedHypervisors.add(HypervisorType.XenServer);
 
+        jsInterpretationEnabled = JsInterpretationEnabled.value();
+
         return true;
     }
 
@@ -4220,8 +4224,10 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         cmdList.add(ListGuestVlansCmd.class);
         cmdList.add(AssignVolumeCmd.class);
         cmdList.add(ListSecondaryStorageSelectorsCmd.class);
-        cmdList.add(CreateSecondaryStorageSelectorCmd.class);
+        if (jsInterpretationEnabled) {
-        cmdList.add(UpdateSecondaryStorageSelectorCmd.class);
+            cmdList.add(CreateSecondaryStorageSelectorCmd.class);
+            cmdList.add(UpdateSecondaryStorageSelectorCmd.class);
+        }
         cmdList.add(RemoveSecondaryStorageSelectorCmd.class);
         cmdList.add(ListAffectedVmsForStorageScopeChangeCmd.class);
         cmdList.add(ListGuiThemesCmd.class);
@@ -4275,7 +4281,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] {exposeCloudStackVersionInApiXmlResponse, exposeCloudStackVersionInApiListCapabilities, vmPasswordLength, sshKeyLength, humanReadableSizes, customCsIdentifier};
+        return new ConfigKey<?>[] {exposeCloudStackVersionInApiXmlResponse, exposeCloudStackVersionInApiListCapabilities, vmPasswordLength, sshKeyLength, humanReadableSizes, customCsIdentifier,
+        JsInterpretationEnabled};
     }
 
     protected class EventPurgeTask extends ManagedContextRunnable {
@@ -5780,4 +5787,13 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
     public Answer getExternalVmConsole(VirtualMachine vm, Host host) {
         return extensionsManager.getInstanceConsole(vm, host);
     }
+    @Override
+    public void checkJsInterpretationAllowedIfNeededForParameterValue(String paramName, boolean paramValue) {
+        if (!paramValue || jsInterpretationEnabled) {
+            return;
+        }
+        throw new InvalidParameterValueException(String.format(
+                "The parameter %s cannot be set to true as JS interpretation is disabled",
+                paramName));
+    }
 }

server/src/main/java/com/cloud/storage/StorageManagerImpl.java

@@ -221,6 +221,7 @@ import com.cloud.org.Grouping.AllocationState;
 import com.cloud.resource.ResourceState;
 import com.cloud.server.ConfigurationServer;
 import com.cloud.server.ManagementServer;
+import com.cloud.server.ManagementService;
 import com.cloud.server.StatsCollector;
 import com.cloud.service.dao.ServiceOfferingDetailsDao;
 import com.cloud.storage.Storage.ImageFormat;
@@ -414,6 +415,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
     ResourceManager _resourceMgr;
     @Inject
     StorageManager storageManager;
+    @Inject
+    ManagementService managementService;
 
     protected List<StoragePoolDiscoverer> _discoverers;
 
@@ -1031,6 +1034,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
             throw new PermissionDeniedException(String.format("Cannot perform this operation, Zone is currently disabled: %s", zone));
         }
 
+        managementService.checkJsInterpretationAllowedIfNeededForParameterValue(ApiConstants.IS_TAG_A_RULE,
+                Boolean.TRUE.equals(cmd.isTagARule()));
+
         Map<String, Object> params = new HashMap<>();
         params.put("zoneId", zone.getId());
         params.put("clusterId", clusterId);
@@ -1214,6 +1220,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         // Input validation
         Long id = cmd.getId();
 
+        managementService.checkJsInterpretationAllowedIfNeededForParameterValue(ApiConstants.IS_TAG_A_RULE,
+                Boolean.TRUE.equals(cmd.isTagARule()));
+
         StoragePoolVO pool = _storagePoolDao.findById(id);
         if (pool == null) {
             throw new IllegalArgumentException("Unable to find storage pool with ID: " + id);

server/src/main/java/com/cloud/tags/ResourceManagerUtilImpl.java

@@ -22,6 +22,7 @@ import java.util.Objects;
 
 import javax.inject.Inject;
 
+import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.api.Identity;
 import org.apache.cloudstack.api.InternalIdentity;
 import org.apache.cloudstack.context.CallContext;
@@ -130,6 +131,11 @@ public class ResourceManagerUtilImpl implements ResourceManagerUtil {
 
     @Override
     public long getResourceId(String resourceId, ResourceTag.ResourceObjectType resourceType) {
+        return getResourceId(resourceId, resourceType, false);
+    }
+
+    @Override
+    public long getResourceId(String resourceId, ResourceTag.ResourceObjectType resourceType, boolean checkAccess) {
         Class<?> clazz = s_typeMap.get(resourceType);
         Object entity = entityMgr.findByUuid(clazz, resourceId);
         if (entity != null) {
@@ -140,6 +146,11 @@ public class ResourceManagerUtilImpl implements ResourceManagerUtil {
         }
         entity = entityMgr.findById(clazz, resourceId);
         if (entity != null) {
+            if (checkAccess && entity instanceof ControlledEntity) {
+                ControlledEntity controlledEntity = (ControlledEntity)entity;
+                Account caller = CallContext.current().getCallingAccount();
+                accountMgr.checkAccess(caller, null, false, controlledEntity);
+            }
             return ((InternalIdentity)entity).getId();
         }
         throw new InvalidParameterValueException("Unable to find resource by id " + resourceId + " and type " + resourceType);

server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java

@@ -496,7 +496,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
 
                 boolean dataDiskDeletetionResult = true;
                 List<VMTemplateVO> dataDiskTemplates = templateDao.listByParentTemplatetId(template.getId());
-                if (dataDiskTemplates != null && dataDiskTemplates.size() > 0) {
+                if (CollectionUtils.isNotEmpty(dataDiskTemplates)) {
                     logger.info("Template: {} has Datadisk template(s) associated with it. Delete Datadisk templates before deleting the template", template);
                     for (VMTemplateVO dataDiskTemplate : dataDiskTemplates) {
                         logger.info("Delete Datadisk template: {} from image store: {}", dataDiskTemplate, imageStore);
@@ -562,6 +562,9 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
         if (success) {
             if ((imageStores != null && imageStores.size() > 1) && (profile.getZoneIdList() != null)) {
                 //if template is stored in more than one image stores, and the zone id is not null, then don't delete other templates.
+                if (templateMgr.TemplateDeleteFromPrimaryStorage.value()) {
+                    templateMgr.evictTemplateFromStoragePoolsForZones(template.getId(), profile.getZoneIdList());
+                }
                 return cleanupTemplate(template, success);
             }
 
@@ -574,7 +577,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
 
             // find all eligible image stores for this template
             List<DataStore> iStores = templateMgr.getImageStoreByTemplate(template.getId(), null);
-            if (iStores == null || iStores.size() == 0) {
+            if (CollectionUtils.isEmpty(iStores)) {
                 // remove any references from template_zone_ref
                 List<VMTemplateZoneVO> templateZones = templateZoneDao.listByTemplateId(template.getId());
                 if (templateZones != null) {
@@ -595,6 +598,10 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
 
             }
 
+            if (templateMgr.TemplateDeleteFromPrimaryStorage.value()) {
+                templateMgr.evictTemplateFromStoragePoolsForZones(template.getId(), profile.getZoneIdList());
+            }
+
             // remove its related ACL permission
             Pair<Class<?>, Long> templateClassForId = new Pair<>(VirtualMachineTemplate.class, template.getId());
             _messageBus.publish(_name, EntityManager.MESSAGE_REMOVE_ENTITY_EVENT, PublishScope.LOCAL, templateClassForId);

server/src/main/java/com/cloud/template/TemplateManagerImpl.java

@@ -1030,33 +1030,53 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
         return adapter.delete(new TemplateProfile(userId, template, zoneId));
     }
 
+    private Boolean templateIsUnusedInPool(VMTemplateStoragePoolVO templatePoolVO) {
+        VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId());
+
+        // If this is a routing template, consider it in use
+        if (template.getTemplateType() == TemplateType.SYSTEM) {
+            return false;
+        }
+
+        // If the template is not yet downloaded to the pool, consider it in use
+        if (templatePoolVO.getDownloadState() != Status.DOWNLOADED) {
+            return false;
+        }
+
+        if (template.getFormat() == ImageFormat.ISO || _volumeDao.isAnyVolumeActivelyUsingTemplateOnPool(template.getId(), templatePoolVO.getPoolId())) {
+            return false;
+        }
+        return true;
+    }
+
     @Override
     public List<VMTemplateStoragePoolVO> getUnusedTemplatesInPool(StoragePoolVO pool) {
         List<VMTemplateStoragePoolVO> unusedTemplatesInPool = new ArrayList<VMTemplateStoragePoolVO>();
         List<VMTemplateStoragePoolVO> allTemplatesInPool = _tmpltPoolDao.listByPoolId(pool.getId());
 
         for (VMTemplateStoragePoolVO templatePoolVO : allTemplatesInPool) {
-            VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId());
+            if (templateIsUnusedInPool(templatePoolVO)) {
-
-            // If this is a routing template, consider it in use
-            if (template.getTemplateType() == TemplateType.SYSTEM) {
-                continue;
-            }
-
-            // If the template is not yet downloaded to the pool, consider it in
-            // use
-            if (templatePoolVO.getDownloadState() != Status.DOWNLOADED) {
-                continue;
-            }
-
-            if (template.getFormat() != ImageFormat.ISO && !_volumeDao.isAnyVolumeActivelyUsingTemplateOnPool(template.getId(), pool.getId())) {
                 unusedTemplatesInPool.add(templatePoolVO);
             }
         }
-
         return unusedTemplatesInPool;
     }
 
+    @Override
+    public void evictTemplateFromStoragePoolsForZones(Long templateId, List<Long> zoneIds) {
+        List<Long> poolIds = new ArrayList<>();
+        if (CollectionUtils.isNotEmpty(zoneIds)) {
+            List<StoragePoolVO> pools = _poolDao.listByDataCenterIds(zoneIds);
+            poolIds = pools.stream().map(StoragePoolVO::getId).collect(Collectors.toList());
+        }
+        List<VMTemplateStoragePoolVO> templateStoragePoolVOS = _tmpltPoolDao.listByPoolIdsAndTemplate(poolIds, templateId);
+        for (VMTemplateStoragePoolVO templateStoragePoolVO: templateStoragePoolVOS) {
+            if (templateIsUnusedInPool(templateStoragePoolVO)) {
+                evictTemplateFromStoragePool(templateStoragePoolVO);
+            }
+        }
+    }
+
     @Override
     @DB
     public void evictTemplateFromStoragePool(VMTemplateStoragePoolVO templatePoolVO) {
@@ -2437,7 +2457,10 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] {AllowPublicUserTemplates, TemplatePreloaderPoolSize, ValidateUrlIsResolvableBeforeRegisteringTemplate};
+        return new ConfigKey<?>[] {AllowPublicUserTemplates,
+                TemplatePreloaderPoolSize,
+                ValidateUrlIsResolvableBeforeRegisteringTemplate,
+                TemplateDeleteFromPrimaryStorage};
     }
 
     public List<TemplateAdapter> getTemplateAdapters() {

server/src/main/java/org/apache/cloudstack/network/RoutedIpv4ManagerImpl.java

@@ -607,10 +607,20 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
     }
 
     protected Ipv4GuestSubnetNetworkMap getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, Long ownerDomainId, Long ownerAccountId, Long zoneId) {
-        validateNetworkCidrSize(ownerAccountId, cidrSize);
+        validateNetworkCidrSize(cidrSize);
         List<DataCenterIpv4GuestSubnetVO> subnets = getZoneSubnetsForAccount(ownerDomainId, ownerAccountId, zoneId);
         for (DataCenterIpv4GuestSubnetVO subnet : subnets) {
-            Ipv4GuestSubnetNetworkMap result = getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(cidrSize, subnet);
+            Ipv4GuestSubnetNetworkMap result = getIpv4SubnetForGuestNetworkOrVpcInternal(cidrSize, subnet);
+            if (result != null) {
+                return result;
+            }
+        }
+        Boolean isAutoAllocationEnabled = RoutedIPv4NetworkCidrAutoAllocationEnabled.valueIn(ownerAccountId);
+        if (!Boolean.TRUE.equals(isAutoAllocationEnabled)) {
+            throw new InvalidParameterValueException("CIDR auto-allocation is disabled for this account");
+        }
+        for (DataCenterIpv4GuestSubnetVO subnet : subnets) {
+            Ipv4GuestSubnetNetworkMap result = createIpv4SubnetForGuestNetworkOrVpcInternal(cidrSize, subnet);
             if (result != null) {
                 return result;
             }
@@ -618,11 +628,11 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         return null;
     }
 
-    protected Ipv4GuestSubnetNetworkMap getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, DataCenterIpv4GuestSubnetVO subnet) {
+    protected Ipv4GuestSubnetNetworkMap getIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, DataCenterIpv4GuestSubnetVO subnet) {
-        Ipv4GuestSubnetNetworkMap map = ipv4GuestSubnetNetworkMapDao.findFirstAvailable(subnet.getId(), cidrSize);
+        return ipv4GuestSubnetNetworkMapDao.findFirstAvailable(subnet.getId(), cidrSize);
-        if (map != null) {
+    }
-            return map;
+
-        }
+    protected Ipv4GuestSubnetNetworkMap createIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, DataCenterIpv4GuestSubnetVO subnet) {
         try {
             return createIpv4SubnetFromParentSubnet(subnet, cidrSize);
         } catch (Exception ex) {
@@ -631,6 +641,14 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         return null;
     }
 
+    protected Ipv4GuestSubnetNetworkMap getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, DataCenterIpv4GuestSubnetVO subnet) {
+        Ipv4GuestSubnetNetworkMap map = getIpv4SubnetForGuestNetworkOrVpcInternal(cidrSize, subnet);
+        if (map != null) {
+            return map;
+        }
+        return createIpv4SubnetForGuestNetworkOrVpcInternal(cidrSize, subnet);
+    }
+
     protected void getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(String networkCidr, Long ownerDomainId, Long ownerAccountId, Long zoneId) {
         Ipv4GuestSubnetNetworkMapVO subnetMap = ipv4GuestSubnetNetworkMapDao.findBySubnet(networkCidr);
         if (subnetMap != null) {
@@ -693,13 +711,9 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         }
     }
 
-    private void validateNetworkCidrSize(long accountId, Integer networkCidrSize) {
+    private void validateNetworkCidrSize(Integer networkCidrSize) {
         if (networkCidrSize == null) {
-            throw new CloudRuntimeException("network/vpc CidrSize is null");
+            throw new InvalidParameterValueException("network/vpc CidrSize is null");
-        }
-        Boolean isAutoAllocationEnabled = RoutedIPv4NetworkCidrAutoAllocationEnabled.valueIn(accountId);
-        if (!Boolean.TRUE.equals(isAutoAllocationEnabled)) {
-            throw new CloudRuntimeException("CIDR auto-allocation is disabled for this account");
         }
     }
 
@@ -755,7 +769,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         // Allocate a subnet automatically
         String networkCidr = getFreeNetworkCidr(subnetsInFreeIpRanges, networkCidrSize);
         if (networkCidr == null) {
-            throw new CloudRuntimeException("Failed to automatically allocate a subnet with specified cidrsize");
+            throw new InvalidParameterValueException("Failed to automatically allocate a subnet with specified cidrsize");
         }
         return networkCidr;
     }

server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java

@@ -71,7 +71,6 @@ import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.hypervisor.HypervisorGuru;
 import com.cloud.hypervisor.HypervisorGuruManager;
-import com.cloud.network.IpAddressManager;
 import com.cloud.network.Network;
 import com.cloud.network.NetworkModel;
 import com.cloud.network.Networks;
@@ -303,8 +302,6 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
     @Inject
     private PhysicalNetworkDao physicalNetworkDao;
     @Inject
-    private IpAddressManager ipAddressManager;
-    @Inject
     private StoragePoolHostDao storagePoolHostDao;
     @Inject
     private HypervisorGuruManager hypervisorGuruManager;
@@ -2766,10 +2763,8 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
         }
 
         String macAddress = networkModel.getNextAvailableMacAddressInNetwork(networkId);
-        String ipAddress = null;
+
-        if (network.getGuestType() != Network.GuestType.L2) {
+        String ipAddress = network.getGuestType() != Network.GuestType.L2 ? "auto" : null;
-            ipAddress = ipAddressManager.acquireGuestIpAddress(network, null);
-        }
 
         Network.IpAddresses requestedIpPair = new Network.IpAddresses(ipAddress, null, macAddress);
 

server/src/test/java/org/apache/cloudstack/network/RoutedIpv4ManagerImplTest.java

@@ -545,12 +545,12 @@ public class RoutedIpv4ManagerImplTest {
         DataCenterIpv4GuestSubnetVO subnet3 = Mockito.mock(DataCenterIpv4GuestSubnetVO.class);
         when(dataCenterIpv4GuestSubnetDao.listNonDedicatedByDataCenterId(zoneId)).thenReturn(Arrays.asList(subnet3));
 
-        doReturn(null).doReturn(null).doReturn(ipv4GuestSubnetNetworkMap).when(routedIpv4Manager).getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(eq(cidrSize), any());
+        doReturn(null).doReturn(null).doReturn(ipv4GuestSubnetNetworkMap).when(routedIpv4Manager).getIpv4SubnetForGuestNetworkOrVpcInternal(eq(cidrSize), any());
 
         Ipv4GuestSubnetNetworkMap result = routedIpv4Manager.getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(cidrSize, domainId, accountId, zoneId);
 
         Assert.assertEquals(ipv4GuestSubnetNetworkMap, result);
-        verify(routedIpv4Manager, times(3)).getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(eq(cidrSize), any());
+        verify(routedIpv4Manager, times(3)).getIpv4SubnetForGuestNetworkOrVpcInternal(eq(cidrSize), any());
     }
 
     @Test

tools/appliance/systemvmtemplate/scripts/configure_locale.sh

@@ -22,13 +22,15 @@ set -x
 function configure_locale() {
   grep LANG=en_US.UTF-8 /etc/default/locale && \
       grep LC_ALL=en_US.UTF-8 /etc/default/locale && \
-      grep "en_US.UTF-8 UTF-8" /etc/locale.gen &&
+      grep "^en_US.UTF-8 UTF-8" /etc/locale.gen &&
       return
 
   cat >> /etc/default/locale  << EOF
 LANG=en_US.UTF-8
 LC_ALL=en_US.UTF-8
 EOF
+
+  grep "^en_US.UTF-8 UTF-8" /etc/locale.gen || \
   cat >> /etc/locale.gen  << EOF
 en_US.UTF-8 UTF-8
 EOF

utils/src/main/java/org/apache/cloudstack/utils/jsinterpreter/JsInterpreter.java

@@ -19,31 +19,52 @@ package org.apache.cloudstack.utils.jsinterpreter;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.StringWriter;
+import java.util.Arrays;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-import org.apache.commons.collections.MapUtils;
+import javax.script.Bindings;
-import org.apache.logging.log4j.Logger;
+import javax.script.Compilable;
-import org.apache.logging.log4j.LogManager;
+import javax.script.CompiledScript;
+import javax.script.ScriptContext;
+import javax.script.ScriptEngine;
+import javax.script.ScriptException;
+import javax.script.SimpleBindings;
+import javax.script.SimpleScriptContext;
 
-import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.commons.collections.MapUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.openjdk.nashorn.api.scripting.ClassFilter;
 import org.openjdk.nashorn.api.scripting.NashornScriptEngineFactory;
 
-import javax.script.ScriptEngine;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 /**
- * A class to execute JavaScript scripts, with the possibility to inject context to the scripts.
+ * Executes JavaScript with strong restrictions to mitigate RCE risks.
+ * - Disables Java interop via --no-java AND a deny-all ClassFilter
+ * - Disables Nashorn syntax extensions
+ * - Uses Bindings instead of string-splicing variables
+ * - Fresh ScriptContext per execution, with timeout on a daemon worker
  */
 public class JsInterpreter implements Closeable {
     protected Logger logger = LogManager.getLogger(JsInterpreter.class);
 
+    protected static final List<String> RESTRICTED_TOKENS = Arrays.asList( "engine", "context", "factory",
+            "Java", "java", "Packages"," javax", "load", "loadWithNewGlobal", "print", "factory", "getClass",
+            "runCommand", "Runtime", "exec", "ProcessBuilder", "Thread", "thread", "Threads", "Class", "class");
+
     protected ScriptEngine interpreter;
     protected String interpreterName;
     private final String injectingLogMessage = "Injecting variable [%s] with value [%s] into the JS interpreter.";
@@ -51,21 +72,40 @@ public class JsInterpreter implements Closeable {
     private TimeUnit defaultTimeUnit = TimeUnit.MILLISECONDS;
     private long timeout;
     private String timeoutDefaultMessage;
-    protected Map<String, String> variables = new LinkedHashMap<>();
+
+    // Store variables as Objects; they go into Bindings (no code splicing)
+    protected Map<String, Object> variables = new LinkedHashMap<>();
+
+    /** Deny-all filter: no Java class is visible from scripts. */
+    static final class DenyAllClassFilter implements ClassFilter {
+        @Override public boolean exposeToScripts(String className) { return false; }
+    }
 
     /**
      * Constructor created exclusively for unit testing.
      */
-    protected JsInterpreter() {
+    protected JsInterpreter() { }
-    }
 
     public JsInterpreter(long timeout) {
         this.timeout = timeout;
-        this.timeoutDefaultMessage = String.format("Timeout (in milliseconds) defined in the global setting [quota.activationrule.timeout]: [%s].", this.timeout);
+        this.timeoutDefaultMessage = String.format(
+                "Timeout (in milliseconds) defined in the global setting [quota.activationrule.timeout]: [%s].", this.timeout);
+
+        if (System.getProperty("nashorn.args") == null) {
+            System.setProperty("nashorn.args", "--no-java --no-syntax-extensions");
+        }
+
+        this.executor = new ThreadPoolExecutor(
+                1, 1, 60L, TimeUnit.SECONDS,
+                new LinkedBlockingQueue<>(),
+                r -> {
+                    Thread t = new Thread(r, "JsInterpreter-worker");
+                    t.setDaemon(true);
+                    return t;
+                }
+        );
 
-        executor = Executors.newSingleThreadExecutor();
         NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
-
         this.interpreterName = factory.getEngineName();
         logger.trace(String.format("Initiating JS interpreter: %s.", interpreterName));
 
@@ -73,49 +113,53 @@ public class JsInterpreter implements Closeable {
     }
 
     protected void setScriptEngineDisablingJavaLanguage(NashornScriptEngineFactory factory) {
-        interpreter = factory.getScriptEngine("--no-java");
+        String[] opts = new String[] {
+                "--no-java",
+                "--no-syntax-extensions",
+        };
+        interpreter = factory.getScriptEngine(
+                opts,
+                JsInterpreter.class.getClassLoader(),
+                new DenyAllClassFilter()
+        );
     }
 
-    /**
+    /** Discards the current variables map and create a new one. */
-     * Discards the current variables map and create a new one.
-     */
     public void discardCurrentVariables() {
         logger.trace("Discarding current variables map and creating a new one.");
         variables = new LinkedHashMap<>();
     }
 
     /**
-     * Adds the parameters to a Map that will be converted to JS variables right before executing the script.
+     * Adds a variable that will be exposed via ENGINE_SCOPE bindings.
-     * @param key The name of the variable.
+     * Safe against code injection (no string concatenation).
-     * @param value The value of the variable.
      */
-    public void injectVariable(String key, String value) {
+    public void injectVariable(String key, Object value) {
-        logger.trace(String.format(injectingLogMessage, key, value));
+        if (key == null) return;
+        logger.trace(String.format(injectingLogMessage, key, String.valueOf(value)));
         variables.put(key, value);
     }
 
     /**
-     * Adds the parameter, surrounded by double quotes, to a Map that will be converted to a JS variable right before executing the script.
+     * @deprecated Not needed when using Bindings; kept for source compatibility.
-     * @param key The name of the variable.
+     *             Prefer {@link #injectVariable(String, Object)}.
-     * @param value The value of the variable.
      */
+    @Deprecated
     public void injectStringVariable(String key, String value) {
         if (value == null) {
             logger.trace(String.format("Not injecting [%s] because its value is null.", key));
             return;
         }
-        value = String.format("\"%s\"", value);
+        injectVariable(key, value);
-        logger.trace(String.format(injectingLogMessage, key, value));
-        variables.put(key, value);
     }
 
     /**
-     * Injects the variables to the script and execute it.
+     * Injects the variables via Bindings and executes the script with a fresh context.
      * @param script Code to be executed.
      * @return The result of the executed script.
      */
     public Object executeScript(String script) {
-        script = addVariablesToScript(script);
+        Objects.requireNonNull(script, "script");
 
         logger.debug(String.format("Executing script [%s].", script));
 
@@ -126,43 +170,60 @@ public class JsInterpreter implements Closeable {
     }
 
     protected Object executeScriptInThread(String script) {
-        Callable<Object> task = () -> interpreter.eval(script);
+        final Callable<Object> task = () -> {
+            final SimpleScriptContext ctx = new SimpleScriptContext();
 
-        Future<Object> future = executor.submit(task);
+            final Bindings engineBindings = new SimpleBindings();
+            if (MapUtils.isNotEmpty(variables)) {
+                engineBindings.putAll(variables);
+            }
+            for (String token : RESTRICTED_TOKENS) {
+                engineBindings.put(token, null);
+            }
+            ctx.setBindings(engineBindings, ScriptContext.ENGINE_SCOPE);
+
+            final StringWriter out = new StringWriter();
+            ctx.setWriter(out);
+
+            try {
+                final CompiledScript compiled = ((Compilable) interpreter).compile(script);
+                Object result = compiled.eval(ctx);
+                if (out.getBuffer().length() > 0) {
+                    logger.info("Script produced output on stdout: [{}]", out);
+                }
+                return result;
+            } catch (ScriptException se) {
+                String msg = se.getMessage() == null ? "Script error" : se.getMessage();
+                throw new ScriptException("Script error: " + msg, se.getFileName(), se.getLineNumber(), se.getColumnNumber());
+            }
+        };
+
+        final Future<Object> future = executor.submit(task);
 
         try {
             return future.get(this.timeout, defaultTimeUnit);
-        } catch (TimeoutException | InterruptedException | ExecutionException e) {
+        } catch (TimeoutException e) {
-            String message = String.format("Unable to execute script [%s] due to [%s]", script, e.getMessage());
+            String message = String.format(
-
+                    "Execution of script [%s] took too long and timed out. %s", script, timeoutDefaultMessage);
-            if (e instanceof TimeoutException) {
-                message = String.format("Execution of script [%s] took too long and timed out. %s", script, timeoutDefaultMessage);
-            }
-
             logger.error(message, e);
             throw new CloudRuntimeException(message, e);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            String message = String.format("Execution of script [%s] was interrupted.", script);
+            logger.error(message, e);
+            throw new CloudRuntimeException(message, e);
+        } catch (ExecutionException e) {
+            Throwable cause = e.getCause() == null ? e : e.getCause();
+            String message = String.format("Unable to execute script [%s] due to [%s]", script, cause.getMessage());
+            logger.error(message, cause);
+            throw new CloudRuntimeException(message, cause);
         } finally {
             future.cancel(true);
         }
     }
 
-    protected String addVariablesToScript(String script) {
-        if (MapUtils.isEmpty(variables)) {
-            logger.trace(String.format("There is no variables to add to script [%s]. Returning.", script));
-            return script;
-        }
-
-        String variablesToString = "";
-        for (Map.Entry<String, String> variable : variables.entrySet()) {
-            variablesToString = String.format("%s %s = %s;", variablesToString, variable.getKey(), variable.getValue());
-        }
-
-        return String.format("%s %s", variablesToString, script);
-    }
-
-
     @Override
     public void close() throws IOException {
-        executor.shutdown();
+        executor.shutdownNow();
     }
 }

utils/src/main/java/org/apache/cloudstack/utils/jsinterpreter/TagAsRuleHelper.java

@@ -16,12 +16,12 @@
 //under the License.
 package org.apache.cloudstack.utils.jsinterpreter;
 
-import com.cloud.utils.exception.CloudRuntimeException;
+import java.io.IOException;
-import org.apache.commons.lang3.StringEscapeUtils;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.io.IOException;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 public class TagAsRuleHelper {
 
@@ -32,7 +32,6 @@ public class TagAsRuleHelper {
 
     public static boolean interpretTagAsRule(String rule, String tags, long timeout) {
         String script = PARSE_TAGS + rule;
-        tags = String.format("'%s'", StringEscapeUtils.escapeEcmaScript(tags));
         try (JsInterpreter jsInterpreter = new JsInterpreter(timeout)) {
             jsInterpreter.injectVariable("tags", tags);
             Object scriptReturn = jsInterpreter.executeScript(script);

utils/src/test/java/org/apache/cloudstack/utils/jsinterpreter/JsInterpreterTest.java

@@ -20,6 +20,7 @@ package org.apache.cloudstack.utils.jsinterpreter;
 import java.io.IOException;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
@@ -27,7 +28,8 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeoutException;
 
-import com.cloud.utils.exception.CloudRuntimeException;
+import javax.script.ScriptEngine;
+
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -36,9 +38,10 @@ import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
+import org.openjdk.nashorn.api.scripting.ClassFilter;
 import org.openjdk.nashorn.api.scripting.NashornScriptEngineFactory;
 
-import javax.script.ScriptEngine;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 @RunWith(MockitoJUnitRunner.class)
 public class JsInterpreterTest {
@@ -61,30 +64,6 @@ public class JsInterpreterTest {
         Assert.assertTrue(executor.isShutdown());
     }
 
-    @Test
-    public void addVariablesToScriptTestVariablesMapIsEmptyReturnScript() {
-        String script = "a + b";
-        jsInterpreterSpy.variables = new LinkedHashMap<>();
-
-        String result = jsInterpreterSpy.addVariablesToScript(script);
-
-        Assert.assertEquals(script, result);
-    }
-
-    @Test
-    public void addVariablesToScriptTestVariablesMapIsNotEmptyInjectVariableToScript() {
-        String script = "a + b";
-        String var1 = "{test: \"test\"}";
-        jsInterpreterSpy.injectVariable("var1", var1);
-        jsInterpreterSpy.injectVariable("var2", var1);
-
-        String expected = String.format(" var1 = %s; var2 = %s; %s", var1, var1, script);
-
-        String result = jsInterpreterSpy.addVariablesToScript(script);
-
-        Assert.assertEquals(expected, result);
-    }
-
     @Test
     public void executeScriptTestReturnResultOfScriptExecution() {
         String script = "5";
@@ -154,7 +133,7 @@ public class JsInterpreterTest {
 
     @Test
     public void discardCurrentVariablesTestInstantiateNewMap() {
-        Map<String, String> variables = new LinkedHashMap<>();
+        Map<String, Object> variables = new LinkedHashMap<>();
         variables.put("a", "b");
         variables.put("b", null);
 
@@ -170,12 +149,14 @@ public class JsInterpreterTest {
         NashornScriptEngineFactory nashornScriptEngineFactoryMock = Mockito.spy(NashornScriptEngineFactory.class);
         ScriptEngine scriptEngineMock = Mockito.mock(ScriptEngine.class);
 
-        Mockito.doReturn(scriptEngineMock).when(nashornScriptEngineFactoryMock).getScriptEngine(Mockito.anyString());
+        Mockito.doReturn(scriptEngineMock).when(nashornScriptEngineFactoryMock).getScriptEngine(Mockito.any(),
+                Mockito.any(ClassLoader.class), Mockito.any(ClassFilter.class));
 
         jsInterpreterSpy.setScriptEngineDisablingJavaLanguage(nashornScriptEngineFactoryMock);
 
         Assert.assertEquals(scriptEngineMock, jsInterpreterSpy.interpreter);
-        Mockito.verify(nashornScriptEngineFactoryMock).getScriptEngine("--no-java");
+        Mockito.verify(nashornScriptEngineFactoryMock).getScriptEngine(Mockito.any(),
+                Mockito.any(ClassLoader.class), Mockito.any(ClassFilter.class));
     }
 
     @Test
@@ -193,6 +174,46 @@ public class JsInterpreterTest {
 
         jsInterpreterSpy.injectStringVariable("a", "b");
 
-        Assert.assertEquals(jsInterpreterSpy.variables.get("a"), "\"b\"");
+        Assert.assertEquals(jsInterpreterSpy.variables.get("a"), "b");
+    }
+
+    @Test
+    public void executeScriptTestValidScriptShouldPassWithMixedVariables() {
+        try (JsInterpreter jsInterpreter = new JsInterpreter(1000)) {
+            jsInterpreter.injectVariable("x", 10);
+            jsInterpreter.injectVariable("y", "hello");
+            jsInterpreter.injectVariable("z", true);
+            String validScript = "var result = x + (z ? 1 : 0); y + '-' + result;";
+            Object result = jsInterpreter.executeScript(validScript);
+            Assert.assertEquals("hello-11", result);
+        } catch (IOException exception) {
+            Assert.fail("IOException not expected here");
+        }
+    }
+
+    private void runMaliciousScriptFileTest(String script, String filename) {
+        try (JsInterpreter jsInterpreter = new JsInterpreter(1000)) {
+            jsInterpreter.executeScript(script);
+        } catch (CloudRuntimeException ex) {
+            Assert.assertTrue(ex.getMessage().contains("Unable to execute script"));
+            java.io.File f = new java.io.File(filename);
+            Assert.assertFalse(f.exists());
+        } catch (IOException exception) {
+            Assert.fail("IOException not expected here");
+        }
+    }
+
+    @Test
+    public void executeScriptTestMaliciousScriptShouldThrowException1() {
+        String filename = "/tmp/hack1-" + UUID.randomUUID();
+        String maliciousScript = "Java.type('java.lang.Runtime').getRuntime().exec('touch " + filename + "')";
+        runMaliciousScriptFileTest(maliciousScript, filename);
+    }
+
+    @Test
+    public void executeScriptTestMaliciousScriptShouldThrowException2() {
+        String filename = "/tmp/hack2-" + UUID.randomUUID();
+        String maliciousScript = "var e=this.engine.getFactory().getScriptEngine('-Dnashorn.args=--no-java=False'); e.eval(\"java.lang.Runtime.getRuntime().exec(['/bin/bash','-c','touch " + filename + "']);\");";
+        runMaliciousScriptFileTest(maliciousScript, filename);
     }
 }