apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-25 09:12:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '4.20'

Browse Source
This commit is contained in:
Harikrishna Patnala 2025-10-16 13:39:40 +05:30
commit 8b9f5fd8f9
36 changed files with 620 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
agent/src/main/java/com/cloud/agent/Agent.java
View File

@ -613,7 +613,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
}
protected String getAgentArch() {
String arch = Script.runSimpleBashScript(Script.getExecutableAbsolutePath("arch"), 1000);
String arch = Script.runSimpleBashScript(Script.getExecutableAbsolutePath("arch"), 2000);
logger.debug("Arch for agent: {} found: {}", _name, arch);
return arch;
}

13
api/src/main/java/com/cloud/server/ManagementService.java
View File

@ -20,7 +20,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.cloud.user.UserData;
import org.apache.cloudstack.api.command.admin.cluster.ListClustersCmd;
import org.apache.cloudstack.api.command.admin.config.ListCfgGroupsByCmd;
import org.apache.cloudstack.api.command.admin.config.ListCfgsByCmd;
@ -72,6 +71,7 @@ import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd;
import org.apache.cloudstack.api.command.user.vmgroup.UpdateVMGroupCmd;
import org.apache.cloudstack.config.Configuration;
import org.apache.cloudstack.config.ConfigurationGroup;
import org.apache.cloudstack.framework.config.ConfigKey;
import com.cloud.alert.Alert;
import com.cloud.capacity.Capacity;
@ -91,6 +91,7 @@ import com.cloud.storage.GuestOSHypervisor;
import com.cloud.storage.GuestOsCategory;
import com.cloud.storage.StoragePool;
import com.cloud.user.SSHKeyPair;
import com.cloud.user.UserData;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
import com.cloud.vm.InstanceGroup;
@ -104,6 +105,14 @@ import com.cloud.vm.VirtualMachine.Type;
public interface ManagementService {
static final String Name = "management-server";
ConfigKey<Boolean> JsInterpretationEnabled = new ConfigKey<>("Hidden"
, Boolean.class
, "js.interpretation.enabled"
, "false"
, "Enable/Disable all JavaScript interpretation related functionalities to create or update Javascript rules."
, false
, ConfigKey.Scope.Global);
/**
* returns the a map of the names/values in the configuration table
*
@ -509,4 +518,6 @@ public interface ManagementService {
boolean removeManagementServer(RemoveManagementServerCmd cmd);
void checkJsInterpretationAllowedIfNeededForParameterValue(String paramName, boolean paramValue);
}

1
api/src/main/java/com/cloud/server/ResourceManagerUtil.java
View File

@ -18,6 +18,7 @@ package com.cloud.server;
public interface ResourceManagerUtil {
long getResourceId(String resourceId, ResourceTag.ResourceObjectType resourceType);
long getResourceId(String resourceId, ResourceTag.ResourceObjectType resourceType, boolean checkAccess);
String getUuid(String resourceId, ResourceTag.ResourceObjectType resourceType);
ResourceTag.ResourceObjectType getResourceType(String resourceTypeStr);
void checkResourceAccessible(Long accountId, Long domainId, String exceptionMessage);

9
engine/components-api/src/main/java/com/cloud/template/TemplateManager.java
View File

@ -57,6 +57,13 @@ public interface TemplateManager {
+ "will validate if the provided URL is resolvable during the register of templates/ISOs before persisting them in the database.",
true);
ConfigKey<Boolean> TemplateDeleteFromPrimaryStorage = new ConfigKey<Boolean>("Advanced",
Boolean.class,
"template.delete.from.primary.storage", "true",
"Template when deleted will be instantly deleted from the Primary Storage",
true,
ConfigKey.Scope.Global);
static final String VMWARE_TOOLS_ISO = "vmware-tools.iso";
static final String XS_TOOLS_ISO = "xs-tools.iso";
@ -104,6 +111,8 @@ public interface TemplateManager {
*/
List<VMTemplateStoragePoolVO> getUnusedTemplatesInPool(StoragePoolVO pool);
void evictTemplateFromStoragePoolsForZones(Long templateId, List<Long> zoneId);
/**
* Deletes a template in the specified storage pool.
*

20
engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java
View File

@ -4799,6 +4799,18 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
}
});
if (selectedIp != null && GuestType.Shared.equals(network.getGuestType())) {
IPAddressVO ipAddressVO = _ipAddressDao.findByIpAndSourceNetworkId(network.getId(), selectedIp);
if (ipAddressVO != null && IpAddress.State.Free.equals(ipAddressVO.getState())) {
ipAddressVO.setState(IPAddressVO.State.Allocated);
ipAddressVO.setAllocatedTime(new Date());
Account account = _accountDao.findById(vm.getAccountId());
ipAddressVO.setAllocatedInDomainId(account.getDomainId());
ipAddressVO.setAllocatedToAccountId(account.getId());
_ipAddressDao.update(ipAddressVO.getId(), ipAddressVO);
}
}
final Integer networkRate = _networkModel.getNetworkRate(network.getId(), vm.getId());
final NicProfile vmNic = new NicProfile(vo, network, vo.getBroadcastUri(), vo.getIsolationUri(), networkRate, _networkModel.isSecurityGroupSupportedInNetwork(network),
_networkModel.getNetworkTag(vm.getHypervisorType(), network));
@ -4810,15 +4822,15 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
if (network.getGuestType() == GuestType.L2) {
return null;
}
return dataCenter.getNetworkType() == NetworkType.Basic ?
getSelectedIpForNicImportOnBasicZone(ipAddresses.getIp4Address(), network, dataCenter):
return GuestType.Shared.equals(network.getGuestType()) ?
getSelectedIpForNicImportOnSharedNetwork(ipAddresses.getIp4Address(), network, dataCenter):
_ipAddrMgr.acquireGuestIpAddress(network, ipAddresses.getIp4Address());
}
protected String getSelectedIpForNicImportOnBasicZone(String requestedIp, Network network, DataCenter dataCenter) {
protected String getSelectedIpForNicImportOnSharedNetwork(String requestedIp, Network network, DataCenter dataCenter) {
IPAddressVO ipAddressVO = StringUtils.isBlank(requestedIp) ?
_ipAddressDao.findBySourceNetworkIdAndDatacenterIdAndState(network.getId(), dataCenter.getId(), IpAddress.State.Free):
_ipAddressDao.findByIp(requestedIp);
_ipAddressDao.findByIpAndSourceNetworkId(network.getId(), requestedIp);
if (ipAddressVO == null || ipAddressVO.getState() != IpAddress.State.Free) {
String msg = String.format("Cannot find a free IP to assign to VM NIC on network %s", network.getName());
logger.error(msg);

2
engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestratorTest.java
View File

@ -822,7 +822,7 @@ public class NetworkOrchestratorTest extends TestCase {
Mockito.when(network.getId()).thenReturn(networkId);
Mockito.when(dataCenter.getId()).thenReturn(dataCenterId);
Mockito.when(ipAddresses.getIp4Address()).thenReturn(requestedIp);
Mockito.when(testOrchestrator._ipAddressDao.findByIp(requestedIp)).thenReturn(ipAddressVO);
Mockito.when(testOrchestrator._ipAddressDao.findByIpAndSourceNetworkId(networkId, requestedIp)).thenReturn(ipAddressVO);
String ipAddress = testOrchestrator.getSelectedIpForNicImport(network, dataCenter, ipAddresses);
Assert.assertEquals(requestedIp, ipAddress);
}

2
engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java
View File

@ -35,6 +35,8 @@ public interface VMTemplatePoolDao extends GenericDao<VMTemplateStoragePoolVO, L
List<VMTemplateStoragePoolVO> listByPoolIdAndState(long poolId, ObjectInDataStoreStateMachine.State state);
List<VMTemplateStoragePoolVO> listByPoolIdsAndTemplate(List<Long> poolIds, Long templateId);
List<VMTemplateStoragePoolVO> listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState);
List<VMTemplateStoragePoolVO> listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState, long poolId);

10
engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java
View File

@ -150,6 +150,16 @@ public class VMTemplatePoolDaoImpl extends GenericDaoBase<VMTemplateStoragePoolV
return findOneIncludingRemovedBy(sc);
}
@Override
public List<VMTemplateStoragePoolVO> listByPoolIdsAndTemplate(List<Long> poolIds, Long templateId) {
SearchCriteria<VMTemplateStoragePoolVO> sc = PoolTemplateSearch.create();
if (CollectionUtils.isNotEmpty(poolIds)) {
sc.setParameters("pool_id", poolIds.toArray());
}
sc.setParameters("template_id", templateId);
return listBy(sc);
}
@Override
public List<VMTemplateStoragePoolVO> listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState) {
SearchCriteria<VMTemplateStoragePoolVO> sc = TemplateStatusSearch.create();

1
engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java
View File

@ -171,4 +171,5 @@ public interface PrimaryDataStoreDao extends GenericDao<StoragePoolVO, Long> {
List<StoragePoolVO> findPoolsByStorageTypeAndZone(Storage.StoragePoolType storageType, Long zoneId);
List<StoragePoolVO> listByDataCenterIds(List<Long> dataCenterIds);
}

14
engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java
View File

@ -65,6 +65,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
private final GenericSearchBuilder<StoragePoolVO, Long> StatusCountSearch;
private final SearchBuilder<StoragePoolVO> ClustersSearch;
private final SearchBuilder<StoragePoolVO> IdsSearch;
private final SearchBuilder<StoragePoolVO> DcsSearch;
@Inject
private StoragePoolDetailsDao _detailsDao;
@ -167,6 +168,9 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
IdsSearch.and("ids", IdsSearch.entity().getId(), SearchCriteria.Op.IN);
IdsSearch.done();
DcsSearch = createSearchBuilder();
DcsSearch.and("dataCenterId", DcsSearch.entity().getDataCenterId(), SearchCriteria.Op.IN);
DcsSearch.done();
}
@Override
@ -927,6 +931,16 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
return listBy(sc);
}
@Override
public List<StoragePoolVO> listByDataCenterIds(List<Long> dataCenterIds) {
if (CollectionUtils.isEmpty(dataCenterIds)) {
return Collections.emptyList();
}
SearchCriteria<StoragePoolVO> sc = DcsSearch.create();
sc.setParameters("dataCenterId", dataCenterIds.toArray());
return listBy(sc);
}
private SearchCriteria<StoragePoolVO> createStoragePoolSearchCriteria(Long storagePoolId, String storagePoolName,
Long zoneId, String path, Long podId, Long clusterId, Long hostId, String address, ScopeType scopeType,
StoragePoolStatus status, String keyword, String storageAccessGroup) {

4
framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java
View File

@ -32,7 +32,6 @@ import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.user.Account;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.quota.activationrule.presetvariables.Configuration;
import org.apache.cloudstack.quota.activationrule.presetvariables.GenericPresetVariable;
@ -63,6 +62,7 @@ import org.springframework.stereotype.Component;
import com.cloud.usage.UsageVO;
import com.cloud.usage.dao.UsageDao;
import com.cloud.user.Account;
import com.cloud.user.AccountVO;
import com.cloud.user.dao.AccountDao;
import com.cloud.utils.DateUtil;
@ -473,7 +473,7 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager {
jsInterpreter.injectVariable("configuration", configuration.toString());
}
jsInterpreter.injectStringVariable("resourceType", presetVariables.getResourceType());
jsInterpreter.injectVariable("resourceType", presetVariables.getResourceType());
jsInterpreter.injectVariable("value", presetVariables.getValue().toString());
jsInterpreter.injectVariable("zone", presetVariables.getZone().toString());
}

5
framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java
View File

@ -270,8 +270,7 @@ public class QuotaManagerImplTest {
Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("account"), Mockito.anyString());
Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("domain"), Mockito.anyString());
Mockito.verify(jsInterpreterMock, Mockito.never()).injectVariable(Mockito.eq("project"), Mockito.anyString());
Mockito.verify(jsInterpreterMock, Mockito.never()).injectVariable(Mockito.eq("configuration"), Mockito.anyString());
Mockito.verify(jsInterpreterMock).injectStringVariable(Mockito.eq("resourceType"), Mockito.anyString());
Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("resourceType"), Mockito.anyString());
Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("value"), Mockito.anyString());
Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("zone"), Mockito.anyString());
}
@ -292,7 +291,7 @@ public class QuotaManagerImplTest {
Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("account"), Mockito.anyString());
Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("domain"), Mockito.anyString());
Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("project"), Mockito.anyString());
Mockito.verify(jsInterpreterMock).injectStringVariable(Mockito.eq("resourceType"), Mockito.anyString());
Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("resourceType"), Mockito.anyString());
Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("value"), Mockito.anyString());
Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("zone"), Mockito.anyString());
}

19
plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java
View File

@ -80,8 +80,8 @@ import org.apache.cloudstack.quota.dao.QuotaCreditsDao;
import org.apache.cloudstack.quota.dao.QuotaEmailConfigurationDao;
import org.apache.cloudstack.quota.dao.QuotaEmailTemplatesDao;
import org.apache.cloudstack.quota.dao.QuotaTariffDao;
import org.apache.cloudstack.quota.vo.QuotaAccountVO;
import org.apache.cloudstack.quota.dao.QuotaUsageDao;
import org.apache.cloudstack.quota.vo.QuotaAccountVO;
import org.apache.cloudstack.quota.vo.QuotaBalanceVO;
import org.apache.cloudstack.quota.vo.QuotaCreditsVO;
import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO;
@ -90,15 +90,17 @@ import org.apache.cloudstack.quota.vo.QuotaTariffVO;
import org.apache.cloudstack.quota.vo.QuotaUsageVO;
import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter;
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.domain.DomainVO;
import com.cloud.domain.dao.DomainDao;
import com.cloud.event.ActionEvent;
import com.cloud.event.EventTypes;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
@ -107,8 +109,6 @@ import com.cloud.user.dao.AccountDao;
import com.cloud.user.dao.UserDao;
import com.cloud.utils.Pair;
import com.cloud.utils.db.Filter;
import com.cloud.event.ActionEvent;
import com.cloud.event.EventTypes;
@Component
public class QuotaResponseBuilderImpl implements QuotaResponseBuilder {
@ -150,6 +150,11 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder {
private final Class<?>[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class};
protected void checkActivationRulesAllowed(String activationRule) {
if (!_quotaService.isJsInterpretationEnabled() && StringUtils.isNotEmpty(activationRule)) {
throw new PermissionDeniedException("Quota Tariff Activation Rule cannot be set, as Javascript interpretation is disabled in the configuration.");
}
}
@Override
public QuotaTariffResponse createQuotaTariffResponse(QuotaTariffVO tariff, boolean returnActivationRule) {
@ -452,6 +457,8 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder {
throw new InvalidParameterValueException(String.format("There is no quota tariffs with name [%s].", name));
}
checkActivationRulesAllowed(activationRule);
Date currentQuotaTariffStartDate = currentQuotaTariff.getEffectiveOn();
currentQuotaTariff.setRemoved(now);
@ -706,6 +713,8 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder {
throw new InvalidParameterValueException(String.format("A quota tariff with name [%s] already exist.", name));
}
checkActivationRulesAllowed(activationRule);
if (startDate.compareTo(now) < 0) {
throw new InvalidParameterValueException(String.format("The value passed as Quota tariff's start date is in the past: [%s]. " +
"Please, inform a date in the future or do not pass the parameter to use the current date and time.", startDate));

12
plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java
View File

@ -16,15 +16,15 @@
//under the License.
package org.apache.cloudstack.quota;
import com.cloud.user.AccountVO;
import com.cloud.utils.component.PluggableService;
import java.math.BigDecimal;
import java.util.Date;
import java.util.List;
import org.apache.cloudstack.quota.vo.QuotaBalanceVO;
import org.apache.cloudstack.quota.vo.QuotaUsageVO;
import java.math.BigDecimal;
import java.util.Date;
import java.util.List;
import com.cloud.user.AccountVO;
import com.cloud.utils.component.PluggableService;
public interface QuotaService extends PluggableService {
@ -40,4 +40,6 @@ public interface QuotaService extends PluggableService {
boolean saveQuotaAccount(AccountVO account, BigDecimal aggrUsage, Date endDate);
boolean isJsInterpretationEnabled();
}

9
plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java
View File

@ -62,6 +62,7 @@ import com.cloud.configuration.Config;
import com.cloud.domain.dao.DomainDao;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.PermissionDeniedException;
import com.cloud.server.ManagementService;
import com.cloud.user.Account;
import com.cloud.user.AccountVO;
import com.cloud.user.dao.AccountDao;
@ -88,6 +89,8 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi
private TimeZone _usageTimezone;
private boolean jsInterpretationEnabled = false;
public QuotaServiceImpl() {
super();
}
@ -99,6 +102,8 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi
String timeZoneStr = ObjectUtils.defaultIfNull(_configDao.getValue(Config.UsageAggregationTimezone.toString()), "GMT");
_usageTimezone = TimeZone.getTimeZone(timeZoneStr);
jsInterpretationEnabled = ManagementService.JsInterpretationEnabled.value();
return true;
}
@ -288,4 +293,8 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi
}
}
@Override
public boolean isJsInterpretationEnabled() {
return jsInterpretationEnabled;
}
}

76
plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java
View File

@ -22,6 +22,7 @@ import static com.cloud.utils.NumbersUtil.toReadableSize;
import java.lang.reflect.InvocationTargetException;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -32,8 +33,6 @@ import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.dc.ClusterVO;
import com.cloud.utils.Ternary;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.ListClustersMetricsCmd;
import org.apache.cloudstack.api.ListDbMetricsCmd;
@ -102,6 +101,7 @@ import com.cloud.capacity.CapacityManager;
import com.cloud.capacity.dao.CapacityDao;
import com.cloud.capacity.dao.CapacityDaoImpl;
import com.cloud.cluster.dao.ManagementServerHostDao;
import com.cloud.dc.ClusterVO;
import com.cloud.dc.DataCenter;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.dc.dao.DataCenterDao;
@ -114,6 +114,7 @@ import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
import com.cloud.network.router.VirtualRouter;
import com.cloud.org.Cluster;
import com.cloud.projects.Project;
import com.cloud.server.DbStatsCollection;
import com.cloud.server.ManagementServerHostStats;
import com.cloud.server.StatsCollector;
@ -126,6 +127,7 @@ import com.cloud.usage.dao.UsageJobDao;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
import com.cloud.utils.db.DbProperties;
import com.cloud.utils.db.DbUtil;
import com.cloud.utils.db.Filter;
@ -188,6 +190,10 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
private static Gson gson = new Gson();
private final List<Account.Type> AccountTypesWithRecursiveUsageAccess = Arrays.asList(
Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN, Account.Type.READ_ONLY_ADMIN
);
protected MetricsServiceImpl() {
super();
}
@ -249,17 +255,30 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
* @return the list of VMs.
*/
protected Pair<List<UserVmVO>, Integer> searchForUserVmsInternal(ListVMsUsageHistoryCmd cmd) {
final Long id = cmd.getId();
Account caller = CallContext.current().getCallingAccount();
List<Long> permittedAccounts = new ArrayList<>();
boolean recursive = AccountTypesWithRecursiveUsageAccess.contains(caller.getType());
Ternary<Long, Boolean, Project.ListProjectResourcesCriteria> domainIdRecursiveListProject = new Ternary<>(null, recursive, null);
accountMgr.buildACLSearchParameters(caller, id, null, null, permittedAccounts, domainIdRecursiveListProject, true, false);
Long domainId = domainIdRecursiveListProject.first();
Boolean isRecursive = domainIdRecursiveListProject.second();
Project.ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third();
Filter searchFilter = new Filter(UserVmVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal());
List<Long> ids = getIdsListFromCmd(cmd.getId(), cmd.getIds());
String name = cmd.getName();
String keyword = cmd.getKeyword();
SearchBuilder<UserVmVO> sb = userVmDao.createSearchBuilder();
sb.select(null, SearchCriteria.Func.DISTINCT, sb.entity().getId()); // select distinct
accountMgr.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
sb.and("idIN", sb.entity().getId(), SearchCriteria.Op.IN);
sb.and("displayName", sb.entity().getDisplayName(), SearchCriteria.Op.LIKE);
sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ);
SearchCriteria<UserVmVO> sc = sb.create();
accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
if (CollectionUtils.isNotEmpty(ids)) {
sc.setParameters("idIN", ids.toArray());
}
@ -273,7 +292,14 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
sc.addAnd("displayName", SearchCriteria.Op.SC, ssc);
}
return userVmDao.searchAndCount(sc, searchFilter);
Pair<List<UserVmVO>, Integer> uniqueVmPair = userVmDao.searchAndCount(sc, searchFilter);
Integer count = uniqueVmPair.second();
if (count == 0) {
return new Pair<>(new ArrayList<>(), count);
}
List<Long> vmIds = uniqueVmPair.first().stream().map(UserVmVO::getId).collect(Collectors.toList());
List<UserVmVO> vms = userVmDao.listByIds(vmIds);
return new Pair<>(vms, count);
}
/**
@ -335,17 +361,49 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
* @return the list of VMs.
*/
protected Pair<List<VolumeVO>, Integer> searchForVolumesInternal(ListVolumesUsageHistoryCmd cmd) {
final Long id = cmd.getId();
Account caller = CallContext.current().getCallingAccount();
List<Long> permittedAccounts = new ArrayList<>();
boolean recursive = AccountTypesWithRecursiveUsageAccess.contains(caller.getType());
Ternary<Long, Boolean, Project.ListProjectResourcesCriteria> domainIdRecursiveListProject = new Ternary<>(null, recursive, null);
accountMgr.buildACLSearchParameters(caller, id, null, null, permittedAccounts, domainIdRecursiveListProject, true, false);
Long domainId = domainIdRecursiveListProject.first();
Boolean isRecursive = domainIdRecursiveListProject.second();
Project.ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third();
Filter searchFilter = new Filter(VolumeVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal());
List<Long> ids = getIdsListFromCmd(cmd.getId(), cmd.getIds());
String name = cmd.getName();
String keyword = cmd.getKeyword();
SearchBuilder<VolumeVO> sb = volumeDao.createSearchBuilder();
sb.select(null, SearchCriteria.Func.DISTINCT, sb.entity().getId()); // select distinct
accountMgr.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
sb.and("idIN", sb.entity().getId(), SearchCriteria.Op.IN);
sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ);
sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ);
boolean shouldListSystemVmVolumes = accountMgr.isRootAdmin(CallContext.current().getCallingAccountId());
List<Long> vmIds = new ArrayList<>();
if (!shouldListSystemVmVolumes) {
SearchBuilder<UserVmVO> vmSearch = userVmDao.createSearchBuilder();
vmSearch.select(null, SearchCriteria.Func.DISTINCT, vmSearch.entity().getId());
accountMgr.buildACLSearchBuilder(vmSearch, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
SearchCriteria<UserVmVO> vmSc = vmSearch.create();
accountMgr.buildACLSearchCriteria(vmSc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
List<UserVmVO> vms = userVmDao.search(vmSc, null);
vmIds = vms.stream().map(UserVmVO::getId).collect(Collectors.toList());
if (vmIds.isEmpty()) {
sb.and("instanceIdNull", sb.entity().getInstanceId(), SearchCriteria.Op.NULL);
} else {
sb.and().op("instanceIdNull", sb.entity().getInstanceId(), SearchCriteria.Op.NULL);
sb.or("instanceIds", sb.entity().getInstanceId(), SearchCriteria.Op.IN);
sb.cp();
}
}
SearchCriteria<VolumeVO> sc = sb.create();
accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria);
if (CollectionUtils.isNotEmpty(ids)) {
sc.setParameters("idIN", ids.toArray());
}
@ -358,8 +416,18 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements
ssc.addOr("state", SearchCriteria.Op.EQ, keyword);
sc.addAnd("name", SearchCriteria.Op.SC, ssc);
}
if (!shouldListSystemVmVolumes && CollectionUtils.isNotEmpty(vmIds)) {
sc.setParameters("instanceIds", vmIds.toArray());
}
return volumeDao.searchAndCount(sc, searchFilter);
Pair<List<VolumeVO>, Integer> uniqueVolumePair = volumeDao.searchAndCount(sc, searchFilter);
Integer count = uniqueVolumePair.second();
if (count == 0) {
return new Pair<>(new ArrayList<>(), count);
}
List<Long> volumeIds = uniqueVolumePair.first().stream().map(VolumeVO::getId).collect(Collectors.toList());
List<VolumeVO> volumes = volumeDao.listByIds(volumeIds);
return new Pair<>(volumes, count);
}
/**

155
plugins/metrics/src/test/java/org/apache/cloudstack/metrics/MetricsServiceImplTest.java
View File

@ -19,13 +19,16 @@ package org.apache.cloudstack.metrics;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.cloudstack.api.ListVMsUsageHistoryCmd;
import org.apache.cloudstack.api.ListVolumesUsageHistoryCmd;
import org.apache.cloudstack.api.response.ListResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.response.VmMetricsStatsResponse;
import org.apache.commons.lang3.time.DateUtils;
import org.junit.Assert;
@ -35,12 +38,18 @@ import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.utils.Pair;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.vm.UserVmVO;
@ -75,6 +84,12 @@ public class MetricsServiceImplTest {
@Mock
VmStatsDao vmStatsDaoMock;
@Mock
AccountManager accountManager;
@Mock
VolumeDao volumeDao;
@Captor
ArgumentCaptor<String> stringCaptor1, stringCaptor2;
@ -95,12 +110,26 @@ public class MetricsServiceImplTest {
@Mock
VmStatsVO vmStatsVOMock;
@Mock
Account mockAccount;
@Mock
ListVolumesUsageHistoryCmd listVolumesUsageHistoryCmdMock;
@Mock
VolumeVO volumeVOMock;
@Mock
SearchBuilder<VolumeVO> volumeSearchBuilderMock;
@Mock
SearchCriteria<VolumeVO> volumeSearchCriteriaMock;
@Mock
Filter filterMock;
private void prepareSearchCriteriaWhenUseSetParameters() {
Mockito.doNothing().when(scMock).setParameters(Mockito.anyString(), Mockito.any());
}
private void preparesearchForUserVmsInternalTest() {
Mockito.when(mockAccount.getType()).thenReturn(Account.Type.NORMAL);
expectedVmListAndCounter = new Pair<>(Arrays.asList(userVmVOMock), 1);
Mockito.doReturn(1L).when(listVMsUsageHistoryCmdMock).getStartIndex();
@ -111,8 +140,10 @@ public class MetricsServiceImplTest {
Mockito.doReturn(userVmVOMock).when(sbMock).entity();
Mockito.doReturn(scMock).when(sbMock).create();
Mockito.doReturn(new Pair<List<UserVmVO>, Integer>(Arrays.asList(userVmVOMock), 1))
Mockito.doReturn(expectedVmListAndCounter)
.when(userVmDaoMock).searchAndCount(Mockito.any(), Mockito.any());
Mockito.doReturn(expectedVmListAndCounter.first())
.when(userVmDaoMock).listByIds(Mockito.anyList());
}
@Test
@ -124,12 +155,17 @@ public class MetricsServiceImplTest {
Mockito.doReturn(null).when(listVMsUsageHistoryCmdMock).getName();
Mockito.doReturn(null).when(listVMsUsageHistoryCmdMock).getKeyword();
Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
CallContext callContextMock = Mockito.mock(CallContext.class);
callContextMocked.when(CallContext::current).thenReturn(callContextMock);
Mockito.when(callContextMock.getCallingAccount()).thenReturn(mockAccount);
Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
Mockito.verify(scMock).setParameters(stringCaptor1.capture(), objectArrayCaptor.capture());
Assert.assertEquals("idIN", stringCaptor1.getValue());
Assert.assertEquals(fakeVmId1, objectArrayCaptor.getAllValues().get(0)[0]);
Assert.assertEquals(expectedVmListAndCounter, result);
Mockito.verify(scMock).setParameters(stringCaptor1.capture(), objectArrayCaptor.capture());
Assert.assertEquals("idIN", stringCaptor1.getValue());
Assert.assertEquals(fakeVmId1, objectArrayCaptor.getAllValues().get(0)[0]);
Assert.assertEquals(expectedVmListAndCounter, result);
}
}
@Test
@ -141,13 +177,17 @@ public class MetricsServiceImplTest {
Mockito.doReturn(expected).when(listVMsUsageHistoryCmdMock).getIds();
Mockito.doReturn(null).when(listVMsUsageHistoryCmdMock).getName();
Mockito.doReturn(null).when(listVMsUsageHistoryCmdMock).getKeyword();
try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
CallContext callContextMock = Mockito.mock(CallContext.class);
callContextMocked.when(CallContext::current).thenReturn(callContextMock);
Mockito.when(callContextMock.getCallingAccount()).thenReturn(mockAccount);
Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
Mockito.verify(scMock).setParameters(stringCaptor1.capture(), objectArrayCaptor.capture());
Assert.assertEquals("idIN", stringCaptor1.getValue());
Assert.assertArrayEquals(expected.toArray(), objectArrayCaptor.getAllValues().get(0));
Assert.assertEquals(expectedVmListAndCounter, result);
Mockito.verify(scMock).setParameters(stringCaptor1.capture(), objectArrayCaptor.capture());
Assert.assertEquals("idIN", stringCaptor1.getValue());
Assert.assertArrayEquals(expected.toArray(), objectArrayCaptor.getAllValues().get(0));
Assert.assertEquals(expectedVmListAndCounter, result);
}
}
@Test
@ -159,12 +199,17 @@ public class MetricsServiceImplTest {
Mockito.doReturn("fakeName").when(listVMsUsageHistoryCmdMock).getName();
Mockito.doReturn(null).when(listVMsUsageHistoryCmdMock).getKeyword();
Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
CallContext callContextMock = Mockito.mock(CallContext.class);
callContextMocked.when(CallContext::current).thenReturn(callContextMock);
Mockito.when(callContextMock.getCallingAccount()).thenReturn(mockAccount);
Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
Mockito.verify(scMock).setParameters(stringCaptor1.capture(), objectArrayCaptor.capture());
Assert.assertEquals("displayName", stringCaptor1.getValue());
Assert.assertEquals("%fakeName%", objectArrayCaptor.getValue()[0]);
Assert.assertEquals(expectedVmListAndCounter, result);
Mockito.verify(scMock).setParameters(stringCaptor1.capture(), objectArrayCaptor.capture());
Assert.assertEquals("displayName", stringCaptor1.getValue());
Assert.assertEquals("%fakeName%", objectArrayCaptor.getValue()[0]);
Assert.assertEquals(expectedVmListAndCounter, result);
}
}
@Test
@ -177,16 +222,21 @@ public class MetricsServiceImplTest {
Mockito.doReturn(null).when(listVMsUsageHistoryCmdMock).getName();
Mockito.doReturn("fakeKeyword").when(listVMsUsageHistoryCmdMock).getKeyword();
Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
CallContext callContextMock = Mockito.mock(CallContext.class);
callContextMocked.when(CallContext::current).thenReturn(callContextMock);
Mockito.when(callContextMock.getCallingAccount()).thenReturn(mockAccount);
Pair<List<UserVmVO>, Integer> result = spy.searchForUserVmsInternal(listVMsUsageHistoryCmdMock);
Mockito.verify(scMock, Mockito.times(2)).addOr(stringCaptor1.capture(), opCaptor.capture(), objectArrayCaptor.capture());
List<String> conditions = stringCaptor1.getAllValues();
List<Object[]> params = objectArrayCaptor.getAllValues();
Assert.assertEquals("displayName", conditions.get(0));
Assert.assertEquals("state", conditions.get(1));
Assert.assertEquals("%fakeKeyword%", params.get(0)[0]);
Assert.assertEquals("fakeKeyword", params.get(1)[0]);
Assert.assertEquals(expectedVmListAndCounter, result);
Mockito.verify(scMock, Mockito.times(2)).addOr(stringCaptor1.capture(), opCaptor.capture(), objectArrayCaptor.capture());
List<String> conditions = stringCaptor1.getAllValues();
List<Object[]> params = objectArrayCaptor.getAllValues();
Assert.assertEquals("displayName", conditions.get(0));
Assert.assertEquals("state", conditions.get(1));
Assert.assertEquals("%fakeKeyword%", params.get(0)[0]);
Assert.assertEquals("fakeKeyword", params.get(1)[0]);
Assert.assertEquals(expectedVmListAndCounter, result);
}
}
@Test
@ -317,4 +367,57 @@ public class MetricsServiceImplTest {
spy.createStatsResponse(Arrays.asList(vmStatsVOMock));
}
@Test
public void searchForVolumesInternalWithValidParameters() {
Mockito.doReturn(null).when(listVolumesUsageHistoryCmdMock).getId();
Mockito.doReturn(Arrays.asList(1L, 2L)).when(listVolumesUsageHistoryCmdMock).getIds();
Mockito.doReturn("volumeName").when(listVolumesUsageHistoryCmdMock).getName();
Mockito.doReturn("keyword").when(listVolumesUsageHistoryCmdMock).getKeyword();
Mockito.doReturn(volumeSearchBuilderMock).when(volumeDao).createSearchBuilder();
Mockito.doReturn(volumeVOMock).when(volumeSearchBuilderMock).entity();
SearchBuilder vmSearchBuilderMock = Mockito.mock(SearchBuilder.class);
Mockito.doReturn(vmSearchBuilderMock).when(userVmDaoMock).createSearchBuilder();
Mockito.doReturn(userVmVOMock).when(vmSearchBuilderMock).entity();
Mockito.doReturn(volumeSearchCriteriaMock).when(volumeSearchBuilderMock).create();
Mockito.doReturn(volumeSearchCriteriaMock).when(volumeDao).createSearchCriteria();
Mockito.doReturn(new Pair<>(Arrays.asList(volumeVOMock), 1)).when(volumeDao).searchAndCount(Mockito.any(), Mockito.any());
Mockito.doReturn(Arrays.asList(volumeVOMock)).when(volumeDao).listByIds(Mockito.anyList());
try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
CallContext callContextMock = Mockito.mock(CallContext.class);
callContextMocked.when(CallContext::current).thenReturn(callContextMock);
Mockito.when(callContextMock.getCallingAccount()).thenReturn(mockAccount);
Pair<List<VolumeVO>, Integer> result = spy.searchForVolumesInternal(listVolumesUsageHistoryCmdMock);
Assert.assertNotNull(result);
Assert.assertEquals(1, result.second().intValue());
Assert.assertEquals(volumeVOMock, result.first().get(0));
}
}
@Test
public void searchForVolumesInternalWithValidParametersNoItem() {
Mockito.doReturn(1L).when(listVolumesUsageHistoryCmdMock).getId();
Mockito.doReturn(volumeSearchBuilderMock).when(volumeDao).createSearchBuilder();
Mockito.doReturn(volumeVOMock).when(volumeSearchBuilderMock).entity();
Mockito.doReturn(volumeSearchCriteriaMock).when(volumeSearchBuilderMock).create();
Mockito.doReturn(new Pair<>(Collections.emptyList(), 0)).when(volumeDao).searchAndCount(Mockito.any(), Mockito.any());
try (MockedStatic<CallContext> callContextMocked = Mockito.mockStatic(CallContext.class)) {
CallContext callContextMock = Mockito.mock(CallContext.class);
callContextMocked.when(CallContext::current).thenReturn(callContextMock);
Mockito.when(callContextMock.getCallingAccount()).thenReturn(mockAccount);
Mockito.when(callContextMock.getCallingAccountId()).thenReturn(1L);
Mockito.when(accountManager.isRootAdmin(1L)).thenReturn(true);
Mockito.when(mockAccount.getType()).thenReturn(Account.Type.ADMIN);
Pair<List<VolumeVO>, Integer> result = spy.searchForVolumesInternal(listVolumesUsageHistoryCmdMock);
Assert.assertNotNull(result);
Assert.assertEquals(0, result.second().intValue());
}
}
}

2
pom.xml
View File

@ -156,7 +156,7 @@
<cs.jakarta.xml.bind.version>2.3.3</cs.jakarta.xml.bind.version>
<cs.jaxws.version>2.3.7</cs.jaxws.version>
<cs.jersey-client.version>2.26</cs.jersey-client.version>
<cs.jetty.version>9.4.51.v20230217</cs.jetty.version>
<cs.jetty.version>9.4.58.v20250814</cs.jetty.version>
<cs.jetty-maven-plugin.version>9.4.27.v20200227</cs.jetty-maven-plugin.version>
<cs.jna.version>5.5.0</cs.jna.version>
<cs.joda-time.version>2.12.5</cs.joda-time.version>

8
python/lib/cloudutils/configFileOps.py
View File

@ -68,14 +68,14 @@ class configFileOps:
for entry in self.entries:
if entry.op == "add":
if entry.separator == "=":
matchString = "^\ *" + entry.name + ".*"
matchString = r"^\ *" + entry.name + ".*"
elif entry.separator == " ":
matchString = "^\ *" + entry.name + "\ *" + entry.value
matchString = r"^\ *" + entry.name + r"\ *" + entry.value
else:
if entry.separator == "=":
matchString = "^\ *" + entry.name + "\ *=\ *" + entry.value
matchString = r"^\ *" + entry.name + r"\ *=\ *" + entry.value
else:
matchString = "^\ *" + entry.name + "\ *" + entry.value
matchString = r"^\ *" + entry.name + r"\ *" + entry.value
match = re.match(matchString, line)
if match is not None:

13
python/lib/cloudutils/networkConfig.py
View File

@ -45,8 +45,11 @@ class networkConfig:
if not cmd.isSuccess():
logging.debug("Failed to get default route")
raise CloudRuntimeException("Failed to get default route")
result = cmd.getStdout().split(" ")
output = cmd.getStdout().strip()
result = output.split(" ")
if len(result) < 2:
logging.debug("Output for the default route incomplete: %s. It should have been '<GATEWAY> <DEVICE>'" % output)
raise CloudRuntimeException("Output for the default route incomplete")
gateway = result[0]
dev = result[1]
@ -150,10 +153,10 @@ class networkConfig:
if line.find("HWaddr") != -1:
macAddr = line.split("HWaddr ")[1].strip(" ")
elif line.find("inet ") != -1:
m = re.search("addr:(.*)\ *Bcast:(.*)\ *Mask:(.*)", line)
m = re.search(r"addr:([^\s]+)\s*Bcast:([^\s]+)\s*Mask:([^\s]+)", line)
if m is not None:
ipAddr = m.group(1).rstrip(" ")
netmask = m.group(3).rstrip(" ")
ipAddr = m.group(1).strip()
netmask = m.group(3).strip()
if networkConfig.isBridgePort(dev):
type = "brport"

2
python/lib/cloudutils/utilities.py
View File

@ -63,7 +63,7 @@ class bash:
return self.stdout.decode('utf-8').strip('\n')
def getLines(self):
return self.stdout.decode('utf-8').strip('\n')
return self.stdout.decode('utf-8').strip('\n').split('\n')
def getStderr(self):
return self.stderr.decode('utf-8').strip('\n')

2
server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
View File

@ -5653,7 +5653,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
//Validation - 1.3
if (resourceIdStr != null) {
resourceId = resourceManagerUtil.getResourceId(resourceIdStr, resourceType);
resourceId = resourceManagerUtil.getResourceId(resourceIdStr, resourceType, true);
}
List<? extends ResourceDetail> detailList = new ArrayList<>();

3
server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java
View File

@ -488,6 +488,8 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
throw new InvalidParameterValueException("Cannot create Network ACL Item. ACL Id or network Id is required");
}
Network network = networkModel.getNetwork(createNetworkACLCmd.getNetworkId());
Account caller = CallContext.current().getCallingAccount();
_accountMgr.checkAccess(caller, null, true, network);
if (network.getVpcId() == null) {
throw new InvalidParameterValueException("Network: " + network.getUuid() + " does not belong to VPC");
}
@ -749,6 +751,7 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
if (networkId != null) {
final Network network = _networkDao.findById(networkId);
_accountMgr.checkAccess(caller, null, true, network);
aclId = network.getNetworkACLId();
if (aclId == null) {
// No aclId associated with the network.

10
server/src/main/java/com/cloud/resource/ResourceManagerImpl.java
View File

@ -179,6 +179,7 @@ import com.cloud.org.Cluster;
import com.cloud.org.Grouping;
import com.cloud.org.Managed;
import com.cloud.serializer.GsonHelper;
import com.cloud.server.ManagementService;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.service.dao.ServiceOfferingDetailsDao;
@ -307,6 +308,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
private UserVmManager userVmManager;
@Inject
private GpuService gpuService;
@Inject
ManagementService managementService;
private List<? extends Discoverer> _discoverers;
@ -2818,8 +2821,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
@Override
public Host updateHost(final UpdateHostCmd cmd) throws NoTransitionException {
return updateHost(cmd.getId(), cmd.getName(), cmd.getOsCategoryId(), cmd.getAllocationState(), cmd.getUrl(),
cmd.getHostTags(), cmd.getIsTagARule(), cmd.getAnnotation(), false,
managementService.checkJsInterpretationAllowedIfNeededForParameterValue(ApiConstants.IS_TAG_A_RULE,
Boolean.TRUE.equals(cmd.getIsTagARule()));
return updateHost(cmd.getId(), cmd.getName(), cmd.getOsCategoryId(),
cmd.getAllocationState(), cmd.getUrl(), cmd.getHostTags(), cmd.getIsTagARule(), cmd.getAnnotation(), false,
cmd.getExternalDetails(), cmd.isCleanupExternalDetails());
}

22
server/src/main/java/com/cloud/server/ManagementServerImpl.java
View File

@ -1067,6 +1067,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
protected List<DeploymentPlanner> _planners;
private boolean jsInterpretationEnabled = false;
private final List<HypervisorType> supportedHypervisors = new ArrayList<>();
public List<DeploymentPlanner> getPlanners() {
@ -1147,6 +1149,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
supportedHypervisors.add(HypervisorType.KVM);
supportedHypervisors.add(HypervisorType.XenServer);
jsInterpretationEnabled = JsInterpretationEnabled.value();
return true;
}
@ -4220,8 +4224,10 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
cmdList.add(ListGuestVlansCmd.class);
cmdList.add(AssignVolumeCmd.class);
cmdList.add(ListSecondaryStorageSelectorsCmd.class);
cmdList.add(CreateSecondaryStorageSelectorCmd.class);
cmdList.add(UpdateSecondaryStorageSelectorCmd.class);
if (jsInterpretationEnabled) {
cmdList.add(CreateSecondaryStorageSelectorCmd.class);
cmdList.add(UpdateSecondaryStorageSelectorCmd.class);
}
cmdList.add(RemoveSecondaryStorageSelectorCmd.class);
cmdList.add(ListAffectedVmsForStorageScopeChangeCmd.class);
cmdList.add(ListGuiThemesCmd.class);
@ -4275,7 +4281,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] {exposeCloudStackVersionInApiXmlResponse, exposeCloudStackVersionInApiListCapabilities, vmPasswordLength, sshKeyLength, humanReadableSizes, customCsIdentifier};
return new ConfigKey<?>[] {exposeCloudStackVersionInApiXmlResponse, exposeCloudStackVersionInApiListCapabilities, vmPasswordLength, sshKeyLength, humanReadableSizes, customCsIdentifier,
JsInterpretationEnabled};
}
protected class EventPurgeTask extends ManagedContextRunnable {
@ -5780,4 +5787,13 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
public Answer getExternalVmConsole(VirtualMachine vm, Host host) {
return extensionsManager.getInstanceConsole(vm, host);
}
@Override
public void checkJsInterpretationAllowedIfNeededForParameterValue(String paramName, boolean paramValue) {
if (!paramValue || jsInterpretationEnabled) {
return;
}
throw new InvalidParameterValueException(String.format(
"The parameter %s cannot be set to true as JS interpretation is disabled",
paramName));
}
}

9
server/src/main/java/com/cloud/storage/StorageManagerImpl.java
View File

@ -221,6 +221,7 @@ import com.cloud.org.Grouping.AllocationState;
import com.cloud.resource.ResourceState;
import com.cloud.server.ConfigurationServer;
import com.cloud.server.ManagementServer;
import com.cloud.server.ManagementService;
import com.cloud.server.StatsCollector;
import com.cloud.service.dao.ServiceOfferingDetailsDao;
import com.cloud.storage.Storage.ImageFormat;
@ -414,6 +415,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
ResourceManager _resourceMgr;
@Inject
StorageManager storageManager;
@Inject
ManagementService managementService;
protected List<StoragePoolDiscoverer> _discoverers;
@ -1031,6 +1034,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
throw new PermissionDeniedException(String.format("Cannot perform this operation, Zone is currently disabled: %s", zone));
}
managementService.checkJsInterpretationAllowedIfNeededForParameterValue(ApiConstants.IS_TAG_A_RULE,
Boolean.TRUE.equals(cmd.isTagARule()));
Map<String, Object> params = new HashMap<>();
params.put("zoneId", zone.getId());
params.put("clusterId", clusterId);
@ -1214,6 +1220,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
// Input validation
Long id = cmd.getId();
managementService.checkJsInterpretationAllowedIfNeededForParameterValue(ApiConstants.IS_TAG_A_RULE,
Boolean.TRUE.equals(cmd.isTagARule()));
StoragePoolVO pool = _storagePoolDao.findById(id);
if (pool == null) {
throw new IllegalArgumentException("Unable to find storage pool with ID: " + id);

11
server/src/main/java/com/cloud/tags/ResourceManagerUtilImpl.java
View File

@ -22,6 +22,7 @@ import java.util.Objects;
import javax.inject.Inject;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.api.InternalIdentity;
import org.apache.cloudstack.context.CallContext;
@ -130,6 +131,11 @@ public class ResourceManagerUtilImpl implements ResourceManagerUtil {
@Override
public long getResourceId(String resourceId, ResourceTag.ResourceObjectType resourceType) {
return getResourceId(resourceId, resourceType, false);
}
@Override
public long getResourceId(String resourceId, ResourceTag.ResourceObjectType resourceType, boolean checkAccess) {
Class<?> clazz = s_typeMap.get(resourceType);
Object entity = entityMgr.findByUuid(clazz, resourceId);
if (entity != null) {
@ -140,6 +146,11 @@ public class ResourceManagerUtilImpl implements ResourceManagerUtil {
}
entity = entityMgr.findById(clazz, resourceId);
if (entity != null) {
if (checkAccess && entity instanceof ControlledEntity) {
ControlledEntity controlledEntity = (ControlledEntity)entity;
Account caller = CallContext.current().getCallingAccount();
accountMgr.checkAccess(caller, null, false, controlledEntity);
}
return ((InternalIdentity)entity).getId();
}
throw new InvalidParameterValueException("Unable to find resource by id " + resourceId + " and type " + resourceType);

11
server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
View File

@ -496,7 +496,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
boolean dataDiskDeletetionResult = true;
List<VMTemplateVO> dataDiskTemplates = templateDao.listByParentTemplatetId(template.getId());
if (dataDiskTemplates != null && dataDiskTemplates.size() > 0) {
if (CollectionUtils.isNotEmpty(dataDiskTemplates)) {
logger.info("Template: {} has Datadisk template(s) associated with it. Delete Datadisk templates before deleting the template", template);
for (VMTemplateVO dataDiskTemplate : dataDiskTemplates) {
logger.info("Delete Datadisk template: {} from image store: {}", dataDiskTemplate, imageStore);
@ -562,6 +562,9 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
if (success) {
if ((imageStores != null && imageStores.size() > 1) && (profile.getZoneIdList() != null)) {
//if template is stored in more than one image stores, and the zone id is not null, then don't delete other templates.
if (templateMgr.TemplateDeleteFromPrimaryStorage.value()) {
templateMgr.evictTemplateFromStoragePoolsForZones(template.getId(), profile.getZoneIdList());
}
return cleanupTemplate(template, success);
}
@ -574,7 +577,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
// find all eligible image stores for this template
List<DataStore> iStores = templateMgr.getImageStoreByTemplate(template.getId(), null);
if (iStores == null || iStores.size() == 0) {
if (CollectionUtils.isEmpty(iStores)) {
// remove any references from template_zone_ref
List<VMTemplateZoneVO> templateZones = templateZoneDao.listByTemplateId(template.getId());
if (templateZones != null) {
@ -595,6 +598,10 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
}
if (templateMgr.TemplateDeleteFromPrimaryStorage.value()) {
templateMgr.evictTemplateFromStoragePoolsForZones(template.getId(), profile.getZoneIdList());
}
// remove its related ACL permission
Pair<Class<?>, Long> templateClassForId = new Pair<>(VirtualMachineTemplate.class, template.getId());
_messageBus.publish(_name, EntityManager.MESSAGE_REMOVE_ENTITY_EVENT, PublishScope.LOCAL, templateClassForId);

55
server/src/main/java/com/cloud/template/TemplateManagerImpl.java
View File

@ -1030,33 +1030,53 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
return adapter.delete(new TemplateProfile(userId, template, zoneId));
}
private Boolean templateIsUnusedInPool(VMTemplateStoragePoolVO templatePoolVO) {
VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId());
// If this is a routing template, consider it in use
if (template.getTemplateType() == TemplateType.SYSTEM) {
return false;
}
// If the template is not yet downloaded to the pool, consider it in use
if (templatePoolVO.getDownloadState() != Status.DOWNLOADED) {
return false;
}
if (template.getFormat() == ImageFormat.ISO || _volumeDao.isAnyVolumeActivelyUsingTemplateOnPool(template.getId(), templatePoolVO.getPoolId())) {
return false;
}
return true;
}
@Override
public List<VMTemplateStoragePoolVO> getUnusedTemplatesInPool(StoragePoolVO pool) {
List<VMTemplateStoragePoolVO> unusedTemplatesInPool = new ArrayList<VMTemplateStoragePoolVO>();
List<VMTemplateStoragePoolVO> allTemplatesInPool = _tmpltPoolDao.listByPoolId(pool.getId());
for (VMTemplateStoragePoolVO templatePoolVO : allTemplatesInPool) {
VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId());
// If this is a routing template, consider it in use
if (template.getTemplateType() == TemplateType.SYSTEM) {
continue;
}
// If the template is not yet downloaded to the pool, consider it in
// use
if (templatePoolVO.getDownloadState() != Status.DOWNLOADED) {
continue;
}
if (template.getFormat() != ImageFormat.ISO && !_volumeDao.isAnyVolumeActivelyUsingTemplateOnPool(template.getId(), pool.getId())) {
if (templateIsUnusedInPool(templatePoolVO)) {
unusedTemplatesInPool.add(templatePoolVO);
}
}
return unusedTemplatesInPool;
}
@Override
public void evictTemplateFromStoragePoolsForZones(Long templateId, List<Long> zoneIds) {
List<Long> poolIds = new ArrayList<>();
if (CollectionUtils.isNotEmpty(zoneIds)) {
List<StoragePoolVO> pools = _poolDao.listByDataCenterIds(zoneIds);
poolIds = pools.stream().map(StoragePoolVO::getId).collect(Collectors.toList());
}
List<VMTemplateStoragePoolVO> templateStoragePoolVOS = _tmpltPoolDao.listByPoolIdsAndTemplate(poolIds, templateId);
for (VMTemplateStoragePoolVO templateStoragePoolVO: templateStoragePoolVOS) {
if (templateIsUnusedInPool(templateStoragePoolVO)) {
evictTemplateFromStoragePool(templateStoragePoolVO);
}
}
}
@Override
@DB
public void evictTemplateFromStoragePool(VMTemplateStoragePoolVO templatePoolVO) {
@ -2437,7 +2457,10 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] {AllowPublicUserTemplates, TemplatePreloaderPoolSize, ValidateUrlIsResolvableBeforeRegisteringTemplate};
return new ConfigKey<?>[] {AllowPublicUserTemplates,
TemplatePreloaderPoolSize,
ValidateUrlIsResolvableBeforeRegisteringTemplate,
TemplateDeleteFromPrimaryStorage};
}
public List<TemplateAdapter> getTemplateAdapters() {

42
server/src/main/java/org/apache/cloudstack/network/RoutedIpv4ManagerImpl.java
View File

@ -607,10 +607,20 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
}
protected Ipv4GuestSubnetNetworkMap getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, Long ownerDomainId, Long ownerAccountId, Long zoneId) {
validateNetworkCidrSize(ownerAccountId, cidrSize);
validateNetworkCidrSize(cidrSize);
List<DataCenterIpv4GuestSubnetVO> subnets = getZoneSubnetsForAccount(ownerDomainId, ownerAccountId, zoneId);
for (DataCenterIpv4GuestSubnetVO subnet : subnets) {
Ipv4GuestSubnetNetworkMap result = getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(cidrSize, subnet);
Ipv4GuestSubnetNetworkMap result = getIpv4SubnetForGuestNetworkOrVpcInternal(cidrSize, subnet);
if (result != null) {
return result;
}
}
Boolean isAutoAllocationEnabled = RoutedIPv4NetworkCidrAutoAllocationEnabled.valueIn(ownerAccountId);
if (!Boolean.TRUE.equals(isAutoAllocationEnabled)) {
throw new InvalidParameterValueException("CIDR auto-allocation is disabled for this account");
}
for (DataCenterIpv4GuestSubnetVO subnet : subnets) {
Ipv4GuestSubnetNetworkMap result = createIpv4SubnetForGuestNetworkOrVpcInternal(cidrSize, subnet);
if (result != null) {
return result;
}
@ -618,11 +628,11 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
return null;
}
protected Ipv4GuestSubnetNetworkMap getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, DataCenterIpv4GuestSubnetVO subnet) {
Ipv4GuestSubnetNetworkMap map = ipv4GuestSubnetNetworkMapDao.findFirstAvailable(subnet.getId(), cidrSize);
if (map != null) {
return map;
}
protected Ipv4GuestSubnetNetworkMap getIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, DataCenterIpv4GuestSubnetVO subnet) {
return ipv4GuestSubnetNetworkMapDao.findFirstAvailable(subnet.getId(), cidrSize);
}
protected Ipv4GuestSubnetNetworkMap createIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, DataCenterIpv4GuestSubnetVO subnet) {
try {
return createIpv4SubnetFromParentSubnet(subnet, cidrSize);
} catch (Exception ex) {
@ -631,6 +641,14 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
return null;
}
protected Ipv4GuestSubnetNetworkMap getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, DataCenterIpv4GuestSubnetVO subnet) {
Ipv4GuestSubnetNetworkMap map = getIpv4SubnetForGuestNetworkOrVpcInternal(cidrSize, subnet);
if (map != null) {
return map;
}
return createIpv4SubnetForGuestNetworkOrVpcInternal(cidrSize, subnet);
}
protected void getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(String networkCidr, Long ownerDomainId, Long ownerAccountId, Long zoneId) {
Ipv4GuestSubnetNetworkMapVO subnetMap = ipv4GuestSubnetNetworkMapDao.findBySubnet(networkCidr);
if (subnetMap != null) {
@ -693,13 +711,9 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
}
}
private void validateNetworkCidrSize(long accountId, Integer networkCidrSize) {
private void validateNetworkCidrSize(Integer networkCidrSize) {
if (networkCidrSize == null) {
throw new CloudRuntimeException("network/vpc CidrSize is null");
}
Boolean isAutoAllocationEnabled = RoutedIPv4NetworkCidrAutoAllocationEnabled.valueIn(accountId);
if (!Boolean.TRUE.equals(isAutoAllocationEnabled)) {
throw new CloudRuntimeException("CIDR auto-allocation is disabled for this account");
throw new InvalidParameterValueException("network/vpc CidrSize is null");
}
}
@ -755,7 +769,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
// Allocate a subnet automatically
String networkCidr = getFreeNetworkCidr(subnetsInFreeIpRanges, networkCidrSize);
if (networkCidr == null) {
throw new CloudRuntimeException("Failed to automatically allocate a subnet with specified cidrsize");
throw new InvalidParameterValueException("Failed to automatically allocate a subnet with specified cidrsize");
}
return networkCidr;
}

9
server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java
View File

@ -71,7 +71,6 @@ import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.hypervisor.HypervisorGuru;
import com.cloud.hypervisor.HypervisorGuruManager;
import com.cloud.network.IpAddressManager;
import com.cloud.network.Network;
import com.cloud.network.NetworkModel;
import com.cloud.network.Networks;
@ -303,8 +302,6 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
@Inject
private PhysicalNetworkDao physicalNetworkDao;
@Inject
private IpAddressManager ipAddressManager;
@Inject
private StoragePoolHostDao storagePoolHostDao;
@Inject
private HypervisorGuruManager hypervisorGuruManager;
@ -2766,10 +2763,8 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager {
}
String macAddress = networkModel.getNextAvailableMacAddressInNetwork(networkId);
String ipAddress = null;
if (network.getGuestType() != Network.GuestType.L2) {
ipAddress = ipAddressManager.acquireGuestIpAddress(network, null);
}
String ipAddress = network.getGuestType() != Network.GuestType.L2 ? "auto" : null;
Network.IpAddresses requestedIpPair = new Network.IpAddresses(ipAddress, null, macAddress);

4
server/src/test/java/org/apache/cloudstack/network/RoutedIpv4ManagerImplTest.java
View File

@ -545,12 +545,12 @@ public class RoutedIpv4ManagerImplTest {
DataCenterIpv4GuestSubnetVO subnet3 = Mockito.mock(DataCenterIpv4GuestSubnetVO.class);
when(dataCenterIpv4GuestSubnetDao.listNonDedicatedByDataCenterId(zoneId)).thenReturn(Arrays.asList(subnet3));
doReturn(null).doReturn(null).doReturn(ipv4GuestSubnetNetworkMap).when(routedIpv4Manager).getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(eq(cidrSize), any());
doReturn(null).doReturn(null).doReturn(ipv4GuestSubnetNetworkMap).when(routedIpv4Manager).getIpv4SubnetForGuestNetworkOrVpcInternal(eq(cidrSize), any());
Ipv4GuestSubnetNetworkMap result = routedIpv4Manager.getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(cidrSize, domainId, accountId, zoneId);
Assert.assertEquals(ipv4GuestSubnetNetworkMap, result);
verify(routedIpv4Manager, times(3)).getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(eq(cidrSize), any());
verify(routedIpv4Manager, times(3)).getIpv4SubnetForGuestNetworkOrVpcInternal(eq(cidrSize), any());
}
@Test

4
tools/appliance/systemvmtemplate/scripts/configure_locale.sh
View File

@ -22,13 +22,15 @@ set -x
function configure_locale() {
grep LANG=en_US.UTF-8 /etc/default/locale && \
grep LC_ALL=en_US.UTF-8 /etc/default/locale && \
grep "en_US.UTF-8 UTF-8" /etc/locale.gen &&
grep "^en_US.UTF-8 UTF-8" /etc/locale.gen &&
return
cat >> /etc/default/locale << EOF
LANG=en_US.UTF-8
LC_ALL=en_US.UTF-8
EOF
grep "^en_US.UTF-8 UTF-8" /etc/locale.gen || \
cat >> /etc/locale.gen << EOF
en_US.UTF-8 UTF-8
EOF

171
utils/src/main/java/org/apache/cloudstack/utils/jsinterpreter/JsInterpreter.java
View File

@ -19,31 +19,52 @@ package org.apache.cloudstack.utils.jsinterpreter;
import java.io.Closeable;
import java.io.IOException;
import java.io.StringWriter;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.commons.collections.MapUtils;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import javax.script.Bindings;
import javax.script.Compilable;
import javax.script.CompiledScript;
import javax.script.ScriptContext;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import javax.script.SimpleBindings;
import javax.script.SimpleScriptContext;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.collections.MapUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.openjdk.nashorn.api.scripting.ClassFilter;
import org.openjdk.nashorn.api.scripting.NashornScriptEngineFactory;
import javax.script.ScriptEngine;
import com.cloud.utils.exception.CloudRuntimeException;
/**
* A class to execute JavaScript scripts, with the possibility to inject context to the scripts.
* Executes JavaScript with strong restrictions to mitigate RCE risks.
* - Disables Java interop via --no-java AND a deny-all ClassFilter
* - Disables Nashorn syntax extensions
* - Uses Bindings instead of string-splicing variables
* - Fresh ScriptContext per execution, with timeout on a daemon worker
*/
public class JsInterpreter implements Closeable {
protected Logger logger = LogManager.getLogger(JsInterpreter.class);
protected static final List<String> RESTRICTED_TOKENS = Arrays.asList( "engine", "context", "factory",
"Java", "java", "Packages"," javax", "load", "loadWithNewGlobal", "print", "factory", "getClass",
"runCommand", "Runtime", "exec", "ProcessBuilder", "Thread", "thread", "Threads", "Class", "class");
protected ScriptEngine interpreter;
protected String interpreterName;
private final String injectingLogMessage = "Injecting variable [%s] with value [%s] into the JS interpreter.";
@ -51,21 +72,40 @@ public class JsInterpreter implements Closeable {
private TimeUnit defaultTimeUnit = TimeUnit.MILLISECONDS;
private long timeout;
private String timeoutDefaultMessage;
protected Map<String, String> variables = new LinkedHashMap<>();
// Store variables as Objects; they go into Bindings (no code splicing)
protected Map<String, Object> variables = new LinkedHashMap<>();
/** Deny-all filter: no Java class is visible from scripts. */
static final class DenyAllClassFilter implements ClassFilter {
@Override public boolean exposeToScripts(String className) { return false; }
}
/**
* Constructor created exclusively for unit testing.
*/
protected JsInterpreter() {
}
protected JsInterpreter() { }
public JsInterpreter(long timeout) {
this.timeout = timeout;
this.timeoutDefaultMessage = String.format("Timeout (in milliseconds) defined in the global setting [quota.activationrule.timeout]: [%s].", this.timeout);
this.timeoutDefaultMessage = String.format(
"Timeout (in milliseconds) defined in the global setting [quota.activationrule.timeout]: [%s].", this.timeout);
if (System.getProperty("nashorn.args") == null) {
System.setProperty("nashorn.args", "--no-java --no-syntax-extensions");
}
this.executor = new ThreadPoolExecutor(
1, 1, 60L, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(),
r -> {
Thread t = new Thread(r, "JsInterpreter-worker");
t.setDaemon(true);
return t;
}
);
executor = Executors.newSingleThreadExecutor();
NashornScriptEngineFactory factory = new NashornScriptEngineFactory();
this.interpreterName = factory.getEngineName();
logger.trace(String.format("Initiating JS interpreter: %s.", interpreterName));
@ -73,49 +113,53 @@ public class JsInterpreter implements Closeable {
}
protected void setScriptEngineDisablingJavaLanguage(NashornScriptEngineFactory factory) {
interpreter = factory.getScriptEngine("--no-java");
String[] opts = new String[] {
"--no-java",
"--no-syntax-extensions",
};
interpreter = factory.getScriptEngine(
opts,
JsInterpreter.class.getClassLoader(),
new DenyAllClassFilter()
);
}
/**
* Discards the current variables map and create a new one.
*/
/** Discards the current variables map and create a new one. */
public void discardCurrentVariables() {
logger.trace("Discarding current variables map and creating a new one.");
variables = new LinkedHashMap<>();
}
/**
* Adds the parameters to a Map that will be converted to JS variables right before executing the script.
* @param key The name of the variable.
* @param value The value of the variable.
* Adds a variable that will be exposed via ENGINE_SCOPE bindings.
* Safe against code injection (no string concatenation).
*/
public void injectVariable(String key, String value) {
logger.trace(String.format(injectingLogMessage, key, value));
public void injectVariable(String key, Object value) {
if (key == null) return;
logger.trace(String.format(injectingLogMessage, key, String.valueOf(value)));
variables.put(key, value);
}
/**
* Adds the parameter, surrounded by double quotes, to a Map that will be converted to a JS variable right before executing the script.
* @param key The name of the variable.
* @param value The value of the variable.
* @deprecated Not needed when using Bindings; kept for source compatibility.
* Prefer {@link #injectVariable(String, Object)}.
*/
@Deprecated
public void injectStringVariable(String key, String value) {
if (value == null) {
logger.trace(String.format("Not injecting [%s] because its value is null.", key));
return;
}
value = String.format("\"%s\"", value);
logger.trace(String.format(injectingLogMessage, key, value));
variables.put(key, value);
injectVariable(key, value);
}
/**
* Injects the variables to the script and execute it.
* Injects the variables via Bindings and executes the script with a fresh context.
* @param script Code to be executed.
* @return The result of the executed script.
*/
public Object executeScript(String script) {
script = addVariablesToScript(script);
Objects.requireNonNull(script, "script");
logger.debug(String.format("Executing script [%s].", script));
@ -126,43 +170,60 @@ public class JsInterpreter implements Closeable {
}
protected Object executeScriptInThread(String script) {
Callable<Object> task = () -> interpreter.eval(script);
final Callable<Object> task = () -> {
final SimpleScriptContext ctx = new SimpleScriptContext();
Future<Object> future = executor.submit(task);
final Bindings engineBindings = new SimpleBindings();
if (MapUtils.isNotEmpty(variables)) {
engineBindings.putAll(variables);
}
for (String token : RESTRICTED_TOKENS) {
engineBindings.put(token, null);
}
ctx.setBindings(engineBindings, ScriptContext.ENGINE_SCOPE);
final StringWriter out = new StringWriter();
ctx.setWriter(out);
try {
final CompiledScript compiled = ((Compilable) interpreter).compile(script);
Object result = compiled.eval(ctx);
if (out.getBuffer().length() > 0) {
logger.info("Script produced output on stdout: [{}]", out);
}
return result;
} catch (ScriptException se) {
String msg = se.getMessage() == null ? "Script error" : se.getMessage();
throw new ScriptException("Script error: " + msg, se.getFileName(), se.getLineNumber(), se.getColumnNumber());
}
};
final Future<Object> future = executor.submit(task);
try {
return future.get(this.timeout, defaultTimeUnit);
} catch (TimeoutException | InterruptedException | ExecutionException e) {
String message = String.format("Unable to execute script [%s] due to [%s]", script, e.getMessage());
if (e instanceof TimeoutException) {
message = String.format("Execution of script [%s] took too long and timed out. %s", script, timeoutDefaultMessage);
}
} catch (TimeoutException e) {
String message = String.format(
"Execution of script [%s] took too long and timed out. %s", script, timeoutDefaultMessage);
logger.error(message, e);
throw new CloudRuntimeException(message, e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
String message = String.format("Execution of script [%s] was interrupted.", script);
logger.error(message, e);
throw new CloudRuntimeException(message, e);
} catch (ExecutionException e) {
Throwable cause = e.getCause() == null ? e : e.getCause();
String message = String.format("Unable to execute script [%s] due to [%s]", script, cause.getMessage());
logger.error(message, cause);
throw new CloudRuntimeException(message, cause);
} finally {
future.cancel(true);
}
}
protected String addVariablesToScript(String script) {
if (MapUtils.isEmpty(variables)) {
logger.trace(String.format("There is no variables to add to script [%s]. Returning.", script));
return script;
}
String variablesToString = "";
for (Map.Entry<String, String> variable : variables.entrySet()) {
variablesToString = String.format("%s %s = %s;", variablesToString, variable.getKey(), variable.getValue());
}
return String.format("%s %s", variablesToString, script);
}
@Override
public void close() throws IOException {
executor.shutdown();
executor.shutdownNow();
}
}

7
utils/src/main/java/org/apache/cloudstack/utils/jsinterpreter/TagAsRuleHelper.java
View File

@ -16,12 +16,12 @@
//under the License.
package org.apache.cloudstack.utils.jsinterpreter;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.lang3.StringEscapeUtils;
import java.io.IOException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import com.cloud.utils.exception.CloudRuntimeException;
public class TagAsRuleHelper {
@ -32,7 +32,6 @@ public class TagAsRuleHelper {
public static boolean interpretTagAsRule(String rule, String tags, long timeout) {
String script = PARSE_TAGS + rule;
tags = String.format("'%s'", StringEscapeUtils.escapeEcmaScript(tags));
try (JsInterpreter jsInterpreter = new JsInterpreter(timeout)) {
jsInterpreter.injectVariable("tags", tags);
Object scriptReturn = jsInterpreter.executeScript(script);

81
utils/src/test/java/org/apache/cloudstack/utils/jsinterpreter/JsInterpreterTest.java
View File

@ -20,6 +20,7 @@ package org.apache.cloudstack.utils.jsinterpreter;
import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@ -27,7 +28,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeoutException;
import com.cloud.utils.exception.CloudRuntimeException;
import javax.script.ScriptEngine;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -36,9 +38,10 @@ import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import org.openjdk.nashorn.api.scripting.ClassFilter;
import org.openjdk.nashorn.api.scripting.NashornScriptEngineFactory;
import javax.script.ScriptEngine;
import com.cloud.utils.exception.CloudRuntimeException;
@RunWith(MockitoJUnitRunner.class)
public class JsInterpreterTest {
@ -61,30 +64,6 @@ public class JsInterpreterTest {
Assert.assertTrue(executor.isShutdown());
}
@Test
public void addVariablesToScriptTestVariablesMapIsEmptyReturnScript() {
String script = "a + b";
jsInterpreterSpy.variables = new LinkedHashMap<>();
String result = jsInterpreterSpy.addVariablesToScript(script);
Assert.assertEquals(script, result);
}
@Test
public void addVariablesToScriptTestVariablesMapIsNotEmptyInjectVariableToScript() {
String script = "a + b";
String var1 = "{test: \"test\"}";
jsInterpreterSpy.injectVariable("var1", var1);
jsInterpreterSpy.injectVariable("var2", var1);
String expected = String.format(" var1 = %s; var2 = %s; %s", var1, var1, script);
String result = jsInterpreterSpy.addVariablesToScript(script);
Assert.assertEquals(expected, result);
}
@Test
public void executeScriptTestReturnResultOfScriptExecution() {
String script = "5";
@ -154,7 +133,7 @@ public class JsInterpreterTest {
@Test
public void discardCurrentVariablesTestInstantiateNewMap() {
Map<String, String> variables = new LinkedHashMap<>();
Map<String, Object> variables = new LinkedHashMap<>();
variables.put("a", "b");
variables.put("b", null);
@ -170,12 +149,14 @@ public class JsInterpreterTest {
NashornScriptEngineFactory nashornScriptEngineFactoryMock = Mockito.spy(NashornScriptEngineFactory.class);
ScriptEngine scriptEngineMock = Mockito.mock(ScriptEngine.class);
Mockito.doReturn(scriptEngineMock).when(nashornScriptEngineFactoryMock).getScriptEngine(Mockito.anyString());
Mockito.doReturn(scriptEngineMock).when(nashornScriptEngineFactoryMock).getScriptEngine(Mockito.any(),
Mockito.any(ClassLoader.class), Mockito.any(ClassFilter.class));
jsInterpreterSpy.setScriptEngineDisablingJavaLanguage(nashornScriptEngineFactoryMock);
Assert.assertEquals(scriptEngineMock, jsInterpreterSpy.interpreter);
Mockito.verify(nashornScriptEngineFactoryMock).getScriptEngine("--no-java");
Mockito.verify(nashornScriptEngineFactoryMock).getScriptEngine(Mockito.any(),
Mockito.any(ClassLoader.class), Mockito.any(ClassFilter.class));
}
@Test
@ -193,6 +174,46 @@ public class JsInterpreterTest {
jsInterpreterSpy.injectStringVariable("a", "b");
Assert.assertEquals(jsInterpreterSpy.variables.get("a"), "\"b\"");
Assert.assertEquals(jsInterpreterSpy.variables.get("a"), "b");
}
@Test
public void executeScriptTestValidScriptShouldPassWithMixedVariables() {
try (JsInterpreter jsInterpreter = new JsInterpreter(1000)) {
jsInterpreter.injectVariable("x", 10);
jsInterpreter.injectVariable("y", "hello");
jsInterpreter.injectVariable("z", true);
String validScript = "var result = x + (z ? 1 : 0); y + '-' + result;";
Object result = jsInterpreter.executeScript(validScript);
Assert.assertEquals("hello-11", result);
} catch (IOException exception) {
Assert.fail("IOException not expected here");
}
}
private void runMaliciousScriptFileTest(String script, String filename) {
try (JsInterpreter jsInterpreter = new JsInterpreter(1000)) {
jsInterpreter.executeScript(script);
} catch (CloudRuntimeException ex) {
Assert.assertTrue(ex.getMessage().contains("Unable to execute script"));
java.io.File f = new java.io.File(filename);
Assert.assertFalse(f.exists());
} catch (IOException exception) {
Assert.fail("IOException not expected here");
}
}
@Test
public void executeScriptTestMaliciousScriptShouldThrowException1() {
String filename = "/tmp/hack1-" + UUID.randomUUID();
String maliciousScript = "Java.type('java.lang.Runtime').getRuntime().exec('touch " + filename + "')";
runMaliciousScriptFileTest(maliciousScript, filename);
}
@Test
public void executeScriptTestMaliciousScriptShouldThrowException2() {
String filename = "/tmp/hack2-" + UUID.randomUUID();
String maliciousScript = "var e=this.engine.getFactory().getScriptEngine('-Dnashorn.args=--no-java=False'); e.eval(\"java.lang.Runtime.getRuntime().exec(['/bin/bash','-c','touch " + filename + "']);\");";
runMaliciousScriptFileTest(maliciousScript, filename);
}
}