apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge release branch 4.20 to main

Browse Source 
* 4.20:
  linstor: Fix ZFS snapshot backup (#10219)
  fix listing of VMs by network (#10204)
  Configure org.eclipse.jetty.server.Request.maxFormKeys from server.properties and increase the default value (#10214)
  api: fix access for listSystemVmUsageHistory (#10032)
  Fix NPE issues during host rolling maintenance, due to host tags and custom constrained/unconstrained service offering (#9844)
This commit is contained in:
Daan Hoogland 2025-01-21 12:00:19 +01:00
commit 81e052cfeb
8 changed files with 144 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
client/conf/server.properties.in
View File

@ -32,6 +32,9 @@ session.timeout=30
# Max allowed API request payload/content size in bytes
request.content.size=1048576
# Max allowed API request form keys
request.max.form.keys=5000
# Options to configure and enable HTTPS on the management server
#
# For the management server to pick up these configuration settings, the configured

10
client/src/main/java/org/apache/cloudstack/ServerDaemon.java
View File

@ -82,6 +82,8 @@ public class ServerDaemon implements Daemon {
private static final String ACCESS_LOG = "access.log";
private static final String REQUEST_CONTENT_SIZE_KEY = "request.content.size";
private static final int DEFAULT_REQUEST_CONTENT_SIZE = 1048576;
private static final String REQUEST_MAX_FORM_KEYS_KEY = "request.max.form.keys";
private static final int DEFAULT_REQUEST_MAX_FORM_KEYS = 5000;
////////////////////////////////////////////////////////
/////////////// Server Configuration ///////////////////
@ -94,6 +96,7 @@ public class ServerDaemon implements Daemon {
private int httpsPort = 8443;
private int sessionTimeout = 30;
private int maxFormContentSize = DEFAULT_REQUEST_CONTENT_SIZE;
private int maxFormKeys = DEFAULT_REQUEST_MAX_FORM_KEYS;
private boolean httpsEnable = false;
private String accessLogFile = "access.log";
private String bindInterface = null;
@ -141,6 +144,7 @@ public class ServerDaemon implements Daemon {
setAccessLogFile(properties.getProperty(ACCESS_LOG, "access.log"));
setSessionTimeout(Integer.valueOf(properties.getProperty(SESSION_TIMEOUT, "30")));
setMaxFormContentSize(Integer.valueOf(properties.getProperty(REQUEST_CONTENT_SIZE_KEY, String.valueOf(DEFAULT_REQUEST_CONTENT_SIZE))));
setMaxFormKeys(Integer.valueOf(properties.getProperty(REQUEST_MAX_FORM_KEYS_KEY, String.valueOf(DEFAULT_REQUEST_MAX_FORM_KEYS))));
} catch (final IOException e) {
logger.warn("Failed to read configuration from server.properties file", e);
} finally {
@ -192,6 +196,7 @@ public class ServerDaemon implements Daemon {
// Extra config options
server.setStopAtShutdown(true);
server.setAttribute(ContextHandler.MAX_FORM_CONTENT_SIZE_KEY, maxFormContentSize);
server.setAttribute(ContextHandler.MAX_FORM_KEYS_KEY, maxFormKeys);
// HTTPS Connector
createHttpsConnector(httpConfig);
@ -264,6 +269,7 @@ public class ServerDaemon implements Daemon {
webApp.setContextPath(contextPath);
webApp.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
webApp.setMaxFormContentSize(maxFormContentSize);
webApp.setMaxFormKeys(maxFormKeys);
// GZIP handler
final GzipHandler gzipHandler = new GzipHandler();
@ -366,4 +372,8 @@ public class ServerDaemon implements Daemon {
public void setMaxFormContentSize(int maxFormContentSize) {
this.maxFormContentSize = maxFormContentSize;
}
public void setMaxFormKeys(int maxFormKeys) {
this.maxFormKeys = maxFormKeys;
}
}

2
plugins/metrics/src/main/java/org/apache/cloudstack/api/ListSystemVMsUsageHistoryCmd.java
View File

@ -26,7 +26,7 @@ import org.apache.cloudstack.response.VmMetricsStatsResponse;
@APICommand(name = "listSystemVmsUsageHistory", description = "Lists System VM stats", responseObject = VmMetricsStatsResponse.class,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.18.0",
authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin})
authorized = {RoleType.Admin})
public class ListSystemVMsUsageHistoryCmd extends BaseResourceUsageHistoryCmd {
/////////////////////////////////////////////////////

6
plugins/storage/volume/linstor/CHANGELOG.md
View File

@ -5,6 +5,12 @@ All notable changes to Linstor CloudStack plugin will be documented in this file
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [2025-01-20]
### Fixed
- Volume snapshots on zfs used the wrong dataset path to hide/unhide snapdev
## [2024-12-13]
### Fixed

14
plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java
View File

@ -46,11 +46,17 @@ public final class LinstorBackupSnapshotCommandWrapper
{
protected static Logger LOGGER = LogManager.getLogger(LinstorBackupSnapshotCommandWrapper.class);
private static String zfsDatasetName(String zfsFullSnapshotUrl) {
String zfsFullPath = zfsFullSnapshotUrl.substring(6);
int atPos = zfsFullPath.indexOf('@');
return atPos >= 0 ? zfsFullPath.substring(0, atPos) : zfsFullPath;
}
private String zfsSnapdev(boolean hide, String zfsUrl) {
Script script = new Script("/usr/bin/zfs", Duration.millis(5000));
Script script = new Script("zfs", Duration.millis(5000));
script.add("set");
script.add("snapdev=" + (hide ? "hidden" : "visible"));
script.add(zfsUrl.substring(6)); // cutting zfs://
script.add(zfsDatasetName(zfsUrl)); // cutting zfs:// and @snapshotname
return script.execute();
}
@ -134,10 +140,10 @@ public final class LinstorBackupSnapshotCommandWrapper
LOGGER.info("Src: " + srcPath + " | " + src.getName());
if (srcPath.startsWith("zfs://")) {
zfsHidden = true;
if (zfsSnapdev(false, srcPath) != null) {
if (zfsSnapdev(false, src.getPath()) != null) {
return new CopyCmdAnswer("Unable to unhide zfs snapshot device.");
}
srcPath = "/dev/" + srcPath.substring(6);
srcPath = "/dev/zvol/" + srcPath.substring(6);
}
secondaryPool = storagePoolMgr.getStoragePoolByURI(dstDataStore.getUrl());

1
server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
View File

@ -1413,6 +1413,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
if (networkId != null || vpcId != null) {
SearchBuilder<NicVO> nicSearch = nicDao.createSearchBuilder();
nicSearch.and("networkId", nicSearch.entity().getNetworkId(), Op.EQ);
nicSearch.and("removed", nicSearch.entity().getRemoved(), Op.NULL);
if (vpcId != null) {
SearchBuilder<NetworkVO> networkSearch = networkDao.createSearchBuilder();
networkSearch.and("vpcId", networkSearch.entity().getVpcId(), Op.EQ);

59
server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java
View File

@ -37,6 +37,7 @@ import org.apache.cloudstack.api.command.admin.resource.StartRollingMaintenanceC
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
@ -64,12 +65,16 @@ import com.cloud.org.Grouping;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.utils.Pair;
import com.cloud.utils.StringUtils;
import com.cloud.utils.Ternary;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.UserVmDetailVO;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine.State;
import com.cloud.vm.VirtualMachineProfileImpl;
import com.cloud.vm.VmDetailConstants;
import com.cloud.vm.dao.UserVmDetailsDao;
import com.cloud.vm.dao.VMInstanceDao;
public class RollingMaintenanceManagerImpl extends ManagerBase implements RollingMaintenanceManager {
@ -85,6 +90,8 @@ public class RollingMaintenanceManagerImpl extends ManagerBase implements Rollin
@Inject
private VMInstanceDao vmInstanceDao;
@Inject
protected UserVmDetailsDao userVmDetailsDao;
@Inject
private ServiceOfferingDao serviceOfferingDao;
@Inject
private ClusterDetailsDao clusterDetailsDao;
@ -619,10 +626,19 @@ public class RollingMaintenanceManagerImpl extends ManagerBase implements Rollin
int successfullyCheckedVmMigrations = 0;
for (VMInstanceVO runningVM : vmsRunning) {
boolean canMigrateVm = false;
Ternary<Integer, Integer, Integer> cpuSpeedAndRamSize = getComputeResourcesCpuSpeedAndRamSize(runningVM);
Integer cpu = cpuSpeedAndRamSize.first();
Integer speed = cpuSpeedAndRamSize.second();
Integer ramSize = cpuSpeedAndRamSize.third();
if (ObjectUtils.anyNull(cpu, speed, ramSize)) {
logger.warn("Cannot fetch compute resources for the VM {}, skipping it from the capacity check", runningVM);
continue;
}
ServiceOfferingVO serviceOffering = serviceOfferingDao.findById(runningVM.getServiceOfferingId());
for (Host hostInCluster : hostsInCluster) {
if (!checkHostTags(hostTags, hostTagsDao.getHostTags(hostInCluster.getId()), serviceOffering.getHostTag())) {
logger.debug(String.format("Host tags mismatch between %s and %s Skipping it from the capacity check", host, hostInCluster));
logger.debug("Host tags mismatch between {} and {} Skipping it from the capacity check", host, hostInCluster);
continue;
}
DeployDestination deployDestination = new DeployDestination(null, null, null, host);
@ -632,13 +648,13 @@ public class RollingMaintenanceManagerImpl extends ManagerBase implements Rollin
affinityChecks = affinityChecks && affinityProcessor.check(vmProfile, deployDestination);
}
if (!affinityChecks) {
logger.debug(String.format("Affinity check failed between %s and %s Skipping it from the capacity check", host, hostInCluster));
logger.debug("Affinity check failed between {} and {} Skipping it from the capacity check", host, hostInCluster);
continue;
}
boolean maxGuestLimit = capacityManager.checkIfHostReachMaxGuestLimit(host);
boolean hostHasCPUCapacity = capacityManager.checkIfHostHasCpuCapability(hostInCluster.getId(), serviceOffering.getCpu(), serviceOffering.getSpeed());
int cpuRequested = serviceOffering.getCpu() * serviceOffering.getSpeed();
long ramRequested = serviceOffering.getRamSize() * 1024L * 1024L;
boolean hostHasCPUCapacity = capacityManager.checkIfHostHasCpuCapability(hostInCluster.getId(), cpu, speed);
int cpuRequested = cpu * speed;
long ramRequested = ramSize * 1024L * 1024L;
ClusterDetailsVO clusterDetailsCpuOvercommit = clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio");
ClusterDetailsVO clusterDetailsRamOvercommmt = clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio");
Float cpuOvercommitRatio = Float.parseFloat(clusterDetailsCpuOvercommit.getValue());
@ -664,11 +680,42 @@ public class RollingMaintenanceManagerImpl extends ManagerBase implements Rollin
return new Pair<>(true, "OK");
}
protected Ternary<Integer, Integer, Integer> getComputeResourcesCpuSpeedAndRamSize(VMInstanceVO runningVM) {
ServiceOfferingVO serviceOffering = serviceOfferingDao.findById(runningVM.getServiceOfferingId());
Integer cpu = serviceOffering.getCpu();
Integer speed = serviceOffering.getSpeed();
Integer ramSize = serviceOffering.getRamSize();
if (!serviceOffering.isDynamic()) {
return new Ternary<>(cpu, speed, ramSize);
}
List<UserVmDetailVO> vmDetails = userVmDetailsDao.listDetails(runningVM.getId());
if (CollectionUtils.isEmpty(vmDetails)) {
return new Ternary<>(cpu, speed, ramSize);
}
for (UserVmDetailVO vmDetail : vmDetails) {
if (StringUtils.isBlank(vmDetail.getName()) || StringUtils.isBlank(vmDetail.getValue())) {
continue;
}
if (cpu == null && VmDetailConstants.CPU_NUMBER.equals(vmDetail.getName())) {
cpu = Integer.valueOf(vmDetail.getValue());
} else if (speed == null && VmDetailConstants.CPU_SPEED.equals(vmDetail.getName())) {
speed = Integer.valueOf(vmDetail.getValue());
} else if (ramSize == null && VmDetailConstants.MEMORY.equals(vmDetail.getName())) {
ramSize = Integer.valueOf(vmDetail.getValue());
}
}
return new Ternary<>(cpu, speed, ramSize);
}
/**
* Check hosts tags
*/
private boolean checkHostTags(List<HostTagVO> hostTags, List<HostTagVO> hostInClusterTags, String offeringTag) {
if (CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) {
if ((CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) || StringUtils.isBlank(offeringTag)) {
return true;
} else if ((CollectionUtils.isNotEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) ||
(CollectionUtils.isEmpty(hostTags) && CollectionUtils.isNotEmpty(hostInClusterTags))) {

60
server/src/test/java/com/cloud/resource/RollingMaintenanceManagerImplTest.java
View File

@ -23,7 +23,15 @@ import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.org.Cluster;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.utils.Ternary;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.UserVmDetailVO;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VmDetailConstants;
import com.cloud.vm.dao.UserVmDetailsDao;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -54,6 +62,12 @@ public class RollingMaintenanceManagerImplTest {
HostVO host4;
@Mock
Cluster cluster;
@Mock
VMInstanceVO vm;
@Mock
ServiceOfferingDao serviceOfferingDao;
@Mock
UserVmDetailsDao userVmDetailsDao;
@Spy
@InjectMocks
@ -172,4 +186,50 @@ public class RollingMaintenanceManagerImplTest {
Assert.assertEquals(1, hosts.size());
}
@Test
public void testGetComputeResourcesCpuSpeedAndRamSize_ForNormalOffering() {
ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class);
Mockito.when(serviceOffering.isDynamic()).thenReturn(false);
Mockito.when(serviceOffering.getCpu()).thenReturn(1);
Mockito.when(serviceOffering.getSpeed()).thenReturn(500);
Mockito.when(serviceOffering.getRamSize()).thenReturn(512);
Mockito.when(vm.getServiceOfferingId()).thenReturn(1L);
Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(serviceOffering);
Ternary<Integer, Integer, Integer> cpuSpeedAndRamSize = manager.getComputeResourcesCpuSpeedAndRamSize(vm);
Assert.assertEquals(1, cpuSpeedAndRamSize.first().intValue());
Assert.assertEquals(500, cpuSpeedAndRamSize.second().intValue());
Assert.assertEquals(512, cpuSpeedAndRamSize.third().intValue());
}
@Test
public void testGetComputeResourcesCpuSpeedAndRamSize_ForCustomOffering() {
ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class);
Mockito.when(serviceOffering.isDynamic()).thenReturn(true);
Mockito.when(serviceOffering.getCpu()).thenReturn(null);
Mockito.when(serviceOffering.getSpeed()).thenReturn(null);
Mockito.when(serviceOffering.getRamSize()).thenReturn(null);
List<UserVmDetailVO> vmDetails = new ArrayList<>();
UserVmDetailVO cpuDetail = new UserVmDetailVO(1L, VmDetailConstants.CPU_NUMBER, "2", false);
vmDetails.add(cpuDetail);
UserVmDetailVO speedDetail = new UserVmDetailVO(1L, VmDetailConstants.CPU_SPEED, "1000", false);
vmDetails.add(speedDetail);
UserVmDetailVO ramSizeDetail = new UserVmDetailVO(1L, VmDetailConstants.MEMORY, "1024", false);
vmDetails.add(ramSizeDetail);
Mockito.when(vm.getId()).thenReturn(1L);
Mockito.when(vm.getServiceOfferingId()).thenReturn(1L);
Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(serviceOffering);
Mockito.when(userVmDetailsDao.listDetails(1L)).thenReturn(vmDetails);
Ternary<Integer, Integer, Integer> cpuSpeedAndRamSize = manager.getComputeResourcesCpuSpeedAndRamSize(vm);
Assert.assertEquals(2, cpuSpeedAndRamSize.first().intValue());
Assert.assertEquals(1000, cpuSpeedAndRamSize.second().intValue());
Assert.assertEquals(1024, cpuSpeedAndRamSize.third().intValue());
}
}