From 9aa1743984de30eb96e8a3fad9690f4d73bc255e Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Mon, 21 May 2018 03:56:31 -0300 Subject: [PATCH 1/3] registerIso: Fixes #2654 register iso in all zones (#2652) Fix to register of iso in all zones. Fixes #2654. --- server/src/com/cloud/template/TemplateAdapterBase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/com/cloud/template/TemplateAdapterBase.java b/server/src/com/cloud/template/TemplateAdapterBase.java index 7467e30aa01..e7f21b15abe 100644 --- a/server/src/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/com/cloud/template/TemplateAdapterBase.java @@ -24,6 +24,7 @@ import java.util.Map; import javax.inject.Inject; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.apache.cloudstack.api.ApiConstants; @@ -325,7 +326,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat Long zoneId = cmd.getZoneId(); // ignore passed zoneId if we are using region wide image store List stores = _imgStoreDao.findRegionImageStores(); - if (!(stores != null && stores.size() > 0)) { + if (CollectionUtils.isEmpty(stores) && zoneId != null && zoneId > 0L) { zoneList = new ArrayList<>(); zoneList.add(zoneId); } From 06f7e495dcc022e935622c24d326b527bbb73f63 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Mon, 21 May 2018 04:19:08 -0300 Subject: [PATCH 2/3] Host Affinity plugin (#2630) This implements a new host-affinity plugin. --- .../cloud/deploy/DataCenterDeployment.java | 14 ++ api/src/com/cloud/deploy/DeploymentPlan.java | 6 + client/pom.xml | 5 + .../spring-core-registry-core-context.xml | 2 +- .../host-affinity/pom.xml | 33 ++++ .../host-affinity/module.properties | 18 ++ .../spring-host-affinity-context.xml | 35 ++++ .../affinity/HostAffinityProcessor.java | 127 +++++++++++++ .../affinity/HostAffinityProcessorTest.java | 176 ++++++++++++++++++ plugins/pom.xml | 1 + .../deploy/DeploymentPlanningManagerImpl.java | 31 ++- .../vm/DeploymentPlanningManagerImplTest.java | 30 ++- .../integration/smoke/test_affinity_groups.py | 95 +++++++++- 13 files changed, 562 insertions(+), 11 deletions(-) create mode 100644 plugins/affinity-group-processors/host-affinity/pom.xml create mode 100644 plugins/affinity-group-processors/host-affinity/resources/META-INF/cloudstack/host-affinity/module.properties create mode 100644 plugins/affinity-group-processors/host-affinity/resources/META-INF/cloudstack/host-affinity/spring-host-affinity-context.xml create mode 100644 plugins/affinity-group-processors/host-affinity/src/org/apache/cloudstack/affinity/HostAffinityProcessor.java create mode 100644 plugins/affinity-group-processors/host-affinity/test/org/apache/cloudstack/affinity/HostAffinityProcessorTest.java diff --git a/api/src/com/cloud/deploy/DataCenterDeployment.java b/api/src/com/cloud/deploy/DataCenterDeployment.java index f046b66ef06..76faf25f726 100644 --- a/api/src/com/cloud/deploy/DataCenterDeployment.java +++ b/api/src/com/cloud/deploy/DataCenterDeployment.java @@ -19,6 +19,9 @@ package com.cloud.deploy; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.vm.ReservationContext; +import java.util.ArrayList; +import java.util.List; + public class DataCenterDeployment implements DeploymentPlan { long _dcId; Long _podId; @@ -29,6 +32,7 @@ public class DataCenterDeployment implements DeploymentPlan { ExcludeList _avoids = null; boolean _recreateDisks; ReservationContext _context; + List preferredHostIds = new ArrayList<>(); public DataCenterDeployment(long dataCenterId) { this(dataCenterId, null, null, null, null, null); @@ -93,4 +97,14 @@ public class DataCenterDeployment implements DeploymentPlan { return _context; } + @Override + public void setPreferredHosts(List hostIds) { + this.preferredHostIds = new ArrayList<>(hostIds); + } + + @Override + public List getPreferredHosts() { + return this.preferredHostIds; + } + } diff --git a/api/src/com/cloud/deploy/DeploymentPlan.java b/api/src/com/cloud/deploy/DeploymentPlan.java index 456d5b85899..b57fec0cf41 100644 --- a/api/src/com/cloud/deploy/DeploymentPlan.java +++ b/api/src/com/cloud/deploy/DeploymentPlan.java @@ -19,6 +19,8 @@ package com.cloud.deploy; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.vm.ReservationContext; +import java.util.List; + /** */ public interface DeploymentPlan { @@ -65,4 +67,8 @@ public interface DeploymentPlan { Long getPhysicalNetworkId(); ReservationContext getReservationContext(); + + void setPreferredHosts(List hostIds); + + List getPreferredHosts(); } diff --git a/client/pom.xml b/client/pom.xml index 9907d8cc2ac..5653f536ac6 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -438,6 +438,11 @@ cloud-plugin-host-anti-affinity ${project.version} + + org.apache.cloudstack + cloud-plugin-host-affinity + ${project.version} + org.apache.cloudstack cloud-plugin-api-solidfire-intg-test diff --git a/core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml b/core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml index 4ec917e3419..1f70e526147 100644 --- a/core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml +++ b/core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml @@ -248,7 +248,7 @@ class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry"> + value="HostAntiAffinityProcessor,ExplicitDedicationProcessor,HostAffinityProcessor" /> diff --git a/plugins/affinity-group-processors/host-affinity/pom.xml b/plugins/affinity-group-processors/host-affinity/pom.xml new file mode 100644 index 00000000000..6b58322270b --- /dev/null +++ b/plugins/affinity-group-processors/host-affinity/pom.xml @@ -0,0 +1,33 @@ + + + 4.0.0 + Apache CloudStack Plugin - Host Affinity Processor + cloud-plugin-host-affinity + + cloudstack-plugins + org.apache.cloudstack + 4.11.1.0-SNAPSHOT + ../../pom.xml + + + install + src + + \ No newline at end of file diff --git a/plugins/affinity-group-processors/host-affinity/resources/META-INF/cloudstack/host-affinity/module.properties b/plugins/affinity-group-processors/host-affinity/resources/META-INF/cloudstack/host-affinity/module.properties new file mode 100644 index 00000000000..fe0d91b7c12 --- /dev/null +++ b/plugins/affinity-group-processors/host-affinity/resources/META-INF/cloudstack/host-affinity/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=host-affinity +parent=planner \ No newline at end of file diff --git a/plugins/affinity-group-processors/host-affinity/resources/META-INF/cloudstack/host-affinity/spring-host-affinity-context.xml b/plugins/affinity-group-processors/host-affinity/resources/META-INF/cloudstack/host-affinity/spring-host-affinity-context.xml new file mode 100644 index 00000000000..3d42e80b277 --- /dev/null +++ b/plugins/affinity-group-processors/host-affinity/resources/META-INF/cloudstack/host-affinity/spring-host-affinity-context.xml @@ -0,0 +1,35 @@ + + + + + + + + \ No newline at end of file diff --git a/plugins/affinity-group-processors/host-affinity/src/org/apache/cloudstack/affinity/HostAffinityProcessor.java b/plugins/affinity-group-processors/host-affinity/src/org/apache/cloudstack/affinity/HostAffinityProcessor.java new file mode 100644 index 00000000000..055a6442e1a --- /dev/null +++ b/plugins/affinity-group-processors/host-affinity/src/org/apache/cloudstack/affinity/HostAffinityProcessor.java @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.affinity; + +import java.util.List; +import java.util.Set; +import java.util.HashSet; +import java.util.ArrayList; + +import javax.inject.Inject; + +import com.cloud.vm.VMInstanceVO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Logger; + +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; + +import com.cloud.deploy.DeployDestination; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.exception.AffinityConflictException; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.VMInstanceDao; + +public class HostAffinityProcessor extends AffinityProcessorBase implements AffinityGroupProcessor { + + private static final Logger s_logger = Logger.getLogger(HostAffinityProcessor.class); + + @Inject + protected VMInstanceDao _vmInstanceDao; + @Inject + protected AffinityGroupDao _affinityGroupDao; + @Inject + protected AffinityGroupVMMapDao _affinityGroupVMMapDao; + + @Override + public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid) throws AffinityConflictException { + VirtualMachine vm = vmProfile.getVirtualMachine(); + List vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType()); + if (CollectionUtils.isNotEmpty(vmGroupMappings)) { + for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) { + processAffinityGroup(vmGroupMapping, plan, vm); + } + } + } + + /** + * Process Affinity Group for VM deployment + */ + protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, DeploymentPlan plan, VirtualMachine vm) { + AffinityGroupVO group = _affinityGroupDao.findById(vmGroupMapping.getAffinityGroupId()); + s_logger.debug("Processing affinity group " + group.getName() + " for VM Id: " + vm.getId()); + + List groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(group.getId()); + groupVMIds.remove(vm.getId()); + + List preferredHosts = getPreferredHostsFromGroupVMIds(groupVMIds); + plan.setPreferredHosts(preferredHosts); + } + + /** + * Get host ids set from vm ids list + */ + protected Set getHostIdSet(List vmIds) { + Set hostIds = new HashSet<>(); + for (Long groupVMId : vmIds) { + VMInstanceVO groupVM = _vmInstanceDao.findById(groupVMId); + hostIds.add(groupVM.getHostId()); + } + return hostIds; + } + + /** + * Get preferred host ids list from the affinity group VMs + */ + protected List getPreferredHostsFromGroupVMIds(List vmIds) { + return new ArrayList<>(getHostIdSet(vmIds)); + } + + @Override + public boolean check(VirtualMachineProfile vmProfile, DeployDestination plannedDestination) throws AffinityConflictException { + if (plannedDestination.getHost() == null) { + return true; + } + long plannedHostId = plannedDestination.getHost().getId(); + VirtualMachine vm = vmProfile.getVirtualMachine(); + List vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType()); + + if (CollectionUtils.isNotEmpty(vmGroupMappings)) { + for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) { + if (!checkAffinityGroup(vmGroupMapping, vm, plannedHostId)) { + return false; + } + } + } + + return true; + } + + /** + * Check Affinity Group + */ + protected boolean checkAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, VirtualMachine vm, long plannedHostId) { + List groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(vmGroupMapping.getAffinityGroupId()); + groupVMIds.remove(vm.getId()); + + Set hostIds = getHostIdSet(groupVMIds); + return CollectionUtils.isEmpty(hostIds) || hostIds.contains(plannedHostId); + } + +} diff --git a/plugins/affinity-group-processors/host-affinity/test/org/apache/cloudstack/affinity/HostAffinityProcessorTest.java b/plugins/affinity-group-processors/host-affinity/test/org/apache/cloudstack/affinity/HostAffinityProcessorTest.java new file mode 100644 index 00000000000..5dc9270e69e --- /dev/null +++ b/plugins/affinity-group-processors/host-affinity/test/org/apache/cloudstack/affinity/HostAffinityProcessorTest.java @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.affinity; + +import com.cloud.deploy.DeployDestination; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.host.Host; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(JUnit4.class) +public class HostAffinityProcessorTest { + + private static final long AFFINITY_GROUP_ID = 2L; + private static final String AFFINITY_GROUP_NAME = "Host affinity group"; + private static final Long VM_ID = 3L; + private static final Long GROUP_VM_1_ID = 1L; + private static final Long GROUP_VM_2_ID = 2L; + private static final Long HOST_ID = 1L; + private static final Long HOST_2_ID = 2L; + + @Mock + AffinityGroupDao affinityGroupDao; + + @Mock + AffinityGroupVMMapDao affinityGroupVMMapDao; + + @Mock + VMInstanceDao vmInstanceDao; + + @Spy + @InjectMocks + HostAffinityProcessor processor = new HostAffinityProcessor(); + + @Mock + DeploymentPlan plan; + + @Mock + VirtualMachine vm; + + @Mock + VMInstanceVO groupVM1; + + @Mock + VMInstanceVO groupVM2; + + @Mock + AffinityGroupVO affinityGroupVO; + + @Mock + AffinityGroupVMMapVO mapVO; + + @Mock + DeployDestination dest; + + @Mock + Host host; + + @Mock + VirtualMachineProfile profile; + + @Before + public void setup() { + MockitoAnnotations.initMocks(this); + + when(groupVM1.getHostId()).thenReturn(HOST_ID); + when(groupVM2.getHostId()).thenReturn(HOST_ID); + when(vmInstanceDao.findById(GROUP_VM_1_ID)).thenReturn(groupVM1); + when(vmInstanceDao.findById(GROUP_VM_2_ID)).thenReturn(groupVM2); + + when(affinityGroupVMMapDao.listVmIdsByAffinityGroup(AFFINITY_GROUP_ID)).thenReturn(new ArrayList<>(Arrays.asList(GROUP_VM_1_ID, GROUP_VM_2_ID, VM_ID))); + + when(vm.getId()).thenReturn(VM_ID); + + when(affinityGroupVO.getId()).thenReturn(AFFINITY_GROUP_ID); + when(affinityGroupVO.getName()).thenReturn(AFFINITY_GROUP_NAME); + when(mapVO.getAffinityGroupId()).thenReturn(AFFINITY_GROUP_ID); + + when(affinityGroupDao.findById(AFFINITY_GROUP_ID)).thenReturn(affinityGroupVO); + + when(dest.getHost()).thenReturn(host); + when(host.getId()).thenReturn(HOST_ID); + when(profile.getVirtualMachine()).thenReturn(vm); + when(affinityGroupVMMapDao.findByVmIdType(eq(VM_ID), any())).thenReturn(new ArrayList<>(Arrays.asList(mapVO))); + } + + @Test + public void testProcessAffinityGroupMultipleVMs() { + processor.processAffinityGroup(mapVO, plan, vm); + verify(plan).setPreferredHosts(Arrays.asList(HOST_ID)); + } + + @Test + public void testProcessAffinityGroupEmptyGroup() { + when(affinityGroupVMMapDao.listVmIdsByAffinityGroup(AFFINITY_GROUP_ID)).thenReturn(new ArrayList<>()); + processor.processAffinityGroup(mapVO, plan, vm); + verify(plan).setPreferredHosts(new ArrayList<>()); + } + + @Test + public void testGetPreferredHostsFromGroupVMIdsMultipleVMs() { + List list = new ArrayList<>(Arrays.asList(GROUP_VM_1_ID, GROUP_VM_2_ID)); + List preferredHosts = processor.getPreferredHostsFromGroupVMIds(list); + assertNotNull(preferredHosts); + assertEquals(1, preferredHosts.size()); + assertEquals(HOST_ID, preferredHosts.get(0)); + } + + @Test + public void testGetPreferredHostsFromGroupVMIdsEmptyVMsList() { + List list = new ArrayList<>(); + List preferredHosts = processor.getPreferredHostsFromGroupVMIds(list); + assertNotNull(preferredHosts); + assertTrue(preferredHosts.isEmpty()); + } + + @Test + public void testCheckAffinityGroup() { + assertTrue(processor.checkAffinityGroup(mapVO, vm, HOST_ID)); + } + + @Test + public void testCheckAffinityGroupWrongHostId() { + assertFalse(processor.checkAffinityGroup(mapVO, vm, HOST_2_ID)); + } + + @Test + public void testCheck() { + assertTrue(processor.check(profile, dest)); + } + + @Test + public void testCheckWrongHostId() { + when(host.getId()).thenReturn(HOST_2_ID); + assertFalse(processor.check(profile, dest)); + } +} diff --git a/plugins/pom.xml b/plugins/pom.xml index 2cf9d3ccf71..67ec0784eec 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -109,6 +109,7 @@ database/quota integrations/cloudian integrations/prometheus + affinity-group-processors/host-affinity diff --git a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java index 5d8ad0a7051..64fabb99dd5 100644 --- a/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -33,6 +33,7 @@ import javax.naming.ConfigurationException; import com.cloud.utils.db.Filter; import com.cloud.utils.fsm.StateMachine2; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -321,7 +322,7 @@ StateListener { suitableHosts.add(host); Pair> potentialResources = findPotentialDeploymentResources( suitableHosts, suitableVolumeStoragePools, avoids, - getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes); + getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes, plan.getPreferredHosts()); if (potentialResources != null) { pod = _podDao.findById(host.getPodId()); cluster = _clusterDao.findById(host.getClusterId()); @@ -461,7 +462,7 @@ StateListener { suitableHosts.add(host); Pair> potentialResources = findPotentialDeploymentResources( suitableHosts, suitableVolumeStoragePools, avoids, - getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes); + getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes, plan.getPreferredHosts()); if (potentialResources != null) { Map storageVolMap = potentialResources.second(); // remove the reused vol<->pool from @@ -1077,7 +1078,7 @@ StateListener { // choose the potential host and pool for the VM if (!suitableVolumeStoragePools.isEmpty()) { Pair> potentialResources = findPotentialDeploymentResources(suitableHosts, suitableVolumeStoragePools, avoid, - resourceUsageRequired, readyAndReusedVolumes); + resourceUsageRequired, readyAndReusedVolumes, plan.getPreferredHosts()); if (potentialResources != null) { Host host = _hostDao.findById(potentialResources.first().getId()); @@ -1217,11 +1218,12 @@ StateListener { } protected Pair> findPotentialDeploymentResources(List suitableHosts, Map> suitableVolumeStoragePools, - ExcludeList avoid, DeploymentPlanner.PlannerResourceUsage resourceUsageRequired, List readyAndReusedVolumes) { + ExcludeList avoid, DeploymentPlanner.PlannerResourceUsage resourceUsageRequired, List readyAndReusedVolumes, List preferredHosts) { s_logger.debug("Trying to find a potenial host and associated storage pools from the suitable host/pool lists for this VM"); boolean hostCanAccessPool = false; boolean haveEnoughSpace = false; + boolean hostAffinityCheck = false; if (readyAndReusedVolumes == null) { readyAndReusedVolumes = new ArrayList(); @@ -1245,6 +1247,7 @@ StateListener { s_logger.debug("Checking if host: " + potentialHost.getId() + " can access any suitable storage pool for volume: " + vol.getVolumeType()); List volumePoolList = suitableVolumeStoragePools.get(vol); hostCanAccessPool = false; + hostAffinityCheck = checkAffinity(potentialHost, preferredHosts); for (StoragePool potentialSPool : volumePoolList) { if (hostCanAccessSPool(potentialHost, potentialSPool)) { hostCanAccessPool = true; @@ -1273,8 +1276,12 @@ StateListener { s_logger.warn("insufficient capacity to allocate all volumes"); break; } + if (!hostAffinityCheck) { + s_logger.debug("Host affinity check failed"); + break; + } } - if (hostCanAccessPool && haveEnoughSpace && checkIfHostFitsPlannerUsage(potentialHost.getId(), resourceUsageRequired)) { + if (hostCanAccessPool && haveEnoughSpace && hostAffinityCheck && checkIfHostFitsPlannerUsage(potentialHost.getId(), resourceUsageRequired)) { s_logger.debug("Found a potential host " + "id: " + potentialHost.getId() + " name: " + potentialHost.getName() + " and associated storage pools for this VM"); return new Pair>(potentialHost, storage); @@ -1286,6 +1293,20 @@ StateListener { return null; } + /** + * True if: + * - Affinity is not enabled (preferred host is empty) + * - Affinity is enabled and potential host is on the preferred hosts list + * + * False if not + */ + @DB + public boolean checkAffinity(Host potentialHost, List preferredHosts) { + boolean hostAffinityEnabled = CollectionUtils.isNotEmpty(preferredHosts); + boolean hostAffinityMatches = hostAffinityEnabled && preferredHosts.contains(potentialHost.getId()); + return !hostAffinityEnabled || hostAffinityMatches; + } + protected boolean hostCanAccessSPool(Host host, StoragePool pool) { boolean hostCanAccessSPool = false; diff --git a/server/test/com/cloud/vm/DeploymentPlanningManagerImplTest.java b/server/test/com/cloud/vm/DeploymentPlanningManagerImplTest.java index 272a4fc300d..5d8f9ad2159 100644 --- a/server/test/com/cloud/vm/DeploymentPlanningManagerImplTest.java +++ b/server/test/com/cloud/vm/DeploymentPlanningManagerImplTest.java @@ -16,15 +16,19 @@ // under the License. package com.cloud.vm; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.host.Host; import org.apache.cloudstack.affinity.dao.AffinityGroupDomainMapDao; import org.junit.Before; import org.junit.BeforeClass; @@ -136,9 +140,12 @@ public class DeploymentPlanningManagerImplTest { @Inject UserVmDetailsDao vmDetailsDao; - private static long domainId = 5L; + @Mock + Host host; + private static long domainId = 5L; private static long dataCenterId = 1L; + private static long hostId = 1l; @BeforeClass public static void setUp() throws ConfigurationException { @@ -172,6 +179,7 @@ public class DeploymentPlanningManagerImplTest { planners.add(_planner); _dpm.setPlanners(planners); + Mockito.when(host.getId()).thenReturn(hostId); } @Test @@ -222,6 +230,26 @@ public class DeploymentPlanningManagerImplTest { assertNull("Planner cannot handle, destination should be null! ", dest); } + @Test + public void testCheckAffinityEmptyPreferredHosts() { + assertTrue(_dpm.checkAffinity(host, new ArrayList<>())); + } + + @Test + public void testCheckAffinityNullPreferredHosts() { + assertTrue(_dpm.checkAffinity(host, null)); + } + + @Test + public void testCheckAffinityNotEmptyPreferredHostsContainingHost() { + assertTrue(_dpm.checkAffinity(host, Arrays.asList(3l, 4l, hostId, 2l))); + } + + @Test + public void testCheckAffinityNotEmptyPreferredHostsNotContainingHost() { + assertFalse(_dpm.checkAffinity(host, Arrays.asList(3l, 4l, 2l))); + } + @Configuration @ComponentScan(basePackageClasses = {DeploymentPlanningManagerImpl.class}, includeFilters = {@Filter(value = TestConfiguration.Library.class, type = FilterType.CUSTOM)}, useDefaultFilters = false) diff --git a/test/integration/smoke/test_affinity_groups.py b/test/integration/smoke/test_affinity_groups.py index 64ec8ae8df3..f58f3d91cdc 100644 --- a/test/integration/smoke/test_affinity_groups.py +++ b/test/integration/smoke/test_affinity_groups.py @@ -21,8 +21,10 @@ from marvin.cloudstackTestCase import * from marvin.cloudstackAPI import * from marvin.lib.utils import * from marvin.lib.base import * -from marvin.lib.common import * -from marvin.sshClient import SshClient +from marvin.lib.common import (get_domain, + get_zone, + get_template, + list_virtual_machines) from nose.plugins.attrib import attr class TestDeployVmWithAffinityGroup(cloudstackTestCase): @@ -42,14 +44,14 @@ class TestDeployVmWithAffinityGroup(cloudstackTestCase): cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) cls.hypervisor = cls.testClient.getHypervisorInfo() - cls.template = get_test_template( + cls.template = get_template( cls.apiclient, cls.zone.id, cls.hypervisor ) if cls.template == FAILED: - assert False, "get_test_template() failed to return template" + assert False, "get_template() failed to return template" cls.services["virtual_machine"]["zoneid"] = cls.zone.id @@ -69,6 +71,16 @@ class TestDeployVmWithAffinityGroup(cloudstackTestCase): cls.ag = AffinityGroup.create(cls.apiclient, cls.services["virtual_machine"]["affinity"], account=cls.account.name, domainid=cls.domain.id) + host_affinity = { + "name": "marvin-host-affinity", + "type": "host affinity", + } + cls.affinity = AffinityGroup.create( + cls.apiclient, + host_affinity, + account=cls.account.name, + domainid=cls.domain.id + ) cls._cleanup = [ cls.service_offering, cls.ag, @@ -152,6 +164,81 @@ class TestDeployVmWithAffinityGroup(cloudstackTestCase): self.assertNotEqual(host_of_vm1, host_of_vm2, msg="Both VMs of affinity group %s are on the same host" % self.ag.name) + @attr(tags=["basic", "advanced", "multihost"], required_hardware="false") + def test_DeployVmAffinityGroup(self): + """ + test DeployVM in affinity groups + + deploy VM1 and VM2 in the same host-affinity groups + Verify that the vms are deployed on the same host + """ + #deploy VM1 in affinity group created in setUp + vm1 = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + templateid=self.template.id, + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + affinitygroupnames=[self.affinity.name] + ) + + list_vm1 = list_virtual_machines( + self.apiclient, + id=vm1.id + ) + self.assertEqual( + isinstance(list_vm1, list), + True, + "Check list response returns a valid list" + ) + self.assertNotEqual( + len(list_vm1), + 0, + "Check VM available in List Virtual Machines" + ) + vm1_response = list_vm1[0] + self.assertEqual( + vm1_response.state, + 'Running', + msg="VM is not in Running state" + ) + host_of_vm1 = vm1_response.hostid + + #deploy VM2 in affinity group created in setUp + vm2 = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + templateid=self.template.id, + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + affinitygroupnames=[self.affinity.name] + ) + list_vm2 = list_virtual_machines( + self.apiclient, + id=vm2.id + ) + self.assertEqual( + isinstance(list_vm2, list), + True, + "Check list response returns a valid list" + ) + self.assertNotEqual( + len(list_vm2), + 0, + "Check VM available in List Virtual Machines" + ) + vm2_response = list_vm2[0] + self.assertEqual( + vm2_response.state, + 'Running', + msg="VM is not in Running state" + ) + host_of_vm2 = vm2_response.hostid + + self.assertEqual(host_of_vm1, host_of_vm2, + msg="Both VMs of affinity group %s are on different hosts" % self.affinity.name) @classmethod def tearDownClass(cls): From 7e6fddb7ab04e16023fa263839067d92576ee948 Mon Sep 17 00:00:00 2001 From: Mike Tutkowski Date: Mon, 21 May 2018 01:24:42 -0600 Subject: [PATCH 3/3] managed-storage: Handle Ceph (#2655) In 4.11.0, I added the ability to online migrate volumes from NFS to managed storage. This actually works for Ceph to managed storage in a private 4.8 branch, as well. I thought I had brought along all of the necessary code from that private 4.8 branch to make Ceph to managed storage functional in 4.11.0, but missed one piece (which is fixed by this PR). --- .../wrapper/LibvirtMigrateCommandWrapper.java | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 67ec1b731af..067e77df3cf 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -45,6 +45,7 @@ import javax.xml.transform.stream.StreamResult; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.libvirt.Connect; import org.libvirt.Domain; @@ -332,9 +333,9 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper paths, String sourceFileDevText) { - if (paths != null && sourceFileDevText != null) { + private String getPathFromSourceText(Set paths, String sourceText) { + if (paths != null && !StringUtils.isBlank(sourceText)) { for (String path : paths) { - if (sourceFileDevText.contains(path)) { + if (sourceText.contains(path)) { return path; } } @@ -395,7 +396,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper