diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 52bcf5033af..c3908795f7c 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -1906,8 +1906,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host)); } } else { - // This might impact other managed storages, grant access for PowerFlex storage pool only - if (pool.getPoolType() == Storage.StoragePoolType.PowerFlex) { + // This might impact other managed storages, grant access for PowerFlex and Iscsi/Solidfire storage pool only + if (pool.getPoolType() == Storage.StoragePoolType.PowerFlex || pool.getPoolType() == Storage.StoragePoolType.Iscsi) { try { volService.grantAccess(volFactory.getVolume(vol.getId()), host, (DataStore)pool); } catch (Exception e) { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java index 80a4362f0bd..7e6b8c19ad4 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java @@ -1107,12 +1107,25 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { return "snapshots/" + accountId + "/" + volumeId; } + protected boolean isManagedStorageDatastorePath(final String datastorePath) { + // ex. [-iqn.2010-01.com.solidfire:3p53.data-9999.97-0] i-2-9999-VM + return datastorePath != null && datastorePath.startsWith("[-iqn."); + } + + protected String getManagedDatastoreName(final String datastorePath) { + // ex. [-iqn.2010-01.com.solidfire:3p53.data-9999.97-0] + return datastorePath == null ? datastorePath : datastorePath.split(" ")[0]; + } + private long getVMSnapshotChainSize(VmwareContext context, VmwareHypervisorHost hyperHost, String fileName, ManagedObjectReference morDs, String exceptFileName, String vmName) throws Exception { long size = 0; DatastoreMO dsMo = new DatastoreMO(context, morDs); HostDatastoreBrowserMO browserMo = dsMo.getHostDatastoreBrowserMO(); String datastorePath = (new DatastoreFile(dsMo.getName(), vmName)).getPath(); + if (isManagedStorageDatastorePath(datastorePath)) { + datastorePath = getManagedDatastoreName(datastorePath); + } HostDatastoreBrowserSearchSpec searchSpec = new HostDatastoreBrowserSearchSpec(); FileQueryFlags fqf = new FileQueryFlags(); fqf.setFileSize(true); @@ -1241,11 +1254,9 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { String vmdkName = null; // if this is managed storage - if (fullPath.startsWith("[-iqn.")) { // ex. [-iqn.2010-01.com.company:3y8w.vol-10.64-0] -iqn.2010-01.com.company:3y8w.vol-10.64-0-000001.vmdk - baseName = fullPath.split(" ")[0]; // ex. [-iqn.2010-01.com.company:3y8w.vol-10.64-0] - - // remove '[' and ']' - baseName = baseName.substring(1, baseName.length() - 1); + if (isManagedStorageDatastorePath(fullPath)) { + baseName = getManagedDatastoreName(fullPath); + baseName = baseName.substring(1, baseName.length() - 1); // remove '[' and ']' vmdkName = fullPath; // for managed storage, vmdkName == fullPath } else { @@ -1288,12 +1299,9 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } } else { Map mapNewDisk = getNewDiskMap(vmMo); - // if this is managed storage - if (path.startsWith("[-iqn.")) { // ex. [-iqn.2010-01.com.company:3y8w.vol-10.64-0] -iqn.2010-01.com.company:3y8w.vol-10.64-0-000001.vmdk - path = path.split(" ")[0]; // ex. [-iqn.2010-01.com.company:3y8w.vol-10.64-0] - - // remove '[' and ']' - baseName = path.substring(1, path.length() - 1); + if (isManagedStorageDatastorePath(path)) { + path = getManagedDatastoreName(path); + baseName = path.substring(1, path.length() - 1); // remove '[' and ']' } else { baseName = VmwareHelper.trimSnapshotDeltaPostfix(path); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 481307de41d..8a99eb30013 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -949,6 +949,11 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes ManagedObjectReference morDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, VmwareResource.getDatastoreName(iScsiName)); DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), morDS); + if (path.startsWith("[-iqn.")) { + // Rescan 1:1 LUN that VMware may not know the LUN was recently resized + _storageProcessor.rescanAllHosts(context, lstHosts, true, true); + } + _storageProcessor.expandDatastore(hostDatastoreSystem, dsMo); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 7b4d5f118b4..ba9d16f8186 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2825,7 +2825,15 @@ public class VmwareStorageProcessor implements StorageProcessor { morDs = firstHostDatastoreSystemMO.findDatastoreByName(datastoreName); if (morDs == null) { - morDs = firstHostDatastoreSystemMO.createVmfsDatastore(datastoreName, hostScsiDisk); + final String hostVersion = firstHostMO.getProductVersion(); + if (hostVersion.compareTo(VmwareHelper.MIN_VERSION_VMFS6) >= 0) { + morDs = firstHostDatastoreSystemMO.createVmfs6Datastore(datastoreName, hostScsiDisk); + } else { + morDs = firstHostDatastoreSystemMO.createVmfs5Datastore(datastoreName, hostScsiDisk); + } + } else { + // in case of iSCSI/solidfire 1:1 VMFS datastore could be inaccessible + mountVmfsDatastore(new DatastoreMO(context, morDs), lstHosts); } if (morDs != null) { @@ -3364,7 +3372,7 @@ public class VmwareStorageProcessor implements StorageProcessor { } } - private void rescanAllHosts(VmwareContext context, List> lstHostPairs, boolean rescanHba, boolean rescanVmfs) throws Exception { + public void rescanAllHosts(VmwareContext context, List> lstHostPairs, boolean rescanHba, boolean rescanVmfs) throws Exception { List hosts = new ArrayList<>(lstHostPairs.size()); for (Pair hostPair : lstHostPairs) { diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImplTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImplTest.java index 1d207e37bca..c72f23c628e 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImplTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImplTest.java @@ -116,4 +116,15 @@ public class VmwareStorageManagerImplTest { public void testSetVolumeToPathAndSizeDatastoreClusterDifferentChildStore() { testCommon(Storage.StoragePoolType.PreSetup, Storage.StoragePoolType.DatastoreCluster, true); } + + @Test + public void testIsManagedStorageDatastorePath() { + Assert.assertTrue("Test if [-iqn... is a managed storage", storageManager.isManagedStorageDatastorePath("[-iqn.2010-01.com.solidfire:3p53.data-9999.97-0] i-2-9999-VM.vmdk")); + Assert.assertFalse("Test if [SomeDS] is not a managed storage", storageManager.isManagedStorageDatastorePath("[SomeDS] i-2-9999-VM/disk.vmdk")); + } + + @Test + public void testGetManagedDatastoreName() { + Assert.assertEquals("[-iqn.2010-01.com.solidfire:3p53.data-9999.97-0]", storageManager.getManagedDatastoreName("[-iqn.2010-01.com.solidfire:3p53.data-9999.97-0] i-2-9999-VM.vmdk")); + } } diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java index 702bdc3669a..4478dc98ca4 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.storage.datastore.driver; import java.text.NumberFormat; +import java.util.Arrays; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -85,6 +86,8 @@ import com.cloud.user.dao.AccountDao; import com.cloud.utils.Pair; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.VirtualMachine; import com.google.common.base.Preconditions; public class SolidFirePrimaryDataStoreDriver implements PrimaryDataStoreDriver { @@ -111,6 +114,7 @@ public class SolidFirePrimaryDataStoreDriver implements PrimaryDataStoreDriver { @Inject private PrimaryDataStoreDao storagePoolDao; @Inject private StoragePoolDetailsDao storagePoolDetailsDao; @Inject private VMTemplatePoolDao vmTemplatePoolDao; + @Inject private VMInstanceDao vmDao; @Inject private VolumeDao volumeDao; @Inject private VolumeDetailsDao volumeDetailsDao; @Inject private VolumeDataFactory volumeFactory; @@ -187,13 +191,33 @@ public class SolidFirePrimaryDataStoreDriver implements PrimaryDataStoreDriver { } } + private boolean isRevokeAccessNotNeeded(DataObject dataObject) { + // Workaround: don't unplug iscsi lun when volume is attached to a VM + // This is regression workaround from upper layers which are calling + // a releaseVmResources() method that calls the revoke on an attached disk + if (dataObject.getType() == DataObjectType.VOLUME) { + Volume volume = volumeDao.findById(dataObject.getId()); + if (volume.getInstanceId() != null) { + VirtualMachine vm = vmDao.findById(volume.getInstanceId()); + if (vm != null && !Arrays.asList(VirtualMachine.State.Destroyed, VirtualMachine.State.Expunging, VirtualMachine.State.Error).contains(vm.getState())) { + return true; + } + } + } + return false; + } + @Override - public void revokeAccess(DataObject dataObject, Host host, DataStore dataStore) - { + public void revokeAccess(DataObject dataObject, Host host, DataStore dataStore) { if (dataObject == null || host == null || dataStore == null) { return; } + if (isRevokeAccessNotNeeded(dataObject)) { + LOGGER.debug("Skipping revoke access for Solidfire data object type:" + dataObject.getType() + " id:" + dataObject.getId()); + return; + } + long sfVolumeId = getSolidFireVolumeId(dataObject, false); long clusterId = host.getClusterId(); long storagePoolId = dataStore.getId(); @@ -210,6 +234,8 @@ public class SolidFirePrimaryDataStoreDriver implements PrimaryDataStoreDriver { throw new CloudRuntimeException(errMsg); } + LOGGER.debug("Revoking access for Solidfire data object type:" + dataObject.getType() + " id:" + dataObject.getId()); + try { SolidFireUtil.SolidFireConnection sfConnection = SolidFireUtil.getSolidFireConnection(storagePoolId, storagePoolDetailsDao); diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index aa98b1fba35..cbf23cde8c1 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -910,7 +910,7 @@ public class ApiResponseHelper implements ResponseGenerator { Long networkId = vlan.getNetworkId(); if (networkId != null) { Network network = _ntwkModel.getNetwork(networkId); - if (network != null) { + if (network != null && TrafficType.Guest.equals(network.getTrafficType())) { Long accountId = network.getAccountId(); populateAccount(vlanResponse, accountId); populateDomain(vlanResponse, ApiDBUtils.findAccountById(accountId).getDomainId()); diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 6722b0a1e8b..ef617ed4cb7 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -5414,10 +5414,11 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati // Check if any of the Public IP addresses is allocated to another // account - boolean forSystemVms = false; final List ips = _publicIpAddressDao.listByVlanId(vlanDbId); for (final IPAddressVO ip : ips) { - forSystemVms = ip.isForSystemVms(); + if (ip.isForSystemVms()) { + throw new InvalidParameterValueException(ip.getAddress() + " Public IP address in range is dedicated to system vms "); + } final Long allocatedToAccountId = ip.getAllocatedToAccountId(); if (allocatedToAccountId != null) { if (vlanOwner != null && allocatedToAccountId != vlanOwner.getId()) { @@ -5442,7 +5443,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NET_IP_ASSIGN, vlanOwner.getId(), ip.getDataCenterId(), ip.getId(), ip.getAddress().toString(), ip.isSourceNat(), vlan.getVlanType().toString(), ip.getSystem(), usageHidden, ip.getClass().getName(), ip.getUuid()); } - } else if (domain != null && !forSystemVms) { + } else if (domain != null) { // Create an DomainVlanMapVO entry DomainVlanMapVO domainVlanMapVO = new DomainVlanMapVO(domain.getId(), vlan.getId()); _domainVlanMapDao.persist(domainVlanMapVO); @@ -7276,7 +7277,6 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati @Override public Domain getVlanDomain(long vlanId) { Vlan vlan = _vlanDao.findById(vlanId); - Long domainId = null; // if vlan is Virtual Domain specific, get vlan information from the // accountVlanMap; otherwise get account information diff --git a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java index 04e9ec51ad5..3d752138be1 100644 --- a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java +++ b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java @@ -334,19 +334,10 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage @DB private IPAddressVO assignAndAllocateIpAddressEntry(final Account owner, final VlanType vlanUse, final Long guestNetworkId, final boolean sourceNat, final boolean allocate, final boolean isSystem, - final Long vpcId, final Boolean displayIp, final boolean fetchFromDedicatedRange, + final Long vpcId, final Boolean displayIp, final List addressVOS) throws CloudRuntimeException { return Transaction.execute((TransactionCallbackWithException) status -> { IPAddressVO finalAddress = null; - if (!fetchFromDedicatedRange && VlanType.VirtualNetwork.equals(vlanUse)) { - // Check that the maximum number of public IPs for the given accountId will not be exceeded - try { - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.public_ip); - } catch (ResourceAllocationException ex) { - s_logger.warn("Failed to allocate resource of type " + ex.getResourceType() + " for account " + owner); - throw new AccountLimitException("Maximum number of public IP addresses for account: " + owner.getAccountName() + " has been exceeded."); - } - } for (IPAddressVO possibleAddr : addressVOS) { if (possibleAddr.getState() != State.Free) { @@ -496,9 +487,8 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage AssignIpAddressSearch.and("dc", AssignIpAddressSearch.entity().getDataCenterId(), Op.EQ); AssignIpAddressSearch.and("allocated", AssignIpAddressSearch.entity().getAllocatedTime(), Op.NULL); AssignIpAddressSearch.and("vlanId", AssignIpAddressSearch.entity().getVlanId(), Op.IN); - if (SystemVmPublicIpReservationModeStrictness.value()) { - AssignIpAddressSearch.and("forSystemVms", AssignIpAddressSearch.entity().isForSystemVms(), Op.EQ); - } + AssignIpAddressSearch.and("forSystemVms", AssignIpAddressSearch.entity().isForSystemVms(), Op.EQ); + SearchBuilder vlanSearch = _vlanDao.createSearchBuilder(); vlanSearch.and("type", vlanSearch.entity().getVlanType(), Op.EQ); vlanSearch.and("networkId", vlanSearch.entity().getNetworkId(), Op.EQ); @@ -827,6 +817,10 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage throws InsufficientAddressCapacityException { List addrs = listAvailablePublicIps(dcId, podId, vlanDbIds, owner, vlanUse, guestNetworkId, sourceNat, assign, allocate, requestedIp, requestedGateway, isSystem, vpcId, displayIp, forSystemVms, true); IPAddressVO addr = addrs.get(0); + if (assign) { + addr = assignAndAllocateIpAddressEntry(owner, vlanUse, guestNetworkId, sourceNat, allocate, + isSystem,vpcId, displayIp, addrs); + } if (vlanUse == VlanType.VirtualNetwork) { _firewallMgr.addSystemFirewallRules(addr, owner); } @@ -838,128 +832,99 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage public List listAvailablePublicIps(final long dcId, final Long podId, final List vlanDbIds, final Account owner, final VlanType vlanUse, final Long guestNetworkId, final boolean sourceNat, final boolean assign, final boolean allocate, final String requestedIp, final String requestedGateway, final boolean isSystem, final Long vpcId, final Boolean displayIp, final boolean forSystemVms, final boolean lockOneRow) throws InsufficientAddressCapacityException { - return Transaction.execute(new TransactionCallbackWithException, InsufficientAddressCapacityException>() { - @Override - public List doInTransaction(TransactionStatus status) throws InsufficientAddressCapacityException { - StringBuilder errorMessage = new StringBuilder("Unable to get ip address in "); - boolean fetchFromDedicatedRange = false; - List dedicatedVlanDbIds = new ArrayList(); - List nonDedicatedVlanDbIds = new ArrayList(); - DataCenter zone = _entityMgr.findById(DataCenter.class, dcId); - SearchCriteria sc = null; - if (podId != null) { - sc = AssignIpAddressFromPodVlanSearch.create(); - sc.setJoinParameters("podVlanMapSB", "podId", podId); - errorMessage.append(" pod id=" + podId); - } else { - sc = AssignIpAddressSearch.create(); - errorMessage.append(" zone id=" + dcId); - } + StringBuilder errorMessage = new StringBuilder("Unable to get ip address in "); + boolean fetchFromDedicatedRange = false; + List dedicatedVlanDbIds = new ArrayList(); + List nonDedicatedVlanDbIds = new ArrayList(); + DataCenter zone = _entityMgr.findById(DataCenter.class, dcId); - // If owner has dedicated Public IP ranges, fetch IP from the dedicated range - // Otherwise fetch IP from the system pool - Network network = _networksDao.findById(guestNetworkId); - //Checking if network is null in the case of system VM's. At the time of allocation of IP address to systemVm, no network is present. - if(network == null || !(network.getGuestType() == GuestType.Shared && zone.getNetworkType() == NetworkType.Advanced)) { - List maps = _accountVlanMapDao.listAccountVlanMapsByAccount(owner.getId()); - for (AccountVlanMapVO map : maps) { - if (vlanDbIds == null || vlanDbIds.contains(map.getVlanDbId())) - dedicatedVlanDbIds.add(map.getVlanDbId()); - } - } - List domainMaps = _domainVlanMapDao.listDomainVlanMapsByDomain(owner.getDomainId()); - for (DomainVlanMapVO map : domainMaps) { + SearchCriteria sc = null; + if (podId != null) { + sc = AssignIpAddressFromPodVlanSearch.create(); + sc.setJoinParameters("podVlanMapSB", "podId", podId); + errorMessage.append(" pod id=" + podId); + } else { + sc = AssignIpAddressSearch.create(); + errorMessage.append(" zone id=" + dcId); + } + + sc.setParameters("dc", dcId); + + // for direct network take ip addresses only from the vlans belonging to the network + if (vlanUse == VlanType.DirectAttached) { + sc.setJoinParameters("vlan", "networkId", guestNetworkId); + errorMessage.append(", network id=" + guestNetworkId); + } + if (requestedGateway != null) { + sc.setJoinParameters("vlan", "vlanGateway", requestedGateway); + errorMessage.append(", requested gateway=" + requestedGateway); + } + sc.setJoinParameters("vlan", "type", vlanUse); + + Network network = _networksDao.findById(guestNetworkId); + String routerIpAddress = null; + if (network != null) { + NetworkDetailVO routerIpDetail = _networkDetailsDao.findDetail(network.getId(), ApiConstants.ROUTER_IP); + routerIpAddress = routerIpDetail != null ? routerIpDetail.getValue() : null; + } + if (requestedIp != null) { + sc.addAnd("address", SearchCriteria.Op.EQ, requestedIp); + errorMessage.append(": requested ip " + requestedIp + " is not available"); + } else if (routerIpAddress != null) { + sc.addAnd("address", Op.NEQ, routerIpAddress); + } + + boolean ascOrder = ! forSystemVms; + Filter filter = new Filter(IPAddressVO.class, "forSystemVms", ascOrder, 0l, 1l); + + filter.addOrderBy(IPAddressVO.class,"vlanId", true); + + List addrs = new ArrayList<>(); + + if (forSystemVms) { + // Get Public IPs for system vms in dedicated ranges + sc.setParameters("forSystemVms", true); + if (lockOneRow) { + addrs = _ipAddressDao.lockRows(sc, filter, true); + } else { + addrs = new ArrayList<>(_ipAddressDao.search(sc, null)); + } + } + if ((!lockOneRow || (lockOneRow && CollectionUtils.isEmpty(addrs))) && + !(forSystemVms && SystemVmPublicIpReservationModeStrictness.value())) { + sc.setParameters("forSystemVms", false); + // If owner has dedicated Public IP ranges, fetch IP from the dedicated range + // Otherwise fetch IP from the system pool + // Checking if network is null in the case of system VM's. At the time of allocation of IP address to systemVm, no network is present. + if (network == null || !(network.getGuestType() == GuestType.Shared && zone.getNetworkType() == NetworkType.Advanced)) { + List maps = _accountVlanMapDao.listAccountVlanMapsByAccount(owner.getId()); + for (AccountVlanMapVO map : maps) { if (vlanDbIds == null || vlanDbIds.contains(map.getVlanDbId())) dedicatedVlanDbIds.add(map.getVlanDbId()); } - List nonDedicatedVlans = _vlanDao.listZoneWideNonDedicatedVlans(dcId); - for (VlanVO nonDedicatedVlan : nonDedicatedVlans) { - if (vlanDbIds == null || vlanDbIds.contains(nonDedicatedVlan.getId())) - nonDedicatedVlanDbIds.add(nonDedicatedVlan.getId()); - } - - if (vlanUse == VlanType.VirtualNetwork) { - if (!dedicatedVlanDbIds.isEmpty()) { - fetchFromDedicatedRange = true; - sc.setParameters("vlanId", dedicatedVlanDbIds.toArray()); - errorMessage.append(", vlanId id=" + Arrays.toString(dedicatedVlanDbIds.toArray())); - } else if (!nonDedicatedVlanDbIds.isEmpty()) { - sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray()); - errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray())); - } else { - if (podId != null) { - InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", Pod.class, podId); - ex.addProxyObject(ApiDBUtils.findPodById(podId).getUuid()); - throw ex; - } - s_logger.warn(errorMessage.toString()); - InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", DataCenter.class, dcId); - ex.addProxyObject(ApiDBUtils.findZoneById(dcId).getUuid()); - throw ex; - } - } - - sc.setParameters("dc", dcId); - - // for direct network take ip addresses only from the vlans belonging to the network - if (vlanUse == VlanType.DirectAttached) { - sc.setJoinParameters("vlan", "networkId", guestNetworkId); - errorMessage.append(", network id=" + guestNetworkId); - } - if (requestedGateway != null) { - sc.setJoinParameters("vlan", "vlanGateway", requestedGateway); - errorMessage.append(", requested gateway=" + requestedGateway); - } - sc.setJoinParameters("vlan", "type", vlanUse); - String routerIpAddress = null; - if (network != null) { - NetworkDetailVO routerIpDetail = _networkDetailsDao.findDetail(network.getId(), ApiConstants.ROUTER_IP); - routerIpAddress = routerIpDetail != null ? routerIpDetail.getValue() : null; - } - if (requestedIp != null) { - sc.addAnd("address", SearchCriteria.Op.EQ, requestedIp); - errorMessage.append(": requested ip " + requestedIp + " is not available"); - } else if (routerIpAddress != null) { - sc.addAnd("address", Op.NEQ, routerIpAddress); - } - - boolean ascOrder = ! forSystemVms; - Filter filter = new Filter(IPAddressVO.class, "forSystemVms", ascOrder, 0l, 1l); - if (SystemVmPublicIpReservationModeStrictness.value()) { - sc.setParameters("forSystemVms", forSystemVms); - } - - filter.addOrderBy(IPAddressVO.class,"vlanId", true); - - List addrs; - - if (lockOneRow) { - addrs = _ipAddressDao.lockRows(sc, filter, true); + } + List domainMaps = _domainVlanMapDao.listDomainVlanMapsByDomain(owner.getDomainId()); + for (DomainVlanMapVO map : domainMaps) { + if (vlanDbIds == null || vlanDbIds.contains(map.getVlanDbId())) + dedicatedVlanDbIds.add(map.getVlanDbId()); + } + List nonDedicatedVlans = _vlanDao.listZoneWideNonDedicatedVlans(dcId); + for (VlanVO nonDedicatedVlan : nonDedicatedVlans) { + if (vlanDbIds == null || vlanDbIds.contains(nonDedicatedVlan.getId())) + nonDedicatedVlanDbIds.add(nonDedicatedVlan.getId()); + } + if (vlanUse == VlanType.VirtualNetwork) { + if (!dedicatedVlanDbIds.isEmpty()) { + fetchFromDedicatedRange = true; + sc.setParameters("vlanId", dedicatedVlanDbIds.toArray()); + errorMessage.append(", vlanId id=" + Arrays.toString(dedicatedVlanDbIds.toArray())); + } else if (!nonDedicatedVlanDbIds.isEmpty()) { + sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray()); + errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray())); } else { - addrs = new ArrayList<>(_ipAddressDao.search(sc, null)); - } - - // If all the dedicated IPs of the owner are in use fetch an IP from the system pool - if ((!lockOneRow || (lockOneRow && addrs.size() == 0)) && fetchFromDedicatedRange && vlanUse == VlanType.VirtualNetwork) { - // Verify if account is allowed to acquire IPs from the system - boolean useSystemIps = UseSystemPublicIps.valueIn(owner.getId()); - if (useSystemIps && !nonDedicatedVlanDbIds.isEmpty()) { - fetchFromDedicatedRange = false; - sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray()); - errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray())); - if (lockOneRow) { - addrs = _ipAddressDao.lockRows(sc, filter, true); - } else { - addrs.addAll(_ipAddressDao.search(sc, null)); - } - } - } - - if (lockOneRow && addrs.size() == 0) { if (podId != null) { InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", Pod.class, podId); - // for now, we hardcode the table names, but we should ideally do a lookup for the tablename from the VO object. ex.addProxyObject(ApiDBUtils.findPodById(podId).getUuid()); throw ex; } @@ -968,17 +933,58 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage ex.addProxyObject(ApiDBUtils.findZoneById(dcId).getUuid()); throw ex; } - - if (lockOneRow) { - assert (addrs.size() == 1) : "Return size is incorrect: " + addrs.size(); - } - if (assign) { - assignAndAllocateIpAddressEntry(owner, vlanUse, guestNetworkId, sourceNat, allocate, - isSystem,vpcId, displayIp, fetchFromDedicatedRange, addrs); - } - return addrs; } - }); + if (lockOneRow) { + addrs = _ipAddressDao.lockRows(sc, filter, true); + } else { + addrs = new ArrayList<>(_ipAddressDao.search(sc, null)); + } + + // If all the dedicated IPs of the owner are in use fetch an IP from the system pool + if ((!lockOneRow || (lockOneRow && addrs.size() == 0)) && fetchFromDedicatedRange && vlanUse == VlanType.VirtualNetwork) { + // Verify if account is allowed to acquire IPs from the system + boolean useSystemIps = UseSystemPublicIps.valueIn(owner.getId()); + if (useSystemIps && !nonDedicatedVlanDbIds.isEmpty()) { + fetchFromDedicatedRange = false; + sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray()); + errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray())); + if (lockOneRow) { + addrs = _ipAddressDao.lockRows(sc, filter, true); + } else { + addrs.addAll(_ipAddressDao.search(sc, null)); + } + } + } + } + + if (lockOneRow && addrs.size() == 0) { + if (podId != null) { + InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", Pod.class, podId); + // for now, we hardcode the table names, but we should ideally do a lookup for the tablename from the VO object. + ex.addProxyObject(ApiDBUtils.findPodById(podId).getUuid()); + throw ex; + } + s_logger.warn(errorMessage.toString()); + InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", DataCenter.class, dcId); + ex.addProxyObject(ApiDBUtils.findZoneById(dcId).getUuid()); + throw ex; + } + + if (lockOneRow) { + assert (addrs.size() == 1) : "Return size is incorrect: " + addrs.size(); + } + + if (assign && !fetchFromDedicatedRange && VlanType.VirtualNetwork.equals(vlanUse)) { + // Check that the maximum number of public IPs for the given accountId will not be exceeded + try { + _resourceLimitMgr.checkResourceLimit(owner, ResourceType.public_ip); + } catch (ResourceAllocationException ex) { + s_logger.warn("Failed to allocate resource of type " + ex.getResourceType() + " for account " + owner); + throw new AccountLimitException("Maximum number of public IP addresses for account: " + owner.getAccountName() + " has been exceeded."); + } + } + + return addrs; } @DB diff --git a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java index 99d3e7d9492..85bd43617b0 100644 --- a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java +++ b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java @@ -21,6 +21,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -35,6 +36,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.network.router.deployment.RouterDeploymentDefinition; import org.apache.cloudstack.utils.CloudStackVersion; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; @@ -42,6 +44,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.manager.Commands; import com.cloud.alert.AlertManager; +import com.cloud.capacity.CapacityManager; import com.cloud.configuration.Config; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenter; @@ -167,6 +170,8 @@ public class NetworkHelperImpl implements NetworkHelper { RouterHealthCheckResultDao _routerHealthCheckResultDao; @Inject Ipv6Service ipv6Service; + @Inject + CapacityManager capacityMgr; protected final Map> hypervisorsMap = new HashMap<>(); @@ -502,12 +507,12 @@ public class NetworkHelperImpl implements NetworkHelper { // failed both times, throw the exception up final List hypervisors = getHypervisors(routerDeploymentDefinition); - int allocateRetry = 0; - int startRetry = 0; DomainRouterVO router = null; for (final Iterator iter = hypervisors.iterator(); iter.hasNext();) { final HypervisorType hType = iter.next(); try { + checkIfZoneHasCapacity(routerDeploymentDefinition.getDest().getDataCenter(), hType, routerOffering); + final long id = _routerDao.getNextInSequence(Long.class, "id"); if (s_logger.isDebugEnabled()) { s_logger.debug(String.format("Allocating the VR with id=%s in datacenter %s with the hypervisor type %s", id, routerDeploymentDefinition.getDest() @@ -548,14 +553,12 @@ public class NetworkHelperImpl implements NetworkHelper { reallocateRouterNetworks(routerDeploymentDefinition, router, template, null); router = _routerDao.findById(router.getId()); } catch (final InsufficientCapacityException ex) { - if (allocateRetry < 2 && iter.hasNext()) { + if (iter.hasNext()) { s_logger.debug("Failed to allocate the VR with hypervisor type " + hType + ", retrying one more time"); continue; } else { throw ex; } - } finally { - allocateRetry++; } if (startRouter) { @@ -563,7 +566,7 @@ public class NetworkHelperImpl implements NetworkHelper { router = startVirtualRouter(router, _accountMgr.getSystemUser(), _accountMgr.getSystemAccount(), routerDeploymentDefinition.getParams()); break; } catch (final InsufficientCapacityException ex) { - if (startRetry < 2 && iter.hasNext()) { + if (iter.hasNext()) { s_logger.debug("Failed to start the VR " + router + " with hypervisor type " + hType + ", " + "destroying it and recreating one more time"); // destroy the router destroyRouter(router.getId(), _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM), User.UID_SYSTEM); @@ -571,8 +574,6 @@ public class NetworkHelperImpl implements NetworkHelper { } else { throw ex; } - } finally { - startRetry++; } } else { // return stopped router @@ -583,6 +584,25 @@ public class NetworkHelperImpl implements NetworkHelper { return router; } + private void checkIfZoneHasCapacity(final DataCenter zone, final HypervisorType hypervisorType, final ServiceOfferingVO routerOffering) throws InsufficientServerCapacityException { + List hosts = _hostDao.listByDataCenterIdAndHypervisorType(zone.getId(), hypervisorType); + if (CollectionUtils.isEmpty(hosts)) { + String msg = String.format("Zone %s has no %s host available which is enabled and in Up state", zone.getName(), hypervisorType); + s_logger.debug(msg); + throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); + } + for (HostVO host : hosts) { + Pair cpuCapabilityAndCapacity = capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host, routerOffering, false); + if (cpuCapabilityAndCapacity.first() && cpuCapabilityAndCapacity.second()) { + s_logger.debug("Host " + host + " has enough capacity for the router"); + return; + } + } + String msg = String.format("Zone %s has no %s host which has enough capacity", zone.getName(), hypervisorType); + s_logger.debug(msg); + throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); + } + protected void filterSupportedHypervisors(final List hypervisors) { // For non vpc we keep them all assuming all types in the list are // supported @@ -619,7 +639,7 @@ public class NetworkHelperImpl implements NetworkHelper { throw new InsufficientServerCapacityException("Unable to create virtual router, there are no clusters in the zone." + getNoHypervisorsErrMsgDetails(), DataCenter.class, dest.getDataCenter().getId()); } - return hypervisors; + return new ArrayList(new LinkedHashSet<>(hypervisors)); } /* diff --git a/test/integration/smoke/test_public_ip_range.py b/test/integration/smoke/test_public_ip_range.py index 69baf5f2fc2..19edc4c164f 100644 --- a/test/integration/smoke/test_public_ip_range.py +++ b/test/integration/smoke/test_public_ip_range.py @@ -128,9 +128,8 @@ class TestDedicatePublicIPRange(cloudstackTestCase): id=self.public_ip_range.vlan.id ) public_ip_response = list_public_ip_range_response[0] - self.assertEqual( + self.assertIsNone( public_ip_response.account, - "system", "Check account name is system account in listVlanIpRanges" ) return diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 0068a12d21a..7b7bb0810b5 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1942,6 +1942,7 @@ "label.suspend.project": "Suspend project", "label.switch.type": "Switch type", "label.sync.storage": "Sync storage pool", +"label.system.ip.pool": "System Pool", "label.system.offering": "System offering", "label.system.offerings": "System offerings", "label.system.service.offering": "System service offering", diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 99634e330c2..6602ecb884c 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -32,9 +32,9 @@ export default { permission: ['listVirtualMachinesMetrics'], resourceType: 'UserVm', params: () => { - var params = {} + var params = { details: 'servoff,tmpl,nics' } if (store.getters.metrics) { - params = { state: 'running' } + params = { details: 'servoff,tmpl,nics,stats' } } return params }, diff --git a/ui/src/store/mutation-types.js b/ui/src/store/mutation-types.js index bbc27ef1b1a..0fe4a104958 100644 --- a/ui/src/store/mutation-types.js +++ b/ui/src/store/mutation-types.js @@ -37,6 +37,7 @@ export const DOMAIN_STORE = 'DOMAIN_STORE' export const DARK_MODE = 'DARK_MODE' export const VUE_VERSION = 'VUE_VERSION' export const CUSTOM_COLUMNS = 'CUSTOM_COLUMNS' +export const RELOAD_ALL_PROJECTS = 'RELOAD_ALL_PROJECTS' export const CONTENT_WIDTH_TYPE = { Fluid: 'Fluid', diff --git a/ui/src/utils/request.js b/ui/src/utils/request.js index 5f22307082e..c2fe04ab9d1 100644 --- a/ui/src/utils/request.js +++ b/ui/src/utils/request.js @@ -192,8 +192,12 @@ const sourceToken = { }, cancel: () => { if (!source) sourceToken.init() - source.cancel() - source = null + if (source) { + source.cancel() + source = null + } else { + console.log('Source token failed to be cancelled') + } } } diff --git a/ui/src/views/AutogenView.vue b/ui/src/views/AutogenView.vue index 2cd4c31d070..09a8e9eb285 100644 --- a/ui/src/views/AutogenView.vue +++ b/ui/src/views/AutogenView.vue @@ -448,6 +448,7 @@ import { ref, reactive, toRaw } from 'vue' import { api } from '@/api' import { mixinDevice } from '@/utils/mixin.js' import { genericCompare } from '@/utils/sort.js' +import { sourceToken } from '@/utils/request' import store from '@/store' import eventBus from '@/config/eventBus' @@ -627,6 +628,9 @@ export default { next() }, beforeRouteLeave (to, from, next) { + console.log('DEBUG - Due to route change, ignoring results for any on-going API request', this.apiName) + sourceToken.cancel() + sourceToken.init() this.currentPath = this.$route.fullPath next() }, @@ -884,6 +888,10 @@ export default { delete params.showunique } + if (['listVirtualMachinesMetrics'].includes(this.apiName) && this.dataView) { + delete params.details + } + this.loading = true if (this.$route.params && this.$route.params.id) { params.id = this.$route.params.id @@ -946,19 +954,30 @@ export default { break } } - this.itemCount = 0 + var apiItemCount = 0 for (const key in json[responseName]) { if (key === 'count') { - this.itemCount = json[responseName].count + apiItemCount = json[responseName].count continue } objectName = key break } + + if ('id' in this.$route.params && this.$route.params.id !== params.id) { + console.log('DEBUG - Discarding API response as its `id` does not match the uuid on the browser path') + return + } + if (this.dataView && apiItemCount > 1) { + console.log('DEBUG - Discarding API response as got more than one item in data view', this.$route.params, this.items) + return + } + this.items = json[responseName][objectName] if (!this.items || this.items.length === 0) { this.items = [] } + this.itemCount = apiItemCount if (['listTemplates', 'listIsos'].includes(this.apiName) && this.items.length > 1) { this.items = [...new Map(this.items.map(x => [x.id, x])).values()] @@ -1008,6 +1027,10 @@ export default { } } }).catch(error => { + if (!error || !error.message) { + console.log('API request likely got cancelled due to route change:', this.apiName) + return + } if ([401].includes(error.response.status)) { return } diff --git a/ui/src/views/infra/network/IpRangesTabPublic.vue b/ui/src/views/infra/network/IpRangesTabPublic.vue index c5505da913f..bfaf6605123 100644 --- a/ui/src/views/infra/network/IpRangesTabPublic.vue +++ b/ui/src/views/infra/network/IpRangesTabPublic.vue @@ -48,21 +48,21 @@ {{ record.endip || record.endipv6 }}