diff --git a/.travis.yml b/.travis.yml index 8b2859a2a10..0648b27e95c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -164,6 +164,7 @@ env: - TESTS="component/test_project_usage component/test_protocol_number_security_group + component/test_public_ip component/test_resource_limits" - TESTS="component/test_regions_accounts diff --git a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java index 92a664f7f95..c47500c7849 100644 --- a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java +++ b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java @@ -27,6 +27,7 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.network.router.VirtualRouter; import com.cloud.user.Account; import com.cloud.utils.Pair; +import com.cloud.vm.Nic; public interface VirtualNetworkApplianceService { /** @@ -75,4 +76,6 @@ public interface VirtualNetworkApplianceService { * @return */ Pair performRouterHealthChecks(long routerId); + + void collectNetworkStatistics(T router, Nic nic); } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java index 8501f71eda1..4ed0cdd8d74 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java @@ -18,8 +18,6 @@ package org.apache.cloudstack.api.response; import java.util.Date; -import com.google.gson.annotations.SerializedName; - import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; @@ -28,6 +26,7 @@ import com.cloud.host.Host; import com.cloud.host.Status; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; @EntityReference(value = Host.class) public class HostForMigrationResponse extends BaseResponse { @@ -452,6 +451,10 @@ public class HostForMigrationResponse extends BaseResponse { this.hypervisorVersion = hypervisorVersion; } + public Boolean getHaHost() { + return haHost; + } + public void setHaHost(Boolean haHost) { this.haHost = haHost; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java index 7204d5a60f7..4d27f004b7d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java @@ -209,7 +209,7 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co private Long memoryKBs; @SerializedName("memoryintfreekbs") - @Param(description = "the internal memory thats free in vm") + @Param(description = "the internal memory that's free in vm or zero if it can not be calculated") private Long memoryIntFreeKBs; @SerializedName("memorytargetkbs") diff --git a/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java b/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java index b62a8e9a838..0a50e4b29df 100644 --- a/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java +++ b/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java @@ -208,5 +208,22 @@ public interface IpAddressManager { void releasePodIp(Long id) throws CloudRuntimeException; boolean isUsageHidden(IPAddressVO address); + + List listAvailablePublicIps(final long dcId, + final Long podId, + final List vlanDbIds, + final Account owner, + final VlanType vlanUse, + final Long guestNetworkId, + final boolean sourceNat, + final boolean assign, + final boolean allocate, + final String requestedIp, + final boolean isSystem, + final Long vpcId, + final Boolean displayIp, + final boolean forSystemVms, + final boolean lockOneRow) + throws InsufficientAddressCapacityException; } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/DomainRouterDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/DomainRouterDaoImpl.java index 135f96f28f3..dc71f898970 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/DomainRouterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/DomainRouterDaoImpl.java @@ -383,12 +383,14 @@ public class DomainRouterDaoImpl extends GenericDaoBase im final RouterNetworkVO routerNtwkMap = new RouterNetworkVO(router.getId(), guestNetwork.getId(), guestNetwork.getGuestType()); _routerNetworkDao.persist(routerNtwkMap); //2) create user stats entry for the network - UserStatisticsVO stats = - _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), guestNetwork.getId(), null, router.getId(), router.getType().toString()); - if (stats == null) { - stats = - new UserStatisticsVO(router.getAccountId(), router.getDataCenterId(), null, router.getId(), router.getType().toString(), guestNetwork.getId()); - _userStatsDao.persist(stats); + if (router.getVpcId() == null) { + UserStatisticsVO stats = + _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), guestNetwork.getId(), null, router.getId(), router.getType().toString()); + if (stats == null) { + stats = + new UserStatisticsVO(router.getAccountId(), router.getDataCenterId(), null, router.getId(), router.getType().toString(), guestNetwork.getId()); + _userStatsDao.persist(stats); + } } txn.commit(); } diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 0d396c6b598..f1bab19c8db 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -27,6 +27,8 @@ import java.util.Set; import javax.annotation.PostConstruct; import javax.inject.Inject; +import com.cloud.vm.NicVO; +import com.cloud.vm.dao.NicDao; import org.apache.cloudstack.acl.Role; import org.apache.cloudstack.acl.RoleService; import org.apache.cloudstack.affinity.AffinityGroup; @@ -459,6 +461,7 @@ public class ApiDBUtils { static BackupDao s_backupDao; static BackupScheduleDao s_backupScheduleDao; static BackupOfferingDao s_backupOfferingDao; + static NicDao s_nicDao; @Inject private ManagementServer ms; @@ -703,6 +706,8 @@ public class ApiDBUtils { private BackupOfferingDao backupOfferingDao; @Inject private BackupScheduleDao backupScheduleDao; + @Inject + private NicDao nicDao; @PostConstruct void init() { @@ -812,6 +817,7 @@ public class ApiDBUtils { s_hostDetailsDao = hostDetailsDao; s_clusterDetailsDao = clusterDetailsDao; s_vmSnapshotDao = vmSnapshotDao; + s_nicDao = nicDao; s_nicSecondaryIpDao = nicSecondaryIpDao; s_vpcProvSvc = vpcProvSvc; s_affinityGroupDao = affinityGroupDao; @@ -2090,4 +2096,12 @@ public class ApiDBUtils { public static BackupOfferingResponse newBackupOfferingResponse(BackupOffering policy) { return s_backupOfferingDao.newBackupOfferingResponse(policy); } + + public static NicVO findByIp4AddressAndNetworkId(String ip4Address, long networkId) { + return s_nicDao.findByIp4AddressAndNetworkId(ip4Address, networkId); + } + + public static NicSecondaryIpVO findSecondaryIpByIp4AddressAndNetworkId(String ip4Address, long networkId) { + return s_nicSecondaryIpDao.findByIp4AddressAndNetworkId(ip4Address, networkId); + } } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 27e0d58b48e..ad8672bbd08 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -921,6 +921,9 @@ public class ApiResponseHelper implements ResponseGenerator { } } + // show vm info for shared networks + showVmInfoForSharedNetworks(forVirtualNetworks, ipAddr, ipResponse); + // show this info to full view only if (view == ResponseView.Full) { VlanVO vl = ApiDBUtils.findVlanById(ipAddr.getVlanId()); @@ -955,6 +958,42 @@ public class ApiResponseHelper implements ResponseGenerator { return ipResponse; } + private void showVmInfoForSharedNetworks(boolean forVirtualNetworks, IpAddress ipAddr, IPAddressResponse ipResponse) { + if (!forVirtualNetworks) { + NicVO nic = ApiDBUtils.findByIp4AddressAndNetworkId(ipAddr.getAddress().toString(), ipAddr.getNetworkId()); + + if (nic == null) { // find in nic_secondary_ips, user vm only + NicSecondaryIpVO secondaryIp = + ApiDBUtils.findSecondaryIpByIp4AddressAndNetworkId(ipAddr.getAddress().toString(), ipAddr.getNetworkId()); + if (secondaryIp != null) { + UserVm vm = ApiDBUtils.findUserVmById(secondaryIp.getVmId()); + if (vm != null) { + ipResponse.setVirtualMachineId(vm.getUuid()); + ipResponse.setVirtualMachineName(vm.getHostName()); + if (vm.getDisplayName() != null) { + ipResponse.setVirtualMachineDisplayName(vm.getDisplayName()); + } else { + ipResponse.setVirtualMachineDisplayName(vm.getHostName()); + } + } + } + } else if (nic.getVmType() == VirtualMachine.Type.User) { + UserVm vm = ApiDBUtils.findUserVmById(nic.getInstanceId()); + if (vm != null) { + ipResponse.setVirtualMachineId(vm.getUuid()); + ipResponse.setVirtualMachineName(vm.getHostName()); + if (vm.getDisplayName() != null) { + ipResponse.setVirtualMachineDisplayName(vm.getDisplayName()); + } else { + ipResponse.setVirtualMachineDisplayName(vm.getHostName()); + } + } + } else if (nic.getVmType() == VirtualMachine.Type.DomainRouter) { + ipResponse.setIsSystem(true); + } + } + } + @Override public LoadBalancerResponse createLoadBalancerResponse(LoadBalancer loadBalancer) { LoadBalancerResponse lbResponse = new LoadBalancerResponse(); diff --git a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java index f7373a7687a..de87a0a0d5f 100644 --- a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java @@ -26,16 +26,18 @@ import java.util.Set; import javax.inject.Inject; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - import org.apache.cloudstack.api.ApiConstants.HostDetails; import org.apache.cloudstack.api.response.GpuResponse; import org.apache.cloudstack.api.response.HostForMigrationResponse; import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.VgpuResponse; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.ha.HAResource; +import org.apache.cloudstack.ha.dao.HAConfigDao; import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.vo.HostJoinVO; @@ -52,9 +54,6 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import org.apache.cloudstack.ha.HAResource; -import org.apache.cloudstack.ha.dao.HAConfigDao; - @Component public class HostJoinDaoImpl extends GenericDaoBase implements HostJoinDao { public static final Logger s_logger = Logger.getLogger(HostJoinDaoImpl.class); @@ -178,17 +177,13 @@ public class HostJoinDaoImpl extends GenericDaoBase implements hostResponse.setMemoryAllocatedPercentage(memoryAllocatedPercentage); String hostTags = host.getTag(); - hostResponse.setHostTags(host.getTag()); + hostResponse.setHostTags(hostTags); + hostResponse.setHaHost(false); String haTag = ApiDBUtils.getHaTag(); - if (haTag != null && !haTag.isEmpty() && hostTags != null && !hostTags.isEmpty()) { - if (haTag.equalsIgnoreCase(hostTags)) { - hostResponse.setHaHost(true); - } else { - hostResponse.setHaHost(false); - } - } else { - hostResponse.setHaHost(false); + if (StringUtils.isNotEmpty(haTag) && StringUtils.isNotEmpty(hostTags) && + haTag.equalsIgnoreCase(hostTags)) { + hostResponse.setHaHost(true); } hostResponse.setHypervisorVersion(host.getHypervisorVersion()); @@ -274,12 +269,19 @@ public class HostJoinDaoImpl extends GenericDaoBase implements @Override public HostResponse setHostResponse(HostResponse response, HostJoinVO host) { String tag = host.getTag(); - if (tag != null) { - if (response.getHostTags() != null && response.getHostTags().length() > 0) { + if (StringUtils.isNotEmpty(tag)) { + if (StringUtils.isNotEmpty(response.getHostTags())) { response.setHostTags(response.getHostTags() + "," + tag); } else { response.setHostTags(tag); } + + if (Boolean.FALSE.equals(response.getHaHost())) { + String haTag = ApiDBUtils.getHaTag(); + if (StringUtils.isNotEmpty(haTag) && haTag.equalsIgnoreCase(tag)) { + response.setHaHost(true); + } + } } return response; } @@ -334,17 +336,13 @@ public class HostJoinDaoImpl extends GenericDaoBase implements hostResponse.setMemoryAllocatedBytes(mem); String hostTags = host.getTag(); - hostResponse.setHostTags(host.getTag()); + hostResponse.setHostTags(hostTags); + hostResponse.setHaHost(false); String haTag = ApiDBUtils.getHaTag(); - if (haTag != null && !haTag.isEmpty() && hostTags != null && !hostTags.isEmpty()) { - if (haTag.equalsIgnoreCase(hostTags)) { - hostResponse.setHaHost(true); - } else { - hostResponse.setHaHost(false); - } - } else { - hostResponse.setHaHost(false); + if (StringUtils.isNotEmpty(haTag) && StringUtils.isNotEmpty(hostTags) && + haTag.equalsIgnoreCase(hostTags)) { + hostResponse.setHaHost(true); } hostResponse.setHypervisorVersion(host.getHypervisorVersion()); @@ -419,6 +417,13 @@ public class HostJoinDaoImpl extends GenericDaoBase implements } else { response.setHostTags(tag); } + + if (Boolean.FALSE.equals(response.getHaHost())) { + String haTag = ApiDBUtils.getHaTag(); + if (StringUtils.isNotEmpty(haTag) && haTag.equalsIgnoreCase(tag)) { + response.setHaHost(true); + } + } } return response; } diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java index ff2b66cfc7e..15b0f5cafae 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java @@ -222,8 +222,11 @@ public class UserVmJoinDaoImpl extends GenericDaoBaseWithTagInformation= totalMemory ? 0 : freeMemory; + userVmResponse.setMemoryKBs(totalMemory); + userVmResponse.setMemoryIntFreeKBs(correctedFreeMemory); userVmResponse.setMemoryTargetKBs((long)vmStats.getTargetMemoryKBs()); } diff --git a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java index 51af56a887a..3a6e1842b18 100644 --- a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java +++ b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java @@ -323,31 +323,30 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage } } - for (final IPAddressVO possibleAddr : addressVOS) { + for (IPAddressVO possibleAddr : addressVOS) { if (possibleAddr.getState() != State.Free) { continue; } - final IPAddressVO addressVO = possibleAddr; - addressVO.setSourceNat(sourceNat); - addressVO.setAllocatedTime(new Date()); - addressVO.setAllocatedInDomainId(owner.getDomainId()); - addressVO.setAllocatedToAccountId(owner.getId()); - addressVO.setSystem(isSystem); + possibleAddr.setSourceNat(sourceNat); + possibleAddr.setAllocatedTime(new Date()); + possibleAddr.setAllocatedInDomainId(owner.getDomainId()); + possibleAddr.setAllocatedToAccountId(owner.getId()); + possibleAddr.setSystem(isSystem); if (displayIp != null) { - addressVO.setDisplay(displayIp); + possibleAddr.setDisplay(displayIp); } if (vlanUse != VlanType.DirectAttached) { - addressVO.setAssociatedWithNetworkId(guestNetworkId); - addressVO.setVpcId(vpcId); + possibleAddr.setAssociatedWithNetworkId(guestNetworkId); + possibleAddr.setVpcId(vpcId); } if (_ipAddressDao.lockRow(possibleAddr.getId(), true) != null) { - final IPAddressVO userIp = _ipAddressDao.findById(addressVO.getId()); + final IPAddressVO userIp = _ipAddressDao.findById(possibleAddr.getId()); if (userIp.getState() == State.Free) { - addressVO.setState(State.Allocating); - if (_ipAddressDao.update(addressVO.getId(), addressVO)) { - finalAddress = addressVO; + possibleAddr.setState(State.Allocating); + if (_ipAddressDao.update(possibleAddr.getId(), possibleAddr)) { + finalAddress = possibleAddr; break; } } @@ -784,9 +783,22 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage public PublicIp fetchNewPublicIp(final long dcId, final Long podId, final List vlanDbIds, final Account owner, final VlanType vlanUse, final Long guestNetworkId, final boolean sourceNat, final boolean assign, final boolean allocate, final String requestedIp, final boolean isSystem, final Long vpcId, final Boolean displayIp, final boolean forSystemVms) throws InsufficientAddressCapacityException { - IPAddressVO addr = Transaction.execute(new TransactionCallbackWithException() { + List addrs = listAvailablePublicIps(dcId, podId, vlanDbIds, owner, vlanUse, guestNetworkId, sourceNat, assign, allocate, requestedIp, isSystem, vpcId, displayIp, forSystemVms, true); + IPAddressVO addr = addrs.get(0); + if (vlanUse == VlanType.VirtualNetwork) { + _firewallMgr.addSystemFirewallRules(addr, owner); + } + + return PublicIp.createFromAddrAndVlan(addr, _vlanDao.findById(addr.getVlanId())); + } + + @Override + public List listAvailablePublicIps(final long dcId, final Long podId, final List vlanDbIds, final Account owner, final VlanType vlanUse, final Long guestNetworkId, + final boolean sourceNat, final boolean assign, final boolean allocate, final String requestedIp, final boolean isSystem, + final Long vpcId, final Boolean displayIp, final boolean forSystemVms, final boolean lockOneRow) throws InsufficientAddressCapacityException { + return Transaction.execute(new TransactionCallbackWithException, InsufficientAddressCapacityException>() { @Override - public IPAddressVO doInTransaction(TransactionStatus status) throws InsufficientAddressCapacityException { + public List doInTransaction(TransactionStatus status) throws InsufficientAddressCapacityException { StringBuilder errorMessage = new StringBuilder("Unable to get ip address in "); boolean fetchFromDedicatedRange = false; List dedicatedVlanDbIds = new ArrayList(); @@ -824,23 +836,26 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage if (vlanDbIds == null || vlanDbIds.contains(nonDedicatedVlan.getId())) nonDedicatedVlanDbIds.add(nonDedicatedVlan.getId()); } - if (dedicatedVlanDbIds != null && !dedicatedVlanDbIds.isEmpty()) { - fetchFromDedicatedRange = true; - sc.setParameters("vlanId", dedicatedVlanDbIds.toArray()); - errorMessage.append(", vlanId id=" + Arrays.toString(dedicatedVlanDbIds.toArray())); - } else if (nonDedicatedVlanDbIds != null && !nonDedicatedVlanDbIds.isEmpty()) { - sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray()); - errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray())); - } else { - if (podId != null) { - InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", Pod.class, podId); - ex.addProxyObject(ApiDBUtils.findPodById(podId).getUuid()); + + if (vlanUse == VlanType.VirtualNetwork) { + if (dedicatedVlanDbIds != null && !dedicatedVlanDbIds.isEmpty()) { + fetchFromDedicatedRange = true; + sc.setParameters("vlanId", dedicatedVlanDbIds.toArray()); + errorMessage.append(", vlanId id=" + Arrays.toString(dedicatedVlanDbIds.toArray())); + } else if (nonDedicatedVlanDbIds != null && !nonDedicatedVlanDbIds.isEmpty()) { + sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray()); + errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray())); + } else { + if (podId != null) { + InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", Pod.class, podId); + ex.addProxyObject(ApiDBUtils.findPodById(podId).getUuid()); + throw ex; + } + s_logger.warn(errorMessage.toString()); + InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", DataCenter.class, dcId); + ex.addProxyObject(ApiDBUtils.findZoneById(dcId).getUuid()); throw ex; } - s_logger.warn(errorMessage.toString()); - InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", DataCenter.class, dcId); - ex.addProxyObject(ApiDBUtils.findZoneById(dcId).getUuid()); - throw ex; } sc.setParameters("dc", dcId); @@ -871,21 +886,31 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage filter.addOrderBy(IPAddressVO.class,"vlanId", true); - List addrs = _ipAddressDao.search(sc, filter, false); + List addrs; + + if (lockOneRow) { + addrs = _ipAddressDao.lockRows(sc, filter, true); + } else { + addrs = new ArrayList<>(_ipAddressDao.search(sc, null)); + } // If all the dedicated IPs of the owner are in use fetch an IP from the system pool - if (addrs.size() == 0 && fetchFromDedicatedRange) { + if ((!lockOneRow || (lockOneRow && addrs.size() == 0)) && fetchFromDedicatedRange && vlanUse == VlanType.VirtualNetwork) { // Verify if account is allowed to acquire IPs from the system boolean useSystemIps = UseSystemPublicIps.valueIn(owner.getId()); if (useSystemIps && nonDedicatedVlanDbIds != null && !nonDedicatedVlanDbIds.isEmpty()) { fetchFromDedicatedRange = false; sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray()); errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray())); - addrs = _ipAddressDao.search(sc, filter, false); + if (lockOneRow) { + addrs = _ipAddressDao.lockRows(sc, filter, true); + } else { + addrs.addAll(_ipAddressDao.search(sc, null)); + } } } - if (addrs.size() == 0) { + if (lockOneRow && addrs.size() == 0) { if (podId != null) { InsufficientAddressCapacityException ex = new InsufficientAddressCapacityException("Insufficient address capacity", Pod.class, podId); // for now, we hardcode the table names, but we should ideally do a lookup for the tablename from the VO object. @@ -898,23 +923,16 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage throw ex; } - assert(addrs.size() == 1) : "Return size is incorrect: " + addrs.size(); - IPAddressVO finalAddr = null; - if (assign) { - finalAddr = assignAndAllocateIpAddressEntry(owner, vlanUse, guestNetworkId, sourceNat, allocate, - isSystem,vpcId, displayIp, fetchFromDedicatedRange, addrs); - } else { - finalAddr = addrs.get(0); + if (lockOneRow) { + assert (addrs.size() == 1) : "Return size is incorrect: " + addrs.size(); } - return finalAddr; + if (assign) { + assignAndAllocateIpAddressEntry(owner, vlanUse, guestNetworkId, sourceNat, allocate, + isSystem,vpcId, displayIp, fetchFromDedicatedRange, addrs); + } + return addrs; } }); - - if (vlanUse == VlanType.VirtualNetwork) { - _firewallMgr.addSystemFirewallRules(addr, owner); - } - - return PublicIp.createFromAddrAndVlan(addr, _vlanDao.findById(addr.getVlanId())); } @DB diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 2657b3f1812..60aa8df98b1 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -734,100 +734,7 @@ Configurable, StateListener routerNics = _nicDao.listByVmId(router.getId()); - for (final Nic routerNic : routerNics) { - final Network network = _networkModel.getNetwork(routerNic.getNetworkId()); - // Send network usage command for public nic in VPC VR - // Send network usage command for isolated guest nic of non) VPC VR - - //[TODO] Avoiding the NPE now, but I have to find out what is going on with the network. - Wilder Rodrigues - if (network == null) { - s_logger.error("Could not find a network with ID => " + routerNic.getNetworkId() + ". It might be a problem!"); - continue; - } - if (forVpc && network.getTrafficType() == TrafficType.Public || !forVpc && network.getTrafficType() == TrafficType.Guest - && network.getGuestType() == Network.GuestType.Isolated) { - final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, routerNic.getIPv4Address()); - final String routerType = router.getType().toString(); - final UserStatisticsVO previousStats = _userStatsDao.findBy(router.getAccountId(), router.getDataCenterId(), network.getId(), - forVpc ? routerNic.getIPv4Address() : null, router.getId(), routerType); - NetworkUsageAnswer answer = null; - try { - answer = (NetworkUsageAnswer) _agentMgr.easySend(router.getHostId(), usageCmd); - } catch (final Exception e) { - s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId(), e); - continue; - } - - if (answer != null) { - if (!answer.getResult()) { - s_logger.warn("Error while collecting network stats from router: " + router.getInstanceName() + " from host: " + router.getHostId() - + "; details: " + answer.getDetails()); - continue; - } - try { - if (answer.getBytesReceived() == 0 && answer.getBytesSent() == 0) { - s_logger.debug("Recieved and Sent bytes are both 0. Not updating user_statistics"); - continue; - } - final NetworkUsageAnswer answerFinal = answer; - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(final TransactionStatus status) { - final UserStatisticsVO stats = _userStatsDao.lock(router.getAccountId(), router.getDataCenterId(), network.getId(), - forVpc ? routerNic.getIPv4Address() : null, router.getId(), routerType); - if (stats == null) { - s_logger.warn("unable to find stats for account: " + router.getAccountId()); - return; - } - - if (previousStats != null - && (previousStats.getCurrentBytesReceived() != stats.getCurrentBytesReceived() || previousStats.getCurrentBytesSent() != stats - .getCurrentBytesSent())) { - s_logger.debug("Router stats changed from the time NetworkUsageCommand was sent. " + "Ignoring current answer. Router: " - + answerFinal.getRouterName() + " Rcvd: " + answerFinal.getBytesReceived() + "Sent: " + answerFinal.getBytesSent()); - return; - } - - if (stats.getCurrentBytesReceived() > answerFinal.getBytesReceived()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " - + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: " - + toHumanReadableSize(answerFinal.getBytesReceived()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesReceived())); - } - stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - stats.setCurrentBytesReceived(answerFinal.getBytesReceived()); - if (stats.getCurrentBytesSent() > answerFinal.getBytesSent()) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Received # of bytes that's less than the last one. " - + "Assuming something went wrong and persisting it. Router: " + answerFinal.getRouterName() + " Reported: " - + toHumanReadableSize(answerFinal.getBytesSent()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesSent())); - } - stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - } - stats.setCurrentBytesSent(answerFinal.getBytesSent()); - if (!_dailyOrHourly) { - // update agg bytes - stats.setAggBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); - stats.setAggBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); - } - _userStatsDao.update(stats.getId(), stats); - } - }); - - } catch (final Exception e) { - s_logger.warn("Unable to update user statistics for account: " + router.getAccountId() + " Rx: " + toHumanReadableSize(answer.getBytesReceived()) + "; Tx: " - + toHumanReadableSize(answer.getBytesSent())); - } - } - } - } - } + collectNetworkStatistics(router, null); } } catch (final Exception e) { s_logger.warn("Error while collecting network stats", e); @@ -3121,6 +3028,11 @@ Configurable, StateListener void collectNetworkStatistics(final T router, final Nic nic) { if (router == null) { return; } @@ -3129,12 +3041,23 @@ Configurable, StateListener routerNics = _nicDao.listByVmId(router.getId()); + List routerNics = new ArrayList(); + if (nic != null) { + routerNics.add(nic); + } else { + routerNics.addAll(_nicDao.listByVmId(router.getId())); + } for (final Nic routerNic : routerNics) { final Network network = _networkModel.getNetwork(routerNic.getNetworkId()); // Send network usage command for public nic in VPC VR // Send network usage command for isolated guest nic of non VPC // VR + + //[TODO] Avoiding the NPE now, but I have to find out what is going on with the network. - Wilder Rodrigues + if (network == null) { + s_logger.error("Could not find a network with ID => " + routerNic.getNetworkId() + ". It might be a problem!"); + continue; + } if (forVpc && network.getTrafficType() == TrafficType.Public || !forVpc && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == Network.GuestType.Isolated) { final NetworkUsageCommand usageCmd = new NetworkUsageCommand(privateIP, router.getHostName(), forVpc, routerNic.getIPv4Address()); diff --git a/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java b/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java index 6ee5e85f271..b671e33df08 100644 --- a/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java +++ b/server/src/main/java/com/cloud/network/rules/NicPlugInOutRules.java @@ -40,6 +40,7 @@ import com.cloud.network.NetworkModel; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.network.Networks.IsolationType; import com.cloud.network.PublicIpAddress; +import com.cloud.network.VpcVirtualNetworkApplianceService; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; @@ -84,6 +85,7 @@ public class NicPlugInOutRules extends RuleApplier { NetworkModel networkModel = visitor.getVirtualNetworkApplianceFactory().getNetworkModel(); VirtualMachineManager itMgr = visitor.getVirtualNetworkApplianceFactory().getItMgr(); NicDao nicDao = visitor.getVirtualNetworkApplianceFactory().getNicDao(); + VpcVirtualNetworkApplianceService routerService = visitor.getVirtualNetworkApplianceFactory().getRouterService(); // de-associate IPs before unplugging nics if (!nicsToUnplug.isEmpty()) { @@ -107,6 +109,12 @@ public class NicPlugInOutRules extends RuleApplier { // 1) Unplug the nics for (Entry entry : nicsToUnplug.entrySet()) { + PublicIpAddress ip = entry.getValue(); + NicVO nic = nicDao.findByIp4AddressAndNetworkIdAndInstanceId(ip.getNetworkId(), _router.getId(), ip.getAddress().addr()); + if (nic != null) { + s_logger.info("Collect network statistics for nic " + nic + " from router " + _router); + routerService.collectNetworkStatistics(_router, nic); + } Network publicNtwk = null; try { publicNtwk = networkModel.getNetwork(entry.getValue().getNetworkId()); diff --git a/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java b/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java index 9d4660da557..d96673d42d3 100644 --- a/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java +++ b/server/src/main/java/com/cloud/network/rules/VirtualNetworkApplianceFactory.java @@ -26,6 +26,7 @@ import com.cloud.dc.dao.HostPodDao; import com.cloud.dc.dao.VlanDao; import com.cloud.network.IpAddressManager; import com.cloud.network.NetworkModel; +import com.cloud.network.VpcVirtualNetworkApplianceService; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.LoadBalancerDao; @@ -98,6 +99,8 @@ public class VirtualNetworkApplianceFactory { @Inject private NetworkTopologyContext _networkTopologyContext; + @Inject + private VpcVirtualNetworkApplianceService _routerService; public NetworkModel getNetworkModel() { return _networkModel; @@ -190,4 +193,8 @@ public class VirtualNetworkApplianceFactory { public FirewallRulesDao getFirewallRulesDao() { return _rulesDao; } + + public VpcVirtualNetworkApplianceService getRouterService() { + return _routerService; + } } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 18afeb42a92..8a1834f7e16 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -613,6 +613,7 @@ import com.cloud.event.EventTypes; import com.cloud.event.EventVO; import com.cloud.event.dao.EventDao; import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientAddressCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ManagementServerException; import com.cloud.exception.OperationTimedoutException; @@ -637,12 +638,20 @@ import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.hypervisor.kvm.dpdk.DpdkHelper; import com.cloud.info.ConsoleProxyInfo; import com.cloud.network.IpAddress; +import com.cloud.network.IpAddressManager; +import com.cloud.network.Network; +import com.cloud.network.NetworkModel; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVO; +import com.cloud.network.dao.NetworkAccountDao; +import com.cloud.network.dao.NetworkAccountVO; +import com.cloud.network.dao.NetworkDomainDao; +import com.cloud.network.dao.NetworkDomainVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.vpc.dao.VpcDao; import com.cloud.org.Cluster; import com.cloud.org.Grouping.AllocationState; import com.cloud.projects.Project; @@ -858,6 +867,16 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe private VolumeDataStoreDao _volumeStoreDao; @Inject private TemplateDataStoreDao _vmTemplateStoreDao; + @Inject + private IpAddressManager _ipAddressMgr; + @Inject + private NetworkAccountDao _networkAccountDao; + @Inject + private NetworkDomainDao _networkDomainDao; + @Inject + private NetworkModel _networkMgr; + @Inject + private VpcDao _vpcDao; private LockMasterListener _lockMasterListener; private final ScheduledExecutorService _eventExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("EventChecker")); @@ -2024,29 +2043,90 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Override public Pair, Integer> searchForIPAddresses(final ListPublicIpAddressesCmd cmd) { - final Object keyword = cmd.getKeyword(); - final Long physicalNetworkId = cmd.getPhysicalNetworkId(); final Long associatedNetworkId = cmd.getAssociatedNetworkId(); - final Long sourceNetworkId = cmd.getNetworkId(); final Long zone = cmd.getZoneId(); - final String address = cmd.getIpAddress(); final Long vlan = cmd.getVlanId(); final Boolean forVirtualNetwork = cmd.isForVirtualNetwork(); - final Boolean forLoadBalancing = cmd.isForLoadBalancing(); final Long ipId = cmd.getId(); - final Boolean sourceNat = cmd.isSourceNat(); - final Boolean staticNat = cmd.isStaticNat(); + final Long networkId = cmd.getNetworkId(); final Long vpcId = cmd.getVpcId(); - final Boolean forDisplay = cmd.getDisplay(); - final Map tags = cmd.getTags(); final String state = cmd.getState(); Boolean isAllocated = cmd.isAllocatedOnly(); if (isAllocated == null) { - isAllocated = Boolean.TRUE; - - if (state != null) { + if (state != null && state.equalsIgnoreCase(IpAddress.State.Free.name())) { isAllocated = Boolean.FALSE; + } else { + isAllocated = Boolean.TRUE; // default + } + } else { + if (state != null && state.equalsIgnoreCase(IpAddress.State.Free.name())) { + if (isAllocated) { + throw new InvalidParameterValueException("Conflict: allocatedonly is true but state is Free"); + } + } else if (state != null && state.equalsIgnoreCase(IpAddress.State.Allocated.name())) { + isAllocated = Boolean.TRUE; + } + } + + VlanType vlanType = null; + if (forVirtualNetwork != null) { + vlanType = forVirtualNetwork ? VlanType.VirtualNetwork : VlanType.DirectAttached; + } else { + vlanType = VlanType.VirtualNetwork; + } + + final Account caller = getCaller(); + List addrs = new ArrayList<>(); + + if (vlanType == VlanType.DirectAttached && networkId == null && ipId == null) { // only root admin can list public ips in all shared networks + if (caller.getType() != Account.ACCOUNT_TYPE_ADMIN) { + isAllocated = true; + } + } else if (vlanType == VlanType.DirectAttached) { + // list public ip address on shared network + // access control. admin: all Ips, domain admin/user: all Ips in shared network in the domain/sub-domain/user + NetworkVO network = null; + if (networkId == null) { + IPAddressVO ip = _publicIpAddressDao.findById(ipId); + if (ip == null) { + throw new InvalidParameterValueException("Please specify a valid ipaddress id"); + } + network = _networkDao.findById(ip.getSourceNetworkId()); + } else { + network = _networkDao.findById(networkId); + } + if (network == null || network.getGuestType() != Network.GuestType.Shared) { + throw new InvalidParameterValueException("Please specify a valid network id"); + } + if (network.getAclType() == ControlledEntity.ACLType.Account) { + NetworkAccountVO networkMap = _networkAccountDao.getAccountNetworkMapByNetworkId(network.getId()); + if (networkMap == null) { + return new Pair<>(addrs, 0); + } + _accountMgr.checkAccess(caller, null, false, _accountDao.findById(networkMap.getAccountId())); + } else { // Domain level + NetworkDomainVO networkMap = _networkDomainDao.getDomainNetworkMapByNetworkId(network.getId()); + if (networkMap == null) { + return new Pair<>(addrs, 0); + } + if (caller.getType() == Account.ACCOUNT_TYPE_NORMAL || caller.getType() == Account.ACCOUNT_TYPE_PROJECT) { + if (_networkMgr.isNetworkAvailableInDomain(network.getId(), caller.getDomainId())) { + isAllocated = Boolean.TRUE; + } else { + return new Pair<>(addrs, 0); + } + } else if (caller.getType() == Account.ACCOUNT_TYPE_DOMAIN_ADMIN || caller.getType() == Account.ACCOUNT_TYPE_RESOURCE_DOMAIN_ADMIN) { + if (caller.getDomainId() == networkMap.getDomainId() || _domainDao.isChildDomain(caller.getDomainId(), networkMap.getDomainId())) { + s_logger.debug("Caller " + caller.getUuid() + " has permission to access the network : " + network.getUuid()); + } else { + if (_networkMgr.isNetworkAvailableInDomain(network.getId(), caller.getDomainId())) { + isAllocated = Boolean.TRUE; + } else { + return new Pair<>(addrs, 0); + } + } + } } } @@ -2054,12 +2134,10 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe final SearchBuilder sb = _publicIpAddressDao.createSearchBuilder(); Long domainId = null; Boolean isRecursive = null; - final List permittedAccounts = new ArrayList(); + final List permittedAccounts = new ArrayList<>(); ListProjectResourcesCriteria listProjectResourcesCriteria = null; - if (isAllocated) { - final Account caller = getCaller(); - - final Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), cmd.isRecursive(), + if (isAllocated || (vlanType == VlanType.VirtualNetwork && (caller.getType() != Account.ACCOUNT_TYPE_ADMIN || cmd.getDomainId() != null))) { + final Ternary domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); _accountMgr.buildACLSearchParameters(caller, cmd.getId(), cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); domainId = domainIdRecursiveListProject.first(); @@ -2068,12 +2146,93 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe _accountMgr.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); } + buildParameters(sb, cmd); + + SearchCriteria sc = sb.create(); + setParameters(sc, cmd, vlanType); + + if (isAllocated || (vlanType == VlanType.VirtualNetwork && (caller.getType() != Account.ACCOUNT_TYPE_ADMIN || cmd.getDomainId() != null))) { + _accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + } + + if (associatedNetworkId != null) { + _accountMgr.checkAccess(caller, null, false, _networkDao.findById(associatedNetworkId)); + sc.setParameters("associatedNetworkIdEq", associatedNetworkId); + } + if (vpcId != null) { + _accountMgr.checkAccess(caller, null, false, _vpcDao.findById(vpcId)); + sc.setParameters("vpcId", vpcId); + } + + addrs = _publicIpAddressDao.search(sc, searchFilter); // Allocated + + // Free IP addresses in system IP ranges + List freeAddrIds = new ArrayList<>(); + if (!(isAllocated || vlanType == VlanType.DirectAttached)) { + Long zoneId = zone; + Account owner = _accountMgr.finalizeOwner(CallContext.current().getCallingAccount(), cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId()); + if (associatedNetworkId != null) { + NetworkVO guestNetwork = _networkDao.findById(associatedNetworkId); + if (zoneId == null) { + zoneId = guestNetwork.getDataCenterId(); + } else if (zoneId != guestNetwork.getDataCenterId()) { + InvalidParameterValueException ex = new InvalidParameterValueException("Please specify a valid associated network id in the specified zone."); + throw ex; + } + owner = _accountDao.findById(guestNetwork.getAccountId()); + } + List dcList = new ArrayList<>(); + if (zoneId == null){ + dcList = ApiDBUtils.listZones(); + } else { + dcList.add(ApiDBUtils.findZoneById(zoneId)); + } + List vlanDbIds = null; + if (vlan != null) { + vlanDbIds = new ArrayList<>(); + vlanDbIds.add(vlan); + } + List freeAddrs = new ArrayList<>(); + for (DataCenterVO dc : dcList) { + long dcId = dc.getId(); + try { + freeAddrs.addAll(_ipAddressMgr.listAvailablePublicIps(dcId, null, vlanDbIds, owner, VlanType.VirtualNetwork, associatedNetworkId, + false, false, false, null, false, cmd.getVpcId(), cmd.isDisplay(), false, false)); // Free + } catch (InsufficientAddressCapacityException e) { + s_logger.warn("no free address is found in zone " + dcId); + } + } + for (IPAddressVO addr: freeAddrs) { + freeAddrIds.add(addr.getId()); + } + } + if (freeAddrIds.size() > 0) { + final SearchBuilder sb2 = _publicIpAddressDao.createSearchBuilder(); + buildParameters(sb2, cmd); + sb2.and("ids", sb2.entity().getId(), SearchCriteria.Op.IN); + + SearchCriteria sc2 = sb2.create(); + setParameters(sc2, cmd, vlanType); + sc2.setParameters("ids", freeAddrIds.toArray()); + addrs.addAll(_publicIpAddressDao.search(sc2, searchFilter)); // Allocated + Free + } + + return new Pair<>(addrs, addrs.size()); + } + + private void buildParameters(final SearchBuilder sb, final ListPublicIpAddressesCmd cmd) { + final Object keyword = cmd.getKeyword(); + final String address = cmd.getIpAddress(); + final Boolean forLoadBalancing = cmd.isForLoadBalancing(); + Boolean isAllocated = cmd.isAllocatedOnly(); + final Map tags = cmd.getTags(); + sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); sb.and("address", sb.entity().getAddress(), SearchCriteria.Op.EQ); sb.and("vlanDbId", sb.entity().getVlanId(), SearchCriteria.Op.EQ); sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); sb.and("physicalNetworkId", sb.entity().getPhysicalNetworkId(), SearchCriteria.Op.EQ); - sb.and("associatedNetworkId", sb.entity().getAssociatedWithNetworkId(), SearchCriteria.Op.EQ); + sb.and("associatedNetworkIdEq", sb.entity().getAssociatedWithNetworkId(), SearchCriteria.Op.EQ); sb.and("sourceNetworkId", sb.entity().getSourceNetworkId(), SearchCriteria.Op.EQ); sb.and("isSourceNat", sb.entity().isSourceNat(), SearchCriteria.Op.EQ); sb.and("isStaticNat", sb.entity().isOneToOneNat(), SearchCriteria.Op.EQ); @@ -2109,21 +2268,24 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe vlanSearch.and("removed", vlanSearch.entity().getRemoved(), SearchCriteria.Op.NULL); sb.join("vlanSearch", vlanSearch, sb.entity().getVlanId(), vlanSearch.entity().getId(), JoinBuilder.JoinType.INNER); - if (isAllocated != null && isAllocated == true) { + if (isAllocated != null && isAllocated) { sb.and("allocated", sb.entity().getAllocatedTime(), SearchCriteria.Op.NNULL); } + } - VlanType vlanType = null; - if (forVirtualNetwork != null) { - vlanType = forVirtualNetwork ? VlanType.VirtualNetwork : VlanType.DirectAttached; - } else { - vlanType = VlanType.VirtualNetwork; - } - - final SearchCriteria sc = sb.create(); - if (isAllocated) { - _accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); - } + private void setParameters(SearchCriteria sc, final ListPublicIpAddressesCmd cmd, VlanType vlanType) { + final Object keyword = cmd.getKeyword(); + final Long physicalNetworkId = cmd.getPhysicalNetworkId(); + final Long sourceNetworkId = cmd.getNetworkId(); + final Long zone = cmd.getZoneId(); + final String address = cmd.getIpAddress(); + final Long vlan = cmd.getVlanId(); + final Long ipId = cmd.getId(); + final Boolean sourceNat = cmd.isSourceNat(); + final Boolean staticNat = cmd.isStaticNat(); + final Boolean forDisplay = cmd.getDisplay(); + final String state = cmd.getState(); + final Map tags = cmd.getTags(); sc.setJoinParameters("vlanSearch", "vlanType", vlanType); @@ -2141,10 +2303,6 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe sc.setParameters("dataCenterId", zone); } - if (vpcId != null) { - sc.setParameters("vpcId", vpcId); - } - if (ipId != null) { sc.setParameters("id", ipId); } @@ -2173,10 +2331,6 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe sc.setParameters("physicalNetworkId", physicalNetworkId); } - if (associatedNetworkId != null) { - sc.setParameters("associatedNetworkId", associatedNetworkId); - } - if (sourceNetworkId != null) { sc.setParameters("sourceNetworkId", sourceNetworkId); } @@ -2190,8 +2344,6 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe } sc.setParameters( "forsystemvms", false); - final Pair, Integer> result = _publicIpAddressDao.searchAndCount(sc, searchFilter); - return new Pair, Integer>(result.first(), result.second()); } @Override diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 550f53b37f8..3bdea7c89c1 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -920,8 +920,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (template.isCrossZones()) { // sync template from cache store to region store if it is not there, for cases where we are going to migrate existing NFS to S3. _tmpltSvr.syncTemplateToRegionStore(templateId, srcSecStore); - s_logger.debug("Template " + templateId + " is cross-zone, don't need to copy"); - return template; } for (Long destZoneId : destZoneIds) { DataStore dstSecStore = getImageStore(destZoneId, templateId); diff --git a/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java b/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java index e734fdfc8ad..fa14f81a489 100644 --- a/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java +++ b/server/src/test/java/com/cloud/vpc/MockVpcVirtualNetworkApplianceManager.java @@ -42,6 +42,7 @@ import com.cloud.user.User; import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; import com.cloud.vm.DomainRouterVO; +import com.cloud.vm.Nic; import com.cloud.vm.VirtualMachineProfile; @Component @@ -281,4 +282,10 @@ public class MockVpcVirtualNetworkApplianceManager extends ManagerBase implement // TODO Auto-generated method stub return false; } -} \ No newline at end of file + + @Override + public void collectNetworkStatistics(T router, Nic nic) { + // TODO Auto-generated method stub + return; + } +} diff --git a/systemvm/debian/etc/logrotate.conf b/systemvm/debian/etc/logrotate.conf index d213d4e9ce8..d33f2dae2e6 100644 --- a/systemvm/debian/etc/logrotate.conf +++ b/systemvm/debian/etc/logrotate.conf @@ -11,15 +11,3 @@ size 50M compress # RPM packages drop log rotation information into this directory include /etc/logrotate.d -# no packages own wtmp and btmp -- we'll rotate them here -/var/log/wtmp { - monthly - create 0664 root utmp - rotate 1 -} -/var/log/btmp { - missingok - monthly - create 0600 root utmp - rotate 1 -} diff --git a/systemvm/debian/etc/logrotate.d/btmp b/systemvm/debian/etc/logrotate.d/btmp new file mode 100644 index 00000000000..0aa1ae1ade1 --- /dev/null +++ b/systemvm/debian/etc/logrotate.d/btmp @@ -0,0 +1,7 @@ +# no packages own btmp -- we'll rotate it here +/var/log/btmp { + missingok + monthly + create 0660 root utmp + rotate 1 +} diff --git a/systemvm/debian/etc/logrotate.d/rsyslog b/systemvm/debian/etc/logrotate.d/rsyslog index 9291494481e..cef88191606 100644 --- a/systemvm/debian/etc/logrotate.d/rsyslog +++ b/systemvm/debian/etc/logrotate.d/rsyslog @@ -6,7 +6,7 @@ notifempty compress postrotate - /usr/sbin/invoke-rc.d rsyslog rotate > /dev/null + /usr/lib/rsyslog/rsyslog-rotate endscript } @@ -30,6 +30,6 @@ compress sharedscripts postrotate - /usr/sbin/invoke-rc.d rsyslog rotate > /dev/null + /usr/lib/rsyslog/rsyslog-rotate endscript } diff --git a/systemvm/debian/etc/logrotate.d/wtmp b/systemvm/debian/etc/logrotate.d/wtmp new file mode 100644 index 00000000000..cc8a151e00b --- /dev/null +++ b/systemvm/debian/etc/logrotate.d/wtmp @@ -0,0 +1,8 @@ +# no packages own wtmp -- we'll rotate it here +/var/log/wtmp { + missingok + monthly + create 0664 root utmp + minsize 1M + rotate 1 +} diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py index 44b69500b4c..dfea7019f7a 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -476,11 +476,6 @@ class CsIP: "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" % (self.dev, guestNetworkCidr, self.address['gateway'], self.dev)]) - self.fw.append(["", "front", "-A NETWORK_STATS_%s -i %s -d %s" % - ("eth1", "eth1", guestNetworkCidr)]) - self.fw.append(["", "front", "-A NETWORK_STATS_%s -o %s -s %s" % - ("eth1", "eth1", guestNetworkCidr)]) - if self.is_private_gateway(): self.fw.append(["filter", "", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" % (self.address['network'], self.dev, self.dev)]) @@ -518,6 +513,10 @@ class CsIP: ["mangle", "", "-A VPN_STATS_%s -i %s -m mark --mark 0x524/0xffffffff" % (self.dev, self.dev)]) self.fw.append( ["", "front", "-A FORWARD -j NETWORK_STATS_%s" % self.dev]) + self.fw.append( + ["", "front", "-A NETWORK_STATS_%s -s %s -o %s" % (self.dev, self.cl.get_vpccidr(), self.dev)]) + self.fw.append( + ["", "front", "-A NETWORK_STATS_%s -d %s -i %s" % (self.dev, self.cl.get_vpccidr(), self.dev)]) self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"]) self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"]) diff --git a/test/integration/component/test_public_ip.py b/test/integration/component/test_public_ip.py new file mode 100644 index 00000000000..a37226f32fc --- /dev/null +++ b/test/integration/component/test_public_ip.py @@ -0,0 +1,870 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" Tests for acquiring/listing public ip address +""" +from nose.plugins.attrib import attr +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.utils import cleanup_resources +from marvin.lib.base import (Account, + Configurations, + Domain, + PublicIpRange, + PublicIPAddress, + Network, + NetworkOffering, + ServiceOffering, + VPC, + VpcOffering, + Zone) +from marvin.lib.common import (get_zone, + get_domain, + get_template, + get_free_vlan + ) +import logging +import random + +class TestPublicIp(cloudstackTestCase): + @classmethod + def setUpClass(cls): + cls.testClient = super(TestPublicIp, cls).getClsTestClient() + cls.apiclient = cls.testClient.getApiClient() + cls.services = cls.testClient.getParsedTestDataConfig() + + # Get Zone, Domain and templates + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + cls.logger = logging.getLogger("TestPublicIp") + cls.domain = get_domain(cls.apiclient) + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + + cls.template = get_template( + cls.apiclient, + cls.zone.id, + cls.services["ostype"] + ) + + cls.use_system_ips_config_name = "use.system.public.ips" + cls.use_system_ips_config = Configurations.list( + cls.apiclient, + name=cls.use_system_ips_config_name + ) + cls.use_system_ips_config_value = cls.use_system_ips_config[0].value + Configurations.update( + cls.apiclient, + name=cls.use_system_ips_config_name, + value="false" + ) + + cls._cleanup = [] + cls.unsupportedHypervisor = False + cls.hypervisor = cls.testClient.getHypervisorInfo() + if cls.hypervisor.lower() in ['lxc']: + cls.unsupportedHypervisor = True + return + + # Create new domain1 + cls.domain1 = Domain.create( + cls.apiclient, + services=cls.services["acl"]["domain1"], + parentdomainid=cls.domain.id) + + # Create account1 + cls.account1 = Account.create( + cls.apiclient, + cls.services["account"], + # cls.services["acl"]["accountD1"], + admin=True, + domainid=cls.domain1.id + ) + + # Create new sub-domain + cls.sub_domain = Domain.create( + cls.apiclient, + services=cls.services["acl"]["domain11"], + parentdomainid=cls.domain1.id) + + # Create account for sub-domain + cls.sub_account = Account.create( + cls.apiclient, + cls.services["acl"]["accountD11"], + domainid=cls.sub_domain.id + ) + + # Create new domain2 + cls.domain2 = Domain.create( + cls.apiclient, + services=cls.services["acl"]["domain2"], + parentdomainid=cls.domain.id) + + # Create account2 + cls.account2 = Account.create( + cls.apiclient, + cls.services["acl"]["accountD2"], + domainid=cls.domain2.id + ) + + cls.services["publiciprange"]["zoneid"] = cls.zone.id + cls.services["publiciprange"]["forvirtualnetwork"] = "true" + + # Create public ip range 1 + cls.services["publiciprange"]["vlan"] = get_free_vlan( + cls.apiclient, + cls.zone.id)[1] + random_subnet_number = random.randrange(10,20) + cls.services["publiciprange"]["gateway"] = "10.100." + \ + str(random_subnet_number) + ".254" + cls.services["publiciprange"]["startip"] = "10.100." + \ + str(random_subnet_number) + ".1" + cls.services["publiciprange"]["endip"] = "10.100." + \ + str(random_subnet_number) + ".10" + cls.services["publiciprange"]["netmask"] = "255.255.255.0" + cls.public_ip_range1 = PublicIpRange.create( + cls.apiclient, + cls.services["publiciprange"], + account=cls.account1.name, + domainid=cls.account1.domainid + ) + + # dedicate ip range to sub domain + cls.services["publiciprange"]["vlan"] = get_free_vlan( + cls.apiclient, + cls.zone.id)[1] + random_subnet_number = random.randrange(10,20) + cls.services["publiciprange"]["gateway"] = "10.110." + \ + str(random_subnet_number) + ".254" + cls.services["publiciprange"]["startip"] = "10.110." + \ + str(random_subnet_number) + ".1" + cls.services["publiciprange"]["endip"] = "10.110." + \ + str(random_subnet_number) + ".10" + cls.services["publiciprange"]["netmask"] = "255.255.255.0" + cls.public_ip_range2 = PublicIpRange.create( + cls.apiclient, + cls.services["publiciprange"], + account=cls.sub_account.name, + domainid=cls.sub_account.domainid + ) + + # dedicate ip range to second domain + cls.services["publiciprange"]["vlan"] = get_free_vlan( + cls.apiclient, + cls.zone.id)[1] + random_subnet_number = random.randrange(10,20) + cls.services["publiciprange"]["gateway"] = "10.120." + \ + str(random_subnet_number) + ".254" + cls.services["publiciprange"]["startip"] = "10.120." + \ + str(random_subnet_number) + ".1" + cls.services["publiciprange"]["endip"] = "10.120." + \ + str(random_subnet_number) + ".10" + cls.services["publiciprange"]["netmask"] = "255.255.255.0" + cls.public_ip_range3 = PublicIpRange.create( + cls.apiclient, + cls.services["publiciprange"], + account=cls.account2.name, + domainid=cls.account2.domainid + ) + + # create vpc offering and VPC + cls.vpc_off = VpcOffering.create( + cls.apiclient, + cls.services["vpc_offering"] + ) + cls.vpc_off.update(cls.apiclient, state='Enabled') + + # create network offering + cls.isolated_network_offering = NetworkOffering.create( + cls.apiclient, + cls.services["isolated_network_offering"], + conservemode=False + ) + + NetworkOffering.update( + cls.isolated_network_offering, + cls.apiclient, + id=cls.isolated_network_offering.id, + state="enabled" + ) + + physical_network, shared_vlan = get_free_vlan( + cls.apiclient, cls.zone.id) + if shared_vlan is None: + cls.fail("Failed to get free vlan id for shared network") + + cls.services["shared_network_offering"]["specifyVlan"] = "True" + cls.services["shared_network_offering"]["specifyIpRanges"] = "True" + + cls.shared_network_offering = NetworkOffering.create( + cls.apiclient, + cls.services["shared_network_offering"], + conservemode=False + ) + + NetworkOffering.update( + cls.shared_network_offering, + cls.apiclient, + id=cls.shared_network_offering.id, + state="enabled" + ) + + # create network using the shared network offering created + cls.services["shared_network"]["acltype"] = "Domain" + cls.services["shared_network"][ + "networkofferingid"] = cls.shared_network_offering.id + cls.services["shared_network"][ + "physicalnetworkid"] = physical_network.id + cls.services["shared_network"]["vlan"] = shared_vlan + shared_network_subnet_number = random.randrange(1, 254) + + cls.services["shared_network"]["netmask"] = "255.255.255.0" + cls.services["shared_network"]["gateway"] = "172.16." + \ + str(shared_network_subnet_number) + ".254" + cls.services["shared_network"]["startip"] = "172.16." + \ + str(shared_network_subnet_number) + ".1" + cls.services["shared_network"]["endip"] = "172.16." + \ + str(shared_network_subnet_number) + ".10" + + cls.guest_network = Network.create( + cls.apiclient, + cls.services["shared_network"], + networkofferingid=cls.shared_network_offering.id, + zoneid=cls.zone.id + ) + + cls._cleanup.append(cls.guest_network) + cls._cleanup.append(cls.shared_network_offering) + cls._cleanup.append(cls.account1) + cls._cleanup.append(cls.account2) + cls._cleanup.append(cls.sub_account) + cls._cleanup.append(cls.sub_domain) + cls._cleanup.append(cls.domain1) + cls._cleanup.append(cls.domain2) + cls._cleanup.append(cls.public_ip_range1) + cls._cleanup.append(cls.public_ip_range2) + cls._cleanup.append(cls.public_ip_range3) + + @classmethod + def tearDownClass(cls): + try: + cls.apiclient = super( + TestPublicIp, + cls).getClsTestClient().getApiClient() + + Configurations.update( + cls.apiclient, + name=cls.use_system_ips_config_name, + value=cls.use_system_ips_config_value + ) + # Cleanup resources used + cleanup_resources(cls.apiclient, cls._cleanup) + + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + return + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.cleanup = [] + return + + def tearDown(self): + try: + # Clean up, terminate the resources created + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags=["advanced", "basic", "sg"], required_hardware="false") + def test_01_list_publicip_root_domain(self): + """ + ROOT domain should be able to list public IP address of all + sub domains + + Step 1: List all public ip address + Step 2: Ensure that the count is greater than or equal to 30 + Step 3: Display only allocated ip address + Step 4: Ensure that the count is greater than or equal to 0 + Step 5: Display ip address from shared networks + Step 6: Ensure that the count is greater than or equal to 30 + :return: + """ + # Step 1 + ipAddresses = PublicIPAddress.list( + self.apiclient, + allocatedonly=False, + isrecursive=True, + listall=True, + forvirtualnetwork=True) + + # Step 2 + self.assertGreaterEqual( + len(ipAddresses), + 30, + "Unable to display all public ip for ROOT domain" + ) + + # Step 4 + ipAddresses = PublicIPAddress.list( + self.apiclient, + allocatedonly=True, + isrecursive=True, + listall=True, + forvirtualnetwork=True) + + self.assertGreaterEqual( + len(ipAddresses), + 0, + "Unable to display all public ip for ROOT domain" + ) + + # Step 5 + ipAddresses = PublicIPAddress.list( + self.apiclient, + allocatedonly=False, + isrecursive=True, + listall=True, + forvirtualnetwork=False) + + # Step 6 + self.assertGreaterEqual( + len(ipAddresses), + 10, + "Unable to display all public ip for ROOT domain" + ) + + @attr(tags=["advanced", "basic", "sg"], required_hardware="false") + def test_02_list_publicip_domain_admin(self): + """ + A domain admin should be able to list public IP address of + his/her domain and all sub domains + + Step 1: Create an isolated network in the user domain and sub domain + Step 2: Associate IP in the range, dedicated to domain1 + Step 3: Display all public ip address in domain1 + Step 4: Ensure that the count is greater than or equal to 10 + Step 5: Display only the allocated Ip address in domain1 + Step 6: Ensure that the count is 1 + Step 7: Try to display public ip address from shared networks + Step 8: It should not return any result + Step 9: Try to display ip address of domain1 from domain2 + Step 10: Ensure that it doesnt not return any result + :return: + """ + # Step 1. Create isolated network + self.isolated_network1 = Network.create( + self.apiclient, + self.services["isolated_network"], + accountid=self.account1.name, + domainid=self.account1.domainid, + networkofferingid=self.isolated_network_offering.id, + zoneid=self.zone.id + ) + + # Step 2. Associate IP in range dedicated to domain1 + ip_address_1 = self.get_free_ipaddress(self.public_ip_range1.vlan.id) + ipaddress = PublicIPAddress.create( + self.apiclient, + zoneid=self.zone.id, + networkid=self.isolated_network1.id, + ipaddress=ip_address_1 + ) + self.assertIsNotNone( + ipaddress, + "Failed to Associate IP Address" + ) + + # Step 3: Display all public ip address in domain1 + user = self.account1.user[0] + user_api_client = self.testClient.getUserApiClient(user.username, self.domain1.name) + + ipAddresses = PublicIPAddress.list( + user_api_client, + allocatedonly=False, + isrecursive=True, + listall=True, + forvirtualnetwork=True) + + # Step 4: Ensure that the count is equal to 10 + self.assertEqual( + len(ipAddresses), + 10, + "Failed to display all public ip address is domain %s" % self.domain1.name + ) + + # Display public ip address using network id + ipAddresses = PublicIPAddress.list( + self.apiclient, + associatednetworkid=self.isolated_network1.id, + account=self.account1.name, + domainid=self.account1.domainid, + allocatedonly=False, + listall=True, + forvirtualnetwork=True) + + # Step 4: Ensure that the count is greater than or equal to 10 + self.assertEqual( + len(ipAddresses), + 10, + "Failed to display all public ip address using network id" + ) + + # Step 5: Display all allocated public ip address in domain1 + ipAddresses = PublicIPAddress.list( + user_api_client, + allocatedonly=True, + isrecursive=True, + listall=True, + forvirtualnetwork=True) + + # Step 6: Ensure that the count is greater than or equal to 1 + self.assertEqual( + len(ipAddresses), + 1, + "Allocated IP address is greater than 1" + ) + + # Step 7: Display public ip address from shared networks + ipAddresses = PublicIPAddress.list( + user_api_client, + allocatedonly=True, + isrecursive=True, + listall=True, + forvirtualnetwork=False) + + # Step 8: Ensure that the result is empty + self.assertIsNone( + ipAddresses, + "Users should not display ip from shared networks" + ) + + try: + # Step 9 + user = self.account2.user[0] + user_api_client = self.testClient.getUserApiClient(user.username, self.domain2.name) + + ipAddresses = PublicIPAddress.list( + user_api_client, + allocatedonly=False, + listall=True, + associatednetworkid=self.isolated_network1.id, + forvirtualnetwork=True) + + # Step 10 + self.fail("Domain should not access public ip of sibling domain") + except Exception as e: + self.info("Got exception as expected since domain2 cant access network of domain1") + + @attr(tags=["advanced", "basic", "sg"], required_hardware="false") + def test_03_list_publicip_user_domain(self): + """ + A regular user should be able to display public ip address + only in his domain + + Step 1: Create an isolated network in the user domain + Step 2: Display all public ip address in that domain + Step 3: Ensure that the count is 10 + Step 4: Associate IP in the range, dedicated to domain2 + Step 5: Display only the allocated Ip address in domain2 + Step 6: Ensure that the count is 1 + Step 7: Try to display public ip address from shared networks + Step 8: It should not return any result + Step 9: Try to display allocated public ip address of child domain using networkid + Step 10: It should display all allocated ip address from child domain + Step 11: Try to display all public ip address from child domain + Step 12: It should display all public ip of child domain + Step 13: Try to display ip of domain2 from a different domain + Step 14: Ensure that we get exception when trying to do so + :return: + """ + user = self.sub_account.user[0] + sub_user_api_client = self.testClient.getUserApiClient(user.username, self.sub_domain.name) + + # Step 1: create network in child domain + self.isolated_network2 = Network.create( + self.apiclient, + self.services["isolated_network"], + accountid=self.sub_account.name, + domainid=self.sub_account.domainid, + networkofferingid=self.isolated_network_offering.id, + zoneid=self.zone.id + ) + + # Step 2: Display all public ip address in sub domain + ipAddresses = PublicIPAddress.list( + sub_user_api_client, + allocatedonly=False, + listall=True, + forvirtualnetwork=True) + + # Step 3: Ensure that sub domain can list only the ip address in his domain + self.assertEqual( + len(ipAddresses), + 10, + "Allocated IP address is greater than 1" + ) + + # Step 4: Associate IP in range dedicated to sub domain + ip_address_1 = self.get_free_ipaddress(self.public_ip_range2.vlan.id) + ipaddress = PublicIPAddress.create( + sub_user_api_client, + zoneid=self.zone.id, + networkid=self.isolated_network2.id, + ipaddress=ip_address_1 + ) + + # Step 5: Display all allocated ip address in sub domain + ipAddresses = PublicIPAddress.list( + sub_user_api_client, + allocatedonly=True, + listall=True, + forvirtualnetwork=True) + + # Step 6: Ensure that the count is 1 + self.assertEqual( + len(ipAddresses), + 1, + "Allocated IP address is greater than 1" + ) + + # Step 7: Display ip address from shared networks + ipAddresses = PublicIPAddress.list( + sub_user_api_client, + allocatedonly=True, + listall=True, + forvirtualnetwork=False) + + # Step 8: It should not return any result + self.assertIsNone( + ipAddresses, + "Users should not display ip from shared networks" + ) + + user = self.account1.user[0] + user_api_client = self.testClient.getUserApiClient(user.username, self.domain1.name) + + # Step 9: display ip of child domain using network id + ipAddresses = PublicIPAddress.list( + user_api_client, + allocatedonly=True, + listall=True, + isrecursive=True, + associatednetworkid=self.isolated_network2.id, + forvirtualnetwork=True) + + # Step 10: Ensure that the count is 1 as only 1 ip is acquired in test 3 + self.assertEqual( + len(ipAddresses), + 1, + "Unable to display IP address of child domain using network id" + ) + + # Step 11: display ip of child domain using network id + ipAddresses = PublicIPAddress.list( + user_api_client, + allocatedonly=False, + listall=True, + isrecursive=True, + associatednetworkid=self.isolated_network2.id, + forvirtualnetwork=True) + + # Step 12: Ensure that the count is 1 as only 1 ip is acquired in test 3 + self.assertEqual( + len(ipAddresses), + 10, + "Unable to display IP address of child domain using network id" + ) + + try: + # Step 13 + user = self.account2.user[0] + user_api_client = self.testClient.getUserApiClient(user.username, self.domain2.name) + + PublicIPAddress.list( + user_api_client, + allocatedonly=False, + listall=True, + associatednetworkid=self.isolated_network2.id, + forvirtualnetwork=True) + + # Step 14 + self.fail("Domain should not access public ip of sibling domain") + except Exception as e: + self.info("Got exception as expected since domain2 cant access network of domain1") + + @attr(tags=["advanced", "basic", "sg"], required_hardware="false") + def test_04_list_publicip_all_subdomains(self): + """ + A domain admin should be able to display public ip address + in his domain and also all child domains + + Step 1: Display all public ip address in that domain and sub domain + Step 2: Ensure that the count is 11 (all ip from parent domain and allocated from sub domain) + Step 3: Display only the allocated Ip address + Step 4: Ensure that the count is 2 + Step 5: Display public ip of child domain using domain/account + Step 6: Ensure that the count is 10 + Step 7: Display public ip of child domain using network id + Step 8: Ensure that the count is 1 as only one IP is allocated + :return: + """ + user = self.account1.user[0] + user_api_client = self.testClient.getUserApiClient(user.username, self.domain1.name) + + # Step 1: Display all public ip + ipAddresses = PublicIPAddress.list( + user_api_client, + allocatedonly=False, + listall=True, + isrecursive=True, + forvirtualnetwork=True) + + # Step 2: Ensure that it can display all ip from current domain and + # only allocated ip from child domains + self.assertEqual( + len(ipAddresses), + 11, + "Unable to display all IP address is domain %s" % self.domain1.name + ) + + # Step 3: display only allocated ip for parent and sub domain + ipAddresses = PublicIPAddress.list( + user_api_client, + allocatedonly=True, + listall=True, + isrecursive=True, + forvirtualnetwork=True) + + # Step 4: Ensure that the count is 2 + self.assertEqual( + len(ipAddresses), + 2, + "Unable to display all allocated IP address is domain %s" % self.domain1.name + ) + + # Step 5: display ip of child domain using domainid/account + ipAddresses = PublicIPAddress.list( + user_api_client, + allocatedonly=False, + listall=True, + isrecursive=True, + domainid=self.sub_domain.id, + account=self.sub_account.name, + forvirtualnetwork=True) + + # Step 6: Ensure that the count is 10 + self.assertEqual( + len(ipAddresses), + 10, + "Unable to display IP address of child domain" + ) + + @attr(tags=["advanced", "basic", "sg"], required_hardware="false") + def test_05_list_publicip_user_domain(self): + """ + A domain admin should be able to display public ip address + in his domain and also all child domains + + Step 1: Display all public ip address in that domain and sub domain + Step 2: Ensure that the count is 20 + Step 3: Display only the allocated Ip address + Step 4: Ensure that the count is 2 + Step 5: Try to display public ip of vpc from different domain + Step 6: Ensure that we get exception when trying to do so + :return: + """ + user = self.account2.user[0] + user_api_client = self.testClient.getUserApiClient(user.username, self.domain2.name) + + self.services["vpc"]["cidr"] = "10.1.1.1/16" + vpc_1 = VPC.create( + user_api_client, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account2.name, + domainid=self.account2.domainid + ) + self.validate_vpc_network(vpc_1) + + ipAddresses = PublicIPAddress.list( + user_api_client, + allocatedonly=False, + listall=True, + forvirtualnetwork=True) + + self.assertGreaterEqual( + len(ipAddresses), + 10, + "Unable to display all public ip in VPC" + ) + + # List ip address using vpcid + ipAddresses = PublicIPAddress.list( + self.apiclient, + vpcid=vpc_1.id, + allocatedonly=False, + account=self.account2.name, + domainid=self.domain2.id, + forvirtualnetwork=True) + + self.assertGreaterEqual( + len(ipAddresses), + 10, + "Unable to display all public ip in VPC" + ) + + # Acquire public ip address from VPC + ip_address_1 = self.get_free_ipaddress(self.public_ip_range3.vlan.id) + PublicIPAddress.create( + user_api_client, + zoneid=self.zone.id, + vpcid=vpc_1.id, + ipaddress=ip_address_1 + ) + + ipAddresses = PublicIPAddress.list( + user_api_client, + allocatedonly=True, + listall=True, + forvirtualnetwork=True) + + self.assertGreaterEqual( + len(ipAddresses), + 2, + "Unable to display allocated public ip in VPC" + ) + + try: + # Step 5 + user = self.account1.user[0] + user_api_client = self.testClient.getUserApiClient(user.username, self.domain1.name) + + PublicIPAddress.list( + user_api_client, + allocatedonly=False, + listall=True, + vpcid=vpc_1.id, + forvirtualnetwork=True) + + # Step 6 + self.fail("Domain should not access public ip of sibling domain") + except Exception as e: + self.info("Got exception as expected since domain2 cant access network of domain1") + + self.vpc_off.update(self.apiclient, state='Disabled') + self.cleanup.append(vpc_1) + self.cleanup.append(self.vpc_off) + + @attr(tags=["advanced", "basic", "sg"], required_hardware="false") + def test_06_list_publicip_user_domain(self): + """ + Display public ip address from shared networks + 1. List public ip address of shared network as root admin + 2. Ensure that it can display all public ip address + 3. List public ip address of shared networks as domain admin + 4. It should not return any result + 5. Try to display public ip by passing network id + 6. Ensure that we get exception when trying to do so + :return: + """ + # Step 1 + ipAddresses = PublicIPAddress.list( + self.apiclient, + allocatedonly=False, + listall=True, + networkid=self.guest_network.id, + forvirtualnetwork=False) + + # Step 2 + self.assertGreaterEqual( + len(ipAddresses), + 10, + "Unable to display allocated public ip in VPC" + ) + + user = self.account1.user[0] + user_api_client = self.testClient.getUserApiClient(user.username, self.domain1.name) + + # Step 3 + ipAddresses = PublicIPAddress.list( + user_api_client, + allocatedonly=False, + listall=True, + networkid=self.guest_network.id, + forvirtualnetwork=False) + + # Step 4 + self.assertIsNone( + ipAddresses, + "Unable to display allocated public ip in VPC" + ) + + try: + # Step 5 + PublicIPAddress.list( + user_api_client, + allocatedonly=False, + listall=True, + associatednetworkid=self.guest_network.id, + forvirtualnetwork=True) + + # Step 6 + self.fail("Domain should not access public ip of sibling domain") + except Exception as e: + self.info("Got exception as expected since domain2 cant access network of domain1") + + def get_free_ipaddress(self, vlanId): + ipaddresses = PublicIPAddress.list( + self.apiclient, + vlanid=vlanId, + state='Free' + ) + self.assertEqual( + isinstance(ipaddresses, list), + True, + "List ipaddresses should return a valid response for Free ipaddresses" + ) + random.shuffle(ipaddresses) + return ipaddresses[0].ipaddress + + def validate_vpc_network(self, network, state=None): + """Validates the VPC network""" + + self.debug("Check if the VPC network is created successfully?") + vpc_networks = VPC.list( + self.apiclient, + id=network.id + ) + self.assertEqual( + isinstance(vpc_networks, list), + True, + "List VPC network should return a valid list" + ) + self.assertEqual( + network.name, + vpc_networks[0].name, + "Name of the VPC network should match with listVPC data" + ) + if state: + self.assertEqual( + vpc_networks[0].state, + state, + "VPC state should be '%s'" % state + ) + self.debug("VPC network validated - %s" % network.name) + return diff --git a/test/integration/smoke/test_network.py b/test/integration/smoke/test_network.py index fd1e4e6c7e0..3799ff7648b 100644 --- a/test/integration/smoke/test_network.py +++ b/test/integration/smoke/test_network.py @@ -227,7 +227,8 @@ class TestPublicIP(cloudstackTestCase): # 1.listPublicIpAddresses should no more return the released address list_pub_ip_addr_resp = list_publicIP( self.apiclient, - id=ip_address.ipaddress.id + id=ip_address.ipaddress.id, + allocatedonly=True ) if list_pub_ip_addr_resp is None: return @@ -279,7 +280,8 @@ class TestPublicIP(cloudstackTestCase): list_pub_ip_addr_resp = list_publicIP( self.apiclient, - id=ip_address.ipaddress.id + id=ip_address.ipaddress.id, + allocatedonly=True ) self.assertEqual( @@ -883,7 +885,8 @@ class TestReleaseIP(cloudstackTestCase): while retriesCount > 0: listResponse = list_publicIP( self.apiclient, - id=self.ip_addr.id + id=self.ip_addr.id, + state="Allocated" ) if listResponse is None: isIpAddressDisassociated = True diff --git a/ui/public/config.json b/ui/public/config.json index f4fda411426..9f3b723c27f 100644 --- a/ui/public/config.json +++ b/ui/public/config.json @@ -46,5 +46,7 @@ "jp": "label.japanese.keyboard", "sc": "label.simplified.chinese.keyboard" }, - "plugins": [] + "docHelpMappings": {}, + "plugins": [], + "basicZoneEnabled": true } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 28e99b0c141..6a2dcdf15a9 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1817,6 +1817,7 @@ "label.rolename": "Role", "label.roles": "Roles", "label.roletype": "Role Type", +"label.rootdisksize": "Root disk size (GB)", "label.root.certificate": "Root certificate", "label.root.disk.offering": "Root Disk Offering", "label.root.disk.size": "Root disk size (GB)", diff --git a/ui/src/components/view/ActionButton.vue b/ui/src/components/view/ActionButton.vue index fd25c70d557..bf9f63b0fdd 100644 --- a/ui/src/components/view/ActionButton.vue +++ b/ui/src/components/view/ActionButton.vue @@ -134,6 +134,9 @@ export default { methods: { execAction (action) { action.resource = this.resource + if (action.docHelp) { + action.docHelp = this.$applyDocHelpMappings(action.docHelp) + } this.$emit('exec-action', action) }, handleShowBadge () { diff --git a/ui/src/components/view/DetailsTab.vue b/ui/src/components/view/DetailsTab.vue index 64809b2546a..e47f3c58a3f 100644 --- a/ui/src/components/view/DetailsTab.vue +++ b/ui/src/components/view/DetailsTab.vue @@ -33,6 +33,11 @@ {{ volume.type }} - {{ volume.path }} ({{ parseFloat(volume.size / (1024.0 * 1024.0 * 1024.0)).toFixed(1) }} GB) +
+
+ {{ parseFloat( resource.rootdisksize / (1024.0 * 1024.0 * 1024.0)).toFixed(1) }} GB +
+
{{ $t(resource[item].toLowerCase()) }} {{ resource[item] }} diff --git a/ui/src/config/router.js b/ui/src/config/router.js index 1b1ad0c2a06..a07f0d68bb5 100644 --- a/ui/src/config/router.js +++ b/ui/src/config/router.js @@ -42,7 +42,7 @@ function generateRouterMap (section) { name: section.name, path: '/' + section.name, hidden: section.hidden, - meta: { title: section.title, icon: section.icon, docHelp: section.docHelp, searchFilters: section.searchFilters }, + meta: { title: section.title, icon: section.icon, docHelp: Vue.prototype.$applyDocHelpMappings(section.docHelp), searchFilters: section.searchFilters }, component: RouteView } @@ -63,7 +63,7 @@ function generateRouterMap (section) { title: child.title, name: child.name, icon: child.icon, - docHelp: child.docHelp, + docHelp: Vue.prototype.$applyDocHelpMappings(child.docHelp), permission: child.permission, resourceType: child.resourceType, filters: child.filters, @@ -85,7 +85,7 @@ function generateRouterMap (section) { title: child.title, name: child.name, icon: child.icon, - docHelp: child.docHelp, + docHelp: Vue.prototype.$applyDocHelpMappings(child.docHelp), permission: child.permission, resourceType: child.resourceType, params: child.params ? child.params : {}, @@ -140,7 +140,7 @@ function generateRouterMap (section) { title: section.title, name: section.name, icon: section.icon, - docHelp: section.docHelp, + docHelp: Vue.prototype.$applyDocHelpMappings(section.docHelp), hidden: section.hidden, permission: section.permission, resourceType: section.resourceType, diff --git a/ui/src/config/section/offering.js b/ui/src/config/section/offering.js index 085d85989f6..1ec2d40d181 100644 --- a/ui/src/config/section/offering.js +++ b/ui/src/config/section/offering.js @@ -36,6 +36,10 @@ export default { store.getters.apis.createServiceOffering.params.filter(x => x.name === 'storagepolicy').length > 0) { fields.splice(6, 0, 'vspherestoragepolicy') } + if (store.getters.apis.createServiceOffering && + store.getters.apis.createServiceOffering.params.filter(x => x.name === 'rootdisksize').length > 0) { + fields.splice(12, 0, 'rootdisksize') + } return fields }, related: [{ diff --git a/ui/src/main.js b/ui/src/main.js index acc41d2de33..064314b98de 100644 --- a/ui/src/main.js +++ b/ui/src/main.js @@ -26,7 +26,7 @@ import './core/lazy_use' import './core/ext' import './permission' // permission control import './utils/filter' // global filter -import { pollJobPlugin, notifierPlugin, toLocaleDatePlugin } from './utils/plugins' +import { pollJobPlugin, notifierPlugin, toLocaleDatePlugin, configUtilPlugin } from './utils/plugins' import { VueAxios } from './utils/request' Vue.config.productionTip = false @@ -49,3 +49,5 @@ fetch('config.json').then(response => response.json()).then(config => { }).$mount('#app') }) }) + +Vue.use(configUtilPlugin) diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index d32217f6ba1..905f0b1bbfd 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -167,3 +167,21 @@ export const toLocaleDatePlugin = { } } } + +export const configUtilPlugin = { + install (Vue) { + Vue.prototype.$applyDocHelpMappings = function (docHelp) { + var docHelpMappings = this.$config.docHelpMappings + if (docHelp && docHelpMappings && + docHelpMappings.constructor === Object && Object.keys(docHelpMappings).length > 0) { + for (var key in docHelpMappings) { + if (docHelp.includes(key)) { + docHelp = docHelp.replace(key, docHelpMappings[key]) + break + } + } + } + return docHelp + } + } +} diff --git a/ui/src/views/compute/AssignInstance.vue b/ui/src/views/compute/AssignInstance.vue index 000dee5b2be..e77cfc4d0a1 100644 --- a/ui/src/views/compute/AssignInstance.vue +++ b/ui/src/views/compute/AssignInstance.vue @@ -134,6 +134,10 @@ export default { this.selectedDomain = this.domains[0].id this.fetchAccounts() this.fetchProjects() + }).catch(error => { + this.$notifyError(error) + }).finally(() => { + this.loading = false }) }, fetchAccounts () { @@ -142,9 +146,12 @@ export default { response: 'json', domainId: this.selectedDomain, state: 'Enabled', - listAll: true + isrecursive: false }).then(response => { this.accounts = response.listaccountsresponse.account + }).catch(error => { + this.$notifyError(error) + }).finally(() => { this.loading = false }) }, @@ -155,9 +162,12 @@ export default { domainId: this.selectedDomain, state: 'Active', details: 'min', - listAll: true + isrecursive: false }).then(response => { this.projects = response.listprojectsresponse.project + }).catch(error => { + this.$notifyError(error) + }).finally(() => { this.loading = false }) }, @@ -172,6 +182,9 @@ export default { projectid: this.selectedProject }).then(response => { this.networks = response.listnetworksresponse.network + }).catch(error => { + this.$notifyError(error) + }).finally(() => { this.loading = false }) }, diff --git a/ui/src/views/infra/zone/AdvancedGuestTrafficForm.vue b/ui/src/views/infra/zone/AdvancedGuestTrafficForm.vue index 8a96677859d..ca2cb0ae6cb 100644 --- a/ui/src/views/infra/zone/AdvancedGuestTrafficForm.vue +++ b/ui/src/views/infra/zone/AdvancedGuestTrafficForm.vue @@ -194,11 +194,10 @@ export default { min-height: 200px; text-align: center; vertical-align: center; - padding-top: 16px; - padding-top: 16px; margin-top: 8px; max-height: 300px; overflow-y: auto; + padding: 16px 20px 0; /deep/.has-error { .ant-form-explain { diff --git a/ui/src/views/infra/zone/IpAddressRangeForm.vue b/ui/src/views/infra/zone/IpAddressRangeForm.vue index f1082a37d9f..07b2ef9c956 100644 --- a/ui/src/views/infra/zone/IpAddressRangeForm.vue +++ b/ui/src/views/infra/zone/IpAddressRangeForm.vue @@ -24,79 +24,94 @@ + style="margin-bottom: 24px; width: 100%" > @@ -126,7 +141,10 @@