Merge branch '4.9'

engine/schema/src/com/cloud/configuration/dao/ResourceCountDaoImpl.java

@@ -21,8 +21,12 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import javax.annotation.PostConstruct;
 import javax.inject.Inject;
 
+import com.cloud.domain.DomainVO;
+import com.cloud.user.AccountVO;
+import com.cloud.utils.db.JoinBuilder;
 import org.springframework.stereotype.Component;
 
 import com.cloud.configuration.Resource;
@@ -59,11 +63,21 @@ public class ResourceCountDaoImpl extends GenericDaoBase<ResourceCountVO, Long>
         TypeSearch.done();
 
         AccountSearch = createSearchBuilder();
+        DomainSearch = createSearchBuilder();
+    }
+
+    @PostConstruct
+    protected void configure() {
         AccountSearch.and("accountId", AccountSearch.entity().getAccountId(), SearchCriteria.Op.NNULL);
+        SearchBuilder<AccountVO> joinAccount = _accountDao.createSearchBuilder();
+        joinAccount.and("notremoved", joinAccount.entity().getRemoved(), SearchCriteria.Op.NULL);
+        AccountSearch.join("account", joinAccount, AccountSearch.entity().getAccountId(), joinAccount.entity().getId(), JoinBuilder.JoinType.INNER);
         AccountSearch.done();
 
-        DomainSearch = createSearchBuilder();
         DomainSearch.and("domainId", DomainSearch.entity().getDomainId(), SearchCriteria.Op.NNULL);
+        SearchBuilder<DomainVO> joinDomain = _domainDao.createSearchBuilder();
+        joinDomain.and("notremoved", joinDomain.entity().getRemoved(), SearchCriteria.Op.NULL);
+        DomainSearch.join("domain", joinDomain, DomainSearch.entity().getDomainId(), joinDomain.entity().getId(), JoinBuilder.JoinType.INNER);
         DomainSearch.done();
     }
 

engine/schema/src/com/cloud/network/dao/NetworkVO.java

@@ -104,7 +104,7 @@ public class NetworkVO implements Network {
     State state;
 
     @Column(name = "redundant")
-    boolean isRedundant;
+    boolean redundant;
 
     @Column(name = "dns1")
     String dns1;
@@ -205,7 +205,7 @@ public class NetworkVO implements Network {
         this.networkOfferingId = networkOfferingId;
         this.dataCenterId = dataCenterId;
         this.physicalNetworkId = physicalNetworkId;
-        this.isRedundant = isRedundant;
+        this.redundant = isRedundant;
         if (state == null) {
             this.state = State.Allocated;
         } else {
@@ -301,7 +301,7 @@ public class NetworkVO implements Network {
 
     @Override
     public boolean isRedundant() {
-        return this.isRedundant;
+        return this.redundant;
     }
 
     // don't use this directly when possible, use Network state machine instead
@@ -632,7 +632,7 @@ public class NetworkVO implements Network {
     }
 
     public void setIsReduntant(boolean reduntant) {
-        this.isRedundant = reduntant;
+        this.redundant = reduntant;
     }
 
 }

plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java

@@ -1153,11 +1153,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
          *  so we assume that it's VLAN for now
          */
         if (VirtualSwitchType.StandardVirtualSwitch == vSwitchType) {
-            synchronized (vmMo.getRunningHost().getMor().getValue().intern()) {
+            networkInfo = HypervisorHostHelper.prepareNetwork(_publicTrafficInfo.getVirtualSwitchName(),
-                networkInfo =
+                    "cloud.public", vmMo.getRunningHost(), vlanId, null, null,
-                        HypervisorHostHelper.prepareNetwork(_publicTrafficInfo.getVirtualSwitchName(), "cloud.public", vmMo.getRunningHost(), vlanId, null, null,
+                    _opsTimeout, true, BroadcastDomainType.Vlan, null);
-                                _opsTimeout, true, BroadcastDomainType.Vlan, null);
-            }
         } else {
             networkInfo =
                     HypervisorHostHelper.prepareNetwork(_publicTrafficInfo.getVirtualSwitchName(), "cloud.public", vmMo.getRunningHost(), vlanId, null, null, null,
@@ -2876,11 +2874,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
         s_logger.info("Prepare network on " + switchType + " " + switchName + " with name prefix: " + namePrefix);
 
         if (VirtualSwitchType.StandardVirtualSwitch == switchType) {
-            synchronized(hostMo.getMor().getValue().intern()) {
+            networkInfo = HypervisorHostHelper.prepareNetwork(switchName, namePrefix, hostMo,
-                networkInfo = HypervisorHostHelper.prepareNetwork(switchName, namePrefix, hostMo, getVlanInfo(nicTo, vlanToken), nicTo.getNetworkRateMbps(),
+                    getVlanInfo(nicTo, vlanToken), nicTo.getNetworkRateMbps(), nicTo.getNetworkRateMulticastMbps(),
-                        nicTo.getNetworkRateMulticastMbps(), _opsTimeout,
+                    _opsTimeout, !namePrefix.startsWith("cloud.private"), nicTo.getBroadcastType(), nicTo.getUuid());
-                        !namePrefix.startsWith("cloud.private"), nicTo.getBroadcastType(), nicTo.getUuid());
-            }
         }
         else {
             String vlanId = getVlanInfo(nicTo, vlanToken);

server/src/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -3113,12 +3113,12 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
             // Check if the new VLAN's subnet conflicts with the guest network
             // in
             // the specified zone (guestCidr is null for basic zone)
+            // when adding shared network with same cidr of zone guest cidr,
+            // if the specified vlan is not present in zone, physical network, allow to create the network as the isolation is based on VLAN.
             final String guestNetworkCidr = zone.getGuestNetworkCidr();
-            if (guestNetworkCidr != null) {
+            if (guestNetworkCidr != null && NetUtils.isNetworksOverlap(newCidr, guestNetworkCidr) && _zoneDao.findVnet(zoneId, physicalNetworkId, vlanId).isEmpty() != true) {
-                if (NetUtils.isNetworksOverlap(newCidr, guestNetworkCidr)) {
+                throw new InvalidParameterValueException("The new IP range you have specified has  overlapped with the guest network in zone: " + zone.getName()
-                    throw new InvalidParameterValueException("The new IP range you have specified has  overlapped with the guest network in zone: " + zone.getName()
+                        + "along with existing Vlan also. Please specify a different gateway/netmask");
-                            + ". Please specify a different gateway/netmask.");
-                }
             }
 
             // Check if there are any errors with the IP range

server/src/com/cloud/network/firewall/FirewallManagerImpl.java

@@ -429,7 +429,13 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
                 boolean allowStaticNat =
                     (rule.getPurpose() == Purpose.StaticNat && newRule.getPurpose() == Purpose.StaticNat && !newRule.getProtocol().equalsIgnoreCase(rule.getProtocol()));
 
-                if (!(allowPf || allowStaticNat || oneOfRulesIsFirewall)) {
+                boolean allowVpnPf =
+                        (rule.getPurpose() == Purpose.PortForwarding && newRule.getPurpose() == Purpose.Vpn && !newRule.getProtocol().equalsIgnoreCase(rule.getProtocol()));
+
+                boolean allowVpnLb =
+                        (rule.getPurpose() == Purpose.LoadBalancing && newRule.getPurpose() == Purpose.Vpn && !newRule.getProtocol().equalsIgnoreCase(rule.getProtocol()));
+
+                if (!(allowPf || allowStaticNat || oneOfRulesIsFirewall || allowVpnPf || allowVpnLb)) {
                     throw new NetworkRuleConflictException("The range specified, " + newRule.getSourcePortStart() + "-" + newRule.getSourcePortEnd() +
                         ", conflicts with rule " + rule.getId() + " which has " + rule.getSourcePortStart() + "-" + rule.getSourcePortEnd());
                 }

server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -1209,10 +1209,9 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
                 if (!Boolean.parseBoolean(serviceMonitoringFlag) || router.getVpcId() != null) {
                     continue;
                 }
+                String controlIP = getRouterControlIP(router);
 
-                final String privateIP = router.getPrivateIpAddress();
+                if (controlIP != null && !controlIP.equals("0.0.0.0")) {
-
-                if (privateIP != null) {
                     OpRouterMonitorServiceVO opRouterMonitorServiceVO = _opRouterMonitorServiceDao.findById(router.getId());
 
                     GetRouterAlertsCommand command = null;
@@ -1226,7 +1225,7 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
                         command = new GetRouterAlertsCommand(opRouterMonitorServiceVO.getLastAlertTimestamp());
                     }
 
-                    command.setAccessDetail(NetworkElementCommand.ROUTER_IP, router.getPrivateIpAddress());
+                    command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlIP);
 
                     try {
                         final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command);
@@ -1281,6 +1280,29 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
         }
     }
 
+    private String getRouterControlIP(DomainRouterVO router){
+        final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId());
+        String controlIP = null;
+
+        if(router.getHypervisorType() == HypervisorType.VMware  && dcVo.getNetworkType() == NetworkType.Basic ){
+
+            final List<NicVO> nics = _nicDao.listByVmId(router.getId());
+            for (final NicVO nic : nics) {
+                final NetworkVO nc = _networkDao.findById(nic.getNetworkId());
+                if (nc.getTrafficType() == TrafficType.Guest && nic.getIPv4Address() != null) {
+                    controlIP = nic.getIPv4Address();
+                    break;
+                }
+            }
+            s_logger.debug("Vmware with Basic network selected Guest NIC ip as control IP " + controlIP );
+        }else{
+            controlIP = _routerControlHelper.getRouterControlIp(router.getId());
+        }
+
+        s_logger.debug("IP of control NIC " + controlIP );
+        return controlIP;
+    }
+
     @Override
     public boolean finalizeVirtualMachineProfile(final VirtualMachineProfile profile, final DeployDestination dest, final ReservationContext context) {
 

systemvm/patches/debian/config/opt/cloud/bin/configure.py

@@ -122,10 +122,10 @@ class CsAcl(CsDataBag):
             rnge = ''
             if "first_port" in self.rule.keys() and \
                self.rule['first_port'] == self.rule['last_port']:
-                    rnge = self.rule['first_port']
+                    rnge = " --dport %s " %self.rule['first_port']
             if "first_port" in self.rule.keys() and \
                self.rule['first_port'] != self.rule['last_port']:
-                    rnge = "%s:%s" % (rule['first_port'], rule['last_port'])
+                    rnge = " --dport %s:%s" % (rule['first_port'], rule['last_port'])
             if self.direction == 'ingress':
                 if rule['protocol'] == "icmp":
                     self.fw.append(["mangle", "front",
@@ -140,7 +140,7 @@ class CsAcl(CsDataBag):
                                     " -s %s " % cidr +
                                     " -p %s " % rule['protocol'] +
                                     " -m %s " % rule['protocol'] +
-                                    " --dport %s -j RETURN" % rnge])
+                                    "  %s -j RETURN" % rnge])
 
             logging.debug("Current ACL IP direction is ==> %s", self.direction)
             if self.direction == 'egress':
@@ -174,7 +174,7 @@ class CsAcl(CsDataBag):
                     fwr += " -s %s " % cidr + \
                            " -p %s " % rule['protocol'] + \
                            " -m %s " % rule['protocol'] + \
-                           " --dport %s" % rnge
+                           "  %s" % rnge
                 elif rule['protocol'] == "all":
                     fwr += " -s %s " % cidr
 

test/integration/smoke/test_volumes.py

@@ -378,6 +378,11 @@ class TestVolumes(cloudstackTestCase):
         #Clean up, terminate the created volumes
         if self.attached:
             self.virtual_machine.detach_volume(self.apiClient, self.volume)
+
+        if self.virtual_machine.hypervisor == "KVM":
+            self.virtual_machine.stop(self.apiClient)
+            self.virtual_machine.start(self.apiClient)
+
         cleanup_resources(self.apiClient, self.cleanup)
         return
 

test/integration/smoke/test_vpc_vpn.py

@@ -168,7 +168,7 @@ class Services:
             "vpc2": {
                 "name": "TestVPC",
                 "displaytext": "VPC2",
-                "cidr": '10.2.0.0/16'
+                "cidr": '10.3.0.0/16'
             },
             "network_1": {
                 "name": "Test Network",
@@ -180,13 +180,13 @@ class Services:
                 "name": "Test Network",
                 "displaytext": "Test Network",
                 "netmask": '255.255.255.0',
-                "gateway": "10.2.1.1"
+                "gateway": "10.3.1.1"
             },
             "vpn": {
                 "vpn_user": "root",
                 "vpn_pass": "Md1s#dc",
                 "vpn_pass_fail": "abc!123",  # too short
-                "iprange": "10.2.2.1-10.2.2.10",
+                "iprange": "10.3.2.1-10.3.2.10",
                 "fordisplay": "true"
             },
             "vpncustomergateway": {

vmware-base/src/com/cloud/hypervisor/vmware/mo/HostMO.java

@@ -1110,4 +1110,58 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost {
         }
         return networkName;
     }
+
+    public void createPortGroup(HostVirtualSwitch vSwitch, String portGroupName, Integer vlanId,
+            HostNetworkSecurityPolicy secPolicy, HostNetworkTrafficShapingPolicy shapingPolicy, long timeOutMs)
+            throws Exception {
+        assert (portGroupName != null);
+
+        // Prepare lock to avoid simultaneous execution of the synchronized block for
+        // duplicate port groups on the ESXi host it's being created on.
+        String hostPortGroup = _mor.getValue() + "-" + portGroupName;
+        synchronized (hostPortGroup.intern()) {
+            // Check if port group exists already
+            if (hasPortGroup(vSwitch, portGroupName)) {
+                if (s_logger.isDebugEnabled()) {
+                    s_logger.debug("Found port group " + portGroupName + " in vSwitch " + vSwitch.getName()
+                        + ". Not attempting to create port group as it already exists.");
+                }
+                return;
+            } else {
+                if (s_logger.isDebugEnabled()) {
+                    s_logger.debug("Port group " + portGroupName + " doesn't exist in vSwitch " + vSwitch.getName()
+                        + ". Attempting to create port group in this vSwitch.");
+                }
+            }
+            // Create port group if not exists already
+            createPortGroup(vSwitch, portGroupName, vlanId, secPolicy, shapingPolicy);
+
+            // Wait for port group to turn up ready on vCenter upto timeout of timeOutMs milli seconds
+            waitForPortGroup(portGroupName, timeOutMs);
+        }
+
+        if (s_logger.isDebugEnabled()) {
+            s_logger.debug("Successfully created port group " + portGroupName + " in vSwitch " + vSwitch.getName()
+                + " on host " + getHostName());
+        }
+    }
+
+    public ManagedObjectReference waitForPortGroup(String networkName, long timeOutMs) throws Exception {
+        ManagedObjectReference morNetwork = null;
+        // if portGroup is just created, getNetwork may fail to retrieve it, we
+        // need to retry
+        long startTick = System.currentTimeMillis();
+        while (System.currentTimeMillis() - startTick <= timeOutMs) {
+            morNetwork = getNetworkMor(networkName);
+            if (morNetwork != null) {
+                break;
+            }
+
+            if (s_logger.isInfoEnabled()) {
+                s_logger.info("Waiting for network " + networkName + " to be ready");
+            }
+            Thread.sleep(1000);
+        }
+        return morNetwork;
+    }
 }

vmware-base/src/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java

@@ -1082,8 +1082,11 @@ public class HypervisorHostHelper {
             }
         } else {
             if (!hostMo.hasPortGroup(vSwitch, networkName)) {
-                hostMo.createPortGroup(vSwitch, networkName, vid, secPolicy, shapingPolicy);
+                hostMo.createPortGroup(vSwitch, networkName, vid, secPolicy, shapingPolicy, timeOutMs);
-                bWaitPortGroupReady = true;
+                // Setting flag "bWaitPortGroupReady" to false.
+                // This flag indicates whether we need to wait for portgroup on vCenter.
+                // Above createPortGroup() method itself ensures creation of portgroup as well as wait for portgroup.
+                bWaitPortGroupReady = false;
             } else {
                 HostPortGroupSpec spec = hostMo.getPortGroupSpec(networkName);
                 if (!isSpecMatch(spec, vid, shapingPolicy)) {

vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java

@@ -661,7 +661,14 @@ public class VirtualMachineMO extends BaseMO {
     public boolean hasSnapshot() throws Exception {
         VirtualMachineSnapshotInfo info = getSnapshotInfo();
         if (info != null) {
-            return info.getCurrentSnapshot() != null;
+            ManagedObjectReference currentSnapshot = info.getCurrentSnapshot();
+            if (currentSnapshot != null) {
+                return true;
+            }
+            List<VirtualMachineSnapshotTree> rootSnapshotList = info.getRootSnapshotList();
+            if (rootSnapshotList != null && rootSnapshotList.size() > 0) {
+                return true;
+            }
         }
         return false;
     }