Merge pull request #1827 from sateesh-chodapuneedi/pr-cloudstack-9673

CLOUDSTACK-9673 : Exception occured while creating the CPVM in VMware setup over standard vSwitchesJira
===
CLOUDSTACK-9673 : Exception occured while creating the CPVM in VMware setup over standard vSwitches

Issue
====
Exception occured while creating the CPVM in the VmWare Setup using standard vswitches.
```
StartCommand failed due to Exception: com.vmware.vim25.AlreadyExists
message: [] com.vmware.vim25.AlreadyExistsFaultMsg: The specified key, name, or identifier already exists
```

Fix
===
Ensure synchronization while attempting to create port group such that simultaneous attempts are not made with same port group name on same ESXi host.

Testing
======
Successfully ran manual tests (deploy user instance) on top of latest master commit `17653a86fad67447a4f13e455e336694ad5c1735`.This code change is involved in virtual network creation over VMware standard vSwitches. Existing functional tests covers this functionality.

* pr/1827:
  CLOUDSTACK-9673 Exception occured while creating the CPVM in the VmWare Setup over standard vSwitches

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java

@@ -1125,11 +1125,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
          *  so we assume that it's VLAN for now
          */
         if (VirtualSwitchType.StandardVirtualSwitch == vSwitchType) {
-            synchronized (vmMo.getRunningHost().getMor().getValue().intern()) {
-                networkInfo =
-                        HypervisorHostHelper.prepareNetwork(_publicTrafficInfo.getVirtualSwitchName(), "cloud.public", vmMo.getRunningHost(), vlanId, null, null,
-                                _opsTimeout, true, BroadcastDomainType.Vlan, null);
-            }
+            networkInfo = HypervisorHostHelper.prepareNetwork(_publicTrafficInfo.getVirtualSwitchName(),
+                    "cloud.public", vmMo.getRunningHost(), vlanId, null, null,
+                    _opsTimeout, true, BroadcastDomainType.Vlan, null);
         } else {
             networkInfo =
                     HypervisorHostHelper.prepareNetwork(_publicTrafficInfo.getVirtualSwitchName(), "cloud.public", vmMo.getRunningHost(), vlanId, null, null, null,
@@ -2852,11 +2850,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
         s_logger.info("Prepare network on " + switchType + " " + switchName + " with name prefix: " + namePrefix);
 
         if (VirtualSwitchType.StandardVirtualSwitch == switchType) {
-            synchronized(hostMo.getMor().getValue().intern()) {
-                networkInfo = HypervisorHostHelper.prepareNetwork(switchName, namePrefix, hostMo, getVlanInfo(nicTo, vlanToken), nicTo.getNetworkRateMbps(),
-                        nicTo.getNetworkRateMulticastMbps(), _opsTimeout,
-                        !namePrefix.startsWith("cloud.private"), nicTo.getBroadcastType(), nicTo.getUuid());
-            }
+            networkInfo = HypervisorHostHelper.prepareNetwork(switchName, namePrefix, hostMo,
+                    getVlanInfo(nicTo, vlanToken), nicTo.getNetworkRateMbps(), nicTo.getNetworkRateMulticastMbps(),
+                    _opsTimeout, !namePrefix.startsWith("cloud.private"), nicTo.getBroadcastType(), nicTo.getUuid());
         }
         else {
             String vlanId = getVlanInfo(nicTo, vlanToken);

vmware-base/src/com/cloud/hypervisor/vmware/mo/HostMO.java

@@ -1110,4 +1110,58 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost {
         }
         return networkName;
     }
+
+    public void createPortGroup(HostVirtualSwitch vSwitch, String portGroupName, Integer vlanId,
+            HostNetworkSecurityPolicy secPolicy, HostNetworkTrafficShapingPolicy shapingPolicy, long timeOutMs)
+            throws Exception {
+        assert (portGroupName != null);
+
+        // Prepare lock to avoid simultaneous execution of the synchronized block for
+        // duplicate port groups on the ESXi host it's being created on.
+        String hostPortGroup = _mor.getValue() + "-" + portGroupName;
+        synchronized (hostPortGroup.intern()) {
+            // Check if port group exists already
+            if (hasPortGroup(vSwitch, portGroupName)) {
+                if (s_logger.isDebugEnabled()) {
+                    s_logger.debug("Found port group " + portGroupName + " in vSwitch " + vSwitch.getName()
+                        + ". Not attempting to create port group as it already exists.");
+                }
+                return;
+            } else {
+                if (s_logger.isDebugEnabled()) {
+                    s_logger.debug("Port group " + portGroupName + " doesn't exist in vSwitch " + vSwitch.getName()
+                        + ". Attempting to create port group in this vSwitch.");
+                }
+            }
+            // Create port group if not exists already
+            createPortGroup(vSwitch, portGroupName, vlanId, secPolicy, shapingPolicy);
+
+            // Wait for port group to turn up ready on vCenter upto timeout of timeOutMs milli seconds
+            waitForPortGroup(portGroupName, timeOutMs);
+        }
+
+        if (s_logger.isDebugEnabled()) {
+            s_logger.debug("Successfully created port group " + portGroupName + " in vSwitch " + vSwitch.getName()
+                + " on host " + getHostName());
+        }
+    }
+
+    public ManagedObjectReference waitForPortGroup(String networkName, long timeOutMs) throws Exception {
+        ManagedObjectReference morNetwork = null;
+        // if portGroup is just created, getNetwork may fail to retrieve it, we
+        // need to retry
+        long startTick = System.currentTimeMillis();
+        while (System.currentTimeMillis() - startTick <= timeOutMs) {
+            morNetwork = getNetworkMor(networkName);
+            if (morNetwork != null) {
+                break;
+            }
+
+            if (s_logger.isInfoEnabled()) {
+                s_logger.info("Waiting for network " + networkName + " to be ready");
+            }
+            Thread.sleep(1000);
+        }
+        return morNetwork;
+    }
 }

vmware-base/src/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java

@@ -1082,8 +1082,11 @@ public class HypervisorHostHelper {
             }
         } else {
             if (!hostMo.hasPortGroup(vSwitch, networkName)) {
-                hostMo.createPortGroup(vSwitch, networkName, vid, secPolicy, shapingPolicy);
-                bWaitPortGroupReady = true;
+                hostMo.createPortGroup(vSwitch, networkName, vid, secPolicy, shapingPolicy, timeOutMs);
+                // Setting flag "bWaitPortGroupReady" to false.
+                // This flag indicates whether we need to wait for portgroup on vCenter.
+                // Above createPortGroup() method itself ensures creation of portgroup as well as wait for portgroup.
+                bWaitPortGroupReady = false;
             } else {
                 HostPortGroupSpec spec = hostMo.getPortGroupSpec(networkName);
                 if (!isSpecMatch(spec, vid, shapingPolicy)) {