Fix for race when automatically assigning IP to Vms (#9240)

* Fix for race when automatically assigning IP to Vms

* code refactor

api/src/main/java/com/cloud/vm/NicProfile.java

@@ -62,6 +62,7 @@ public class NicProfile implements InternalIdentity, Serializable {
     String iPv4Dns1;
     String iPv4Dns2;
     String requestedIPv4;
+    boolean ipv4AllocationRaceCheck;
 
     // IPv6
     String iPv6Address;
@@ -405,6 +406,13 @@ public class NicProfile implements InternalIdentity, Serializable {
         this.mtu = mtu;
     }
 
+    public boolean getIpv4AllocationRaceCheck() {
+        return this.ipv4AllocationRaceCheck;
+    }
+
+    public void setIpv4AllocationRaceCheck(boolean ipv4AllocationRaceCheck) {
+        this.ipv4AllocationRaceCheck = ipv4AllocationRaceCheck;
+    }
 
     //
     // OTHER METHODS

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -1020,18 +1020,32 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         }
     }
 
-    @DB
+    private NicVO persistNicAfterRaceCheck(final NicVO nic, final Long networkId, final NicProfile profile, int deviceId) {
+        return Transaction.execute(new TransactionCallback<NicVO>() {
             @Override
-    public Pair<NicProfile, Integer> allocateNic(final NicProfile requested, final Network network, final Boolean isDefaultNic, int deviceId, final VirtualMachineProfile vm)
+            public NicVO doInTransaction(TransactionStatus status) {
-            throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException {
+                NicVO vo = _nicDao.findByIp4AddressAndNetworkId(profile.getIPv4Address(), networkId);
+                if (vo == null) {
+                    applyProfileToNic(nic, profile, deviceId);
+                    vo = _nicDao.persist(nic);
+                    return vo;
+                } else {
+                    return null;
+                }
+            }
+        });
+    }
 
+    private NicVO checkForRaceAndAllocateNic(final NicProfile requested, final Network network, final Boolean isDefaultNic, int deviceId, final VirtualMachineProfile vm)
+            throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException {
         final NetworkVO ntwkVO = _networksDao.findById(network.getId());
         s_logger.debug("Allocating nic for vm " + vm.getVirtualMachine() + " in network " + network + " with requested profile " + requested);
         final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, ntwkVO.getGuruName());
 
-        if (requested != null && requested.getMode() == null) {
+        NicVO vo = null;
-            requested.setMode(network.getMode());
+        boolean retryIpAllocation;
-        }
+        do {
+            retryIpAllocation = false;
             final NicProfile profile = guru.allocate(network, requested, vm);
             if (profile == null) {
                 return null;
@@ -1047,15 +1061,43 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
                 profile.setMode(network.getMode());
             }
 
-        NicVO vo = new NicVO(guru.getName(), vm.getId(), network.getId(), vm.getType());
+            vo = new NicVO(guru.getName(), vm.getId(), network.getId(), vm.getType());
 
             DataCenterVO dcVo = _dcDao.findById(network.getDataCenterId());
             if (dcVo.getNetworkType() == NetworkType.Basic) {
                 configureNicProfileBasedOnRequestedIp(requested, profile, network);
             }
 
-        deviceId = applyProfileToNic(vo, profile, deviceId);
+            if (profile.getIpv4AllocationRaceCheck()) {
+                vo = persistNicAfterRaceCheck(vo, network.getId(), profile, deviceId);
+            } else {
+                applyProfileToNic(vo, profile, deviceId);
                 vo = _nicDao.persist(vo);
+            }
+
+            if (vo == null) {
+                if (requested.getRequestedIPv4() != null) {
+                    throw new InsufficientVirtualNetworkCapacityException("Unable to acquire requested Guest IP address " + requested.getRequestedIPv4() + " for network " + network, DataCenter.class, dcVo.getId());
+                } else {
+                    requested.setIPv4Address(null);
+                }
+                retryIpAllocation = true;
+            }
+        } while (retryIpAllocation);
+
+        return vo;
+    }
+
+    @DB
+    @Override
+    public Pair<NicProfile, Integer> allocateNic(final NicProfile requested, final Network network, final Boolean isDefaultNic, int deviceId, final VirtualMachineProfile vm)
+            throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException {
+
+        if (requested != null && requested.getMode() == null) {
+            requested.setMode(network.getMode());
+        }
+
+        NicVO vo = checkForRaceAndAllocateNic(requested, network, isDefaultNic, deviceId, vm);
 
         final Integer networkRate = _networkModel.getNetworkRate(network.getId(), vm.getId());
         final NicProfile vmNic = new NicProfile(vo, network, vo.getBroadcastUri(), vo.getIsolationUri(), networkRate, _networkModel.isSecurityGroupSupportedInNetwork(network),

server/src/main/java/com/cloud/network/guru/GuestNetworkGuru.java

@@ -441,6 +441,7 @@ public abstract class GuestNetworkGuru extends AdapterBase implements NetworkGur
                         } else {
                             guestIp = _ipAddrMgr.acquireGuestIpAddress(network, nic.getRequestedIPv4());
                         }
+                        nic.setIpv4AllocationRaceCheck(true);
                     }
                     if (guestIp == null && network.getGuestType() != GuestType.L2 && !_networkModel.listNetworkOfferingServices(network.getNetworkOfferingId()).isEmpty()) {
                         throw new InsufficientVirtualNetworkCapacityException("Unable to acquire Guest IP" + " address for network " + network, DataCenter.class,