Handle public IP race conditions (#9234)

* Lock public IP

* Release IP if ID is not null

* Fix NPEs

Co-authored-by: Henrique Sato <henrique.sato@scclouds.com.br>

server/src/main/java/com/cloud/network/IpAddressManagerImpl.java

@@ -725,9 +725,15 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
     @Override
     @DB
     public boolean disassociatePublicIpAddress(long addrId, long userId, Account caller) {
-
         boolean success = true;
-        IPAddressVO ipToBeDisassociated = _ipAddressDao.findById(addrId);
+
+        try {
+            IPAddressVO ipToBeDisassociated = _ipAddressDao.acquireInLockTable(addrId);
+
+            if (ipToBeDisassociated == null) {
+                s_logger.error(String.format("Unable to acquire lock on public IP %s.", addrId));
+                throw new CloudRuntimeException("Unable to acquire lock on public IP.");
+            }
 
             PublicIpQuarantine publicIpQuarantine = null;
             // Cleanup all ip address resources - PF/LB/Static nat rules
@@ -770,6 +776,9 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
             } else if (publicIpQuarantine != null) {
                 removePublicIpAddressFromQuarantine(publicIpQuarantine.getId(), "Public IP address removed from quarantine as there was an error while disassociating it.");
             }
+        } finally {
+            _ipAddressDao.releaseFromLockTable(addrId);
+        }
 
         return success;
     }

server/src/main/java/com/cloud/network/NetworkModelImpl.java

@@ -1612,6 +1612,10 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi
         }
 
         NetworkVO network = _networksDao.findById(networkId);
+        if (network == null) {
+            throw new CloudRuntimeException("Could not find network associated with public IP.");
+        }
+
         NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId());
         if (offering.getGuestType() != GuestType.Isolated) {
             return true;

server/src/main/java/com/cloud/network/firewall/FirewallManagerImpl.java

@@ -194,19 +194,19 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
         return createFirewallRule(sourceIpAddressId, caller, rule.getXid(), rule.getSourcePortStart(), rule.getSourcePortEnd(), rule.getProtocol(),
             rule.getSourceCidrList(), null, rule.getIcmpCode(), rule.getIcmpType(), null, rule.getType(), rule.getNetworkId(), rule.getTrafficType(), rule.isDisplay());
     }
+
     //Destination CIDR capability is currently implemented for egress rules only. For others, the field is passed as null.
     @DB
-    protected FirewallRule createFirewallRule(final Long ipAddrId, Account caller, final String xId, final Integer portStart, final Integer portEnd,
+    protected FirewallRule createFirewallRule(final Long ipAddrId, Account caller, final String xId, final Integer portStart, final Integer portEnd, final String protocol,
-        final String protocol, final List<String> sourceCidrList, final List<String> destCidrList, final Integer icmpCode, final Integer icmpType, final Long relatedRuleId,
+                                              final List<String> sourceCidrList, final List<String> destCidrList, final Integer icmpCode, final Integer icmpType, final Long relatedRuleId,
- final FirewallRule.FirewallRuleType type,
+                                              final FirewallRule.FirewallRuleType type, final Long networkId, final FirewallRule.TrafficType trafficType, final Boolean forDisplay) throws NetworkRuleConflictException {
-            final Long networkId, final FirewallRule.TrafficType trafficType, final Boolean forDisplay) throws NetworkRuleConflictException {
-
         IPAddressVO ipAddress = null;
+        try {
+            // Validate ip address
             if (ipAddrId != null) {
                 // this for ingress firewall rule, for egress id is null
-             ipAddress = _ipAddressDao.findById(ipAddrId);
+                ipAddress = _ipAddressDao.acquireInLockTable(ipAddrId);
-        // Validate ip address
+                if (ipAddress == null) {
-        if (ipAddress == null && type == FirewallRule.FirewallRuleType.User) {
                     throw new InvalidParameterValueException("Unable to create firewall rule; " + "couldn't locate IP address by id in the system");
                 }
                 _networkModel.checkIpForService(ipAddress, Service.Firewall, null);
@@ -239,11 +239,8 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
 
             final Long accountIdFinal = accountId;
             final Long domainIdFinal = domainId;
-        return Transaction.execute(new TransactionCallbackWithException<FirewallRuleVO, NetworkRuleConflictException>() {
+            return Transaction.execute((TransactionCallbackWithException<FirewallRuleVO, NetworkRuleConflictException>) status -> {
-            @Override
+                FirewallRuleVO newRule = new FirewallRuleVO(xId, ipAddrId, portStart, portEnd, protocol.toLowerCase(), networkId, accountIdFinal, domainIdFinal, Purpose.Firewall,
-            public FirewallRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException {
-                FirewallRuleVO newRule =
-                        new FirewallRuleVO(xId, ipAddrId, portStart, portEnd, protocol.toLowerCase(), networkId, accountIdFinal, domainIdFinal, Purpose.Firewall,
                         sourceCidrList, destCidrList, icmpCode, icmpType, relatedRuleId, trafficType);
                 newRule.setType(type);
                 if (forDisplay != null) {
@@ -261,8 +258,12 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
                 CallContext.current().putContextParameter(FirewallRule.class, newRule.getId());
 
                 return newRule;
-            }
             });
+        } finally {
+            if (ipAddrId != null) {
+                _ipAddressDao.releaseFromLockTable(ipAddrId);
+            }
+        }
     }
 
     @Override
@@ -668,9 +669,19 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
     }
 
     @Override
+    @DB
     public boolean applyIngressFirewallRules(long ipId, Account caller) throws ResourceUnavailableException {
+        try {
+            IPAddressVO ipAddress = _ipAddressDao.acquireInLockTable(ipId);
+            if (ipAddress == null) {
+                s_logger.error(String.format("Unable to acquire lock for public IP [%s].", ipId));
+                throw new CloudRuntimeException("Unable to acquire lock for public IP.");
+            }
             List<FirewallRuleVO> rules = _firewallDao.listByIpAndPurpose(ipId, Purpose.Firewall);
             return applyFirewallRules(rules, false, caller);
+        } finally {
+            _ipAddressDao.releaseFromLockTable(ipId);
+        }
     }
 
     @Override

server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java

@@ -1814,13 +1814,12 @@ public class LoadBalancingRulesManagerImpl<Type> extends ManagerBase implements
         }
         return cidr;
     }
+
     @DB
     @Override
-    public LoadBalancer createPublicLoadBalancer(final String xId, final String name, final String description, final int srcPort, final int destPort,
+    public LoadBalancer createPublicLoadBalancer(final String xId, final String name, final String description, final int srcPort, final int destPort, final long sourceIpId,
- final long sourceIpId,
+                                                 final String protocol, final String algorithm, final boolean openFirewall, final CallContext caller, final String lbProtocol,
-            final String protocol, final String algorithm, final boolean openFirewall, final CallContext caller, final String lbProtocol, final Boolean forDisplay, String cidrList)
+                                                 final Boolean forDisplay, String cidrList) throws NetworkRuleConflictException {
-            throws NetworkRuleConflictException {
-
         if (!NetUtils.isValidPort(destPort)) {
             throw new InvalidParameterValueException("privatePort is an invalid value: " + destPort);
         }
@@ -1829,7 +1828,9 @@ public class LoadBalancingRulesManagerImpl<Type> extends ManagerBase implements
             throw new InvalidParameterValueException("Invalid algorithm: " + algorithm);
         }
 
-        final IPAddressVO ipAddr = _ipAddressDao.findById(sourceIpId);
+        try {
+            final IPAddressVO ipAddr = _ipAddressDao.acquireInLockTable(sourceIpId);
+
             // make sure ip address exists
             if (ipAddr == null || !ipAddr.readyToUse()) {
                 InvalidParameterValueException ex = new InvalidParameterValueException("Unable to create load balancer rule, invalid IP address id specified");
@@ -1860,24 +1861,8 @@ public class LoadBalancingRulesManagerImpl<Type> extends ManagerBase implements
 
             _firewallMgr.validateFirewallRule(caller.getCallingAccount(), ipAddr, srcPort, srcPort, protocol, Purpose.LoadBalancing, FirewallRuleType.User, networkId, null);
 
-        LoadBalancerVO newRule =
+            return Transaction.execute((TransactionCallbackWithException<LoadBalancerVO, NetworkRuleConflictException>) status -> {
-            new LoadBalancerVO(xId, name, description, sourceIpId, srcPort, destPort, algorithm, networkId, ipAddr.getAllocatedToAccountId(),
+                LoadBalancerVO newRule = new LoadBalancerVO(xId, name, description, sourceIpId, srcPort, destPort, algorithm, networkId, ipAddr.getAllocatedToAccountId(),
-                ipAddr.getAllocatedInDomainId(), lbProtocol, cidrList);
-
-        // verify rule is supported by Lb provider of the network
-        Ip sourceIp = getSourceIp(newRule);
-        LoadBalancingRule loadBalancing =
-            new LoadBalancingRule(newRule, new ArrayList<LbDestination>(), new ArrayList<LbStickinessPolicy>(), new ArrayList<LbHealthCheckPolicy>(), sourceIp, null,
-                lbProtocol);
-        if (!validateLbRule(loadBalancing)) {
-            throw new InvalidParameterValueException("LB service provider cannot support this rule");
-        }
-
-        return Transaction.execute(new TransactionCallbackWithException<LoadBalancerVO, NetworkRuleConflictException>() {
-            @Override
-            public LoadBalancerVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException {
-                LoadBalancerVO newRule =
-                    new LoadBalancerVO(xId, name, description, sourceIpId, srcPort, destPort, algorithm, networkId, ipAddr.getAllocatedToAccountId(),
                         ipAddr.getAllocatedInDomainId(), lbProtocol, cidrList);
 
                 if (forDisplay != null) {
@@ -1886,9 +1871,7 @@ public class LoadBalancingRulesManagerImpl<Type> extends ManagerBase implements
 
                 // verify rule is supported by Lb provider of the network
                 Ip sourceIp = getSourceIp(newRule);
-                LoadBalancingRule loadBalancing =
+                LoadBalancingRule loadBalancing = new LoadBalancingRule(newRule, new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), sourceIp, null, lbProtocol);
-                    new LoadBalancingRule(newRule, new ArrayList<LbDestination>(), new ArrayList<LbStickinessPolicy>(), new ArrayList<LbHealthCheckPolicy>(), sourceIp,
-                        null, lbProtocol);
                 if (!validateLbRule(loadBalancing)) {
                     throw new InvalidParameterValueException("LB service provider cannot support this rule");
                 }
@@ -1926,9 +1909,10 @@ public class LoadBalancingRulesManagerImpl<Type> extends ManagerBase implements
                         removeLBRule(newRule);
                     }
                 }
-            }
             });
-
+        } finally {
+            _ipAddressDao.releaseFromLockTable(sourceIpId);
+        }
     }
 
     @Override

server/src/main/java/com/cloud/network/rules/RulesManagerImpl.java

@@ -207,7 +207,8 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
 
         final Long ipAddrId = rule.getSourceIpAddressId();
 
-        IPAddressVO ipAddress = _ipAddressDao.findById(ipAddrId);
+        try {
+            IPAddressVO ipAddress = _ipAddressDao.acquireInLockTable(ipAddrId);
 
             // Validate ip address
             if (ipAddress == null) {
@@ -309,9 +310,7 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
 
                 final Ip dstIpFinal = dstIp;
                 final IPAddressVO ipAddressFinal = ipAddress;
-            return Transaction.execute(new TransactionCallbackWithException<PortForwardingRuleVO, NetworkRuleConflictException>() {
+                return Transaction.execute((TransactionCallbackWithException<PortForwardingRuleVO, NetworkRuleConflictException>) status -> {
-                @Override
-                public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException {
                     PortForwardingRuleVO newRule =
                             new PortForwardingRuleVO(rule.getXid(), rule.getSourceIpAddressId(), rule.getSourcePortStart(), rule.getSourcePortEnd(), dstIpFinal,
                                     rule.getDestinationPortStart(), rule.getDestinationPortEnd(), rule.getProtocol().toLowerCase(), networkId, accountId, domainId, vmId);
@@ -320,7 +319,6 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
                         newRule.setDisplay(forDisplay);
                     }
                     newRule = _portForwardingDao.persist(newRule);
-
                     // create firewallRule for 0.0.0.0/0 cidr
                     if (openFirewall) {
                         _firewallMgr.createRuleForAllCidrs(ipAddrId, caller, rule.getSourcePortStart(), rule.getSourcePortEnd(), rule.getProtocol(), null, null,
@@ -349,9 +347,7 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
 
                         throw new CloudRuntimeException("Unable to add rule for the ip id=" + ipAddrId, e);
                     }
-                }
                 });
-
             } finally {
                 // release ip address if ipassoc was perfored
                 if (performedIpAssoc) {
@@ -360,6 +356,9 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
                     _vpcMgr.unassignIPFromVpcNetwork(ip.getId(), networkId);
                 }
             }
+        } finally {
+            _ipAddressDao.releaseFromLockTable(ipAddrId);
+        }
     }
 
     @Override
@@ -370,7 +369,8 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
 
         final Long ipAddrId = rule.getSourceIpAddressId();
 
-        IPAddressVO ipAddress = _ipAddressDao.findById(ipAddrId);
+        try {
+            IPAddressVO ipAddress = _ipAddressDao.acquireInLockTable(ipAddrId);
 
             // Validate ip address
             if (ipAddress == null) {
@@ -396,10 +396,7 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
 
             //String dstIp = _networkModel.getIpInNetwork(ipAddress.getAssociatedWithVmId(), networkId);
             final String dstIp = ipAddress.getVmIp();
-        return Transaction.execute(new TransactionCallbackWithException<StaticNatRule, NetworkRuleConflictException>() {
+            return Transaction.execute((TransactionCallbackWithException<StaticNatRule, NetworkRuleConflictException>) status -> {
-            @Override
-            public StaticNatRule doInTransaction(TransactionStatus status) throws NetworkRuleConflictException {
-
                 FirewallRuleVO newRule =
                         new FirewallRuleVO(rule.getXid(), rule.getSourceIpAddressId(), rule.getSourcePortStart(), rule.getSourcePortEnd(), rule.getProtocol().toLowerCase(),
                                 networkId, accountId, domainId, rule.getPurpose(), null, null, null, null, null);
@@ -421,9 +418,7 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
                     UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NET_RULE_ADD, newRule.getAccountId(), 0, newRule.getId(), null, FirewallRule.class.getName(),
                             newRule.getUuid());
 
-                    StaticNatRule staticNatRule = new StaticNatRuleImpl(newRule, dstIp);
+                    return new StaticNatRuleImpl(newRule, dstIp);
-
-                    return staticNatRule;
                 } catch (Exception e) {
                     if (newRule != null) {
                         // no need to apply the rule as it wasn't programmed on the backend yet
@@ -436,9 +431,10 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
                     }
                     throw new CloudRuntimeException("Unable to add static nat rule for the ip id=" + newRule.getSourceIpAddressId(), e);
                 }
-            }
             });
-
+        } finally {
+            _ipAddressDao.releaseFromLockTable(ipAddrId);
+        }
     }
 
     @Override

server/src/main/java/com/cloud/network/vpn/RemoteAccessVpnManagerImpl.java

@@ -155,6 +155,7 @@ public class RemoteAccessVpnManagerImpl extends ManagerBase implements RemoteAcc
         return vpns;
     }
 
+
     @Override
     @DB
     public RemoteAccessVpn createRemoteAccessVpn(final long publicIpId, String ipRange, boolean openFirewall, final Boolean forDisplay) throws NetworkRuleConflictException {
@@ -172,7 +173,13 @@ public class RemoteAccessVpnManagerImpl extends ManagerBase implements RemoteAcc
             throw new InvalidParameterValueException("The Ip address is not ready to be used yet: " + ipAddr.getAddress());
         }
 
-        IPAddressVO ipAddress = _ipAddressDao.findById(publicIpId);
+        try {
+            IPAddressVO ipAddress = _ipAddressDao.acquireInLockTable(publicIpId);
+
+            if (ipAddress == null) {
+                s_logger.error(String.format("Unable to acquire lock on public IP %s.", publicIpId));
+                throw new CloudRuntimeException("Unable to acquire lock on public IP.");
+            }
 
             Long networkId = ipAddress.getAssociatedWithNetworkId();
             if (networkId != null) {
@@ -241,23 +248,22 @@ public class RemoteAccessVpnManagerImpl extends ManagerBase implements RemoteAcc
             final String newIpRange = NetUtils.long2Ip(++startIp) + "-" + range[1];
             final String sharedSecret = PasswordGenerator.generatePresharedKey(_pskLength);
 
-        return Transaction.execute(new TransactionCallbackWithException<RemoteAccessVpn, NetworkRuleConflictException>() {
+            return Transaction.execute((TransactionCallbackWithException<RemoteAccessVpn, NetworkRuleConflictException>) status -> {
-            @Override
-            public RemoteAccessVpn doInTransaction(TransactionStatus status) throws NetworkRuleConflictException {
                 if (vpcId == null) {
                     _rulesMgr.reservePorts(ipAddr, NetUtils.UDP_PROTO, Purpose.Vpn, openFirewallFinal, caller, NetUtils.VPN_PORT, NetUtils.VPN_L2TP_PORT,
                             NetUtils.VPN_NATT_PORT);
                 }
-                RemoteAccessVpnVO vpnVO =
+                RemoteAccessVpnVO remoteAccessVpnVO = new RemoteAccessVpnVO(ipAddr.getAccountId(), ipAddr.getDomainId(), ipAddr.getAssociatedWithNetworkId(),
-                    new RemoteAccessVpnVO(ipAddr.getAccountId(), ipAddr.getDomainId(), ipAddr.getAssociatedWithNetworkId(), publicIpId, vpcId, range[0], newIpRange,
+                        publicIpId, vpcId, range[0], newIpRange, sharedSecret);
-                        sharedSecret);
 
                 if (forDisplay != null) {
-                    vpnVO.setDisplay(forDisplay);
+                    remoteAccessVpnVO.setDisplay(forDisplay);
-                }
-                return _remoteAccessVpnDao.persist(vpnVO);
                 }
+                return _remoteAccessVpnDao.persist(remoteAccessVpnVO);
             });
+        } finally {
+            _ipAddressDao.releaseFromLockTable(publicIpId);
+        }
     }
 
     private void validateRemoteAccessVpnConfiguration() throws ConfigurationException {