Handle public IP race conditions (#9234)

* Lock public IP

* Release IP if ID is not null

* Fix NPEs

Co-authored-by: Henrique Sato <henrique.sato@scclouds.com.br>
This commit is contained in:
Henrique Sato 2024-06-29 01:58:01 -03:00 committed by GitHub
parent 063dc60114
commit d79735606b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 340 additions and 330 deletions

View File

@ -725,9 +725,15 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
@Override
@DB
public boolean disassociatePublicIpAddress(long addrId, long userId, Account caller) {
boolean success = true;
IPAddressVO ipToBeDisassociated = _ipAddressDao.findById(addrId);
try {
IPAddressVO ipToBeDisassociated = _ipAddressDao.acquireInLockTable(addrId);
if (ipToBeDisassociated == null) {
s_logger.error(String.format("Unable to acquire lock on public IP %s.", addrId));
throw new CloudRuntimeException("Unable to acquire lock on public IP.");
}
PublicIpQuarantine publicIpQuarantine = null;
// Cleanup all ip address resources - PF/LB/Static nat rules
@ -770,6 +776,9 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
} else if (publicIpQuarantine != null) {
removePublicIpAddressFromQuarantine(publicIpQuarantine.getId(), "Public IP address removed from quarantine as there was an error while disassociating it.");
}
} finally {
_ipAddressDao.releaseFromLockTable(addrId);
}
return success;
}

View File

@ -1612,6 +1612,10 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi
}
NetworkVO network = _networksDao.findById(networkId);
if (network == null) {
throw new CloudRuntimeException("Could not find network associated with public IP.");
}
NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId());
if (offering.getGuestType() != GuestType.Isolated) {
return true;

View File

@ -194,19 +194,19 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
return createFirewallRule(sourceIpAddressId, caller, rule.getXid(), rule.getSourcePortStart(), rule.getSourcePortEnd(), rule.getProtocol(),
rule.getSourceCidrList(), null, rule.getIcmpCode(), rule.getIcmpType(), null, rule.getType(), rule.getNetworkId(), rule.getTrafficType(), rule.isDisplay());
}
//Destination CIDR capability is currently implemented for egress rules only. For others, the field is passed as null.
@DB
protected FirewallRule createFirewallRule(final Long ipAddrId, Account caller, final String xId, final Integer portStart, final Integer portEnd,
final String protocol, final List<String> sourceCidrList, final List<String> destCidrList, final Integer icmpCode, final Integer icmpType, final Long relatedRuleId,
final FirewallRule.FirewallRuleType type,
final Long networkId, final FirewallRule.TrafficType trafficType, final Boolean forDisplay) throws NetworkRuleConflictException {
protected FirewallRule createFirewallRule(final Long ipAddrId, Account caller, final String xId, final Integer portStart, final Integer portEnd, final String protocol,
final List<String> sourceCidrList, final List<String> destCidrList, final Integer icmpCode, final Integer icmpType, final Long relatedRuleId,
final FirewallRule.FirewallRuleType type, final Long networkId, final FirewallRule.TrafficType trafficType, final Boolean forDisplay) throws NetworkRuleConflictException {
IPAddressVO ipAddress = null;
try {
// Validate ip address
if (ipAddrId != null) {
// this for ingress firewall rule, for egress id is null
ipAddress = _ipAddressDao.findById(ipAddrId);
// Validate ip address
if (ipAddress == null && type == FirewallRule.FirewallRuleType.User) {
ipAddress = _ipAddressDao.acquireInLockTable(ipAddrId);
if (ipAddress == null) {
throw new InvalidParameterValueException("Unable to create firewall rule; " + "couldn't locate IP address by id in the system");
}
_networkModel.checkIpForService(ipAddress, Service.Firewall, null);
@ -239,11 +239,8 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
final Long accountIdFinal = accountId;
final Long domainIdFinal = domainId;
return Transaction.execute(new TransactionCallbackWithException<FirewallRuleVO, NetworkRuleConflictException>() {
@Override
public FirewallRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException {
FirewallRuleVO newRule =
new FirewallRuleVO(xId, ipAddrId, portStart, portEnd, protocol.toLowerCase(), networkId, accountIdFinal, domainIdFinal, Purpose.Firewall,
return Transaction.execute((TransactionCallbackWithException<FirewallRuleVO, NetworkRuleConflictException>) status -> {
FirewallRuleVO newRule = new FirewallRuleVO(xId, ipAddrId, portStart, portEnd, protocol.toLowerCase(), networkId, accountIdFinal, domainIdFinal, Purpose.Firewall,
sourceCidrList, destCidrList, icmpCode, icmpType, relatedRuleId, trafficType);
newRule.setType(type);
if (forDisplay != null) {
@ -261,8 +258,12 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
CallContext.current().putContextParameter(FirewallRule.class, newRule.getId());
return newRule;
}
});
} finally {
if (ipAddrId != null) {
_ipAddressDao.releaseFromLockTable(ipAddrId);
}
}
}
@Override
@ -668,9 +669,19 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
}
@Override
@DB
public boolean applyIngressFirewallRules(long ipId, Account caller) throws ResourceUnavailableException {
try {
IPAddressVO ipAddress = _ipAddressDao.acquireInLockTable(ipId);
if (ipAddress == null) {
s_logger.error(String.format("Unable to acquire lock for public IP [%s].", ipId));
throw new CloudRuntimeException("Unable to acquire lock for public IP.");
}
List<FirewallRuleVO> rules = _firewallDao.listByIpAndPurpose(ipId, Purpose.Firewall);
return applyFirewallRules(rules, false, caller);
} finally {
_ipAddressDao.releaseFromLockTable(ipId);
}
}
@Override

View File

@ -1814,13 +1814,12 @@ public class LoadBalancingRulesManagerImpl<Type> extends ManagerBase implements
}
return cidr;
}
@DB
@Override
public LoadBalancer createPublicLoadBalancer(final String xId, final String name, final String description, final int srcPort, final int destPort,
final long sourceIpId,
final String protocol, final String algorithm, final boolean openFirewall, final CallContext caller, final String lbProtocol, final Boolean forDisplay, String cidrList)
throws NetworkRuleConflictException {
public LoadBalancer createPublicLoadBalancer(final String xId, final String name, final String description, final int srcPort, final int destPort, final long sourceIpId,
final String protocol, final String algorithm, final boolean openFirewall, final CallContext caller, final String lbProtocol,
final Boolean forDisplay, String cidrList) throws NetworkRuleConflictException {
if (!NetUtils.isValidPort(destPort)) {
throw new InvalidParameterValueException("privatePort is an invalid value: " + destPort);
}
@ -1829,7 +1828,9 @@ public class LoadBalancingRulesManagerImpl<Type> extends ManagerBase implements
throw new InvalidParameterValueException("Invalid algorithm: " + algorithm);
}
final IPAddressVO ipAddr = _ipAddressDao.findById(sourceIpId);
try {
final IPAddressVO ipAddr = _ipAddressDao.acquireInLockTable(sourceIpId);
// make sure ip address exists
if (ipAddr == null || !ipAddr.readyToUse()) {
InvalidParameterValueException ex = new InvalidParameterValueException("Unable to create load balancer rule, invalid IP address id specified");
@ -1860,24 +1861,8 @@ public class LoadBalancingRulesManagerImpl<Type> extends ManagerBase implements
_firewallMgr.validateFirewallRule(caller.getCallingAccount(), ipAddr, srcPort, srcPort, protocol, Purpose.LoadBalancing, FirewallRuleType.User, networkId, null);
LoadBalancerVO newRule =
new LoadBalancerVO(xId, name, description, sourceIpId, srcPort, destPort, algorithm, networkId, ipAddr.getAllocatedToAccountId(),
ipAddr.getAllocatedInDomainId(), lbProtocol, cidrList);
// verify rule is supported by Lb provider of the network
Ip sourceIp = getSourceIp(newRule);
LoadBalancingRule loadBalancing =
new LoadBalancingRule(newRule, new ArrayList<LbDestination>(), new ArrayList<LbStickinessPolicy>(), new ArrayList<LbHealthCheckPolicy>(), sourceIp, null,
lbProtocol);
if (!validateLbRule(loadBalancing)) {
throw new InvalidParameterValueException("LB service provider cannot support this rule");
}
return Transaction.execute(new TransactionCallbackWithException<LoadBalancerVO, NetworkRuleConflictException>() {
@Override
public LoadBalancerVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException {
LoadBalancerVO newRule =
new LoadBalancerVO(xId, name, description, sourceIpId, srcPort, destPort, algorithm, networkId, ipAddr.getAllocatedToAccountId(),
return Transaction.execute((TransactionCallbackWithException<LoadBalancerVO, NetworkRuleConflictException>) status -> {
LoadBalancerVO newRule = new LoadBalancerVO(xId, name, description, sourceIpId, srcPort, destPort, algorithm, networkId, ipAddr.getAllocatedToAccountId(),
ipAddr.getAllocatedInDomainId(), lbProtocol, cidrList);
if (forDisplay != null) {
@ -1886,9 +1871,7 @@ public class LoadBalancingRulesManagerImpl<Type> extends ManagerBase implements
// verify rule is supported by Lb provider of the network
Ip sourceIp = getSourceIp(newRule);
LoadBalancingRule loadBalancing =
new LoadBalancingRule(newRule, new ArrayList<LbDestination>(), new ArrayList<LbStickinessPolicy>(), new ArrayList<LbHealthCheckPolicy>(), sourceIp,
null, lbProtocol);
LoadBalancingRule loadBalancing = new LoadBalancingRule(newRule, new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), sourceIp, null, lbProtocol);
if (!validateLbRule(loadBalancing)) {
throw new InvalidParameterValueException("LB service provider cannot support this rule");
}
@ -1926,9 +1909,10 @@ public class LoadBalancingRulesManagerImpl<Type> extends ManagerBase implements
removeLBRule(newRule);
}
}
}
});
} finally {
_ipAddressDao.releaseFromLockTable(sourceIpId);
}
}
@Override

View File

@ -207,7 +207,8 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
final Long ipAddrId = rule.getSourceIpAddressId();
IPAddressVO ipAddress = _ipAddressDao.findById(ipAddrId);
try {
IPAddressVO ipAddress = _ipAddressDao.acquireInLockTable(ipAddrId);
// Validate ip address
if (ipAddress == null) {
@ -309,9 +310,7 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
final Ip dstIpFinal = dstIp;
final IPAddressVO ipAddressFinal = ipAddress;
return Transaction.execute(new TransactionCallbackWithException<PortForwardingRuleVO, NetworkRuleConflictException>() {
@Override
public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException {
return Transaction.execute((TransactionCallbackWithException<PortForwardingRuleVO, NetworkRuleConflictException>) status -> {
PortForwardingRuleVO newRule =
new PortForwardingRuleVO(rule.getXid(), rule.getSourceIpAddressId(), rule.getSourcePortStart(), rule.getSourcePortEnd(), dstIpFinal,
rule.getDestinationPortStart(), rule.getDestinationPortEnd(), rule.getProtocol().toLowerCase(), networkId, accountId, domainId, vmId);
@ -320,7 +319,6 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
newRule.setDisplay(forDisplay);
}
newRule = _portForwardingDao.persist(newRule);
// create firewallRule for 0.0.0.0/0 cidr
if (openFirewall) {
_firewallMgr.createRuleForAllCidrs(ipAddrId, caller, rule.getSourcePortStart(), rule.getSourcePortEnd(), rule.getProtocol(), null, null,
@ -349,9 +347,7 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
throw new CloudRuntimeException("Unable to add rule for the ip id=" + ipAddrId, e);
}
}
});
} finally {
// release ip address if ipassoc was perfored
if (performedIpAssoc) {
@ -360,6 +356,9 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
_vpcMgr.unassignIPFromVpcNetwork(ip.getId(), networkId);
}
}
} finally {
_ipAddressDao.releaseFromLockTable(ipAddrId);
}
}
@Override
@ -370,7 +369,8 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
final Long ipAddrId = rule.getSourceIpAddressId();
IPAddressVO ipAddress = _ipAddressDao.findById(ipAddrId);
try {
IPAddressVO ipAddress = _ipAddressDao.acquireInLockTable(ipAddrId);
// Validate ip address
if (ipAddress == null) {
@ -396,10 +396,7 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
//String dstIp = _networkModel.getIpInNetwork(ipAddress.getAssociatedWithVmId(), networkId);
final String dstIp = ipAddress.getVmIp();
return Transaction.execute(new TransactionCallbackWithException<StaticNatRule, NetworkRuleConflictException>() {
@Override
public StaticNatRule doInTransaction(TransactionStatus status) throws NetworkRuleConflictException {
return Transaction.execute((TransactionCallbackWithException<StaticNatRule, NetworkRuleConflictException>) status -> {
FirewallRuleVO newRule =
new FirewallRuleVO(rule.getXid(), rule.getSourceIpAddressId(), rule.getSourcePortStart(), rule.getSourcePortEnd(), rule.getProtocol().toLowerCase(),
networkId, accountId, domainId, rule.getPurpose(), null, null, null, null, null);
@ -421,9 +418,7 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NET_RULE_ADD, newRule.getAccountId(), 0, newRule.getId(), null, FirewallRule.class.getName(),
newRule.getUuid());
StaticNatRule staticNatRule = new StaticNatRuleImpl(newRule, dstIp);
return staticNatRule;
return new StaticNatRuleImpl(newRule, dstIp);
} catch (Exception e) {
if (newRule != null) {
// no need to apply the rule as it wasn't programmed on the backend yet
@ -436,9 +431,10 @@ public class RulesManagerImpl extends ManagerBase implements RulesManager, Rules
}
throw new CloudRuntimeException("Unable to add static nat rule for the ip id=" + newRule.getSourceIpAddressId(), e);
}
}
});
} finally {
_ipAddressDao.releaseFromLockTable(ipAddrId);
}
}
@Override

View File

@ -155,6 +155,7 @@ public class RemoteAccessVpnManagerImpl extends ManagerBase implements RemoteAcc
return vpns;
}
@Override
@DB
public RemoteAccessVpn createRemoteAccessVpn(final long publicIpId, String ipRange, boolean openFirewall, final Boolean forDisplay) throws NetworkRuleConflictException {
@ -172,7 +173,13 @@ public class RemoteAccessVpnManagerImpl extends ManagerBase implements RemoteAcc
throw new InvalidParameterValueException("The Ip address is not ready to be used yet: " + ipAddr.getAddress());
}
IPAddressVO ipAddress = _ipAddressDao.findById(publicIpId);
try {
IPAddressVO ipAddress = _ipAddressDao.acquireInLockTable(publicIpId);
if (ipAddress == null) {
s_logger.error(String.format("Unable to acquire lock on public IP %s.", publicIpId));
throw new CloudRuntimeException("Unable to acquire lock on public IP.");
}
Long networkId = ipAddress.getAssociatedWithNetworkId();
if (networkId != null) {
@ -241,23 +248,22 @@ public class RemoteAccessVpnManagerImpl extends ManagerBase implements RemoteAcc
final String newIpRange = NetUtils.long2Ip(++startIp) + "-" + range[1];
final String sharedSecret = PasswordGenerator.generatePresharedKey(_pskLength);
return Transaction.execute(new TransactionCallbackWithException<RemoteAccessVpn, NetworkRuleConflictException>() {
@Override
public RemoteAccessVpn doInTransaction(TransactionStatus status) throws NetworkRuleConflictException {
return Transaction.execute((TransactionCallbackWithException<RemoteAccessVpn, NetworkRuleConflictException>) status -> {
if (vpcId == null) {
_rulesMgr.reservePorts(ipAddr, NetUtils.UDP_PROTO, Purpose.Vpn, openFirewallFinal, caller, NetUtils.VPN_PORT, NetUtils.VPN_L2TP_PORT,
NetUtils.VPN_NATT_PORT);
}
RemoteAccessVpnVO vpnVO =
new RemoteAccessVpnVO(ipAddr.getAccountId(), ipAddr.getDomainId(), ipAddr.getAssociatedWithNetworkId(), publicIpId, vpcId, range[0], newIpRange,
sharedSecret);
RemoteAccessVpnVO remoteAccessVpnVO = new RemoteAccessVpnVO(ipAddr.getAccountId(), ipAddr.getDomainId(), ipAddr.getAssociatedWithNetworkId(),
publicIpId, vpcId, range[0], newIpRange, sharedSecret);
if (forDisplay != null) {
vpnVO.setDisplay(forDisplay);
}
return _remoteAccessVpnDao.persist(vpnVO);
remoteAccessVpnVO.setDisplay(forDisplay);
}
return _remoteAccessVpnDao.persist(remoteAccessVpnVO);
});
} finally {
_ipAddressDao.releaseFromLockTable(publicIpId);
}
}
private void validateRemoteAccessVpnConfiguration() throws ConfigurationException {