apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-9595: Avoiding the deadlocks in the code (#1762)

Browse Source 
MySQLTransactionRollbackException is seen frequently in logs
Root Cause
Attempts to lock rows in the core data access layer of database fails if there is a possibility of deadlock. However Operations are not getting retried in case of deadlock. So introducing retries here
Solution
Operations would be retried after some wait time in case of dead lock exception.
This commit is contained in:
subhash yedugundla 2017-12-19 00:06:21 +05:30 committed by Rohit Yadav
parent 3c6df7c970
commit 1edb3e8a45
2 changed files with 66 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
engine/schema/src/com/cloud/host/dao/HostDaoImpl.java
View File

@ -553,7 +553,6 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
public List<HostVO> findAndUpdateDirectAgentToLoad(long lastPingSecondsAfter, Long limit, long managementServerId) { public List<HostVO> findAndUpdateDirectAgentToLoad(long lastPingSecondsAfter, Long limit, long managementServerId) {
TransactionLegacy txn = TransactionLegacy.currentTxn(); TransactionLegacy txn = TransactionLegacy.currentTxn();
txn.start();
if (s_logger.isDebugEnabled()) { if (s_logger.isDebugEnabled()) {
s_logger.debug("Resetting hosts suitable for reconnect"); s_logger.debug("Resetting hosts suitable for reconnect");
} }
@ -569,6 +568,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
s_logger.debug("Acquiring hosts for clusters already owned by this management server"); s_logger.debug("Acquiring hosts for clusters already owned by this management server");
} }
List<Long> clusters = findClustersOwnedByManagementServer(managementServerId); List<Long> clusters = findClustersOwnedByManagementServer(managementServerId);
txn.start();
if (clusters.size() > 0) { if (clusters.size() > 0) {
// handle clusters already owned by @managementServerId // handle clusters already owned by @managementServerId
SearchCriteria<HostVO> sc = UnmanagedDirectConnectSearch.create(); SearchCriteria<HostVO> sc = UnmanagedDirectConnectSearch.create();

127
server/src/com/cloud/network/IpAddressManagerImpl.java
View File

@ -291,6 +291,8 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
SearchBuilder<IPAddressVO> AssignIpAddressSearch; SearchBuilder<IPAddressVO> AssignIpAddressSearch;
SearchBuilder<IPAddressVO> AssignIpAddressFromPodVlanSearch; SearchBuilder<IPAddressVO> AssignIpAddressFromPodVlanSearch;
private final Object _allocatedLock = new Object();
private final Object _allocatingLock = new Object();
static Boolean rulesContinueOnErrFlag = true; static Boolean rulesContinueOnErrFlag = true;
@ -759,7 +761,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
Filter filter = new Filter(IPAddressVO.class, "vlanId", true, 0l, 1l); Filter filter = new Filter(IPAddressVO.class, "vlanId", true, 0l, 1l);
List<IPAddressVO> addrs = _ipAddressDao.lockRows(sc, filter, true); List<IPAddressVO> addrs = _ipAddressDao.search(sc, filter, false);
// If all the dedicated IPs of the owner are in use fetch an IP from the system pool // If all the dedicated IPs of the owner are in use fetch an IP from the system pool
if (addrs.size() == 0 && fetchFromDedicatedRange) { if (addrs.size() == 0 && fetchFromDedicatedRange) {
@ -769,7 +771,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
fetchFromDedicatedRange = false; fetchFromDedicatedRange = false;
sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray()); sc.setParameters("vlanId", nonDedicatedVlanDbIds.toArray());
errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray())); errorMessage.append(", vlanId id=" + Arrays.toString(nonDedicatedVlanDbIds.toArray()));
addrs = _ipAddressDao.lockRows(sc, filter, true); addrs = _ipAddressDao.search(sc, filter, false);
} }
} }
@ -804,24 +806,21 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
addr.setAllocatedInDomainId(owner.getDomainId()); addr.setAllocatedInDomainId(owner.getDomainId());
addr.setAllocatedToAccountId(owner.getId()); addr.setAllocatedToAccountId(owner.getId());
addr.setSystem(isSystem); addr.setSystem(isSystem);
if (displayIp != null) { if (displayIp != null) {
addr.setDisplay(displayIp); addr.setDisplay(displayIp);
} }
if (assign) {
markPublicIpAsAllocated(addr);
} else {
addr.setState(IpAddress.State.Allocating);
}
addr.setState(assign ? IpAddress.State.Allocated : IpAddress.State.Allocating);
if (vlanUse != VlanType.DirectAttached) { if (vlanUse != VlanType.DirectAttached) {
addr.setAssociatedWithNetworkId(guestNetworkId); addr.setAssociatedWithNetworkId(guestNetworkId);
addr.setVpcId(vpcId); addr.setVpcId(vpcId);
} }
_ipAddressDao.update(addr.getId(), addr); if (assign) {
markPublicIpAsAllocated(addr);
} else {
markPublicIpAsAllocating(addr);
}
return addr; return addr;
} }
}); });
@ -836,35 +835,51 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
@DB @DB
@Override @Override
public void markPublicIpAsAllocated(final IPAddressVO addr) { public void markPublicIpAsAllocated(final IPAddressVO addr) {
synchronized (_allocatedLock) {
Transaction.execute(new TransactionCallbackNoReturn() { Transaction.execute(new TransactionCallbackNoReturn() {
@Override @Override
public void doInTransactionWithoutResult(TransactionStatus status) { public void doInTransactionWithoutResult(TransactionStatus status) {
Account owner = _accountMgr.getAccount(addr.getAllocatedToAccountId()); Account owner = _accountMgr.getAccount(addr.getAllocatedToAccountId());
synchronized (this) {
if (_ipAddressDao.lockRow(addr.getId(), true) != null) { if (_ipAddressDao.lockRow(addr.getId(), true) != null) {
IPAddressVO userIp = _ipAddressDao.findById(addr.getId()); IPAddressVO userIp = _ipAddressDao.findById(addr.getId());
if (userIp.getState() == IpAddress.State.Allocating || addr.getState() == IpAddress.State.Free) { if (userIp.getState() == IpAddress.State.Allocating || addr.getState() == IpAddress.State.Free) {
addr.setState(IpAddress.State.Allocated); addr.setState(IpAddress.State.Allocated);
_ipAddressDao.update(addr.getId(), addr); if (_ipAddressDao.update(addr.getId(), addr)) {
// Save usage event // Save usage event
if (owner.getAccountId() != Account.ACCOUNT_ID_SYSTEM) { if (owner.getAccountId() != Account.ACCOUNT_ID_SYSTEM) {
VlanVO vlan = _vlanDao.findById(addr.getVlanId()); VlanVO vlan = _vlanDao.findById(addr.getVlanId());
String guestType = vlan.getVlanType().toString(); String guestType = vlan.getVlanType().toString();
if (!isIpDedicated(addr)) { if (!isIpDedicated(addr)) {
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NET_IP_ASSIGN, owner.getId(), addr.getDataCenterId(), addr.getId(), UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NET_IP_ASSIGN, owner.getId(), addr.getDataCenterId(), addr.getId(),
addr.getAddress().toString(), addr.getAddress().toString(),
addr.isSourceNat(), guestType, addr.getSystem(), addr.getClass().getName(), addr.getUuid()); addr.isSourceNat(), guestType, addr.getSystem(), addr.getClass().getName(), addr.getUuid());
} }
if (updateIpResourceCount(addr)) { if (updateIpResourceCount(addr)) {
_resourceLimitMgr.incrementResourceCount(owner.getId(), ResourceType.public_ip); _resourceLimitMgr.incrementResourceCount(owner.getId(), ResourceType.public_ip);
}
} }
} }
} }
} }
} }
} });
}); }
}
@DB
private void markPublicIpAsAllocating(final IPAddressVO addr) {
synchronized (_allocatingLock) {
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
if (_ipAddressDao.lockRow(addr.getId(), true) != null) {
addr.setState(IpAddress.State.Allocating);
_ipAddressDao.update(addr.getId(), addr);
}
}
});
}
} }
private boolean isIpDedicated(IPAddressVO addr) { private boolean isIpDedicated(IPAddressVO addr) {
@ -901,40 +916,28 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage
PublicIp ip = null; PublicIp ip = null;
try { try {
ip = Transaction.execute(new TransactionCallbackWithException<PublicIp, InsufficientAddressCapacityException>() { Account ownerAccount = _accountDao.acquireInLockTable(ownerId);
@Override
public PublicIp doInTransaction(TransactionStatus status) throws InsufficientAddressCapacityException {
Account owner = _accountDao.acquireInLockTable(ownerId);
if (owner == null) { if (ownerAccount == null) {
// this ownerId comes from owner or type Account. See the class "AccountVO" and the annotations in that class // this ownerId comes from owner or type Account. See the class "AccountVO" and the annotations in that class
// to get the table name and field name that is queried to fill this ownerid. // to get the table name and field name that is queried to fill this ownerid.
ConcurrentOperationException ex = new ConcurrentOperationException("Unable to lock account"); ConcurrentOperationException ex = new ConcurrentOperationException("Unable to lock account");
throw ex; throw ex;
} }
if (s_logger.isDebugEnabled()) { if (s_logger.isDebugEnabled()) {
s_logger.debug("lock account " + ownerId + " is acquired"); s_logger.debug("lock account " + ownerId + " is acquired");
} }
boolean displayIp = true; boolean displayIp = true;
if (guestNtwkId != null) { if (guestNtwkId != null) {
Network ntwk = _networksDao.findById(guestNtwkId); Network ntwk = _networksDao.findById(guestNtwkId);
displayIp = ntwk.getDisplayNetwork(); displayIp = ntwk.getDisplayNetwork();
} else if (vpcId != null) { } else if (vpcId != null) {
VpcVO vpc = _vpcDao.findById(vpcId); VpcVO vpc = _vpcDao.findById(vpcId);
displayIp = vpc.isDisplay(); displayIp = vpc.isDisplay();
} }
PublicIp ip = fetchNewPublicIp(dcId, null, null, owner, VlanType.VirtualNetwork, guestNtwkId, isSourceNat, false, null, false, vpcId, displayIp); return fetchNewPublicIp(dcId, null, null, owner, VlanType.VirtualNetwork, guestNtwkId, isSourceNat, false, null, false, vpcId, displayIp);
IPAddressVO publicIp = ip.ip();
markPublicIpAsAllocated(publicIp);
_ipAddressDao.update(publicIp.getId(), publicIp);
return ip;
}
});
return ip;
} finally { } finally {
if (owner != null) { if (owner != null) {
if (s_logger.isDebugEnabled()) { if (s_logger.isDebugEnabled()) {