apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-12-16 10:32:34 +01:00
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-7260: Management server not responding after some time for

Browse Source 
Vmware due to Oom (cannot create native thread).
This commit is contained in:
Min Chen 2014-08-05 17:26:10 -07:00
parent b9d834e838
commit 9866c648eb
2 changed files with 61 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java
View File

@ -3847,18 +3847,22 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
@Override @Override
public PingCommand getCurrentStatus(long id) { public PingCommand getCurrentStatus(long id) {
gcAndKillHungWorkerVMs();
VmwareContext context = getServiceContext();
VmwareHypervisorHost hyperHost = getHyperHost(context);
try { try {
if (!hyperHost.isHyperHostConnected()) { gcAndKillHungWorkerVMs();
VmwareContext context = getServiceContext();
VmwareHypervisorHost hyperHost = getHyperHost(context);
try {
if (!hyperHost.isHyperHostConnected()) {
return null;
}
} catch (Exception e) {
s_logger.error("Unexpected exception", e);
return null; return null;
} }
} catch (Exception e) { return new PingRoutingCommand(getType(), id, syncHostVmStates());
s_logger.error("Unexpected exception", e); } finally {
return null; recycleServiceContext();
} }
return new PingRoutingCommand(getType(), id, syncHostVmStates());
} }
private void gcAndKillHungWorkerVMs() { private void gcAndKillHungWorkerVMs() {
@ -3926,8 +3930,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
s_logger.warn("Encounter remote exception to vCenter, invalidate VMware session context"); s_logger.warn("Encounter remote exception to vCenter, invalidate VMware session context");
invalidateServiceContext(); invalidateServiceContext();
} }
} finally {
recycleServiceContext();
} }
} }
@ -4661,37 +4663,32 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
_guestTrafficInfo = (VmwareTrafficLabel)params.get("guestTrafficInfo"); _guestTrafficInfo = (VmwareTrafficLabel)params.get("guestTrafficInfo");
_publicTrafficInfo = (VmwareTrafficLabel)params.get("publicTrafficInfo"); _publicTrafficInfo = (VmwareTrafficLabel)params.get("publicTrafficInfo");
VmwareContext context = getServiceContext(); VmwareContext context = getServiceContext();
VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
if (mgr == null) {
throw new ConfigurationException("Invalid vmwareContext: vmwareMgr stock object is not set or cleared.");
}
mgr.setupResourceStartupParams(params);
// TODO ??? this is an invalid usage pattern. need to fix the reference to VolumeManagerImp here at resource file CustomFieldsManagerMO cfmMo = new CustomFieldsManagerMO(context, context.getServiceContent().getCustomFieldsManager());
// volMgr = ComponentContext.inject(VolumeManagerImpl.class); cfmMo.ensureCustomFieldDef("Datastore", CustomFieldConstants.CLOUD_UUID);
try { if (_publicTrafficInfo != null && _publicTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch || _guestTrafficInfo != null &&
VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME); _guestTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch) {
mgr.setupResourceStartupParams(params); cfmMo.ensureCustomFieldDef("DistributedVirtualPortgroup", CustomFieldConstants.CLOUD_GC_DVP);
}
cfmMo.ensureCustomFieldDef("Network", CustomFieldConstants.CLOUD_GC);
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_UUID);
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_NIC_MASK);
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME);
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_WORKER);
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_WORKER_TAG);
CustomFieldsManagerMO cfmMo = new CustomFieldsManagerMO(context, context.getServiceContent().getCustomFieldsManager()); VmwareHypervisorHost hostMo = this.getHyperHost(context);
cfmMo.ensureCustomFieldDef("Datastore", CustomFieldConstants.CLOUD_UUID); _hostName = hostMo.getHyperHostName();
if (_publicTrafficInfo != null && _publicTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch || _guestTrafficInfo != null &&
_guestTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch) {
cfmMo.ensureCustomFieldDef("DistributedVirtualPortgroup", CustomFieldConstants.CLOUD_GC_DVP);
}
cfmMo.ensureCustomFieldDef("Network", CustomFieldConstants.CLOUD_GC);
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_UUID);
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_NIC_MASK);
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME);
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_WORKER);
cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_WORKER_TAG);
VmwareHypervisorHost hostMo = this.getHyperHost(context); if (_guestTrafficInfo.getVirtualSwitchType() == VirtualSwitchType.NexusDistributedVirtualSwitch ||
_hostName = hostMo.getHyperHostName(); _publicTrafficInfo.getVirtualSwitchType() == VirtualSwitchType.NexusDistributedVirtualSwitch) {
_privateNetworkVSwitchName = mgr.getPrivateVSwitchName(Long.parseLong(_dcId), HypervisorType.VMware);
if (_guestTrafficInfo.getVirtualSwitchType() == VirtualSwitchType.NexusDistributedVirtualSwitch || _vsmCredentials = mgr.getNexusVSMCredentialsByClusterId(Long.parseLong(_cluster));
_publicTrafficInfo.getVirtualSwitchType() == VirtualSwitchType.NexusDistributedVirtualSwitch) {
_privateNetworkVSwitchName = mgr.getPrivateVSwitchName(Long.parseLong(_dcId), HypervisorType.VMware);
_vsmCredentials = mgr.getNexusVSMCredentialsByClusterId(Long.parseLong(_cluster));
}
} catch (Exception e) {
s_logger.error("Unexpected Exception ", e);
} }
if (_privateNetworkVSwitchName == null) { if (_privateNetworkVSwitchName == null) {
@ -4724,15 +4721,15 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
_publicTrafficInfo.getVirtualSwitchType() + " : " + _publicTrafficInfo.getVirtualSwitchName() + ", guest traffic over " + _publicTrafficInfo.getVirtualSwitchType() + " : " + _publicTrafficInfo.getVirtualSwitchName() + ", guest traffic over " +
_guestTrafficInfo.getVirtualSwitchType() + " : " + _guestTrafficInfo.getVirtualSwitchName()); _guestTrafficInfo.getVirtualSwitchType() + " : " + _guestTrafficInfo.getVirtualSwitchName());
value = params.get("vmware.create.full.clone").toString(); Boolean boolObj = (Boolean)params.get("vmware.create.full.clone");
if (value != null && value.equalsIgnoreCase("true")) { if (boolObj != null && boolObj.booleanValue()) {
_fullCloneFlag = true; _fullCloneFlag = true;
} else { } else {
_fullCloneFlag = false; _fullCloneFlag = false;
} }
value = params.get("vm.instancename.flag").toString(); boolObj = (Boolean)params.get("vm.instancename.flag");
if (value != null && value.equalsIgnoreCase("true")) { if (boolObj != null && boolObj.booleanValue()) {
_instanceNameFlag = true; _instanceNameFlag = true;
} else { } else {
_instanceNameFlag = false; _instanceNameFlag = false;
@ -4740,7 +4737,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
value = (String)params.get("scripts.timeout"); value = (String)params.get("scripts.timeout");
int timeout = NumbersUtil.parseInt(value, 1440) * 1000; int timeout = NumbersUtil.parseInt(value, 1440) * 1000;
VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
_storageProcessor = new VmwareStorageProcessor((VmwareHostService)this, _fullCloneFlag, (VmwareStorageMount)mgr, timeout, this, _shutdownWaitMs, null); _storageProcessor = new VmwareStorageProcessor((VmwareHostService)this, _fullCloneFlag, (VmwareStorageMount)mgr, timeout, this, _shutdownWaitMs, null);
storageHandler = new VmwareStorageSubsystemCommandHandler(_storageProcessor); storageHandler = new VmwareStorageSubsystemCommandHandler(_storageProcessor);
@ -4748,7 +4744,14 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
if (!_vrResource.configure(name, params)) { if (!_vrResource.configure(name, params)) {
throw new ConfigurationException("Unable to configure VirtualRoutingResource"); throw new ConfigurationException("Unable to configure VirtualRoutingResource");
} }
if (s_logger.isTraceEnabled()) {
s_logger.trace("Successfully configured VmwareResource.");
}
return true; return true;
} catch (Exception e) {
s_logger.error("Unexpected Exception ", e);
throw new ConfigurationException("Failed to configure VmwareResource due to unexpect exception.");
} finally { } finally {
recycleServiceContext(); recycleServiceContext();
} }
@ -4787,6 +4790,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
if(s_serviceContext.get() != null) { if(s_serviceContext.get() != null) {
context = s_serviceContext.get(); context = s_serviceContext.get();
if (context.validate()) { if (context.validate()) {
if (s_logger.isTraceEnabled()) {
s_logger.trace("ThreadLocal context is still valid, just reuse");
}
return context; return context;
} else { } else {
s_logger.info("Validation of the context failed, dispose and use a new one"); s_logger.info("Validation of the context failed, dispose and use a new one");
@ -4814,10 +4820,16 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
private static void recycleServiceContext() { private static void recycleServiceContext() {
VmwareContext context = s_serviceContext.get(); VmwareContext context = s_serviceContext.get();
if (s_logger.isTraceEnabled()) {
s_logger.trace("Reset threadlocal context to null");
}
s_serviceContext.set(null); s_serviceContext.set(null);
if (context != null) { if (context != null) {
assert (context.getPool() != null); assert (context.getPool() != null);
if (s_logger.isTraceEnabled()) {
s_logger.trace("Recycling threadlocal context to pool");
}
context.getPool().returnContext(context); context.getPool().returnContext(context);
} }
} }

12
vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareContextPool.java
View File

@ -83,8 +83,8 @@ public class VmwareContextPool {
VmwareContext context = l.remove(0); VmwareContext context = l.remove(0);
context.setPoolInfo(this, poolKey); context.setPoolInfo(this, poolKey);
if (s_logger.isTraceEnabled()) if (s_logger.isInfoEnabled())
s_logger.trace("Return a VmwareContext from the idle pool: " + poolKey + ". current pool size: " + l.size() + ", outstanding count: " + s_logger.info("Return a VmwareContext from the idle pool: " + poolKey + ". current pool size: " + l.size() + ", outstanding count: " +
VmwareContext.getOutstandingContextCount()); VmwareContext.getOutstandingContextCount());
return context; return context;
} }
@ -108,12 +108,12 @@ public class VmwareContextPool {
context.clearStockObjects(); context.clearStockObjects();
l.add(context); l.add(context);
if (s_logger.isTraceEnabled()) if (s_logger.isInfoEnabled())
s_logger.trace("Recycle VmwareContext into idle pool: " + context.getPoolKey() + ", current idle pool size: " + l.size() + ", outstanding count: " + s_logger.info("Recycle VmwareContext into idle pool: " + context.getPoolKey() + ", current idle pool size: " + l.size() + ", outstanding count: " +
VmwareContext.getOutstandingContextCount()); VmwareContext.getOutstandingContextCount());
} else { } else {
if (s_logger.isTraceEnabled()) if (s_logger.isInfoEnabled())
s_logger.trace("VmwareContextPool queue exceeds limits, queue size: " + l.size()); s_logger.info("VmwareContextPool queue exceeds limits, queue size: " + l.size());
context.close(); context.close();
} }
} }