Moved over the new jobs framework from vmsync. This has not been integrated into the server package yet. Will do that next

api/src/org/apache/cloudstack/jobs/JobInfo.java

@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.jobs;
+
+import java.util.Date;
+
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+
+public interface JobInfo extends Identity, InternalIdentity {
+    public enum Status {
+        IN_PROGRESS(false),
+        SUCCEEDED(true),
+        FAILED(true),
+        CANCELLED(true);
+        
+        private final boolean done;
+
+        private Status(boolean done) {
+            this.done = done;
+        }
+        
+        public boolean done() {
+            return done;
+        }
+    }
+
+    String getType();
+
+    String getDispatcher();
+
+    int getPendingSignals();
+
+    long getUserId();
+
+    long getAccountId();
+
+    String getCmd();
+
+    int getCmdVersion();
+
+    String getCmdInfo();
+
+    Status getStatus();
+
+    int getProcessStatus();
+
+    int getResultCode();
+
+    String getResult();
+
+    Long getInitMsid();
+
+    Long getExecutingMsid();
+
+    Long getCompleteMsid();
+
+    Date getCreated();
+
+    Date getLastUpdated();
+
+    Date getLastPolled();
+
+    String getInstanceType();
+
+    Long getInstanceId();
+}

framework/cluster/src/com/cloud/cluster/ClusterFenceManagerImpl.java

@@ -43,11 +43,11 @@ public class ClusterFenceManagerImpl extends ManagerBase implements ClusterFence
 	}
 
 	@Override
-	public void onManagementNodeJoined(List<ManagementServerHostVO> nodeList, long selfNodeId) {
+    public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
 	}
 
 	@Override
-	public void onManagementNodeLeft(List<ManagementServerHostVO> nodeList, long selfNodeId) {
+    public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
 	}
 
 	@Override

framework/cluster/src/com/cloud/cluster/ClusterManagerListener.java

@@ -19,7 +19,8 @@ package com.cloud.cluster;
 import java.util.List;
 
 public interface ClusterManagerListener {
-	void onManagementNodeJoined(List<ManagementServerHostVO> nodeList, long selfNodeId);
-	void onManagementNodeLeft(List<ManagementServerHostVO> nodeList, long selfNodeId);
+    void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId);
+
+    void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId);
 	void onManagementNodeIsolated();
 }

framework/ipc/src/org/apache/cloudstack/framework/async/AsyncCallbackDispatcher.java

@@ -22,20 +22,20 @@ package org.apache.cloudstack.framework.async;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 
-import org.apache.log4j.Logger;
-
-import net.sf.cglib.proxy.CallbackFilter;
 import net.sf.cglib.proxy.Callback;
+import net.sf.cglib.proxy.CallbackFilter;
 import net.sf.cglib.proxy.Enhancer;
 import net.sf.cglib.proxy.MethodInterceptor;
 import net.sf.cglib.proxy.MethodProxy;
 
+import org.apache.log4j.Logger;
+
 @SuppressWarnings("rawtypes")
 public class AsyncCallbackDispatcher<T, R> implements AsyncCompletionCallback {
     private static final Logger s_logger = Logger.getLogger(AsyncCallbackDispatcher.class);
 
     private Method _callbackMethod;
-	private T _targetObject;
+	private final T _targetObject;
 	private Object _contextObject;
 	private Object _resultObject;
 	private AsyncCallbackDriver _driver = new InplaceAsyncCallbackDriver();
@@ -84,6 +84,7 @@ public class AsyncCallbackDispatcher<T, R> implements AsyncCompletionCallback {
         }
 	    });
 	    en.setCallbackFilter(new CallbackFilter() {
+	        @Override
 	        public int accept(Method method) {
 	            if (method.getParameterTypes().length == 0 && method.getName().equals("finalize")) {
 	                return 1;
@@ -115,6 +116,7 @@ public class AsyncCallbackDispatcher<T, R> implements AsyncCompletionCallback {
 		return (P)_contextObject;
 	}
 
+	@Override
 	public void complete(Object resultObject) {
 		_resultObject = resultObject;
 		_driver.performCompletionCallback(this);

framework/ipc/src/org/apache/cloudstack/framework/client/ClientMessageBus.java

@@ -21,7 +21,7 @@ package org.apache.cloudstack.framework.client;
 import org.apache.cloudstack.framework.messagebus.MessageBusBase;
 import org.apache.cloudstack.framework.transport.TransportMultiplexier;
 
-public class ClientEventBus extends MessageBusBase implements TransportMultiplexier {
+public class ClientMessageBus extends MessageBusBase implements TransportMultiplexier {
 
 	@Override
 	public void onTransportMessage(String senderEndpointAddress,

framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageBusBase.java

@@ -30,10 +30,10 @@ import org.apache.cloudstack.framework.serializer.MessageSerializer;
 
 public class MessageBusBase implements MessageBus {
 
-	private Gate _gate;
-	private List<ActionRecord> _pendingActions;
+	private final Gate _gate;
+	private final List<ActionRecord> _pendingActions;
 	
-	private SubscriptionNode _subscriberRoot;
+	private final SubscriptionNode _subscriberRoot;
 	private MessageSerializer _messageSerializer; 
 	
 	public MessageBusBase() {
@@ -77,7 +77,7 @@ public class MessageBusBase implements MessageBus {
 				if(current != null)
 					current.removeSubscriber(subscriber, false);
 			} else {
-				this._subscriberRoot.removeSubscriber(subscriber, true);
+				_subscriberRoot.removeSubscriber(subscriber, true);
 			}
 			_gate.leave();
 		} else {
@@ -151,11 +151,10 @@ public class MessageBusBase implements MessageBus {
 	private void onGateOpen() {
 		synchronized(_pendingActions) {
 			ActionRecord record = null;
-			if(_pendingActions.size() > 0) {
-				while((record = _pendingActions.remove(0)) != null) {
+            while (_pendingActions.size() > 0) {
+                record = _pendingActions.remove(0);
 					switch(record.getType()) {
-					case Subscribe :
-						{
+                case Subscribe: {
 							SubscriptionNode current = locate(record.getSubject(), null, true);
 							assert(current != null);
 							current.addSubscriber(record.getSubscriber());
@@ -168,7 +167,7 @@ public class MessageBusBase implements MessageBus {
 							if(current != null)
 								current.removeSubscriber(record.getSubscriber(), false);
 						} else {
-							this._subscriberRoot.removeSubscriber(record.getSubscriber(), true);
+                        _subscriberRoot.removeSubscriber(record.getSubscriber(), true);
 						}
 						break;
 					
@@ -188,7 +187,6 @@ public class MessageBusBase implements MessageBus {
 				}
 			}
 		}
-	}
 	
 	private SubscriptionNode locate(String subject, List<SubscriptionNode> chainFromTop,
 		boolean createPath) {
@@ -223,7 +221,7 @@ public class MessageBusBase implements MessageBus {
 		}
 		
 		if(subjectPathTokens.length > 1) {
-			return locate((String[])Arrays.copyOfRange(subjectPathTokens, 1, subjectPathTokens.length),
+			return locate(Arrays.copyOfRange(subjectPathTokens, 1, subjectPathTokens.length),
 				next, chainFromTop, createPath);
 		} else {
 			return next;
@@ -242,9 +240,9 @@ public class MessageBusBase implements MessageBus {
 	}
 	
 	private static class ActionRecord {
-		private ActionType _type;
-		private String _subject;
-		private MessageSubscriber _subscriber;
+		private final ActionType _type;
+		private final String _subject;
+		private final MessageSubscriber _subscriber;
 		
 		public ActionRecord(ActionType type, String subject, MessageSubscriber subscriber) {
 			_type = type;
@@ -320,10 +318,10 @@ public class MessageBusBase implements MessageBus {
 	}
 	
 	private static class SubscriptionNode {
-		private String _nodeKey;
-		private List<MessageSubscriber> _subscribers;
-		private Map<String, SubscriptionNode> _children;
-		private SubscriptionNode _parent;
+		private final String _nodeKey;
+		private final List<MessageSubscriber> _subscribers;
+		private final Map<String, SubscriptionNode> _children;
+		private final SubscriptionNode _parent;
 		
 		public SubscriptionNode(SubscriptionNode parent, String nodeKey, MessageSubscriber subscriber) {
 			assert(nodeKey != null);

framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.framework.messagebus;
+
+public class MessageDetector implements MessageSubscriber {
+	
+	private MessageBus _messageBus;
+	private String[] _subjects;
+	
+	private volatile boolean _signalled = false;
+	
+	public MessageDetector() {
+		_messageBus = null;
+		_subjects = null;
+	}
+	
+	public boolean waitAny(long timeoutInMiliseconds) {
+		_signalled = false;
+		synchronized(this) {
+			try {
+				wait(timeoutInMiliseconds);
+			} catch (InterruptedException e) {
+			}
+		}
+		return _signalled;
+	}
+	
+	public void open(MessageBus messageBus, String[] subjects) {
+		assert(messageBus != null);
+		assert(subjects != null);
+		
+		_messageBus = messageBus;
+		_subjects = subjects;
+		
+		if(subjects != null) {
+			for(String subject : subjects) {
+				messageBus.subscribe(subject, this);
+			}
+		}
+	}
+	
+	public void close() {
+		if(_subjects != null) {
+			assert(_messageBus != null);
+			
+			for(String subject : _subjects) {
+				_messageBus.unsubscribe(subject, this);
+			}
+		}
+	}
+
+	@Override
+	public void onPublishMessage(String senderAddress, String subject, Object args) {
+		synchronized(this) {
+			_signalled = true;
+			notifyAll();
+		}
+	}
+}

framework/ipc/src/org/apache/cloudstack/framework/server/ServerMessageBus.java

@@ -21,7 +21,7 @@ package org.apache.cloudstack.framework.server;
 import org.apache.cloudstack.framework.messagebus.MessageBusBase;
 import org.apache.cloudstack.framework.transport.TransportMultiplexier;
 
-public class ServerEventBus extends MessageBusBase implements TransportMultiplexier {
+public class ServerMessageBus extends MessageBusBase implements TransportMultiplexier {
 
 	@Override
 	public void onTransportMessage(String senderEndpointAddress,

framework/ipc/test/org/apache/cloudstack/framework/sampleserver/SampleManagementServer.java

@@ -18,9 +18,6 @@
  */
 package org.apache.cloudstack.framework.sampleserver;
 
-import org.springframework.stereotype.Component;
-
-@Component
 public class SampleManagementServer {
 
 	public void mainLoop() {

framework/ipc/test/org/apache/cloudstack/messagebus/TestMessageBus.java

@@ -23,6 +23,7 @@ import javax.inject.Inject;
 import junit.framework.TestCase;
 
 import org.apache.cloudstack.framework.messagebus.MessageBus;
+import org.apache.cloudstack.framework.messagebus.MessageDetector;
 import org.apache.cloudstack.framework.messagebus.MessageSubscriber;
 import org.apache.cloudstack.framework.messagebus.PublishScope;
 import org.junit.Assert;
@@ -113,4 +114,42 @@ public class TestMessageBus extends TestCase {
 		
 		_messageBus.clearAll();
 	}
+	
+	public void testMessageDetector() {
+		MessageDetector detector = new MessageDetector();
+		detector.open(_messageBus, new String[] {"VM", "Host"});
+		
+		Thread thread = new Thread(new Runnable() {
+			@Override
+			public void run() {
+				for(int i = 0; i < 2; i++) {
+					try {
+						Thread.sleep(3000);
+					} catch (InterruptedException e) {
+					}
+					_messageBus.publish(null, "Host", PublishScope.GLOBAL, null);
+				}
+			}
+		});
+		thread.start();
+		
+		try {
+			int count = 0;
+			while(count < 2) {
+				if(detector.waitAny(1000)) {
+					System.out.println("Detected signal on bus");
+					count++;
+				} else {
+					System.out.println("Waiting timed out");
+				}
+			}
+		} finally {
+			detector.close();
+		}
+		
+		try {
+			thread.join();
+		} catch (InterruptedException e) {
+		}
+	}
 }

framework/jobs/pom.xml

@@ -27,11 +27,6 @@
     <relativePath>../pom.xml</relativePath>
   </parent>  
   <dependencies>
-    <dependency>
-      <groupId>org.quartz-scheduler</groupId>
-      <artifactId>quartz</artifactId>
-      <version>2.1.6</version>
-    </dependency>
     <dependency>
       <groupId>org.apache.cloudstack</groupId>
       <artifactId>cloud-utils</artifactId>
@@ -42,8 +37,20 @@
       <artifactId>cloud-api</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-framework-ipc</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-framework-db</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-framework-cluster</artifactId>
+      <version>${project.version}</version>
+    </dependency>
   </dependencies>
-  <build>
-    <defaultGoal>install</defaultGoal>
-  </build>
 </project>

framework/jobs/src/org/apache/cloudstack/framework/job/Job.java

@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cloudstack.framework.job;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Retention(RetentionPolicy.RUNTIME)
-@Target(ElementType.METHOD)
-public @interface Job {
-
-
-
-}

framework/jobs/src/org/apache/cloudstack/framework/job/JobInterceptor.java

@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cloudstack.framework.job;
-
-public class JobInterceptor {
-
-}

framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJob.java

@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs;
+
+import java.util.Date;
+
+import org.apache.cloudstack.framework.jobs.impl.SyncQueueItem;
+import org.apache.cloudstack.jobs.JobInfo;
+
+public interface AsyncJob extends JobInfo {
+
+    public enum JournalType {
+        SUCCESS, FAILURE
+    };
+
+    public static interface Topics {
+        public static final String JOB_HEARTBEAT = "job.heartbeat";
+        public static final String JOB_STATE = "job.state";
+    }
+    
+    public static interface Constants {
+
+    	// Although we may have detailed masks for each individual wakeup event, i.e.
+        // periodical timer, matched topic from message bus, it seems that we don't
+        // need to distinguish them to such level. Therefore, only one wakeup signal
+        // is defined
+        public static final int SIGNAL_MASK_WAKEUP = 1;
+        
+        public static final String SYNC_LOCK_NAME = "SyncLock";
+    }
+	
+    @Override
+    String getType();
+    
+    @Override
+    String getDispatcher();
+
+    @Override
+    int getPendingSignals();
+    
+    @Override
+    long getUserId();
+
+    @Override
+    long getAccountId();
+
+    @Override
+    String getCmd();
+
+    @Override
+    int getCmdVersion();
+
+    @Override
+    String getCmdInfo();
+    
+    @Override
+    Status getStatus();
+
+    @Override
+    int getProcessStatus();
+
+    @Override
+    int getResultCode();
+
+    @Override
+    String getResult();
+
+    @Override
+    Long getInitMsid();
+
+    void setInitMsid(Long msid);
+
+    @Override
+    Long getExecutingMsid();
+    
+    @Override
+    Long getCompleteMsid();
+
+    void setCompleteMsid(Long msid);
+
+    @Override
+    Date getCreated();
+
+    @Override
+    Date getLastUpdated();
+
+    @Override
+    Date getLastPolled();
+
+    @Override
+    String getInstanceType();
+
+    @Override
+    Long getInstanceId();
+
+    String getShortUuid();
+
+    SyncQueueItem getSyncSource();
+
+    void setSyncSource(SyncQueueItem item);
+
+    String getRelated();
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobDispatcher.java

@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs;
+
+import com.cloud.utils.component.Adapter;
+
+//
+// We extend it from Adapter interface for
+//	1)	getName()/setName()
+//	2)	Confirming to general adapter pattern used across CloudStack
+//
+public interface AsyncJobDispatcher extends Adapter {
+	void runJob(AsyncJob job);
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java

@@ -0,0 +1,167 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs;
+
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.framework.jobs.dao.AsyncJobJoinMapDao;
+import org.apache.cloudstack.framework.jobs.impl.AsyncJobJoinMapVO;
+import org.apache.cloudstack.framework.jobs.impl.JobSerializerHelper;
+import org.apache.cloudstack.framework.jobs.impl.SyncQueueItem;
+import org.apache.cloudstack.jobs.JobInfo;
+
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.ResourceUnavailableException;
+
+public class AsyncJobExecutionContext  {
+    private AsyncJob _job;
+	
+    static private AsyncJobManager _jobMgr;
+    static private AsyncJobJoinMapDao _joinMapDao;
+
+    public static void init(AsyncJobManager jobMgr, AsyncJobJoinMapDao joinMapDao) {
+        _jobMgr = jobMgr;
+        _joinMapDao = joinMapDao;
+    }
+	
+	private static ThreadLocal<AsyncJobExecutionContext> s_currentExectionContext = new ThreadLocal<AsyncJobExecutionContext>();
+
+	public AsyncJobExecutionContext() {
+	}
+	
+    public AsyncJobExecutionContext(AsyncJob job) {
+		_job = job;
+	}
+	
+	public SyncQueueItem getSyncSource() {
+		return _job.getSyncSource();
+	}
+	
+	public void resetSyncSource() {
+		_job.setSyncSource(null);
+	}
+	
+    public AsyncJob getJob() {
+		return _job;
+	}
+	
+    public void setJob(AsyncJob job) {
+		_job = job;
+	}
+	
+    public void completeAsyncJob(JobInfo.Status jobStatus, int resultCode, String resultObject) {
+    	assert(_job != null);
+    	_jobMgr.completeAsyncJob(_job.getId(), jobStatus, resultCode, resultObject);
+    }
+    
+    public void updateAsyncJobStatus(int processStatus, String resultObject) {
+    	assert(_job != null);
+    	_jobMgr.updateAsyncJobStatus(_job.getId(), processStatus, resultObject);
+    }
+    
+    public void updateAsyncJobAttachment(String instanceType, Long instanceId) {
+    	assert(_job != null);
+    	_jobMgr.updateAsyncJobAttachment(_job.getId(), instanceType, instanceId);
+    }
+	
+    public void logJobJournal(AsyncJob.JournalType journalType, String journalText, String journalObjJson) {
+		assert(_job != null);
+		_jobMgr.logJobJournal(_job.getId(), journalType, journalText, journalObjJson);
+	}
+
+    public void log(Logger logger, String journalText) {
+        _jobMgr.logJobJournal(_job.getId(), AsyncJob.JournalType.SUCCESS, journalText, null);
+        logger.debug(journalText);
+    }
+
+    public void joinJob(long joinJobId) {
+    	assert(_job != null);
+    	_jobMgr.joinJob(_job.getId(), joinJobId);
+    }
+	
+    public void joinJob(long joinJobId, String wakeupHandler, String wakeupDispatcher,
+    		String[] wakeupTopcisOnMessageBus, long wakeupIntervalInMilliSeconds, long timeoutInMilliSeconds) {
+    	assert(_job != null);
+    	_jobMgr.joinJob(_job.getId(), joinJobId, wakeupHandler, wakeupDispatcher, wakeupTopcisOnMessageBus,
+    		wakeupIntervalInMilliSeconds, timeoutInMilliSeconds);
+    }
+    
+    //
+	// check failure exception before we disjoin the worker job
+	// TODO : it is ugly and this will become unnecessary after we switch to full-async mode
+	//
+    public void disjoinJob(long joinedJobId) throws InsufficientCapacityException,
+		ConcurrentOperationException, ResourceUnavailableException {
+    	assert(_job != null);
+    	
+    	AsyncJobJoinMapVO record = _joinMapDao.getJoinRecord(_job.getId(), joinedJobId);
+    	if(record.getJoinStatus() == JobInfo.Status.FAILED && record.getJoinResult() != null) {
+    		Object exception = JobSerializerHelper.fromObjectSerializedString(record.getJoinResult());
+    		if(exception != null && exception instanceof Exception) {
+    			if(exception instanceof InsufficientCapacityException)
+    				throw (InsufficientCapacityException)exception;
+    			else if(exception instanceof ConcurrentOperationException)
+    				throw (ConcurrentOperationException)exception;
+    			else if(exception instanceof ResourceUnavailableException)
+    				throw (ResourceUnavailableException)exception;
+    			else
+    				throw new RuntimeException((Exception)exception);
+    		}
+    	}
+    	
+    	_jobMgr.disjoinJob(_job.getId(), joinedJobId);
+    }
+    
+    public void completeJoin(JobInfo.Status joinStatus, String joinResult) {
+    	assert(_job != null);
+    	_jobMgr.completeJoin(_job.getId(), joinStatus, joinResult);
+    }
+    
+    public void completeJobAndJoin(JobInfo.Status joinStatus, String joinResult) {
+    	assert(_job != null);
+    	_jobMgr.completeJoin(_job.getId(), joinStatus, joinResult);
+    	_jobMgr.completeAsyncJob(_job.getId(), joinStatus, 0, null);
+    }
+
+	public static AsyncJobExecutionContext getCurrentExecutionContext() {
+		AsyncJobExecutionContext context = s_currentExectionContext.get();
+		return context;
+	}
+	
+    public static AsyncJobExecutionContext registerPseudoExecutionContext(long accountId, long userId) {
+        AsyncJobExecutionContext context = s_currentExectionContext.get();
+        if (context == null) {
+            context = new AsyncJobExecutionContext();
+            context.setJob(_jobMgr.getPseudoJob(accountId, userId));
+            setCurrentExecutionContext(context);
+        }
+
+        return context;
+    }
+
+    public static AsyncJobExecutionContext unregister() {
+        AsyncJobExecutionContext context = s_currentExectionContext.get();
+        setCurrentExecutionContext(null);
+        return context;
+    }
+
+    // This is intended to be package level access for AsyncJobManagerImpl only.
+    public static void setCurrentExecutionContext(AsyncJobExecutionContext currentContext) {
+		s_currentExectionContext.set(currentContext);
+	}
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobMBean.java

@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs;
+
+public interface AsyncJobMBean {
+	public long getAccountId();
+	public long getUserId();
+	public String getCmd();
+	public String getCmdInfo();
+	public String getStatus();
+	public int getProcessStatus();
+	public int getResultCode();
+	public String getResult();
+	public String getInstanceType();
+	public String getInstanceId();
+	public String getInitMsid();
+	public String getCreateTime();
+	public String getLastUpdateTime();
+	public String getLastPollTime();
+	public String getSyncQueueId();
+	public String getSyncQueueContentType();
+	public String getSyncQueueContentId();
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobManager.java

@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under ones
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs;
+
+import java.util.List;
+
+import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
+import org.apache.cloudstack.jobs.JobInfo;
+
+import com.cloud.utils.Predicate;
+import com.cloud.utils.component.Manager;
+
+public interface AsyncJobManager extends Manager {
+    
+	public static final String JOB_POOL_THREAD_PREFIX = "Job-Executor";
+
+    AsyncJobVO getAsyncJob(long jobId);
+	
+	List<? extends AsyncJob> findInstancePendingAsyncJobs(String instanceType, Long accountId);
+	
+	long submitAsyncJob(AsyncJob job);
+	long submitAsyncJob(AsyncJob job, String syncObjType, long syncObjId);
+
+    void completeAsyncJob(long jobId, JobInfo.Status jobStatus, int resultCode, String result);
+
+    void updateAsyncJobStatus(long jobId, int processStatus, String resultObject);
+    void updateAsyncJobAttachment(long jobId, String instanceType, Long instanceId);
+    void logJobJournal(long jobId, AsyncJob.JournalType journalType, String
+    	journalText, String journalObjJson);
+    
+	/**
+	 * A running thread inside management server can have a 1:1 linked pseudo job.
+	 * This is to help make some legacy code work without too dramatic changes.
+	 * 
+	 * All pseudo jobs should be expunged upon management start event
+	 *
+	 * @return pseudo job for the thread
+	 */
+    AsyncJob getPseudoJob(long accountId, long userId);
+
+    /**
+     * Used by upper level job to wait for completion of a down-level job (usually VmWork jobs)
+     * in synchronous way. Caller needs to use waitAndCheck() to check the completion status
+     * of the down-level job
+     * 
+     * Due to the amount of legacy code that relies on synchronous-call semantics, this form of joinJob
+     * is used mostly
+     * 
+     * 
+     * @param jobId upper job that is going to wait the completion of a down-level job
+     * @param joinJobId down-level job
+	 */
+	void joinJob(long jobId, long joinJobId);
+	
+    /**
+     * Used by upper level job to wait for completion of a down-level job (usually VmWork jobs)
+     * in asynchronous way, it will cause upper job to cease current execution, upper job will be
+     * rescheduled to execute periodically or on wakeup events detected from message bus
+     * 
+     * @param jobId upper job that is going to wait the completion of a down-level job
+     * @param joinJobId down-level job
+     * @Param wakeupHandler	wake-up handler
+     * @Param wakeupDispatcher wake-up dispatcher
+     * @param wakeupTopicsOnMessageBus
+     * @param wakeupIntervalInMilliSeconds
+     * @param timeoutInMilliSeconds
+     */
+    void joinJob(long jobId, long joinJobId, String wakeupHandler, String wakupDispatcher,
+    		String[] wakeupTopicsOnMessageBus, long wakeupIntervalInMilliSeconds, long timeoutInMilliSeconds);
+    
+    /**
+     * Dis-join two related jobs
+     * 
+     * @param jobId
+     * @param joinedJobId
+     */
+    void disjoinJob(long jobId, long joinedJobId);
+    
+    /**
+     * Used by down-level job to notify its completion to upper level jobs
+     * 
+     * @param joinJobId down-level job for upper level job to join with
+     * @param joinStatus AsyncJobConstants status code to indicate success or failure of the
+     * 					down-level job
+     * @param joinResult object-stream serialized result object
+     * 					this is primarily used by down-level job to pass error exception objects
+     * 					for legacy code to work. To help pass exception object easier, we use
+     * 					object-stream based serialization instead of GSON
+     */
+    void completeJoin(long joinJobId, JobInfo.Status joinStatus, String joinResult);
+   
+    void releaseSyncSource();
+    void syncAsyncJobExecution(AsyncJob job, String syncObjType, long syncObjId, long queueSizeLimit);
+    
+    /**
+     * This method will be deprecated after all code has been migrated to fully-asynchronous mode
+     * that uses async-feature of joinJob/disjoinJob
+     * 
+     * @param wakupTopicsOnMessageBus topic on message bus to wakeup the wait
+     * @param checkIntervalInMilliSeconds time to break out wait for checking predicate condition
+     * @param timeoutInMiliseconds time out to break out the whole wait process
+     * @param predicate
+     * @return true, predicate condition is satisfied
+     * 			false, wait is timed out
+     */
+    boolean waitAndCheck(AsyncJob job, String[] wakupTopicsOnMessageBus, long checkIntervalInMilliSeconds,
+    	long timeoutInMiliseconds, Predicate predicate);
+
+    AsyncJob queryJob(long jobId, boolean updatePollTime);
+
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/JobCancellationException.java

@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs;
+
+import java.util.concurrent.CancellationException;
+
+import com.cloud.utils.SerialVersionUID;
+
+
+/**
+ * This exception is fired when the job has been cancelled
+ *
+ */
+public class JobCancellationException extends CancellationException {
+    
+    private static final long serialVersionUID = SerialVersionUID.AffinityConflictException;
+
+    public enum Reason {
+        RequestedByUser,
+        RequestedByCaller,
+        TimedOut;
+    }
+
+    Reason reason;
+
+    public JobCancellationException(Reason reason) {
+        super("The job was cancelled due to " + reason.toString());
+        this.reason = reason;
+    }
+
+    public Reason getReason() {
+        return reason;
+    }
+
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/Outcome.java

@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs;
+
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Outcome is returned by clients of jobs framework as a way to wait for the
+ * outcome of a job.  It fully complies with how Future interface is designed.
+ * In addition, it allows the callee to file a task to be scheduled when the
+ * job completes.
+ * 
+ * Note that the callee should schedule a job when using the Task interface.
+ * It shouldn't try to complete the job in the schedule code as that will take
+ * up threads in the jobs framework.
+ * 
+ * For the client of the jobs framework, you can either use the OutcomeImpl
+ * class to implement this interface or you can add to this interface to
+ * allow for your specific exceptions to be thrown.
+ *
+ * @param <T> Object returned to the callee when the job completes
+ */
+public interface Outcome<T> extends Future<T> {
+    AsyncJob getJob();
+
+    /**
+     * In addition to the normal Future methods, Outcome allows the ability
+     * to register a schedule task to be performed when the job is completed.
+     * 
+     * @param listener
+     */
+    void execute(Task<T> task);
+
+    void execute(Task<T> task, long wait, TimeUnit unit);
+
+    /**
+     * Listener is used by Outcome to schedule a task to run when a job
+     * completes.
+     *
+     * @param <T> T result returned
+     */
+    public interface Task<T> extends Runnable {
+        void schedule(AsyncJobExecutionContext context, T result);
+
+        void scheduleOnError(AsyncJobExecutionContext context, Throwable e);
+    }
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java

@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.dao;
+
+import java.util.Date;
+import java.util.List;
+
+import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
+
+import com.cloud.utils.db.GenericDao;
+
+public interface AsyncJobDao extends GenericDao<AsyncJobVO, Long> {
+	AsyncJobVO findInstancePendingAsyncJob(String instanceType, long instanceId);
+	List<AsyncJobVO> findInstancePendingAsyncJobs(String instanceType, Long accountId);
+	
+	AsyncJobVO findPseudoJob(long threadId, long msid);
+	void cleanupPseduoJobs(long msid);
+	
+	List<AsyncJobVO> getExpiredJobs(Date cutTime, int limit);
+	List<AsyncJobVO> getExpiredUnfinishedJobs(Date cutTime, int limit);
+	void resetJobProcess(long msid, int jobResultCode, String jobResultMessage);
+	List<AsyncJobVO> getExpiredCompletedJobs(Date cutTime, int limit);
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java

@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.dao;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
+import org.apache.cloudstack.jobs.JobInfo;
+
+import com.cloud.utils.db.DB;
+import com.cloud.utils.db.Filter;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import com.cloud.utils.db.SearchCriteria.Op;
+import com.cloud.utils.db.Transaction;
+
+public class AsyncJobDaoImpl extends GenericDaoBase<AsyncJobVO, Long> implements AsyncJobDao {
+    private static final Logger s_logger = Logger.getLogger(AsyncJobDaoImpl.class.getName());
+	
+	private final SearchBuilder<AsyncJobVO> pendingAsyncJobSearch;
+	private final SearchBuilder<AsyncJobVO> pendingAsyncJobsSearch;
+    private final SearchBuilder<AsyncJobVO> expiringAsyncJobSearch;
+	private final SearchBuilder<AsyncJobVO> pseudoJobSearch;
+	private final SearchBuilder<AsyncJobVO> pseudoJobCleanupSearch;
+	private final SearchBuilder<AsyncJobVO> expiringUnfinishedAsyncJobSearch;
+	private final SearchBuilder<AsyncJobVO> expiringCompletedAsyncJobSearch;
+
+	
+	public AsyncJobDaoImpl() {
+		pendingAsyncJobSearch = createSearchBuilder();
+		pendingAsyncJobSearch.and("instanceType", pendingAsyncJobSearch.entity().getInstanceType(),
+			SearchCriteria.Op.EQ);
+		pendingAsyncJobSearch.and("instanceId", pendingAsyncJobSearch.entity().getInstanceId(),
+			SearchCriteria.Op.EQ);
+		pendingAsyncJobSearch.and("status", pendingAsyncJobSearch.entity().getStatus(),
+				SearchCriteria.Op.EQ);
+		pendingAsyncJobSearch.done();
+		
+        expiringAsyncJobSearch = createSearchBuilder();
+        expiringAsyncJobSearch.and("created", expiringAsyncJobSearch.entity().getCreated(), SearchCriteria.Op.LTEQ);
+        expiringAsyncJobSearch.done();
+
+		pendingAsyncJobsSearch = createSearchBuilder();
+		pendingAsyncJobsSearch.and("instanceType", pendingAsyncJobsSearch.entity().getInstanceType(),
+			SearchCriteria.Op.EQ);
+		pendingAsyncJobsSearch.and("accountId", pendingAsyncJobsSearch.entity().getAccountId(),
+			SearchCriteria.Op.EQ);
+		pendingAsyncJobsSearch.and("status", pendingAsyncJobsSearch.entity().getStatus(),
+				SearchCriteria.Op.EQ);
+		pendingAsyncJobsSearch.done();
+		
+		expiringUnfinishedAsyncJobSearch = createSearchBuilder();
+		expiringUnfinishedAsyncJobSearch.and("created", expiringUnfinishedAsyncJobSearch.entity().getCreated(),
+			SearchCriteria.Op.LTEQ);
+		expiringUnfinishedAsyncJobSearch.and("completeMsId", expiringUnfinishedAsyncJobSearch.entity().getCompleteMsid(), SearchCriteria.Op.NULL);
+		expiringUnfinishedAsyncJobSearch.and("jobStatus", expiringUnfinishedAsyncJobSearch.entity().getStatus(), SearchCriteria.Op.EQ);
+		expiringUnfinishedAsyncJobSearch.done();
+		
+		expiringCompletedAsyncJobSearch = createSearchBuilder();
+		expiringCompletedAsyncJobSearch.and("created", expiringCompletedAsyncJobSearch.entity().getCreated(),
+			SearchCriteria.Op.LTEQ);
+        expiringCompletedAsyncJobSearch.and("completeMsId", expiringCompletedAsyncJobSearch.entity().getCompleteMsid(), SearchCriteria.Op.NNULL);
+        expiringCompletedAsyncJobSearch.and("jobStatus", expiringCompletedAsyncJobSearch.entity().getStatus(), SearchCriteria.Op.NEQ);
+        expiringCompletedAsyncJobSearch.done();
+		
+		pseudoJobSearch = createSearchBuilder();
+		pseudoJobSearch.and("jobDispatcher", pseudoJobSearch.entity().getDispatcher(), Op.EQ);
+		pseudoJobSearch.and("instanceType", pseudoJobSearch.entity().getInstanceType(), Op.EQ);
+		pseudoJobSearch.and("instanceId", pseudoJobSearch.entity().getInstanceId(), Op.EQ);
+		pseudoJobSearch.done();
+		
+		pseudoJobCleanupSearch = createSearchBuilder();
+		pseudoJobCleanupSearch.and("initMsid", pseudoJobCleanupSearch.entity().getInitMsid(), Op.EQ);
+		pseudoJobCleanupSearch.done();
+		
+	}
+	
+	@Override
+    public AsyncJobVO findInstancePendingAsyncJob(String instanceType, long instanceId) {
+        SearchCriteria<AsyncJobVO> sc = pendingAsyncJobSearch.create();
+        sc.setParameters("instanceType", instanceType);
+        sc.setParameters("instanceId", instanceId);
+        sc.setParameters("status", JobInfo.Status.IN_PROGRESS);
+        
+        List<AsyncJobVO> l = listIncludingRemovedBy(sc);
+        if(l != null && l.size() > 0) {
+        	if(l.size() > 1) {
+        		s_logger.warn("Instance " + instanceType + "-" + instanceId + " has multiple pending async-job");
+        	}
+        	
+        	return l.get(0);
+        }
+        return null;
+	}
+	
+	@Override
+    public List<AsyncJobVO> findInstancePendingAsyncJobs(String instanceType, Long accountId) {
+		SearchCriteria<AsyncJobVO> sc = pendingAsyncJobsSearch.create();
+        sc.setParameters("instanceType", instanceType);
+        
+        if (accountId != null) {
+            sc.setParameters("accountId", accountId);
+        }
+        sc.setParameters("status", JobInfo.Status.IN_PROGRESS);
+        
+        return listBy(sc);
+	}
+	
+	@Override
+    public AsyncJobVO findPseudoJob(long threadId, long msid) {
+		SearchCriteria<AsyncJobVO> sc = pseudoJobSearch.create();
+        sc.setParameters("jobDispatcher", AsyncJobVO.JOB_DISPATCHER_PSEUDO);
+        sc.setParameters("instanceType", AsyncJobVO.PSEUDO_JOB_INSTANCE_TYPE);
+		sc.setParameters("instanceId", threadId);
+		
+		List<AsyncJobVO> result = listBy(sc);
+		if(result != null && result.size() > 0) {
+			assert(result.size() == 1);
+			return result.get(0);
+		}
+		
+		return null;
+	}
+	
+	@Override
+    public void cleanupPseduoJobs(long msid) {
+		SearchCriteria<AsyncJobVO> sc = pseudoJobCleanupSearch.create();
+		sc.setParameters("initMsid", msid);
+		this.expunge(sc);
+	}
+	
+    @Override
+    public List<AsyncJobVO> getExpiredJobs(Date cutTime, int limit) {
+        SearchCriteria<AsyncJobVO> sc = expiringAsyncJobSearch.create();
+        sc.setParameters("created", cutTime);
+        Filter filter = new Filter(AsyncJobVO.class, "created", true, 0L, (long)limit);
+        return listIncludingRemovedBy(sc, filter);
+    }
+    
+    @Override
+	public List<AsyncJobVO> getExpiredUnfinishedJobs(Date cutTime, int limit) {
+		SearchCriteria<AsyncJobVO> sc = expiringUnfinishedAsyncJobSearch.create();
+		sc.setParameters("created", cutTime);
+        sc.setParameters("jobStatus", JobInfo.Status.IN_PROGRESS);
+		Filter filter = new Filter(AsyncJobVO.class, "created", true, 0L, (long)limit);
+		return listIncludingRemovedBy(sc, filter);
+	}
+	
+	@Override
+	public List<AsyncJobVO> getExpiredCompletedJobs(Date cutTime, int limit) {
+		SearchCriteria<AsyncJobVO> sc = expiringCompletedAsyncJobSearch.create();
+		sc.setParameters("created", cutTime);
+        sc.setParameters("jobStatus", JobInfo.Status.IN_PROGRESS);
+		Filter filter = new Filter(AsyncJobVO.class, "created", true, 0L, (long)limit);
+		return listIncludingRemovedBy(sc, filter);
+	}
+
+	@Override
+    @DB
+	public void resetJobProcess(long msid, int jobResultCode, String jobResultMessage) {
+        String sql = "UPDATE async_job SET job_status=" + JobInfo.Status.FAILED.ordinal() + ", job_result_code=" + jobResultCode
+                + ", job_result='" + jobResultMessage + "' where job_status=" + JobInfo.Status.IN_PROGRESS.ordinal()
+                + " AND (job_executing_msid=? OR (job_executing_msid IS NULL AND job_init_msid=?))";
+		
+        Transaction txn = Transaction.currentTxn();
+        PreparedStatement pstmt = null;
+        try {
+            pstmt = txn.prepareAutoCloseStatement(sql);
+            pstmt.setLong(1, msid);
+            pstmt.setLong(2, msid);
+            pstmt.execute();
+        } catch (SQLException e) {
+        	s_logger.warn("Unable to reset job status for management server " + msid, e);
+        } catch (Throwable e) {
+        	s_logger.warn("Unable to reset job status for management server " + msid, e);
+        }
+	}
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDao.java

@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.dao;
+
+import java.util.Date;
+import java.util.List;
+
+import org.apache.cloudstack.framework.jobs.impl.AsyncJobJoinMapVO;
+import org.apache.cloudstack.jobs.JobInfo;
+
+import com.cloud.utils.db.GenericDao;
+
+public interface AsyncJobJoinMapDao extends GenericDao<AsyncJobJoinMapVO, Long> {
+	
+	Long joinJob(long jobId, long joinJobId, long joinMsid,
+		long wakeupIntervalMs, long expirationMs,
+		Long syncSourceId, String wakeupHandler, String wakeupDispatcher);
+	void disjoinJob(long jobId, long joinedJobId);
+	void disjoinAllJobs(long jobId);
+	
+	AsyncJobJoinMapVO getJoinRecord(long jobId, long joinJobId);
+	List<AsyncJobJoinMapVO> listJoinRecords(long jobId);
+	
+    void completeJoin(long joinJobId, JobInfo.Status joinStatus, String joinResult, long completeMsid);
+	
+//	List<Long> wakeupScan();
+
+    List<Long> findJobsToWake(long joinedJobId);
+
+    List<Long> findJobsToWakeBetween(Date cutDate);
+//	List<Long> wakeupByJoinedJobCompletion(long joinedJobId);
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDaoImpl.java

@@ -0,0 +1,303 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.dao;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.framework.jobs.impl.AsyncJobJoinMapVO;
+import org.apache.cloudstack.jobs.JobInfo;
+
+import com.cloud.utils.DateUtil;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import com.cloud.utils.db.SearchCriteria.Op;
+import com.cloud.utils.db.Transaction;
+import com.cloud.utils.db.UpdateBuilder;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class AsyncJobJoinMapDaoImpl extends GenericDaoBase<AsyncJobJoinMapVO, Long> implements AsyncJobJoinMapDao {
+    public static final Logger s_logger = Logger.getLogger(AsyncJobJoinMapDaoImpl.class);
+	
+	private final SearchBuilder<AsyncJobJoinMapVO> RecordSearch;
+	private final SearchBuilder<AsyncJobJoinMapVO> RecordSearchByOwner;
+	private final SearchBuilder<AsyncJobJoinMapVO> CompleteJoinSearch;
+	private final SearchBuilder<AsyncJobJoinMapVO> WakeupSearch;
+
+//    private final GenericSearchBuilder<AsyncJobJoinMapVO, Long> JoinJobSearch;
+	
+    protected AsyncJobJoinMapDaoImpl() {
+		RecordSearch = createSearchBuilder();
+		RecordSearch.and("jobId", RecordSearch.entity().getJobId(), Op.EQ);
+		RecordSearch.and("joinJobId", RecordSearch.entity().getJoinJobId(), Op.EQ);
+		RecordSearch.done();
+
+		RecordSearchByOwner = createSearchBuilder();
+		RecordSearchByOwner.and("jobId", RecordSearchByOwner.entity().getJobId(), Op.EQ);
+		RecordSearchByOwner.done();
+		
+		CompleteJoinSearch = createSearchBuilder();
+		CompleteJoinSearch.and("joinJobId", CompleteJoinSearch.entity().getJoinJobId(), Op.EQ);
+		CompleteJoinSearch.done();
+		
+		WakeupSearch = createSearchBuilder();
+		WakeupSearch.and("nextWakeupTime", WakeupSearch.entity().getNextWakeupTime(), Op.LT);
+		WakeupSearch.and("expiration", WakeupSearch.entity().getExpiration(), Op.GT);
+		WakeupSearch.and("joinStatus", WakeupSearch.entity().getJoinStatus(), Op.EQ);
+		WakeupSearch.done();
+
+//        JoinJobSearch = createSearchBuilder(Long.class);
+//        JoinJobSearch.and(JoinJobSearch.entity().getJoinJobId(), Op.SC, "joinJobId");
+//        JoinJobSearch.done();
+	}
+	
+	@Override
+    public Long joinJob(long jobId, long joinJobId, long joinMsid,
+		long wakeupIntervalMs, long expirationMs,
+		Long syncSourceId, String wakeupHandler, String wakeupDispatcher) {
+		
+		AsyncJobJoinMapVO record = new AsyncJobJoinMapVO();
+		record.setJobId(jobId);
+		record.setJoinJobId(joinJobId);
+		record.setJoinMsid(joinMsid);
+		record.setJoinStatus(JobInfo.Status.IN_PROGRESS);
+		record.setSyncSourceId(syncSourceId);
+		record.setWakeupInterval(wakeupIntervalMs / 1000);		// convert millisecond to second
+		record.setWakeupHandler(wakeupHandler);
+		record.setWakeupDispatcher(wakeupDispatcher);
+		if(wakeupHandler != null) {
+			record.setNextWakeupTime(new Date(DateUtil.currentGMTTime().getTime() + wakeupIntervalMs));
+			record.setExpiration(new Date(DateUtil.currentGMTTime().getTime() + expirationMs));
+		}
+		
+		persist(record);
+		return record.getId();
+	}
+	
+	@Override
+    public void disjoinJob(long jobId, long joinedJobId) {
+		SearchCriteria<AsyncJobJoinMapVO> sc = RecordSearch.create();
+		sc.setParameters("jobId", jobId);
+		sc.setParameters("joinJobId", joinedJobId);
+		
+		this.expunge(sc);
+	}
+	
+	@Override
+    public void disjoinAllJobs(long jobId) {
+		SearchCriteria<AsyncJobJoinMapVO> sc = RecordSearchByOwner.create();
+		sc.setParameters("jobId", jobId);
+		
+		this.expunge(sc);
+	}
+	
+	@Override
+    public AsyncJobJoinMapVO getJoinRecord(long jobId, long joinJobId) {
+		SearchCriteria<AsyncJobJoinMapVO> sc = RecordSearch.create();
+		sc.setParameters("jobId", jobId);
+		sc.setParameters("joinJobId", joinJobId);
+		
+		List<AsyncJobJoinMapVO> result = this.listBy(sc);
+		if(result != null && result.size() > 0) {
+			assert(result.size() == 1);
+			return result.get(0);
+		}
+		
+		return null;
+	}
+	
+	@Override
+    public List<AsyncJobJoinMapVO> listJoinRecords(long jobId) {
+		SearchCriteria<AsyncJobJoinMapVO> sc = RecordSearchByOwner.create();
+		sc.setParameters("jobId", jobId);
+		
+		return this.listBy(sc);
+	}
+	
+    @Override
+    public void completeJoin(long joinJobId, JobInfo.Status joinStatus, String joinResult, long completeMsid) {
+        AsyncJobJoinMapVO record = createForUpdate();
+        record.setJoinStatus(joinStatus);
+        record.setJoinResult(joinResult);
+        record.setCompleteMsid(completeMsid);
+        record.setLastUpdated(DateUtil.currentGMTTime());
+        
+        UpdateBuilder ub = getUpdateBuilder(record);
+        
+        SearchCriteria<AsyncJobJoinMapVO> sc = CompleteJoinSearch.create();
+        sc.setParameters("joinJobId", joinJobId);
+        update(ub, sc, null);
+	}
+
+//	@Override
+//    public List<Long> wakeupScan() {
+//		List<Long> standaloneList = new ArrayList<Long>();
+//
+//		Date cutDate = DateUtil.currentGMTTime();
+//
+//		Transaction txn = Transaction.currentTxn();
+//        PreparedStatement pstmt = null;
+//        try {
+//			txn.start();
+//
+//			//
+//			// performance sensitive processing, do it in plain SQL
+//			//
+//			String sql = "UPDATE async_job SET job_pending_signals=? WHERE id IN " +
+//					"(SELECT job_id FROM async_job_join_map WHERE next_wakeup < ? AND expiration > ?)";
+//			pstmt = txn.prepareStatement(sql);
+//			pstmt.setInt(1, AsyncJob.Constants.SIGNAL_MASK_WAKEUP);
+//	        pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
+//	        pstmt.setString(3, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
+//	        pstmt.executeUpdate();
+//	        pstmt.close();
+//
+//			sql = "UPDATE sync_queue_item SET queue_proc_msid=NULL, queue_proc_number=NULL WHERE content_id IN " +
+//					"(SELECT job_id FROM async_job_join_map WHERE next_wakeup < ? AND expiration > ?)";
+//			pstmt = txn.prepareStatement(sql);
+//	        pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
+//	        pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
+//	        pstmt.executeUpdate();
+//	        pstmt.close();
+//
+//	        sql = "SELECT job_id FROM async_job_join_map WHERE next_wakeup < ? AND expiration > ? AND job_id NOT IN (SELECT content_id FROM sync_queue_item)";
+//			pstmt = txn.prepareStatement(sql);
+//	        pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
+//	        pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
+//	        ResultSet rs = pstmt.executeQuery();
+//	        while(rs.next()) {
+//	        	standaloneList.add(rs.getLong(1));
+//	        }
+//	        rs.close();
+//	        pstmt.close();
+//
+//	        // update for next wake-up
+//	        sql = "UPDATE async_job_join_map SET next_wakeup=DATE_ADD(next_wakeup, INTERVAL wakeup_interval SECOND) WHERE next_wakeup < ? AND expiration > ?";
+//			pstmt = txn.prepareStatement(sql);
+//	        pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
+//	        pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
+//	        pstmt.executeUpdate();
+//	        pstmt.close();
+//
+//	        txn.commit();
+//		} catch (SQLException e) {
+//			s_logger.error("Unexpected exception", e);
+//		}
+//
+//        return standaloneList;
+//	}
+
+    @Override
+    public List<Long> findJobsToWake(long joinedJobId) {
+        // TODO: We should fix this.  We shouldn't be crossing daos in a dao code.
+        List<Long> standaloneList = new ArrayList<Long>();
+        Transaction txn = Transaction.currentTxn();
+        String sql = "SELECT job_id FROM async_job_join_map WHERE join_job_id = ? AND job_id NOT IN (SELECT content_id FROM sync_queue_item)";
+        try {
+            PreparedStatement pstmt = txn.prepareStatement(sql);
+            pstmt.setLong(1, joinedJobId);
+            ResultSet rs = pstmt.executeQuery();
+            while (rs.next()) {
+                standaloneList.add(rs.getLong(1));
+            }
+        } catch (SQLException e) {
+            throw new CloudRuntimeException("Unable to execute " + sql, e);
+        }
+        return standaloneList;
+    }
+
+    @Override
+    public List<Long> findJobsToWakeBetween(Date cutDate) {
+        List<Long> standaloneList = new ArrayList<Long>();
+        Transaction txn = Transaction.currentTxn();
+        try {
+            String sql = "SELECT job_id FROM async_job_join_map WHERE next_wakeup < ? AND expiration > ? AND job_id NOT IN (SELECT content_id FROM sync_queue_item)";
+            PreparedStatement pstmt = txn.prepareStatement(sql);
+            pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
+            pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
+            ResultSet rs = pstmt.executeQuery();
+            while (rs.next()) {
+                standaloneList.add(rs.getLong(1));
+            }
+
+            // update for next wake-up
+            sql = "UPDATE async_job_join_map SET next_wakeup=DATE_ADD(next_wakeup, INTERVAL wakeup_interval SECOND) WHERE next_wakeup < ? AND expiration > ?";
+            pstmt = txn.prepareStatement(sql);
+            pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
+            pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate));
+            pstmt.executeUpdate();
+
+            return standaloneList;
+        } catch (SQLException e) {
+            throw new CloudRuntimeException("Unable to handle SQL exception", e);
+        }
+
+    }
+	
+//    @Override
+//    public List<Long> wakeupByJoinedJobCompletion(long joinedJobId) {
+//        List<Long> standaloneList = new ArrayList<Long>();
+//
+//        Transaction txn = Transaction.currentTxn();
+//        PreparedStatement pstmt = null;
+//        try {
+//            txn.start();
+//
+//            //
+//            // performance sensitive processing, do it in plain SQL
+//            //
+//            String sql = "UPDATE async_job SET job_pending_signals=? WHERE id IN " +
+//                    "(SELECT job_id FROM async_job_join_map WHERE join_job_id = ?)";
+//            pstmt = txn.prepareStatement(sql);
+//            pstmt.setInt(1, AsyncJob.Contants.SIGNAL_MASK_WAKEUP);
+//            pstmt.setLong(2, joinedJobId);
+//            pstmt.executeUpdate();
+//            pstmt.close();
+//
+//            sql = "UPDATE sync_queue_item SET queue_proc_msid=NULL, queue_proc_number=NULL WHERE content_id IN " +
+//                    "(SELECT job_id FROM async_job_join_map WHERE join_job_id = ?)";
+//            pstmt = txn.prepareStatement(sql);
+//            pstmt.setLong(1, joinedJobId);
+//            pstmt.executeUpdate();
+//            pstmt.close();
+//
+//            sql = "SELECT job_id FROM async_job_join_map WHERE join_job_id = ? AND job_id NOT IN (SELECT content_id FROM sync_queue_item)";
+//            pstmt = txn.prepareStatement(sql);
+//            pstmt.setLong(1, joinedJobId);
+//            ResultSet rs = pstmt.executeQuery();
+//            while(rs.next()) {
+//                standaloneList.add(rs.getLong(1));
+//            }
+//            rs.close();
+//            pstmt.close();
+//
+//            txn.commit();
+//        } catch (SQLException e) {
+//            s_logger.error("Unexpected exception", e);
+//        }
+//
+//        return standaloneList;
+//    }
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJournalDao.java

@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.dao;
+
+import java.util.List;
+
+import org.apache.cloudstack.framework.jobs.impl.AsyncJobJournalVO;
+
+import com.cloud.utils.db.GenericDao;
+
+public interface AsyncJobJournalDao extends GenericDao<AsyncJobJournalVO, Long> {
+	List<AsyncJobJournalVO> getJobJournal(long jobId);
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJournalDaoImpl.java

@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.dao;
+
+import java.util.List;
+
+import org.apache.cloudstack.framework.jobs.impl.AsyncJobJournalVO;
+
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import com.cloud.utils.db.SearchCriteria.Op;
+
+public class AsyncJobJournalDaoImpl extends GenericDaoBase<AsyncJobJournalVO, Long> implements AsyncJobJournalDao {
+
+	private final SearchBuilder<AsyncJobJournalVO> JobJournalSearch;	
+
+	public AsyncJobJournalDaoImpl() {
+		JobJournalSearch = createSearchBuilder();
+		JobJournalSearch.and("jobId", JobJournalSearch.entity().getJobId(), Op.EQ);
+		JobJournalSearch.done();
+	}
+	
+	@Override
+	public List<AsyncJobJournalVO> getJobJournal(long jobId) {
+		SearchCriteria<AsyncJobJournalVO> sc = JobJournalSearch.create();
+		sc.setParameters("jobId", jobId);
+		
+		return this.listBy(sc);
+	}
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueDao.java

@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.dao;
+
+import org.apache.cloudstack.framework.jobs.impl.SyncQueueVO;
+
+import com.cloud.utils.db.GenericDao;
+
+public interface SyncQueueDao extends GenericDao<SyncQueueVO, Long>{
+	public void ensureQueue(String syncObjType, long syncObjId);
+	public SyncQueueVO find(String syncObjType, long syncObjId);
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueDaoImpl.java

@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.framework.jobs.dao;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Date;
+import java.util.TimeZone;
+
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.framework.jobs.impl.SyncQueueVO;
+
+import com.cloud.utils.DateUtil;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import com.cloud.utils.db.Transaction;
+
+public class SyncQueueDaoImpl extends GenericDaoBase<SyncQueueVO, Long> implements SyncQueueDao {
+    private static final Logger s_logger = Logger.getLogger(SyncQueueDaoImpl.class.getName());
+    
+    SearchBuilder<SyncQueueVO> TypeIdSearch = createSearchBuilder();
+
+    public SyncQueueDaoImpl() {
+	    super();
+	    TypeIdSearch = createSearchBuilder();
+        TypeIdSearch.and("syncObjType", TypeIdSearch.entity().getSyncObjType(), SearchCriteria.Op.EQ);
+        TypeIdSearch.and("syncObjId", TypeIdSearch.entity().getSyncObjId(), SearchCriteria.Op.EQ);
+        TypeIdSearch.done();
+	}
+	
+	@Override
+	public void ensureQueue(String syncObjType, long syncObjId) {
+		Date dt = DateUtil.currentGMTTime();
+        String sql = "INSERT IGNORE INTO sync_queue(sync_objtype, sync_objid, created, last_updated)" +
+                " values(?, ?, ?, ?)";
+		
+        Transaction txn = Transaction.currentTxn();
+        PreparedStatement pstmt = null;
+        try {
+            pstmt = txn.prepareAutoCloseStatement(sql);
+            pstmt.setString(1, syncObjType);
+            pstmt.setLong(2, syncObjId);
+            pstmt.setString(3, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), dt));
+            pstmt.setString(4, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), dt));
+            pstmt.execute();
+        } catch (SQLException e) {
+        	s_logger.warn("Unable to create sync queue " + syncObjType + "-" + syncObjId + ":" + e.getMessage(), e);
+        } catch (Throwable e) {
+        	s_logger.warn("Unable to create sync queue " + syncObjType + "-" + syncObjId + ":" + e.getMessage(), e);
+        }
+	}
+	
+	@Override
+	public SyncQueueVO find(String syncObjType, long syncObjId) {
+    	SearchCriteria<SyncQueueVO> sc = TypeIdSearch.create();
+    	sc.setParameters("syncObjType", syncObjType);
+    	sc.setParameters("syncObjId", syncObjId);
+        return findOneBy(sc);
+	}
+
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.dao;
+
+import java.util.List;
+
+import org.apache.cloudstack.framework.jobs.impl.SyncQueueItemVO;
+
+import com.cloud.utils.db.GenericDao;
+
+public interface SyncQueueItemDao extends GenericDao<SyncQueueItemVO, Long> {
+	public SyncQueueItemVO getNextQueueItem(long queueId);
+	public List<SyncQueueItemVO> getNextQueueItems(int maxItems);
+	public List<SyncQueueItemVO> getActiveQueueItems(Long msid, boolean exclusive);
+	public List<SyncQueueItemVO> getBlockedQueueItems(long thresholdMs, boolean exclusive);
+	public Long getQueueItemIdByContentIdAndType(long contentId, String contentType);
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java

@@ -0,0 +1,155 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.framework.jobs.dao;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.framework.jobs.impl.SyncQueueItemVO;
+
+import com.cloud.utils.DateUtil;
+import com.cloud.utils.db.DB;
+import com.cloud.utils.db.Filter;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.GenericSearchBuilder;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import com.cloud.utils.db.SearchCriteria.Op;
+import com.cloud.utils.db.Transaction;
+
+@DB
+public class SyncQueueItemDaoImpl extends GenericDaoBase<SyncQueueItemVO, Long> implements SyncQueueItemDao {
+    private static final Logger s_logger = Logger.getLogger(SyncQueueItemDaoImpl.class);
+    final GenericSearchBuilder<SyncQueueItemVO, Long> queueIdSearch;
+    
+    public SyncQueueItemDaoImpl() {
+        super();
+        queueIdSearch = createSearchBuilder(Long.class);
+        queueIdSearch.and("contentId", queueIdSearch.entity().getContentId(), Op.EQ);
+        queueIdSearch.and("contentType", queueIdSearch.entity().getContentType(), Op.EQ);
+        queueIdSearch.selectField(queueIdSearch.entity().getId());
+        queueIdSearch.done();
+    }
+
+	@Override
+	public SyncQueueItemVO getNextQueueItem(long queueId) {
+		
+		SearchBuilder<SyncQueueItemVO> sb = createSearchBuilder();
+        sb.and("queueId", sb.entity().getQueueId(), SearchCriteria.Op.EQ);
+        sb.and("lastProcessNumber", sb.entity().getLastProcessNumber(),	SearchCriteria.Op.NULL);
+        sb.done();
+        
+    	SearchCriteria<SyncQueueItemVO> sc = sb.create();
+    	sc.setParameters("queueId", queueId);
+    	
+    	Filter filter = new Filter(SyncQueueItemVO.class, "created", true, 0L, 1L);
+        List<SyncQueueItemVO> l = listBy(sc, filter);
+        if(l != null && l.size() > 0)
+        	return l.get(0);
+    	
+		return null;
+	}
+
+	@Override
+	public List<SyncQueueItemVO> getNextQueueItems(int maxItems) {
+		List<SyncQueueItemVO> l = new ArrayList<SyncQueueItemVO>();
+		
+		String sql = "SELECT i.id, i.queue_id, i.content_type, i.content_id, i.created " +
+					 " FROM sync_queue AS q JOIN sync_queue_item AS i ON q.id = i.queue_id " +
+                     " WHERE i.queue_proc_number IS NULL " +
+					 " GROUP BY q.id " +
+					 " ORDER BY i.id " +
+					 " LIMIT 0, ?";
+
+        Transaction txn = Transaction.currentTxn();
+        PreparedStatement pstmt = null;
+        try {
+            pstmt = txn.prepareAutoCloseStatement(sql);
+            pstmt.setInt(1, maxItems);
+            ResultSet rs = pstmt.executeQuery();
+            while(rs.next()) {
+            	SyncQueueItemVO item = new SyncQueueItemVO();
+            	item.setId(rs.getLong(1));
+            	item.setQueueId(rs.getLong(2));
+            	item.setContentType(rs.getString(3));
+            	item.setContentId(rs.getLong(4));
+            	item.setCreated(DateUtil.parseDateString(TimeZone.getTimeZone("GMT"), rs.getString(5)));
+            	l.add(item);
+            }
+        } catch (SQLException e) {
+        	s_logger.error("Unexpected sql excetpion, ", e);
+        } catch (Throwable e) {
+        	s_logger.error("Unexpected excetpion, ", e);
+        }
+		return l;
+	}
+	
+	@Override
+	public List<SyncQueueItemVO> getActiveQueueItems(Long msid, boolean exclusive) {
+		SearchBuilder<SyncQueueItemVO> sb = createSearchBuilder();
+        sb.and("lastProcessMsid", sb.entity().getLastProcessMsid(),
+    		SearchCriteria.Op.EQ);
+        sb.done();
+        
+    	SearchCriteria<SyncQueueItemVO> sc = sb.create();
+    	sc.setParameters("lastProcessMsid", msid);
+    	
+    	Filter filter = new Filter(SyncQueueItemVO.class, "created", true, null, null);
+    	
+    	if(exclusive)
+    		return lockRows(sc, filter, true);
+        return listBy(sc, filter);
+	}
+
+    @Override
+    public List<SyncQueueItemVO> getBlockedQueueItems(long thresholdMs, boolean exclusive) {
+        Date cutTime = DateUtil.currentGMTTime();
+        
+        SearchBuilder<SyncQueueItemVO> sbItem = createSearchBuilder();
+        sbItem.and("lastProcessMsid", sbItem.entity().getLastProcessMsid(), SearchCriteria.Op.NNULL);
+        sbItem.and("lastProcessNumber", sbItem.entity().getLastProcessNumber(), SearchCriteria.Op.NNULL);
+        sbItem.and("lastProcessNumber", sbItem.entity().getLastProcessTime(), SearchCriteria.Op.NNULL);
+        sbItem.and("lastProcessTime2", sbItem.entity().getLastProcessTime(), SearchCriteria.Op.LT);
+        
+        sbItem.done();
+        
+        SearchCriteria<SyncQueueItemVO> sc = sbItem.create();
+        sc.setParameters("lastProcessTime2", new Date(cutTime.getTime() - thresholdMs));
+        
+        if(exclusive)
+            return lockRows(sc, null, true);
+        return listBy(sc, null);
+    }
+
+    @Override
+    public Long getQueueItemIdByContentIdAndType(long contentId, String contentType) {
+        SearchCriteria<Long> sc = queueIdSearch.create();
+        sc.setParameters("contentId", contentId);
+        sc.setParameters("contentType", contentType);
+        List<Long> id = customSearch(sc, null);
+
+        return id.size() == 0 ? null : id.get(0);
+    }
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobJoinMapVO.java

@@ -0,0 +1,215 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.impl;
+
+import java.util.Date;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+
+import org.apache.cloudstack.jobs.JobInfo;
+
+import com.cloud.utils.DateUtil;
+import com.cloud.utils.db.GenericDao;
+
+@Entity
+@Table(name="async_job_join_map")
+public class AsyncJobJoinMapVO {
+	@Id
+    @GeneratedValue(strategy=GenerationType.IDENTITY)
+    @Column(name="id")
+    private Long id = null;
+
+    @Column(name="job_id")
+	private long jobId;
+    
+    @Column(name="join_job_id")
+	private long joinJobId;
+
+    @Column(name="join_status")
+    @Enumerated(EnumType.ORDINAL)
+    private JobInfo.Status joinStatus;
+    
+    @Column(name="join_result", length=1024)
+    private String joinResult;
+
+    @Column(name="join_msid")
+    private long joinMsid;
+    
+    @Column(name="complete_msid")
+    private Long completeMsid;
+    
+    @Column(name="sync_source_id")
+    private Long syncSourceId;
+    
+    @Column(name="wakeup_handler")
+    private String wakeupHandler;
+
+    @Column(name="wakeup_dispatcher")
+    private String wakeupDispatcher;
+
+    @Column(name="wakeup_interval")
+    private long wakeupInterval;
+    
+    @Column(name=GenericDao.CREATED_COLUMN)
+    private Date created;
+    
+    @Column(name="last_updated")
+    @Temporal(TemporalType.TIMESTAMP)
+    private Date lastUpdated;
+
+    @Column(name="next_wakeup")
+    @Temporal(TemporalType.TIMESTAMP)
+    private Date nextWakeupTime;
+    
+    @Column(name="expiration")
+    @Temporal(TemporalType.TIMESTAMP)
+    private Date expiration;
+
+    public AsyncJobJoinMapVO() {
+    	created = DateUtil.currentGMTTime();
+    	lastUpdated = DateUtil.currentGMTTime();
+    }
+    
+	public Long getId() {
+		return id;
+	}
+
+	public void setId(Long id) {
+		this.id = id;
+	}
+
+	public long getJobId() {
+		return jobId;
+	}
+
+	public void setJobId(long jobId) {
+		this.jobId = jobId;
+	}
+
+	public long getJoinJobId() {
+		return joinJobId;
+	}
+
+	public void setJoinJobId(long joinJobId) {
+		this.joinJobId = joinJobId;
+	}
+
+    public JobInfo.Status getJoinStatus() {
+		return joinStatus;
+	}
+
+    public void setJoinStatus(JobInfo.Status joinStatus) {
+		this.joinStatus = joinStatus;
+	}
+
+	public String getJoinResult() {
+		return joinResult;
+	}
+
+	public void setJoinResult(String joinResult) {
+		this.joinResult = joinResult;
+	}
+
+	public long getJoinMsid() {
+		return joinMsid;
+	}
+
+	public void setJoinMsid(long joinMsid) {
+		this.joinMsid = joinMsid;
+	}
+
+	public Long getCompleteMsid() {
+		return completeMsid;
+	}
+
+	public void setCompleteMsid(Long completeMsid) {
+		this.completeMsid = completeMsid;
+	}
+
+	public Date getCreated() {
+		return created;
+	}
+
+	public void setCreated(Date created) {
+		this.created = created;
+	}
+
+	public Date getLastUpdated() {
+		return lastUpdated;
+	}
+
+	public void setLastUpdated(Date lastUpdated) {
+		this.lastUpdated = lastUpdated;
+	}
+
+	public Long getSyncSourceId() {
+		return syncSourceId;
+	}
+
+	public void setSyncSourceId(Long syncSourceId) {
+		this.syncSourceId = syncSourceId;
+	}
+
+	public String getWakeupHandler() {
+		return wakeupHandler;
+	}
+
+	public void setWakeupHandler(String wakeupHandler) {
+		this.wakeupHandler = wakeupHandler;
+	}
+
+	public String getWakeupDispatcher() {
+		return wakeupDispatcher;
+	}
+
+	public void setWakeupDispatcher(String wakeupDispatcher) {
+		this.wakeupDispatcher = wakeupDispatcher;
+	}
+
+	public long getWakeupInterval() {
+		return wakeupInterval;
+	}
+
+	public void setWakeupInterval(long wakeupInterval) {
+		this.wakeupInterval = wakeupInterval;
+	}
+
+	public Date getNextWakeupTime() {
+		return nextWakeupTime;
+	}
+
+	public void setNextWakeupTime(Date nextWakeupTime) {
+		this.nextWakeupTime = nextWakeupTime;
+	}
+
+	public Date getExpiration() {
+		return expiration;
+	}
+
+	public void setExpiration(Date expiration) {
+		this.expiration = expiration;
+	}
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobJournalVO.java

@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.impl;
+
+import java.util.Date;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+import org.apache.cloudstack.framework.jobs.AsyncJob;
+import org.apache.cloudstack.framework.jobs.AsyncJob.JournalType;
+
+import com.cloud.utils.DateUtil;
+import com.cloud.utils.db.GenericDao;
+
+@Entity
+@Table(name="async_job_journal")
+public class AsyncJobJournalVO {
+	@Id
+    @GeneratedValue(strategy=GenerationType.IDENTITY)
+    @Column(name="id")
+    private Long id = null;
+
+    @Column(name="job_id")
+	private long jobId;
+    
+    @Column(name="journal_type", updatable=false, nullable=false, length=32)
+    @Enumerated(value=EnumType.STRING)
+    private AsyncJob.JournalType journalType;
+    
+    @Column(name="journal_text", length=1024)
+    private String journalText;
+
+    @Column(name="journal_obj", length=1024)
+    private String journalObjJsonString;
+    
+    @Column(name=GenericDao.CREATED_COLUMN)
+    protected Date created;
+
+    public AsyncJobJournalVO() {
+    	created = DateUtil.currentGMTTime();
+    }
+    
+	public Long getId() {
+		return id;
+	}
+
+	public void setId(Long id) {
+		this.id = id;
+	}
+
+	public long getJobId() {
+		return jobId;
+	}
+
+	public void setJobId(long jobId) {
+		this.jobId = jobId;
+	}
+
+	public AsyncJob.JournalType getJournalType() {
+		return journalType;
+	}
+
+	public void setJournalType(AsyncJob.JournalType journalType) {
+		this.journalType = journalType;
+	}
+
+	public String getJournalText() {
+		return journalText;
+	}
+
+	public void setJournalText(String journalText) {
+		this.journalText = journalText;
+	}
+
+	public String getJournalObjJsonString() {
+		return journalObjJsonString;
+	}
+
+	public void setJournalObjJsonString(String journalObjJsonString) {
+		this.journalObjJsonString = journalObjJsonString;
+	}
+
+	public Date getCreated() {
+		return created;
+	}
+
+	public void setCreated(Date created) {
+		this.created = created;
+	}
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMBeanImpl.java

@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.impl;
+
+import java.util.Date;
+import java.util.TimeZone;
+
+import javax.management.StandardMBean;
+
+import org.apache.cloudstack.framework.jobs.AsyncJob;
+import org.apache.cloudstack.framework.jobs.AsyncJobMBean;
+
+import com.cloud.utils.DateUtil;
+
+public class AsyncJobMBeanImpl extends StandardMBean implements AsyncJobMBean {
+	private final AsyncJob _job;
+	
+	public AsyncJobMBeanImpl(AsyncJob job) {
+		super(AsyncJobMBean.class, false);
+		
+		_job = job;
+	}
+	
+	@Override
+    public long getAccountId() {
+		return _job.getAccountId();
+	}
+	
+	@Override
+    public long getUserId() {
+		return _job.getUserId();
+	}
+	
+	@Override
+    public String getCmd() {
+		return _job.getCmd();
+	}
+	
+	@Override
+    public String getCmdInfo() {
+		return _job.getCmdInfo();
+	}
+	
+	@Override
+    public String getStatus() {
+        switch (_job.getStatus()) {
+        case SUCCEEDED:
+			return "Completed";
+		
+        case IN_PROGRESS:
+			return "In preogress";
+			
+        case FAILED:
+			return "failed";
+			
+        case CANCELLED:
+            return "cancelled";
+		}
+		
+		return "Unknow";
+	}
+	
+	@Override
+    public int getProcessStatus() {
+		return _job.getProcessStatus();
+	}
+	
+	@Override
+    public int getResultCode() {
+		return _job.getResultCode();
+	}
+	
+	@Override
+    public String getResult() {
+		return _job.getResult();
+	}
+	
+	@Override
+    public String getInstanceType() {
+		if(_job.getInstanceType() != null)
+			return _job.getInstanceType().toString();
+		return "N/A";
+	}
+	
+	@Override
+    public String getInstanceId() {
+		if(_job.getInstanceId() != null)
+			return String.valueOf(_job.getInstanceId());
+		return "N/A";
+	}
+	
+	@Override
+    public String getInitMsid() {
+		if(_job.getInitMsid() != null) {
+			return String.valueOf(_job.getInitMsid());
+		}
+		return "N/A";
+	}
+	
+	@Override
+    public String getCreateTime() {
+		Date time = _job.getCreated();
+		if(time != null)
+			return DateUtil.getDateDisplayString(TimeZone.getDefault(), time);
+		return "N/A";
+	}
+	
+	@Override
+    public String getLastUpdateTime() {
+		Date time = _job.getLastUpdated();
+		if(time != null)
+			return DateUtil.getDateDisplayString(TimeZone.getDefault(), time);
+		return "N/A";
+	}
+	
+	@Override
+    public String getLastPollTime() {
+		Date time = _job.getLastPolled();
+	
+		if(time != null)
+			return DateUtil.getDateDisplayString(TimeZone.getDefault(), time);
+		return "N/A";
+	}
+	
+	@Override
+    public String getSyncQueueId() {
+		SyncQueueItem item = _job.getSyncSource();
+		if(item != null && item.getQueueId() != null) {
+			return String.valueOf(item.getQueueId());
+		}
+		return "N/A";
+	}
+	
+	@Override
+    public String getSyncQueueContentType() {
+		SyncQueueItem item = _job.getSyncSource();
+		if(item != null) {
+			return item.getContentType();
+		}
+		return "N/A";
+	}
+	
+	@Override
+    public String getSyncQueueContentId() {
+		SyncQueueItem item = _job.getSyncSource();
+		if(item != null && item.getContentId() != null) {
+			return String.valueOf(item.getContentId());
+		}
+		return "N/A";
+	}
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java

@@ -0,0 +1,988 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.framework.jobs.impl;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import javax.inject.Inject;
+import javax.naming.ConfigurationException;
+
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.config.ConfigDepot;
+import org.apache.cloudstack.config.ConfigKey;
+import org.apache.cloudstack.config.ConfigValue;
+import org.apache.cloudstack.config.Configurable;
+import org.apache.cloudstack.framework.jobs.AsyncJob;
+import org.apache.cloudstack.framework.jobs.AsyncJobDispatcher;
+import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext;
+import org.apache.cloudstack.framework.jobs.AsyncJobManager;
+import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao;
+import org.apache.cloudstack.framework.jobs.dao.AsyncJobJoinMapDao;
+import org.apache.cloudstack.framework.jobs.dao.AsyncJobJournalDao;
+import org.apache.cloudstack.framework.jobs.dao.SyncQueueItemDao;
+import org.apache.cloudstack.framework.messagebus.MessageBus;
+import org.apache.cloudstack.framework.messagebus.MessageDetector;
+import org.apache.cloudstack.framework.messagebus.PublishScope;
+import org.apache.cloudstack.jobs.JobInfo;
+import org.apache.cloudstack.jobs.JobInfo.Status;
+import org.apache.cloudstack.utils.identity.ManagementServerNode;
+
+import com.cloud.cluster.ClusterManagerListener;
+import com.cloud.cluster.ManagementServerHost;
+import com.cloud.utils.DateUtil;
+import com.cloud.utils.Predicate;
+import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.concurrency.NamedThreadFactory;
+import com.cloud.utils.db.DB;
+import com.cloud.utils.db.GenericDao;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.GenericSearchBuilder;
+import com.cloud.utils.db.GlobalLock;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import com.cloud.utils.db.SearchCriteria.Op;
+import com.cloud.utils.db.Transaction;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.exception.ExceptionUtil;
+import com.cloud.utils.mgmt.JmxUtil;
+
+import edu.emory.mathcs.backport.java.util.Collections;
+
+public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, ClusterManagerListener, Configurable {
+    // Advanced
+    private static final ConfigKey<Long> JobExpireMinutes = new ConfigKey<Long>(Long.class, "job.expire.minutes", "Advanced", AsyncJobManager.class, "1440",
+            "Time (in minutes) for async-jobs to be kept in system", true, null);
+    private static final ConfigKey<Long> JobCancelThresholdMinutes = new ConfigKey<Long>(Long.class, "job.cancel.threshold.minutes", "Advanced", AsyncJobManager.class,
+            "60", "Time (in minutes) for async-jobs to be forcely cancelled if it has been in process for long", true, null);
+
+    private static final Logger s_logger = Logger.getLogger(AsyncJobManagerImpl.class);
+
+    private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 3; 	// 3 seconds
+    private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC 		= 60; 	// 60 seconds
+
+    private static final int MAX_ONETIME_SCHEDULE_SIZE = 50;
+    private static final int HEARTBEAT_INTERVAL = 2000;
+    private static final int GC_INTERVAL = 10000;				// 10 seconds
+
+    @Inject
+    private SyncQueueItemDao _queueItemDao;
+    @Inject private SyncQueueManager _queueMgr;
+    @Inject private AsyncJobDao _jobDao;
+    @Inject private AsyncJobJournalDao _journalDao;
+    @Inject private AsyncJobJoinMapDao _joinMapDao;
+    @Inject private List<AsyncJobDispatcher> _jobDispatchers;
+    @Inject private MessageBus _messageBus;
+    @Inject private AsyncJobMonitor _jobMonitor;
+    @Inject
+    private ConfigDepot _configDepot;
+
+    private ConfigValue<Long> _jobExpireSeconds;						// 1 day
+    private ConfigValue<Long> _jobCancelThresholdSeconds;         	// 1 hour (for cancelling the jobs blocking other jobs)
+    
+    private volatile long _executionRunNumber = 1;
+
+    private final ScheduledExecutorService _heartbeatScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("AsyncJobMgr-Heartbeat"));
+    private ExecutorService _executor;
+
+    @Override
+    public ConfigKey<?>[] getConfigKeys() {
+        return new ConfigKey<?>[] {JobExpireMinutes, JobCancelThresholdMinutes};
+    }
+
+    @Override
+    public AsyncJobVO getAsyncJob(long jobId) {
+        return _jobDao.findById(jobId);
+    }
+
+    @Override
+    public List<AsyncJobVO> findInstancePendingAsyncJobs(String instanceType, Long accountId) {
+        return _jobDao.findInstancePendingAsyncJobs(instanceType, accountId);
+    }
+    
+    @Override @DB
+    public AsyncJob getPseudoJob(long accountId, long userId) {
+    	AsyncJobVO job = _jobDao.findPseudoJob(Thread.currentThread().getId(), getMsid());
+    	if(job == null) {
+	    	job = new AsyncJobVO();
+            job.setAccountId(accountId);
+            job.setUserId(userId);
+	    	job.setInitMsid(getMsid());
+            job.setDispatcher(AsyncJobVO.JOB_DISPATCHER_PSEUDO);
+            job.setInstanceType(AsyncJobVO.PSEUDO_JOB_INSTANCE_TYPE);
+	    	job.setInstanceId(Thread.currentThread().getId());
+	    	_jobDao.persist(job);
+    	}
+    	return job;
+    }
+
+    @Override
+    public long submitAsyncJob(AsyncJob job) {
+        return submitAsyncJob(job, false);
+    }
+
+    @SuppressWarnings("unchecked")
+    @DB
+    public long submitAsyncJob(AsyncJob job, boolean scheduleJobExecutionInContext) {
+        @SuppressWarnings("rawtypes")
+        GenericDao dao = GenericDaoBase.getDao(job.getClass());
+        job.setInitMsid(getMsid());
+        job.setSyncSource(null);        // no sync source originally
+        dao.persist(job);
+
+        scheduleExecution(job, scheduleJobExecutionInContext);
+        if (s_logger.isDebugEnabled()) {
+            s_logger.debug("submit async job-" + job.getId() + ", details: " + job.toString());
+        }
+        return job.getId();
+    }
+
+    @SuppressWarnings("unchecked")
+	@Override @DB
+	public long submitAsyncJob(AsyncJob job, String syncObjType, long syncObjId) {
+        Transaction txt = Transaction.currentTxn();
+        try {
+        	@SuppressWarnings("rawtypes")
+			GenericDao dao = GenericDaoBase.getDao(job.getClass());
+        	
+            txt.start();
+            job.setInitMsid(getMsid());
+            dao.persist(job);
+
+            syncAsyncJobExecution(job, syncObjType, syncObjId, 1);
+            txt.commit();
+            return job.getId();
+        } catch(Exception e) {
+            String errMsg = "Unable to schedule async job for command " + job.getCmd() + ", unexpected exception.";
+            s_logger.warn(errMsg, e);
+            throw new CloudRuntimeException(errMsg);
+        }
+	}
+    
+    @Override @DB
+    public void completeAsyncJob(long jobId, Status jobStatus, int resultCode, String resultObject) {
+        if(s_logger.isDebugEnabled()) {
+            s_logger.debug("Complete async job-" + jobId + ", jobStatus: " + jobStatus +
+                    ", resultCode: " + resultCode + ", result: " + resultObject);
+        }
+
+        Transaction txn = Transaction.currentTxn();
+        try {
+            txn.start();
+            AsyncJobVO job = _jobDao.findById(jobId);
+            if(job == null) {
+                if(s_logger.isDebugEnabled()) {
+                    s_logger.debug("job-" + jobId + " no longer exists, we just log completion info here. " + jobStatus +
+                            ", resultCode: " + resultCode + ", result: " + resultObject);
+                }
+
+                txn.rollback();
+                return;
+            }
+            
+            if(job.getStatus() != JobInfo.Status.IN_PROGRESS) {
+                if(s_logger.isDebugEnabled()) {
+                    s_logger.debug("job-" + jobId + " is already completed.");
+                }
+            	
+            	txn.rollback();
+            	return;
+            }
+
+            job.setCompleteMsid(getMsid());
+            job.setStatus(jobStatus);
+            job.setResultCode(resultCode);
+
+            // reset attached object
+            job.setInstanceType(null);
+            job.setInstanceId(null);
+
+            if (resultObject != null) {
+                job.setResult(resultObject);
+            }
+
+            job.setLastUpdated(DateUtil.currentGMTTime());
+            _jobDao.update(jobId, job);
+            
+            List<Long> wakeupList = wakeupByJoinedJobCompletion(jobId);
+            _joinMapDao.disjoinAllJobs(jobId);
+            
+            txn.commit();
+
+            for(Long id : wakeupList) {
+            	// TODO, we assume that all jobs in this category is API job only
+            	AsyncJobVO jobToWakeup = _jobDao.findById(id);
+                if (jobToWakeup != null && (jobToWakeup.getPendingSignals() & AsyncJob.Constants.SIGNAL_MASK_WAKEUP) != 0)
+            	    scheduleExecution(jobToWakeup, false);
+            }
+             
+            _messageBus.publish(null, AsyncJob.Topics.JOB_STATE, PublishScope.GLOBAL, jobId);
+        } catch(Exception e) {
+            s_logger.error("Unexpected exception while completing async job-" + jobId, e);
+            txn.rollback();
+        }
+    }
+
+    @Override @DB
+    public void updateAsyncJobStatus(long jobId, int processStatus, String resultObject) {
+        if(s_logger.isDebugEnabled()) {
+            s_logger.debug("Update async-job progress, job-" + jobId + ", processStatus: " + processStatus +
+                    ", result: " + resultObject);
+        }
+
+        Transaction txt = Transaction.currentTxn();
+        try {
+            txt.start();
+            AsyncJobVO job = _jobDao.findById(jobId);
+            if(job == null) {
+                if(s_logger.isDebugEnabled()) {
+                    s_logger.debug("job-" + jobId + " no longer exists, we just log progress info here. progress status: " + processStatus);
+                }
+
+                txt.rollback();
+                return;
+            }
+
+            job.setProcessStatus(processStatus);
+            if(resultObject != null) {
+                job.setResult(resultObject);
+            }
+            job.setLastUpdated(DateUtil.currentGMTTime());
+            _jobDao.update(jobId, job);
+            txt.commit();
+        } catch(Exception e) {
+            s_logger.error("Unexpected exception while updating async job-" + jobId + " status: ", e);
+            txt.rollback();
+        }
+    }
+
+    @Override @DB
+    public void updateAsyncJobAttachment(long jobId, String instanceType, Long instanceId) {
+        if(s_logger.isDebugEnabled()) {
+            s_logger.debug("Update async-job attachment, job-" + jobId + ", instanceType: " + instanceType +
+                    ", instanceId: " + instanceId);
+        }
+
+        Transaction txt = Transaction.currentTxn();
+        try {
+            txt.start();
+
+            AsyncJobVO job = _jobDao.createForUpdate();
+            job.setInstanceType(instanceType);
+            job.setInstanceId(instanceId);
+            job.setLastUpdated(DateUtil.currentGMTTime());
+            _jobDao.update(jobId, job);
+
+            txt.commit();
+        } catch(Exception e) {
+            s_logger.error("Unexpected exception while updating async job-" + jobId + " attachment: ", e);
+            txt.rollback();
+        }
+    }
+    
+    @Override @DB
+    public void logJobJournal(long jobId, AsyncJob.JournalType journalType, String
+        journalText, String journalObjJson) {
+    	AsyncJobJournalVO journal = new AsyncJobJournalVO();
+    	journal.setJobId(jobId);
+    	journal.setJournalType(journalType);
+    	journal.setJournalText(journalText);
+    	journal.setJournalObjJsonString(journalObjJson);
+    	
+    	_journalDao.persist(journal);
+    }
+    
+    @Override @DB
+	public void joinJob(long jobId, long joinJobId) {
+    	_joinMapDao.joinJob(jobId, joinJobId, getMsid(), 0, 0, null, null, null);
+    }
+    
+    @Override @DB
+    public void joinJob(long jobId, long joinJobId, String wakeupHandler, String wakeupDispatcher,
+    		String[] wakeupTopcisOnMessageBus, long wakeupIntervalInMilliSeconds, long timeoutInMilliSeconds) {
+    	
+    	Long syncSourceId = null;
+    	AsyncJobExecutionContext context = AsyncJobExecutionContext.getCurrentExecutionContext();
+    	assert(context.getJob() != null);
+    	if(context.getJob().getSyncSource() != null) {
+    		syncSourceId = context.getJob().getSyncSource().getQueueId();
+    	}
+    	
+    	_joinMapDao.joinJob(jobId, joinJobId, getMsid(),
+    		wakeupIntervalInMilliSeconds, timeoutInMilliSeconds,
+    		syncSourceId, wakeupHandler, wakeupDispatcher);
+    }
+    
+    @Override @DB
+    public void disjoinJob(long jobId, long joinedJobId) {
+    	_joinMapDao.disjoinJob(jobId, joinedJobId);
+    }
+    
+    @Override @DB
+    public void completeJoin(long joinJobId, JobInfo.Status joinStatus, String joinResult) {
+    	//
+    	// TODO
+    	// this is a temporary solution to solve strange MySQL deadlock issue,
+    	// completeJoin() causes deadlock happens at async_job table
+        // I removed the temporary solution already.  I think my changes should fix the deadlock.
+
+/*
+    	------------------------
+    	LATEST DETECTED DEADLOCK
+    	------------------------
+    	130625 20:03:10
+    	*** (1) TRANSACTION:
+    	TRANSACTION 0 98087127, ACTIVE 0 sec, process no 1489, OS thread id 139837829175040 fetching rows, thread declared inside InnoDB 494
+    	mysql tables in use 2, locked 1
+    	LOCK WAIT 3 lock struct(s), heap size 368, 2 row lock(s), undo log entries 1
+    	MySQL thread id 28408, query id 368571321 localhost 127.0.0.1 cloud preparing
+    	UPDATE async_job SET job_pending_signals=1 WHERE id IN (SELECT job_id FROM async_job_join_map WHERE join_job_id = 9)
+    	*** (1) WAITING FOR THIS LOCK TO BE GRANTED:
+    	RECORD LOCKS space id 0 page no 1275 n bits 80 index `PRIMARY` of table `cloud`.`async_job` trx id 0 98087127 lock_mode X locks rec but not gap waiting
+    	Record lock, heap no 9 PHYSICAL RECORD: n_fields 26; compact format; info bits 0
+    	0: len 8; hex 0000000000000008; asc         ;; 1: len 6; hex 000005d8b0d8; asc       ;; 2: len 7; hex 00000009270110; asc     '  ;; 3: len 8; hex 0000000000000002; asc         ;; 4: len 8; hex 0000000000000002; asc         ;; 5: SQL NULL; 6: SQL NULL; 7: len 30; hex 6f72672e6170616368652e636c6f7564737461636b2e6170692e636f6d6d; asc org.apache.cloudstack.api.comm;...(truncated); 8: len 30; hex 7b226964223a2232222c22706879736963616c6e6574776f726b6964223a; asc {"id":"2","physicalnetworkid":;...(truncated); 9: len 4; hex 80000000; asc     ;; 10: len 4; hex 80000001; asc     ;; 11: len 4; hex 80000000; asc     ;; 12: len 4; hex 80000000; asc     ;; 13: len 30; hex 6f72672e6170616368652e636c6f7564737461636b2e6170692e72657370; asc org.apache.cloudstack.api.resp;...(truncated); 14: len 8; hex 80001a6f7bb0d0a8; asc    o{   ;; 15: len 8; hex 80001a6f7bb0d0a8; asc    o{   ;; 16: len 8; hex 8000124f06cfd5b6; asc    O    ;; 17: len 8; hex 8000124f06cfd5b6; asc    O    ;; 18: SQL NULL; 19: SQL NULL; 20: len 30; hex 66376466396532362d323139622d346338652d393231332d393766653636; asc f7df9e26-219b-4c8e-9213-97fe66;...(truncated); 21: len 30; hex 36623238306364362d663436652d343563322d383833642d333863616439; asc 6b280cd6-f46e-45c2-883d-38cad9;...(truncated); 22: SQL NULL; 23: len 21; hex 4170694173796e634a6f6244697370617463686572; asc ApiAsyncJobDispatcher;; 24: SQL NULL; 25: len 4; hex 80000000; asc     ;;
+
+    	*** (2) TRANSACTION:
+    	TRANSACTION 0 98087128, ACTIVE 0 sec, process no 1489, OS thread id 139837671909120 fetching rows, thread declared inside InnoDB 492
+    	mysql tables in use 2, locked 1
+    	3 lock struct(s), heap size 368, 2 row lock(s), undo log entries 1
+    	MySQL thread id 28406, query id 368571323 localhost 127.0.0.1 cloud preparing
+    	UPDATE async_job SET job_pending_signals=1 WHERE id IN (SELECT job_id FROM async_job_join_map WHERE join_job_id = 8)
+    	*** (2) HOLDS THE LOCK(S):
+    	RECORD LOCKS space id 0 page no 1275 n bits 80 index `PRIMARY` of table `cloud`.`async_job` trx id 0 98087128 lock_mode X locks rec but not gap
+    	Record lock, heap no 9 PHYSICAL RECORD: n_fields 26; compact format; info bits 0
+    	0: len 8; hex 0000000000000008; asc         ;; 1: len 6; hex 000005d8b0d8; asc       ;; 2: len 7; hex 00000009270110; asc     '  ;; 3: len 8; hex 0000000000000002; asc         ;; 4: len 8; hex 0000000000000002; asc         ;; 5: SQL NULL; 6: SQL NULL; 7: len 30; hex 6f72672e6170616368652e636c6f7564737461636b2e6170692e636f6d6d; asc org.apache.cloudstack.api.comm;...(truncated); 8: len 30; hex 7b226964223a2232222c22706879736963616c6e6574776f726b6964223a; asc {"id":"2","physicalnetworkid":;...(truncated); 9: len 4; hex 80000000; asc     ;; 10: len 4; hex 80000001; asc     ;; 11: len 4; hex 80000000; asc     ;; 12: len 4; hex 80000000; asc     ;; 13: len 30; hex 6f72672e6170616368652e636c6f7564737461636b2e6170692e72657370; asc org.apache.cloudstack.api.resp;...(truncated); 14: len 8; hex 80001a6f7bb0d0a8; asc    o{   ;; 15: len 8; hex 80001a6f7bb0d0a8; asc    o{   ;; 16: len 8; hex 8000124f06cfd5b6; asc    O    ;; 17: len 8; hex 8000124f06cfd5b6; asc    O    ;; 18: SQL NULL; 19: SQL NULL; 20: len 30; hex 66376466396532362d323139622d346338652d393231332d393766653636; asc f7df9e26-219b-4c8e-9213-97fe66;...(truncated); 21: len 30; hex 36623238306364362d663436652d343563322d383833642d333863616439; asc 6b280cd6-f46e-45c2-883d-38cad9;...(truncated); 22: SQL NULL; 23: len 21; hex 4170694173796e634a6f6244697370617463686572; asc ApiAsyncJobDispatcher;; 24: SQL NULL; 25: len 4; hex 80000000; asc     ;;
+
+    	*** (2) WAITING FOR THIS LOCK TO BE GRANTED:
+    	RECORD LOCKS space id 0 page no 1275 n bits 80 index `PRIMARY` of table `cloud`.`async_job` trx id 0 98087128 lock_mode X locks rec but not gap waiting
+    	Record lock, heap no 10 PHYSICAL RECORD: n_fields 26; compact format; info bits 0
+    	0: len 8; hex 0000000000000009; asc         ;; 1: len 6; hex 000005d8b0d7; asc       ;; 2: len 7; hex 00000009280110; asc     (  ;; 3: len 8; hex 0000000000000002; asc         ;; 4: len 8; hex 0000000000000002; asc         ;; 5: SQL NULL; 6: SQL NULL; 7: len 30; hex 6f72672e6170616368652e636c6f7564737461636b2e6170692e636f6d6d; asc org.apache.cloudstack.api.comm;...(truncated); 8: len 30; hex 7b226964223a2233222c22706879736963616c6e6574776f726b6964223a; asc {"id":"3","physicalnetworkid":;...(truncated); 9: len 4; hex 80000000; asc     ;; 10: len 4; hex 80000001; asc     ;; 11: len 4; hex 80000000; asc     ;; 12: len 4; hex 80000000; asc     ;; 13: len 30; hex 6f72672e6170616368652e636c6f7564737461636b2e6170692e72657370; asc org.apache.cloudstack.api.resp;...(truncated); 14: len 8; hex 80001a6f7bb0d0a8; asc    o{   ;; 15: len 8; hex 80001a6f7bb0d0a8; asc    o{   ;; 16: len 8; hex 8000124f06cfd5b6; asc    O    ;; 17: len 8; hex 8000124f06cfd5b6; asc    O    ;; 18: SQL NULL; 19: SQL NULL; 20: len 30; hex 62313065306432342d336233352d343663622d386361622d623933623562; asc b10e0d24-3b35-46cb-8cab-b93b5b;...(truncated); 21: len 30; hex 39353664383563632d383336622d346663612d623738622d646238343739; asc 956d85cc-836b-4fca-b78b-db8479;...(truncated); 22: SQL NULL; 23: len 21; hex 4170694173796e634a6f6244697370617463686572; asc ApiAsyncJobDispatcher;; 24: SQL NULL; 25: len 4; hex 80000000; asc     ;;
+
+    	*** WE ROLL BACK TRANSACTION (2)
+*/
+    	
+    			_joinMapDao.completeJoin(joinJobId, joinStatus, joinResult, getMsid());
+    }
+    
+    @Override
+    public void syncAsyncJobExecution(AsyncJob job, String syncObjType, long syncObjId, long queueSizeLimit) {
+        if(s_logger.isDebugEnabled()) {
+            s_logger.debug("Sync job-" + job.getId() + " execution on object " + syncObjType + "." + syncObjId);
+        }
+
+        SyncQueueVO queue = null;
+
+        // to deal with temporary DB exceptions like DB deadlock/Lock-wait time out cased rollbacks
+        // we retry five times until we throw an exception
+        Random random = new Random();
+
+        for(int i = 0; i < 5; i++) {
+            queue = _queueMgr.queue(syncObjType, syncObjId, SyncQueueItem.AsyncJobContentType, job.getId(), queueSizeLimit);
+            if(queue != null) {
+                break;
+            }
+
+            try {
+                Thread.sleep(1000 + random.nextInt(5000));
+            } catch (InterruptedException e) {
+            }
+        }
+
+        if (queue == null)
+            throw new CloudRuntimeException("Unable to insert queue item into database, DB is full?");
+    }
+
+    @Override
+    public AsyncJob queryJob(long jobId, boolean updatePollTime) {
+        AsyncJobVO job = _jobDao.findById(jobId);
+        
+        if (updatePollTime) {
+            job.setLastPolled(DateUtil.currentGMTTime());
+            _jobDao.update(jobId, job);
+        }
+        return job;
+    }
+
+
+    private void scheduleExecution(final AsyncJobVO job) {
+        scheduleExecution(job, false);
+    }
+
+    private void scheduleExecution(final AsyncJob job, boolean executeInContext) {
+        Runnable runnable = getExecutorRunnable(job);
+        if (executeInContext) {
+            runnable.run();
+        } else {
+            _executor.submit(runnable);
+        }
+    }
+    
+    private AsyncJobDispatcher getDispatcher(String dispatcherName) {
+        assert (dispatcherName != null && !dispatcherName.isEmpty()) : "Who's not setting the dispatcher when submitting a job?  Who am I suppose to call if you do that!";
+    	
+        for (AsyncJobDispatcher dispatcher : _jobDispatchers) {
+            if (dispatcherName.equals(dispatcher.getName()))
+                return dispatcher;
+        }
+
+        throw new CloudRuntimeException("Unable to find dispatcher name: " + dispatcherName);
+    }
+    
+    private AsyncJobDispatcher getWakeupDispatcher(AsyncJob job) {
+    	if(_jobDispatchers != null) {
+    		List<AsyncJobJoinMapVO> joinRecords = _joinMapDao.listJoinRecords(job.getId());
+    		if(joinRecords.size() > 0) {
+    			AsyncJobJoinMapVO joinRecord = joinRecords.get(0);
+	    		for(AsyncJobDispatcher dispatcher : _jobDispatchers) {
+	    			if(dispatcher.getName().equals(joinRecord.getWakeupDispatcher()))
+	    				return dispatcher;
+	    		}
+    		} else {
+    			s_logger.warn("job-" + job.getId() + " is scheduled for wakeup run, but there is no joining info anymore");
+    		}
+    	}
+    	return null;
+    }
+    
+    private long getJobRunNumber() {
+    	synchronized(this) {
+    		return _executionRunNumber++;
+    	}
+    }
+    
+    private Runnable getExecutorRunnable(final AsyncJob job) {
+        return new Runnable() {
+            @Override
+            public void run() {
+            	Transaction txn = null;
+            	long runNumber = getJobRunNumber();
+            	
+            	try {
+            		//
+            		// setup execution environment
+            		//
+            		txn = Transaction.open(Transaction.CLOUD_DB);
+            		
+                    try {
+                        JmxUtil.registerMBean("AsyncJobManager", "Active Job " + job.getId(), new AsyncJobMBeanImpl(job));
+                    } catch(Exception e) {
+                		// Due to co-existence of normal-dispatched-job/wakeup-dispatched-job, MBean register() call
+                		// is expected to fail under situations
+                    	if(s_logger.isTraceEnabled())
+                    		s_logger.trace("Unable to register active job " + job.getId() + " to JMX monitoring due to exception " + ExceptionUtil.toString(e));
+                    }
+                    
+                    _jobMonitor.registerActiveTask(runNumber, job.getId());
+                    AsyncJobExecutionContext.setCurrentExecutionContext(new AsyncJobExecutionContext(job));
+                    
+                    // execute the job
+                    if(s_logger.isDebugEnabled()) {
+                        s_logger.debug("Executing " + job);
+                    }
+
+                    if ((getAndResetPendingSignals(job) & AsyncJob.Constants.SIGNAL_MASK_WAKEUP) != 0) {
+                    	AsyncJobDispatcher jobDispatcher = getWakeupDispatcher(job);
+                    	if(jobDispatcher != null) {
+                    		jobDispatcher.runJob(job);
+                    	} else {
+                    		s_logger.error("Unable to find a wakeup dispatcher from the joined job: " + job);
+                    	}
+                    } else {
+	                    AsyncJobDispatcher jobDispatcher = getDispatcher(job.getDispatcher());
+	                    if(jobDispatcher != null) {
+	                    	jobDispatcher.runJob(job);
+	                    } else {
+	                    	s_logger.error("Unable to find job dispatcher, job will be cancelled");
+	                        completeAsyncJob(job.getId(), JobInfo.Status.FAILED, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), null);
+	                    }
+                    }
+                    
+                    if (s_logger.isDebugEnabled()) {
+                        s_logger.debug("Done executing " + job.getCmd() + " for job-" + job.getId());
+                    }
+                   
+            	} catch (Throwable e) {
+            		s_logger.error("Unexpected exception", e);
+                    completeAsyncJob(job.getId(), JobInfo.Status.FAILED, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), null);
+            	} finally {
+            		// guard final clause as well
+                    try {
+                    	AsyncJobVO jobToUpdate = _jobDao.findById(job.getId());
+                    	jobToUpdate.setExecutingMsid(null);
+                    	_jobDao.update(job.getId(), jobToUpdate);
+                    	
+                    	if (job.getSyncSource() != null) {
+                            _queueMgr.purgeItem(job.getSyncSource().getId());
+                            checkQueue(job.getSyncSource().getQueueId());
+                        }
+
+                    	try {
+                    		JmxUtil.unregisterMBean("AsyncJobManager", "Active Job " + job.getId());
+                    	} catch(Exception e) {
+                    		// Due to co-existence of normal-dispatched-job/wakeup-dispatched-job, MBean unregister() call
+                    		// is expected to fail under situations
+                    		if(s_logger.isTraceEnabled())
+                    			s_logger.trace("Unable to unregister job " + job.getId() + " to JMX monitoring due to exception " + ExceptionUtil.toString(e));
+                    	}
+                    	
+	                    if(txn != null)
+	                    	txn.close();
+	                    
+                    	//
+                    	// clean execution environment
+                    	//
+                        AsyncJobExecutionContext.unregister();
+                        _jobMonitor.unregisterActiveTask(runNumber);
+	                    
+                    } catch(Throwable e) {
+                		s_logger.error("Double exception", e);
+                    }
+            	}
+            }
+        };
+    }
+    
+    private int getAndResetPendingSignals(AsyncJob job) {
+    	int signals = job.getPendingSignals();
+    	if(signals != 0) {
+	    	AsyncJobVO jobRecord = _jobDao.findById(job.getId());
+	    	jobRecord.setPendingSignals(0);
+	    	_jobDao.update(job.getId(), jobRecord);
+    	}
+    	return signals;
+    }
+    
+    private void executeQueueItem(SyncQueueItemVO item, boolean fromPreviousSession) {
+        AsyncJobVO job = _jobDao.findById(item.getContentId());
+        if (job != null) {
+            if(s_logger.isDebugEnabled()) {
+                s_logger.debug("Schedule queued job-" + job.getId());
+            }
+
+            job.setSyncSource(item);
+            
+            job.setExecutingMsid(getMsid());
+            _jobDao.update(job.getId(), job);
+
+            try {
+                scheduleExecution(job);
+            } catch(RejectedExecutionException e) {
+                s_logger.warn("Execution for job-" + job.getId() + " is rejected, return it to the queue for next turn");
+                _queueMgr.returnItem(item.getId());
+                
+            	job.setExecutingMsid(null);
+            	_jobDao.update(job.getId(), job);
+            }
+
+        } else {
+            if(s_logger.isDebugEnabled()) {
+                s_logger.debug("Unable to find related job for queue item: " + item.toString());
+            }
+
+            _queueMgr.purgeItem(item.getId());
+        }
+    }
+
+    @Override
+    public void releaseSyncSource() {
+    	AsyncJobExecutionContext executionContext = AsyncJobExecutionContext.getCurrentExecutionContext();
+    	assert(executionContext != null);
+    	
+    	if(executionContext.getSyncSource() != null) {
+            if(s_logger.isDebugEnabled()) {
+                s_logger.debug("Release sync source for job-" + executionContext.getJob().getId() + " sync source: "
+                        + executionContext.getSyncSource().getContentType() + "-"
+                        + executionContext.getSyncSource().getContentId());
+            }
+
+            _queueMgr.purgeItem(executionContext.getSyncSource().getId());
+            checkQueue(executionContext.getSyncSource().getQueueId());
+    	}
+    }
+    
+    @Override
+    public boolean waitAndCheck(AsyncJob job, String[] wakeupTopicsOnMessageBus, long checkIntervalInMilliSeconds,
+        long timeoutInMiliseconds, Predicate predicate) {
+    	
+    	MessageDetector msgDetector = new MessageDetector();
+        String[] topics = Arrays.copyOf(wakeupTopicsOnMessageBus, wakeupTopicsOnMessageBus.length + 1);
+        topics[topics.length - 1] = AsyncJob.Topics.JOB_STATE;
+
+        msgDetector.open(_messageBus, topics);
+    	try {
+    		long startTick = System.currentTimeMillis();
+    		while(System.currentTimeMillis() - startTick < timeoutInMiliseconds) {
+    			msgDetector.waitAny(checkIntervalInMilliSeconds);
+                job = _jobDao.findById(job.getId());
+                if (job.getStatus().done()) {
+                    return true;
+                }
+
+                if (predicate.checkCondition()) {
+    				return true;
+                }
+    		}
+    	} finally {
+    		msgDetector.close();
+    	}
+    	
+    	return false;
+    }
+
+    private void checkQueue(long queueId) {
+        while(true) {
+            try {
+                SyncQueueItemVO item = _queueMgr.dequeueFromOne(queueId, getMsid());
+                if(item != null) {
+                    if(s_logger.isDebugEnabled()) {
+                        s_logger.debug("Executing sync queue item: " + item.toString());
+                    }
+
+                    executeQueueItem(item, false);
+                } else {
+                    break;
+                }
+            } catch(Throwable e) {
+                s_logger.error("Unexpected exception when kicking sync queue-" + queueId, e);
+                break;
+            }
+        }
+    }
+
+    private Runnable getHeartbeatTask() {
+        return new Runnable() {
+            @Override
+            public void run() {
+            	Transaction txn = Transaction.open("AsyncJobManagerImpl.getHeartbeatTask");
+                try {
+                    List<SyncQueueItemVO> l = _queueMgr.dequeueFromAny(getMsid(), MAX_ONETIME_SCHEDULE_SIZE);
+                    if(l != null && l.size() > 0) {
+                        for(SyncQueueItemVO item: l) {
+                            if(s_logger.isDebugEnabled()) {
+                                s_logger.debug("Execute sync-queue item: " + item.toString());
+                            }
+                            executeQueueItem(item, false);
+                        }
+                    }
+              
+                    List<Long> standaloneWakeupJobs = wakeupScan();
+		                    for(Long jobId : standaloneWakeupJobs) {
+		                    	// TODO, we assume that all jobs in this category is API job only
+		                    	AsyncJobVO job = _jobDao.findById(jobId);
+		                        if (job != null && (job.getPendingSignals() & AsyncJob.Constants.SIGNAL_MASK_WAKEUP) != 0)
+		                    	    scheduleExecution(job, false);
+		                    }
+                } catch(Throwable e) {
+                    s_logger.error("Unexpected exception when trying to execute queue item, ", e);
+                } finally {
+                	try {
+                		txn.close();
+                	} catch(Throwable e) {
+                        s_logger.error("Unexpected exception", e);
+                	}
+                }
+            }
+        };
+    }
+
+    @DB
+    private Runnable getGCTask() {
+        return new Runnable() {
+            @Override
+            public void run() {
+                GlobalLock scanLock = GlobalLock.getInternLock("AsyncJobManagerGC");
+                try {
+                    if(scanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) {
+                        try {
+                            reallyRun();
+                        } finally {
+                            scanLock.unlock();
+                        }
+                    }
+                } finally {
+                    scanLock.releaseRef();
+                }
+            }
+
+            public void reallyRun() {
+                try {
+                    s_logger.trace("Begin cleanup expired async-jobs");
+
+                    Date cutTime = new Date(DateUtil.currentGMTTime().getTime() - _jobExpireSeconds.value() * 1000);
+
+                    // limit to 100 jobs per turn, this gives cleanup throughput as 600 jobs per minute
+                    // hopefully this will be fast enough to balance potential growth of job table
+                    //1) Expire unfinished jobs that weren't processed yet
+                    List<AsyncJobVO> l = _jobDao.getExpiredUnfinishedJobs(cutTime, 100);
+                        for(AsyncJobVO job : l) {
+                    	s_logger.trace("Expunging unfinished job " + job);
+                            expungeAsyncJob(job);
+                        }
+                    
+                    //2) Expunge finished jobs
+                    List<AsyncJobVO> completedJobs = _jobDao.getExpiredCompletedJobs(cutTime, 100);
+                    for(AsyncJobVO job : completedJobs) {
+                    	s_logger.trace("Expunging completed job " + job);
+                        expungeAsyncJob(job);
+                    }
+
+                    // forcefully cancel blocking queue items if they've been staying there for too long
+                    List<SyncQueueItemVO> blockItems = _queueMgr.getBlockedQueueItems(_jobCancelThresholdSeconds.value()
+                            * 1000, false);
+                    if(blockItems != null && blockItems.size() > 0) {
+                        for(SyncQueueItemVO item : blockItems) {
+                            if(item.getContentType().equalsIgnoreCase(SyncQueueItem.AsyncJobContentType)) {
+                                completeAsyncJob(item.getContentId(), JobInfo.Status.FAILED, 0, "Job is cancelled as it has been blocking others for too long");
+                            }
+
+                            // purge the item and resume queue processing
+                            _queueMgr.purgeItem(item.getId());
+                        }
+                    }
+
+                    s_logger.trace("End cleanup expired async-jobs");
+                } catch(Throwable e) {
+                    s_logger.error("Unexpected exception when trying to execute queue item, ", e);
+                }
+            }
+        };
+    }
+
+    @DB
+    protected void expungeAsyncJob(AsyncJobVO job) {
+        Transaction txn = Transaction.currentTxn();
+        txn.start();
+        _jobDao.expunge(job.getId());
+        //purge corresponding sync queue item
+        _queueMgr.purgeAsyncJobQueueItemId(job.getId());
+        txn.commit();
+    }
+
+    private long getMsid() {
+        return ManagementServerNode.getManagementServerId();
+    }
+
+    private void cleanupPendingJobs(List<SyncQueueItemVO> l) {
+        for (SyncQueueItemVO item : l) {
+            if (s_logger.isInfoEnabled()) {
+                s_logger.info("Discard left-over queue item: " + item.toString());
+            }
+
+            String contentType = item.getContentType();
+            if (contentType != null && contentType.equalsIgnoreCase(SyncQueueItem.AsyncJobContentType)) {
+                Long jobId = item.getContentId();
+                if (jobId != null) {
+                    s_logger.warn("Mark job as failed as its correspoding queue-item has been discarded. job id: " + jobId);
+                    completeAsyncJob(jobId, JobInfo.Status.FAILED, 0, "Execution was cancelled because of server shutdown");
+                }
+            }
+            _queueMgr.purgeItem(item.getId());
+        }
+    }
+
+    @DB
+    protected List<Long> wakeupByJoinedJobCompletion(long joinedJobId) {
+        SearchCriteria<Long> joinJobSC = JoinJobSearch.create("joinJobId", joinedJobId);
+
+        List<Long> result = _joinMapDao.customSearch(joinJobSC, null);
+        if (result.size() > 0) {
+            Collections.sort(result);
+            Long[] ids = result.toArray(new Long[result.size()]);
+
+            SearchCriteria<AsyncJobVO> jobsSC = JobIdsSearch.create("ids", ids);
+            SearchCriteria<SyncQueueItemVO> queueItemsSC = QueueJobIdsSearch.create("contentIds", ids);
+
+            Transaction txn = Transaction.currentTxn();
+            txn.start();
+            AsyncJobVO job = _jobDao.createForUpdate();
+            job.setPendingSignals(AsyncJob.Constants.SIGNAL_MASK_WAKEUP);
+            _jobDao.update(job, jobsSC);
+
+            SyncQueueItemVO item = _queueItemDao.createForUpdate();
+            item.setLastProcessNumber(null);
+            item.setLastProcessMsid(null);
+            _queueItemDao.update(item, queueItemsSC);
+            txn.commit();
+        }
+        return _joinMapDao.findJobsToWake(joinedJobId);
+    }
+
+    @DB
+    protected List<Long> wakeupScan() {
+        Date cutDate = DateUtil.currentGMTTime();
+        Transaction txn = Transaction.currentTxn();
+
+        SearchCriteria<Long> sc = JoinJobTimeSearch.create();
+        sc.setParameters("beginTime", cutDate);
+        sc.setParameters("endTime", cutDate);
+
+        List<Long> result = _joinMapDao.customSearch(sc, null);
+
+        txn.start();
+        if (result.size() > 0) {
+            Collections.sort(result);
+            Long[] ids = result.toArray(new Long[result.size()]);
+
+            AsyncJobVO job = _jobDao.createForUpdate();
+            job.setPendingSignals(AsyncJob.Constants.SIGNAL_MASK_WAKEUP);
+
+            SearchCriteria<AsyncJobVO> sc2 = JobIdsSearch.create("ids", ids);
+            SearchCriteria<SyncQueueItemVO> queueItemsSC = QueueJobIdsSearch.create("contentIds", ids);
+
+            _jobDao.update(job, sc2);
+
+            SyncQueueItemVO item = _queueItemDao.createForUpdate();
+            item.setLastProcessNumber(null);
+            item.setLastProcessMsid(null);
+            _queueItemDao.update(item, queueItemsSC);
+        }
+
+        List<Long> wakupIds = _joinMapDao.findJobsToWakeBetween(cutDate);
+        txn.commit();
+
+        return wakupIds;
+    }
+
+    @Override
+    public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
+        _jobExpireSeconds = _configDepot.get(JobExpireMinutes).setMultiplier(60);
+        _jobCancelThresholdSeconds = _configDepot.get(JobCancelThresholdMinutes).setMultiplier(60);
+
+        try {
+            final File dbPropsFile = PropertiesUtil.findConfigFile("db.properties");
+            final Properties dbProps = new Properties();
+            dbProps.load(new FileInputStream(dbPropsFile));
+
+            final int cloudMaxActive = Integer.parseInt(dbProps.getProperty("db.cloud.maxActive"));
+
+            int poolSize = (cloudMaxActive * 2) / 3;
+
+            s_logger.info("Start AsyncJobManager thread pool in size " + poolSize);
+            _executor = Executors.newFixedThreadPool(poolSize, new NamedThreadFactory(AsyncJobManager.JOB_POOL_THREAD_PREFIX));
+        } catch (final Exception e) {
+            throw new ConfigurationException("Unable to load db.properties to configure AsyncJobManagerImpl");
+        }
+
+        JoinJobSearch = _joinMapDao.createSearchBuilder(Long.class);
+        JoinJobSearch.and(JoinJobSearch.entity().getJoinJobId(), Op.EQ, "joinJobId");
+        JoinJobSearch.selectField(JoinJobSearch.entity().getJobId());
+        JoinJobSearch.done();
+
+        JoinJobTimeSearch = _joinMapDao.createSearchBuilder(Long.class);
+        JoinJobTimeSearch.and(JoinJobTimeSearch.entity().getNextWakeupTime(), Op.LT, "beginTime");
+        JoinJobTimeSearch.and(JoinJobTimeSearch.entity().getExpiration(), Op.GT, "endTime");
+        JoinJobTimeSearch.selectField(JoinJobTimeSearch.entity().getJobId()).done();
+
+        JobIdsSearch = _jobDao.createSearchBuilder();
+        JobIdsSearch.and(JobIdsSearch.entity().getId(), Op.IN, "ids").done();
+
+        QueueJobIdsSearch = _queueItemDao.createSearchBuilder();
+        QueueJobIdsSearch.and(QueueJobIdsSearch.entity().getContentId(), Op.IN, "contentIds").done();
+
+        JoinJobIdsSearch = _joinMapDao.createSearchBuilder(Long.class);
+        JoinJobIdsSearch.selectField(JoinJobIdsSearch.entity().getJobId());
+        JoinJobIdsSearch.and(JoinJobIdsSearch.entity().getJoinJobId(), Op.EQ, "joinJobId");
+        JoinJobIdsSearch.and(JoinJobIdsSearch.entity().getJobId(), Op.NIN, "jobIds");
+        JoinJobIdsSearch.done();
+
+        ContentIdsSearch = _queueItemDao.createSearchBuilder(Long.class);
+        ContentIdsSearch.selectField(ContentIdsSearch.entity().getContentId()).done();
+
+        AsyncJobExecutionContext.init(this, _joinMapDao);
+        OutcomeImpl.init(this);
+
+        return true;
+    }
+
+    @Override
+    public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
+    }
+
+    @Override
+    public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
+        for (ManagementServerHost msHost : nodeList) {
+            Transaction txn = Transaction.open(Transaction.CLOUD_DB);
+            try {
+                txn.start();
+                List<SyncQueueItemVO> items = _queueMgr.getActiveQueueItems(msHost.getId(), true);
+                cleanupPendingJobs(items);
+                _jobDao.resetJobProcess(msHost.getId(), ApiErrorCode.INTERNAL_ERROR.getHttpCode(), "job cancelled because of management server restart");
+                txn.commit();
+            } catch(Throwable e) {
+                s_logger.warn("Unexpected exception ", e);
+            } finally {
+                txn.close();
+            }
+        }
+    }
+
+    @Override
+    public void onManagementNodeIsolated() {
+    }
+
+    @Override
+    public boolean start() {
+        try {
+        	_jobDao.cleanupPseduoJobs(getMsid());
+        	
+            List<SyncQueueItemVO> l = _queueMgr.getActiveQueueItems(getMsid(), false);
+            cleanupPendingJobs(l);
+            _jobDao.resetJobProcess(getMsid(), ApiErrorCode.INTERNAL_ERROR.getHttpCode(), "job cancelled because of management server restart");
+        } catch(Throwable e) {
+            s_logger.error("Unexpected exception " + e.getMessage(), e);
+        }
+
+        _heartbeatScheduler.scheduleAtFixedRate(getHeartbeatTask(), HEARTBEAT_INTERVAL, HEARTBEAT_INTERVAL, TimeUnit.MILLISECONDS);
+        _heartbeatScheduler.scheduleAtFixedRate(getGCTask(), GC_INTERVAL, GC_INTERVAL, TimeUnit.MILLISECONDS);
+
+        return true;
+    }
+
+    @Override
+    public boolean stop() {
+        _heartbeatScheduler.shutdown();
+        _executor.shutdown();
+        return true;
+    }
+
+    private GenericSearchBuilder<SyncQueueItemVO, Long> ContentIdsSearch;
+    private GenericSearchBuilder<AsyncJobJoinMapVO, Long> JoinJobSearch;
+    private SearchBuilder<AsyncJobVO> JobIdsSearch;
+    private SearchBuilder<SyncQueueItemVO> QueueJobIdsSearch;
+    private GenericSearchBuilder<AsyncJobJoinMapVO, Long> JoinJobIdsSearch;
+    private GenericSearchBuilder<AsyncJobJoinMapVO, Long> JoinJobTimeSearch;
+
+    protected AsyncJobManagerImpl() {
+
+    }
+
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java

@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.impl;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import javax.inject.Inject;
+import javax.naming.ConfigurationException;
+
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.framework.jobs.AsyncJob;
+import org.apache.cloudstack.framework.jobs.AsyncJobManager;
+import org.apache.cloudstack.framework.messagebus.MessageBus;
+import org.apache.cloudstack.framework.messagebus.MessageDispatcher;
+import org.apache.cloudstack.framework.messagebus.MessageHandler;
+
+import com.cloud.utils.component.ManagerBase;
+
+public class AsyncJobMonitor extends ManagerBase {
+    public static final Logger s_logger = Logger.getLogger(AsyncJobMonitor.class);
+    
+    @Inject private MessageBus _messageBus;
+	
+	private final Map<Long, ActiveTaskRecord> _activeTasks = new HashMap<Long, ActiveTaskRecord>();
+	private final Timer _timer = new Timer();
+	
+	private volatile int _activePoolThreads = 0;
+	private volatile int _activeInplaceThreads = 0;
+	
+	// configuration
+	private long _inactivityCheckIntervalMs = 60000;
+	private long _inactivityWarningThresholdMs = 90000;
+	
+	public AsyncJobMonitor() {
+	}
+	
+	public long getInactivityCheckIntervalMs() {
+		return _inactivityCheckIntervalMs;
+	}
+	
+	public void setInactivityCheckIntervalMs(long intervalMs) {
+		_inactivityCheckIntervalMs = intervalMs;
+	}
+	
+	public long getInactivityWarningThresholdMs() {
+		return _inactivityWarningThresholdMs;
+	}
+	
+	public void setInactivityWarningThresholdMs(long thresholdMs) {
+		_inactivityWarningThresholdMs = thresholdMs;
+	}
+	
+    @MessageHandler(topic = AsyncJob.Topics.JOB_HEARTBEAT)
+	public void onJobHeartbeatNotify(String subject, String senderAddress, Object args) {
+		if(args != null && args instanceof Long) {
+			synchronized(this) {
+				ActiveTaskRecord record = _activeTasks.get(args);
+				if(record != null) {
+					record.updateJobHeartbeatTick();
+				}
+			}
+		}
+	}
+	
+	private void heartbeat() {
+		synchronized(this) {
+			for(Map.Entry<Long, ActiveTaskRecord> entry : _activeTasks.entrySet()) {
+				if(entry.getValue().millisSinceLastJobHeartbeat() > _inactivityWarningThresholdMs) {
+					s_logger.warn("Task (job-" + entry.getValue().getJobId() + ") has been pending for "
+						+ entry.getValue().millisSinceLastJobHeartbeat()/1000 + " seconds");
+				}
+			}
+		}
+	}
+	
+	@Override
+	public boolean configure(String name, Map<String, Object> params)
+			throws ConfigurationException {
+		
+        _messageBus.subscribe(AsyncJob.Topics.JOB_HEARTBEAT, MessageDispatcher.getDispatcher(this));
+		_timer.scheduleAtFixedRate(new TimerTask() {
+
+			@Override
+			public void run() {
+				heartbeat();
+			}
+			
+		}, _inactivityCheckIntervalMs, _inactivityCheckIntervalMs);
+		return true;
+	}
+	
+	public void registerActiveTask(long runNumber, long jobId) {
+		synchronized(this) {
+			s_logger.info("Add job-" + jobId + " into job monitoring");
+			
+			assert(_activeTasks.get(runNumber) == null);
+			
+			long threadId = Thread.currentThread().getId();
+            boolean fromPoolThread = Thread.currentThread().getName().contains(AsyncJobManager.JOB_POOL_THREAD_PREFIX);
+            ActiveTaskRecord record = new ActiveTaskRecord(jobId, threadId, fromPoolThread);
+			_activeTasks.put(runNumber, record);
+			if(fromPoolThread)
+				_activePoolThreads++;
+			else
+				_activeInplaceThreads++;
+		}
+	}
+	
+	public void unregisterActiveTask(long runNumber) {
+		synchronized(this) {
+			ActiveTaskRecord record = _activeTasks.get(runNumber);
+			assert(record != null);
+			if(record != null) {
+				s_logger.info("Remove job-" + record.getJobId() + " from job monitoring");
+				
+				if(record.isPoolThread())
+					_activePoolThreads--;
+				else
+					_activeInplaceThreads--;
+				
+				_activeTasks.remove(runNumber);
+			}
+		}
+	}
+	
+	public int getActivePoolThreads() {
+		return _activePoolThreads;
+	}
+	
+	public int getActiveInplaceThread() {
+		return _activeInplaceThreads;
+	}
+	
+	private static class ActiveTaskRecord {
+		long _jobId;
+		long _threadId;
+		boolean _fromPoolThread;
+		long _jobLastHeartbeatTick;
+		
+		public ActiveTaskRecord(long jobId, long threadId, boolean fromPoolThread) {
+			_threadId = threadId;
+			_jobId = jobId;
+			_fromPoolThread = fromPoolThread;
+			_jobLastHeartbeatTick = System.currentTimeMillis();
+		}
+		
+		public long getThreadId() {
+			return _threadId;
+		}
+		
+		public long getJobId() {
+			return _jobId;
+		}
+		
+		public boolean isPoolThread() {
+			return _fromPoolThread;
+		}
+		
+		public void updateJobHeartbeatTick() {
+			_jobLastHeartbeatTick = System.currentTimeMillis();
+		}
+		
+		public long millisSinceLastJobHeartbeat() {
+			return System.currentTimeMillis() - _jobLastHeartbeatTick;
+		}
+	}
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobVO.java

@@ -0,0 +1,398 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.impl;
+
+import java.util.Date;
+import java.util.UUID;
+
+import javax.persistence.Column;
+import javax.persistence.DiscriminatorColumn;
+import javax.persistence.DiscriminatorType;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Inheritance;
+import javax.persistence.InheritanceType;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+import javax.persistence.Transient;
+
+import org.apache.cloudstack.framework.jobs.AsyncJob;
+import org.apache.cloudstack.jobs.JobInfo;
+
+import com.cloud.utils.UuidUtils;
+import com.cloud.utils.db.GenericDao;
+
+@Entity
+@Table(name="async_job")
+@Inheritance(strategy=InheritanceType.JOINED)
+@DiscriminatorColumn(name="job_type", discriminatorType=DiscriminatorType.STRING, length=32)
+public class AsyncJobVO implements AsyncJob, JobInfo {
+
+    public static final String JOB_DISPATCHER_PSEUDO = "pseudoJobDispatcher";
+    public static final String PSEUDO_JOB_INSTANCE_TYPE = "Thread";
+	
+	@Id
+    @GeneratedValue(strategy=GenerationType.IDENTITY)
+    @Column(name="id")
+    private long id;
+	
+    @Column(name="job_type", length=32)
+    protected String type;
+    
+    @Column(name="job_dispatcher", length=64)
+    protected String dispatcher;
+    
+    @Column(name="job_pending_signals")
+    protected int pendingSignals;
+    
+    @Column(name="user_id")
+    private long userId;
+    
+    @Column(name="account_id")
+    private long accountId;
+    
+	@Column(name="job_cmd")
+    private String cmd;
+
+	@Column(name="job_cmd_ver")
+    private int cmdVersion;
+	
+    @Column(name = "related")
+    private String related;
+
+    @Column(name="job_cmd_info", length=65535)
+    private String cmdInfo;
+  
+    @Column(name="job_status")
+    @Enumerated(value = EnumType.ORDINAL)
+    private Status status;
+    
+    @Column(name="job_process_status")
+    private int processStatus;
+    
+    @Column(name="job_result_code")
+    private int resultCode;
+    
+    @Column(name="job_result", length=65535)
+    private String result;
+    
+    @Column(name="instance_type", length=64)
+    private String instanceType;
+    
+	@Column(name="instance_id", length=64)
+    private Long instanceId;
+    
+    @Column(name="job_init_msid")
+    private Long initMsid;
+
+    @Column(name="job_complete_msid")
+    private Long completeMsid;
+    
+    @Column(name="job_executing_msid")
+    private Long executingMsid;
+
+    @Column(name=GenericDao.CREATED_COLUMN)
+    private Date created;
+    
+    @Column(name="last_updated")
+    @Temporal(TemporalType.TIMESTAMP)
+    private Date lastUpdated;
+    
+    @Column(name="last_polled")
+    @Temporal(TemporalType.TIMESTAMP)
+    private Date lastPolled;
+    
+    @Column(name=GenericDao.REMOVED_COLUMN)
+    private Date removed;
+    
+    @Column(name="uuid")
+    private String uuid;
+
+    @Transient
+    private SyncQueueItem syncSource = null;
+
+    public AsyncJobVO() {
+        uuid = UUID.randomUUID().toString();
+        related = UUID.randomUUID().toString();
+        status = Status.IN_PROGRESS;
+    }
+
+    public AsyncJobVO(String related, long userId, long accountId, String cmd, String cmdInfo, Long instanceId, String instanceType) {
+		this.userId = userId;
+		this.accountId = accountId;
+		this.cmd = cmd;
+		this.cmdInfo = cmdInfo;
+        uuid = UUID.randomUUID().toString();
+        this.related = related;
+	    this.instanceId = instanceId;
+	    this.instanceType = instanceType;
+        status = Status.IN_PROGRESS;
+    }
+
+    @Override
+    public long getId() {
+		return id;
+	}
+
+    public void setId(long id) {
+		this.id = id;
+	}
+	
+    @Override
+    public String getShortUuid() {
+        return UuidUtils.first(uuid);
+    }
+
+    public void setRelated(String related) {
+        this.related = related;
+    }
+
+    @Override
+    public String getRelated() {
+        return related;
+    }
+
+	@Override
+	public String getType() {
+		return type;
+	}
+	
+	public void setType(String type) {
+		this.type = type;
+	}
+	
+	@Override
+	public String getDispatcher() {
+		return dispatcher;
+	}
+	
+	public void setDispatcher(String dispatcher) {
+		this.dispatcher = dispatcher;
+	}
+	
+	@Override
+	public int getPendingSignals() {
+		return pendingSignals;
+	}
+	
+	public void setPendingSignals(int signals) {
+		pendingSignals = signals;
+	}
+
+	@Override
+    public long getUserId() {
+		return userId;
+	}
+
+	public void setUserId(long userId) {
+		this.userId = userId;
+	}
+
+	@Override
+    public long getAccountId() {
+		return accountId;
+	}
+
+	public void setAccountId(long accountId) {
+		this.accountId = accountId;
+	}
+
+	@Override
+    public String getCmd() {
+		return cmd;
+	}
+
+	public void setCmd(String cmd) {
+		this.cmd = cmd;
+	}
+	
+	@Override
+    public int getCmdVersion() {
+		return cmdVersion;
+	}
+	
+	public void setCmdVersion(int version) {
+		cmdVersion = version;
+	}
+
+	@Override
+    public String getCmdInfo() {
+		return cmdInfo;
+	}
+
+	public void setCmdInfo(String cmdInfo) {
+		this.cmdInfo = cmdInfo;
+	}
+	
+	@Override
+    public Status getStatus() {
+		return status;
+	}
+
+    public void setStatus(Status status) {
+		this.status = status;
+	}
+	
+	@Override
+    public int getProcessStatus() {
+		return processStatus;
+	}
+	
+	public void setProcessStatus(int status) {
+		processStatus = status;
+	}
+	
+	@Override
+    public int getResultCode() {
+		return resultCode;
+	}
+	
+	public void setResultCode(int resultCode) {
+		this.resultCode = resultCode;
+	}
+
+	@Override
+    public String getResult() {
+		return result;
+	}
+
+	public void setResult(String result) {
+		this.result = result;
+	}
+
+	@Override
+    public Long getInitMsid() {
+		return initMsid;
+	}
+
+	@Override
+	public void setInitMsid(Long initMsid) {
+		this.initMsid = initMsid;
+	}
+	
+	@Override
+	public Long getExecutingMsid() {
+		return executingMsid;
+	}
+	
+	public void setExecutingMsid(Long executingMsid) {
+		this.executingMsid = executingMsid;
+	}
+
+	@Override
+    public Long getCompleteMsid() {
+		return completeMsid;
+	}
+
+	@Override
+	public void setCompleteMsid(Long completeMsid) {
+		this.completeMsid = completeMsid;
+	}
+
+	@Override
+    public Date getCreated() {
+		return created;
+	}
+
+	public void setCreated(Date created) {
+		this.created = created;
+	}
+
+	@Override
+    public Date getLastUpdated() {
+		return lastUpdated;
+	}
+
+	public void setLastUpdated(Date lastUpdated) {
+		this.lastUpdated = lastUpdated;
+	}
+
+	@Override
+    public Date getLastPolled() {
+		return lastPolled;
+	}
+
+	public void setLastPolled(Date lastPolled) {
+		this.lastPolled = lastPolled;
+	}
+
+    @Override
+    public String getInstanceType() {
+		return instanceType;
+	}
+
+	public void setInstanceType(String instanceType) {
+		this.instanceType = instanceType;
+	}
+
+	@Override
+    public Long getInstanceId() {
+		return instanceId;
+	}
+
+	public void setInstanceId(Long instanceId) {
+		this.instanceId = instanceId;
+	}
+	
+	@Override
+    public SyncQueueItem getSyncSource() {
+        return syncSource;
+    }
+    
+	@Override
+    public void setSyncSource(SyncQueueItem syncSource) {
+        this.syncSource = syncSource;
+    }
+    
+    @Override
+    public String getUuid() {
+    	return uuid;
+    }
+    
+    public void setUuid(String uuid) {
+    	this.uuid = uuid;
+    }
+    
+	@Override
+    public String toString() {
+		StringBuffer sb = new StringBuffer();
+		sb.append("AsyncJobVO {id:").append(getId());
+		sb.append(", userId: ").append(getUserId());
+		sb.append(", accountId: ").append(getAccountId());
+		sb.append(", instanceType: ").append(getInstanceType());
+		sb.append(", instanceId: ").append(getInstanceId());
+		sb.append(", cmd: ").append(getCmd());
+		sb.append(", cmdInfo: ").append(getCmdInfo());
+		sb.append(", cmdVersion: ").append(getCmdVersion());
+		sb.append(", status: ").append(getStatus());
+		sb.append(", processStatus: ").append(getProcessStatus());
+		sb.append(", resultCode: ").append(getResultCode());
+		sb.append(", result: ").append(getResult());
+		sb.append(", initMsid: ").append(getInitMsid());
+		sb.append(", completeMsid: ").append(getCompleteMsid());
+		sb.append(", lastUpdated: ").append(getLastUpdated());
+		sb.append(", lastPolled: ").append(getLastPolled());
+		sb.append(", created: ").append(getCreated());
+		sb.append("}");
+		return sb.toString();
+	}
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/JobSerializerHelper.java

@@ -0,0 +1,203 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.impl;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Type;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.log4j.Logger;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+
+/**
+ * Note: toPairList and appendPairList only support simple POJO objects currently
+ */
+public class JobSerializerHelper {
+    private static final Logger s_logger = Logger.getLogger(JobSerializerHelper.class);
+    public static String token = "/";
+
+    private static Gson s_gson;
+    static {
+        GsonBuilder gsonBuilder = new GsonBuilder();
+        gsonBuilder.setVersion(1.5);
+        s_logger.debug("Job GSON Builder initialized.");
+        gsonBuilder.registerTypeAdapter(Class.class, new ClassTypeAdapter());
+        gsonBuilder.registerTypeAdapter(Throwable.class, new ThrowableTypeAdapter());
+        s_gson = gsonBuilder.create();
+    }
+
+    public static String toSerializedString(Object result) {
+        if(result != null) {
+            Class<?> clz = result.getClass();
+            return clz.getName() + token + s_gson.toJson(result);
+        }
+        return null;
+    }
+
+    public static Object fromSerializedString(String result) {
+        try {
+            if(result != null && !result.isEmpty()) {
+
+                String[] serializedParts = result.split(token);
+
+                if (serializedParts.length < 2) {
+                    return null;
+                }
+                String clzName = serializedParts[0];
+                String nameField = null;
+                String content = null;
+                if (serializedParts.length == 2) {
+                    content = serializedParts[1];
+                } else {
+                    nameField = serializedParts[1];
+                    int index = result.indexOf(token + nameField + token);
+                    content = result.substring(index + nameField.length() + 2);
+                }
+
+                Class<?> clz;
+                try {
+                    clz = Class.forName(clzName);
+                } catch (ClassNotFoundException e) {
+                    return null;
+                }
+
+                Object obj = s_gson.fromJson(content, clz);
+                return obj;
+            }
+            return null;
+        } catch(RuntimeException e) {
+            throw new CloudRuntimeException("Unable to deserialize: " + result, e);
+        }
+    }
+    
+    public static String toObjectSerializedString(Serializable object) {
+    	assert(object != null);
+    	
+    	ByteArrayOutputStream bs = new ByteArrayOutputStream();
+    	try {
+    		ObjectOutputStream os = new ObjectOutputStream(bs);
+    		os.writeObject(object);
+    		os.close();
+    		bs.close();
+    		
+    		return Base64.encodeBase64URLSafeString(bs.toByteArray());
+    	} catch(IOException e) {
+            throw new CloudRuntimeException("Unable to serialize: " + object, e);
+    	}
+    }
+    
+    public static Object fromObjectSerializedString(String base64EncodedString) {
+    	if(base64EncodedString == null)
+    		return null;
+    	
+    	byte[] content = Base64.decodeBase64(base64EncodedString);
+    	ByteArrayInputStream bs = new ByteArrayInputStream(content);
+    	try {
+    		ObjectInputStream is = new ObjectInputStream(bs);
+    		Object obj = is.readObject();
+    		is.close();
+    		bs.close();
+    		return obj;
+    	} catch(IOException e) {
+            throw new CloudRuntimeException("Unable to serialize: " + base64EncodedString, e);
+        } catch (ClassNotFoundException e) {
+            throw new CloudRuntimeException("Unable to serialize: " + base64EncodedString, e);
+    	}
+    }
+
+    public static class ClassTypeAdapter implements JsonSerializer<Class<?>>, JsonDeserializer<Class<?>> {
+        @Override
+        public JsonElement serialize(Class<?> clazz, Type typeOfResponseObj, JsonSerializationContext ctx) {
+            return new JsonPrimitive(clazz.getName());
+        }
+
+        @Override
+        public Class<?> deserialize(JsonElement arg0, Type arg1, JsonDeserializationContext arg2) throws JsonParseException {
+            String str = arg0.getAsString();
+            try {
+                return Class.forName(str);
+            } catch (ClassNotFoundException e) {
+                throw new CloudRuntimeException("Unable to find class " + str);
+            }
+        }
+    }
+
+    public static class ThrowableTypeAdapter implements JsonSerializer<Throwable>, JsonDeserializer<Throwable> {
+
+        @Override
+        public Throwable deserialize(JsonElement json, Type type, JsonDeserializationContext ctx) throws JsonParseException {
+            JsonObject obj = (JsonObject)json;
+
+            String className = obj.get("class").getAsString();
+            try {
+                Class<Throwable> clazz  = (Class<Throwable>)Class.forName(className);
+                Throwable cause = s_gson.fromJson(obj.get("cause"), Throwable.class);
+                String msg = obj.get("msg").getAsString();
+                Constructor<Throwable> constructor = clazz.getConstructor(String.class, Throwable.class);
+                Throwable th = constructor.newInstance(msg, cause);
+                return th;
+            } catch (ClassNotFoundException e) {
+                throw new JsonParseException("Unable to find " + className);
+            } catch (NoSuchMethodException e) {
+                throw new JsonParseException("Unable to find constructor for " + className);
+            } catch (SecurityException e) {
+                throw new JsonParseException("Unable to get over security " + className);
+            } catch (InstantiationException e) {
+                throw new JsonParseException("Unable to instantiate " + className);
+            } catch (IllegalAccessException e) {
+                throw new JsonParseException("Illegal access to " + className, e);
+            } catch (IllegalArgumentException e) {
+                throw new JsonParseException("Illegal argument to " + className, e);
+            } catch (InvocationTargetException e) {
+                throw new JsonParseException("Cannot invoke " + className, e);
+            }
+        }
+
+        @Override
+        public JsonElement serialize(Throwable th, Type type, JsonSerializationContext ctx) {
+            JsonObject json = new JsonObject();
+
+            json.add("class", new JsonPrimitive(th.getClass().getName()));
+            json.add("cause", s_gson.toJsonTree(th.getCause()));
+            json.add("msg", new JsonPrimitive(th.getMessage()));
+//            json.add("stack", s_gson.toJsonTree(th.getStackTrace()));
+
+            return json;
+        }
+
+    }
+
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/OutcomeImpl.java

@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.impl;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.cloudstack.framework.jobs.AsyncJob;
+import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext;
+import org.apache.cloudstack.framework.jobs.Outcome;
+
+import com.cloud.utils.Predicate;
+public class OutcomeImpl<T> implements Outcome<T> {
+    protected AsyncJob _job;
+    protected Class<T> _clazz;
+    protected String[] _topics;
+    protected Predicate _predicate;
+    protected long _checkIntervalInMs;
+
+    protected T _result;
+
+    private static AsyncJobManagerImpl s_jobMgr;
+
+    public static void init(AsyncJobManagerImpl jobMgr) {
+        s_jobMgr = jobMgr;
+    }
+
+    public OutcomeImpl(Class<T> clazz, AsyncJob job, long checkIntervalInMs, Predicate predicate, String... topics) {
+        _clazz = clazz;
+        _job = job;
+        _topics = topics;
+        _predicate = predicate;
+        _checkIntervalInMs = checkIntervalInMs;
+    }
+
+    @Override
+    public AsyncJob getJob() {
+        return _job;
+    }
+
+    @Override
+    public boolean cancel(boolean mayInterruptIfRunning) {
+        return false;
+    }
+
+    @Override
+    public T get() throws InterruptedException, ExecutionException {
+        s_jobMgr.waitAndCheck(getJob(), _topics, _checkIntervalInMs, -1, _predicate);
+        try {
+            AsyncJobExecutionContext.getCurrentExecutionContext().disjoinJob(_job.getId());
+        } catch (Throwable e) {
+            throw new ExecutionException("Job task has trouble executing", e);
+        }
+
+        return retrieve();
+    }
+
+    @Override
+    public T get(long timeToWait, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
+        s_jobMgr.waitAndCheck(getJob(), _topics, _checkIntervalInMs, unit.toMillis(timeToWait), _predicate);
+        try {
+            AsyncJobExecutionContext.getCurrentExecutionContext().disjoinJob(_job.getId());
+        } catch (Throwable e) {
+            throw new ExecutionException("Job task has trouble executing", e);
+        }
+        return retrieve();
+    }
+
+    /**
+     * This method can be overridden by children classes to retrieve the
+     * actual object.
+     */
+    protected T retrieve() {
+        return _result;
+    }
+
+    protected Outcome<T> set(T result) {
+        _result = result;
+        return this;
+    }
+
+    @Override
+    public boolean isCancelled() {
+        // TODO Auto-generated method stub
+        return false;
+    }
+
+    @Override
+    public boolean isDone() {
+        // TODO Auto-generated method stub
+        return false;
+    }
+
+    @Override
+    public void execute(Task<T> task) {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public void execute(Task<T> task, long wait, TimeUnit unit) {
+        // TODO Auto-generated method stub
+
+    }
+
+    public Predicate getPredicate() {
+        return _predicate;
+    }
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueItem.java

@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.impl;
+
+public interface SyncQueueItem {
+    public final String AsyncJobContentType = "AsyncJob";
+
+    /**
+     * @return queue item id
+     */
+    long getId();
+
+    /**
+     * @return queue id
+     */
+    Long getQueueId();
+
+    /**
+     * @return subject object type pointed by the queue item
+     */
+    String getContentType();
+    
+    /**
+     * @return subject object id pointed by the queue item
+     */
+    Long getContentId();
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueItemVO.java

@@ -0,0 +1,143 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.impl;
+
+import org.apache.cloudstack.api.InternalIdentity;
+
+
+import java.util.Date;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+
+@Entity
+@Table(name="sync_queue_item")
+public class SyncQueueItemVO implements SyncQueueItem, InternalIdentity {
+
+    @Id
+    @GeneratedValue(strategy=GenerationType.IDENTITY)
+    @Column(name="id")
+    private Long id = null;
+    
+    @Column(name="queue_id")
+    private Long queueId;
+    
+    @Column(name="content_type")
+    private String contentType;
+    
+    @Column(name="content_id")
+    private Long contentId;
+    
+    @Column(name="queue_proc_msid")
+    private Long lastProcessMsid;
+
+    @Column(name="queue_proc_number")
+    private Long lastProcessNumber;
+    
+    @Column(name="queue_proc_time")
+    @Temporal(TemporalType.TIMESTAMP)
+    private Date lastProcessTime;
+    
+    @Column(name="created")
+    private Date created;
+    
+    public long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    @Override
+    public Long getQueueId() {
+        return queueId;
+    }
+
+    public void setQueueId(Long queueId) {
+        this.queueId = queueId;
+    }
+
+    @Override
+    public String getContentType() {
+        return contentType;
+    }
+
+    public void setContentType(String contentType) {
+        this.contentType = contentType;
+    }
+    
+    @Override
+    public Long getContentId() {
+        return contentId;
+    }
+
+    public void setContentId(Long contentId) {
+        this.contentId = contentId;
+    }
+
+    public Long getLastProcessMsid() {
+        return lastProcessMsid;
+    }
+
+    public void setLastProcessMsid(Long lastProcessMsid) {
+        this.lastProcessMsid = lastProcessMsid;
+    }
+
+    public Long getLastProcessNumber() {
+        return lastProcessNumber;
+    }
+
+    public void setLastProcessNumber(Long lastProcessNumber) {
+        this.lastProcessNumber = lastProcessNumber;
+    }
+
+    public Date getCreated() {
+        return created;
+    }
+
+    public void setCreated(Date created) {
+        this.created = created;
+    }
+
+    public String toString() {
+        StringBuffer sb = new StringBuffer();
+        sb.append("SyncQueueItemVO {id:").append(getId()).append(", queueId: ").append(getQueueId());
+        sb.append(", contentType: ").append(getContentType());
+        sb.append(", contentId: ").append(getContentId());
+        sb.append(", lastProcessMsid: ").append(getLastProcessMsid());
+        sb.append(", lastprocessNumber: ").append(getLastProcessNumber());
+        sb.append(", lastProcessTime: ").append(getLastProcessTime());
+        sb.append(", created: ").append(getCreated());
+        sb.append("}");
+        return sb.toString();
+    }
+
+       public Date getLastProcessTime() {
+            return lastProcessTime;
+        }
+
+        public void setLastProcessTime(Date lastProcessTime) {
+            this.lastProcessTime = lastProcessTime;
+        }
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java

@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.impl;
+
+import java.util.List;
+
+import com.cloud.utils.component.Manager;
+
+public interface SyncQueueManager extends Manager {
+    public SyncQueueVO queue(String syncObjType, long syncObjId, String itemType, long itemId, long queueSizeLimit);
+    public SyncQueueItemVO dequeueFromOne(long queueId, Long msid);
+    public List<SyncQueueItemVO> dequeueFromAny(Long msid, int maxItems);
+    public void purgeItem(long queueItemId);
+    public void returnItem(long queueItemId);
+
+	public List<SyncQueueItemVO> getActiveQueueItems(Long msid, boolean exclusive);
+    public List<SyncQueueItemVO> getBlockedQueueItems(long thresholdMs, boolean exclusive);
+
+    void purgeAsyncJobQueueItemId(long asyncJobId);
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java

@@ -0,0 +1,258 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.framework.jobs.impl;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import javax.inject.Inject;
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.framework.jobs.dao.SyncQueueDao;
+import org.apache.cloudstack.framework.jobs.dao.SyncQueueItemDao;
+
+import com.cloud.utils.DateUtil;
+import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.db.DB;
+import com.cloud.utils.db.Transaction;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class SyncQueueManagerImpl extends ManagerBase implements SyncQueueManager {
+    public static final Logger s_logger = Logger.getLogger(SyncQueueManagerImpl.class.getName());
+
+    @Inject private SyncQueueDao _syncQueueDao;
+    @Inject private SyncQueueItemDao _syncQueueItemDao;
+
+    @Override
+    @DB
+    public SyncQueueVO queue(String syncObjType, long syncObjId, String itemType, long itemId, long queueSizeLimit) {
+        Transaction txn = Transaction.currentTxn();
+        try {
+            txn.start();
+
+            _syncQueueDao.ensureQueue(syncObjType, syncObjId);
+            SyncQueueVO queueVO = _syncQueueDao.find(syncObjType, syncObjId);
+            if(queueVO == null)
+                throw new CloudRuntimeException("Unable to queue item into DB, DB is full?");
+
+            queueVO.setQueueSizeLimit(queueSizeLimit);
+            _syncQueueDao.update(queueVO.getId(), queueVO);
+
+            Date dt = DateUtil.currentGMTTime();
+            SyncQueueItemVO item = new SyncQueueItemVO();
+            item.setQueueId(queueVO.getId());
+            item.setContentType(itemType);
+            item.setContentId(itemId);
+            item.setCreated(dt);
+
+            _syncQueueItemDao.persist(item);
+            txn.commit();
+
+            return queueVO;
+        } catch(Exception e) {
+            s_logger.error("Unexpected exception: ", e);
+            txn.rollback();
+        }
+        return null;
+    }
+
+    @Override
+    @DB
+    public SyncQueueItemVO dequeueFromOne(long queueId, Long msid) {
+        Transaction txt = Transaction.currentTxn();
+        try {
+            txt.start();
+
+            SyncQueueVO queueVO = _syncQueueDao.lockRow(queueId, true);
+            if(queueVO == null) {
+                s_logger.error("Sync queue(id: " + queueId + ") does not exist");
+                txt.commit();
+                return null;
+            }
+
+            if(queueReadyToProcess(queueVO)) {
+                SyncQueueItemVO itemVO = _syncQueueItemDao.getNextQueueItem(queueVO.getId());
+                if(itemVO != null) {
+                    Long processNumber = queueVO.getLastProcessNumber();
+                    if(processNumber == null)
+                        processNumber = new Long(1);
+                    else
+                        processNumber = processNumber + 1;
+                    Date dt = DateUtil.currentGMTTime();
+                    queueVO.setLastProcessNumber(processNumber);
+                    queueVO.setLastUpdated(dt);
+                    queueVO.setQueueSize(queueVO.getQueueSize() + 1);
+                    _syncQueueDao.update(queueVO.getId(), queueVO);
+
+                    itemVO.setLastProcessMsid(msid);
+                    itemVO.setLastProcessNumber(processNumber);
+                    itemVO.setLastProcessTime(dt);
+                    _syncQueueItemDao.update(itemVO.getId(), itemVO);
+
+                    txt.commit();
+                    return itemVO;
+                } else {
+                    if(s_logger.isDebugEnabled())
+                        s_logger.debug("Sync queue (" + queueId + ") is currently empty");
+                }
+            } else {
+                if(s_logger.isDebugEnabled())
+                    s_logger.debug("There is a pending process in sync queue(id: " + queueId + ")");
+            }
+            txt.commit();
+        } catch(Exception e) {
+            s_logger.error("Unexpected exception: ", e);
+            txt.rollback();
+        }
+
+        return null;
+    }
+
+    @Override
+    @DB
+    public List<SyncQueueItemVO> dequeueFromAny(Long msid, int maxItems) {
+
+        List<SyncQueueItemVO> resultList = new ArrayList<SyncQueueItemVO>();
+        Transaction txt = Transaction.currentTxn();
+        try {
+            txt.start();
+
+            List<SyncQueueItemVO> l = _syncQueueItemDao.getNextQueueItems(maxItems);
+            if(l != null && l.size() > 0) {
+                for(SyncQueueItemVO item : l) {
+                    SyncQueueVO queueVO = _syncQueueDao.lockRow(item.getQueueId(), true);
+                    SyncQueueItemVO itemVO = _syncQueueItemDao.lockRow(item.getId(), true);
+                    if(queueReadyToProcess(queueVO) && itemVO.getLastProcessNumber() == null) {
+                        Long processNumber = queueVO.getLastProcessNumber();
+                        if(processNumber == null)
+                            processNumber = new Long(1);
+                        else
+                            processNumber = processNumber + 1;
+
+                        Date dt = DateUtil.currentGMTTime();
+                        queueVO.setLastProcessNumber(processNumber);
+                        queueVO.setLastUpdated(dt);
+                        queueVO.setQueueSize(queueVO.getQueueSize() + 1);
+                        _syncQueueDao.update(queueVO.getId(), queueVO);
+
+                        itemVO.setLastProcessMsid(msid);
+                        itemVO.setLastProcessNumber(processNumber);
+                        itemVO.setLastProcessTime(dt);
+                        _syncQueueItemDao.update(item.getId(), itemVO);
+
+                        resultList.add(item);
+                    }
+                }
+            }
+            txt.commit();
+            return resultList;
+        } catch(Exception e) {
+            s_logger.error("Unexpected exception: ", e);
+            txt.rollback();
+        }
+        return null;
+    }
+
+    @Override
+    @DB
+    public void purgeItem(long queueItemId) {
+        Transaction txt = Transaction.currentTxn();
+        try {
+            txt.start();
+
+            SyncQueueItemVO itemVO = _syncQueueItemDao.findById(queueItemId);
+            if(itemVO != null) {
+                SyncQueueVO queueVO = _syncQueueDao.lockRow(itemVO.getQueueId(), true);
+
+                _syncQueueItemDao.expunge(itemVO.getId());
+
+                // if item is active, reset queue information
+                if (itemVO.getLastProcessMsid() != null) {
+                    queueVO.setLastUpdated(DateUtil.currentGMTTime());
+                    // decrement the count
+                    assert (queueVO.getQueueSize() > 0) : "Count reduce happens when it's already <= 0!";
+                    queueVO.setQueueSize(queueVO.getQueueSize() - 1);
+                    _syncQueueDao.update(queueVO.getId(), queueVO);
+                }
+            }
+            txt.commit();
+        } catch(Exception e) {
+            s_logger.error("Unexpected exception: ", e);
+            txt.rollback();
+        }
+    }
+
+    @Override
+    @DB
+    public void returnItem(long queueItemId) {
+        Transaction txt = Transaction.currentTxn();
+        try {
+            txt.start();
+
+            SyncQueueItemVO itemVO = _syncQueueItemDao.findById(queueItemId);
+            if(itemVO != null) {
+                SyncQueueVO queueVO = _syncQueueDao.lockRow(itemVO.getQueueId(), true);
+
+                itemVO.setLastProcessMsid(null);
+                itemVO.setLastProcessNumber(null);
+                itemVO.setLastProcessTime(null);
+                _syncQueueItemDao.update(queueItemId, itemVO);
+
+                queueVO.setLastUpdated(DateUtil.currentGMTTime());
+                _syncQueueDao.update(queueVO.getId(), queueVO);
+            }
+            txt.commit();
+        } catch(Exception e) {
+            s_logger.error("Unexpected exception: ", e);
+            txt.rollback();
+        }
+    }
+
+    @Override
+    public List<SyncQueueItemVO> getActiveQueueItems(Long msid, boolean exclusive) {
+        return _syncQueueItemDao.getActiveQueueItems(msid, exclusive);
+    }
+
+    @Override
+    public List<SyncQueueItemVO> getBlockedQueueItems(long thresholdMs, boolean exclusive) {
+        return _syncQueueItemDao.getBlockedQueueItems(thresholdMs, exclusive);
+    }
+
+    private boolean queueReadyToProcess(SyncQueueVO queueVO) {
+    	return true;
+    	
+    	//
+    	// TODO
+    	//
+    	// Need to disable concurrency disable at queue level due to the need to support
+    	// job wake-up dispatching task
+    	//
+    	// Concurrency control is better done at higher level and leave the job scheduling/serializing simpler
+    	//
+    	
+        // return queueVO.getQueueSize() < queueVO.getQueueSizeLimit();
+    }
+
+    @Override
+    public void purgeAsyncJobQueueItemId(long asyncJobId) {
+        Long itemId = _syncQueueItemDao.getQueueItemIdByContentIdAndType(asyncJobId, SyncQueueItem.AsyncJobContentType);
+        if (itemId != null) {
+            purgeItem(itemId);
+        }
+    }
+}

framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueVO.java

@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.framework.jobs.impl;
+
+import org.apache.cloudstack.api.InternalIdentity;
+
+import java.util.Date;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+
+@Entity
+@Table(name="sync_queue")
+public class SyncQueueVO implements InternalIdentity {
+
+    @Id
+    @GeneratedValue(strategy=GenerationType.IDENTITY)
+    @Column(name="id")
+    private Long id;
+
+    @Column(name="sync_objtype")
+
+    private String syncObjType;
+
+    @Column(name="sync_objid")
+    private Long syncObjId;
+
+    @Column(name="queue_proc_number")
+    private Long lastProcessNumber;
+
+    @Column(name="created")
+    @Temporal(TemporalType.TIMESTAMP)
+    private Date created;
+
+    @Column(name="last_updated")
+    @Temporal(TemporalType.TIMESTAMP)
+    private Date lastUpdated;
+
+    @Column(name="queue_size")
+    private long queueSize = 0;
+
+    @Column(name="queue_size_limit")
+    private long queueSizeLimit = 0;
+
+    public long getId() {
+        return id;
+    }
+
+    public String getSyncObjType() {
+        return syncObjType;
+    }
+
+    public void setSyncObjType(String syncObjType) {
+        this.syncObjType = syncObjType;
+    }
+
+    public Long getSyncObjId() {
+        return syncObjId;
+    }
+
+    public void setSyncObjId(Long syncObjId) {
+        this.syncObjId = syncObjId;
+    }
+    
+    public Long getLastProcessNumber() {
+        return lastProcessNumber;
+    }
+    
+    public void setLastProcessNumber(Long number) {
+        lastProcessNumber = number;
+    }
+
+    public Date getCreated() {
+        return created;
+    }
+
+    public void setCreated(Date created) {
+        this.created = created;
+    }
+
+    public Date getLastUpdated() {
+        return lastUpdated;
+    }
+
+    public void setLastUpdated(Date lastUpdated) {
+        this.lastUpdated = lastUpdated;
+    }
+
+    public String toString() {
+        StringBuffer sb = new StringBuffer();
+        sb.append("SyncQueueVO {id:").append(getId());
+        sb.append(", syncObjType: ").append(getSyncObjType());
+        sb.append(", syncObjId: ").append(getSyncObjId());
+        sb.append(", lastProcessNumber: ").append(getLastProcessNumber());
+        sb.append(", lastUpdated: ").append(getLastUpdated());
+        sb.append(", created: ").append(getCreated());
+        sb.append(", count: ").append(getQueueSize());
+        sb.append("}");
+        return sb.toString();
+    }
+
+    public long getQueueSize() {
+        return queueSize;
+    }
+
+    public void setQueueSize(long queueSize) {
+        this.queueSize = queueSize;
+    }
+
+    public long getQueueSizeLimit() {
+        return queueSizeLimit;
+    }
+
+    public void setQueueSizeLimit(long queueSizeLimit) {
+        this.queueSizeLimit = queueSizeLimit;
+    }
+}

server/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java

@@ -713,12 +713,12 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
     }
 
     @Override
-    public void onManagementNodeJoined(List<ManagementServerHostVO> nodeList, long selfNodeId) {
+    public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
     }
 
     @Override
-    public void onManagementNodeLeft(List<ManagementServerHostVO> nodeList, long selfNodeId) {
-        for (ManagementServerHostVO vo : nodeList) {
+    public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
+        for (ManagementServerHost vo : nodeList) {
             s_logger.info("Marking hosts as disconnected on Management server" + vo.getMsid());
             long lastPing = (System.currentTimeMillis() >> 10) - getTimeout();
             _hostDao.markHostsAsDisconnected(vo.getMsid(), lastPing);

server/src/com/cloud/async/AsyncJobManagerImpl.java

@@ -61,7 +61,7 @@ import com.cloud.api.ApiGsonHelper;
 import com.cloud.api.ApiSerializerHelper;
 import com.cloud.async.dao.AsyncJobDao;
 import com.cloud.cluster.ClusterManagerListener;
-import com.cloud.cluster.ManagementServerHostVO;
+import com.cloud.cluster.ManagementServerHost;
 import com.cloud.configuration.Config;
 import com.cloud.configuration.dao.ConfigurationDao;
 import com.cloud.domain.DomainVO;
@@ -823,12 +823,12 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
     }
 
     @Override
-    public void onManagementNodeJoined(List<ManagementServerHostVO> nodeList, long selfNodeId) {
+    public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
     }
 
     @Override
-    public void onManagementNodeLeft(List<ManagementServerHostVO> nodeList, long selfNodeId) {
-        for(ManagementServerHostVO msHost : nodeList) {
+    public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
+        for (ManagementServerHost msHost : nodeList) {
             Transaction txn = Transaction.open(Transaction.CLOUD_DB);
             try {
                 txn.start();

server/src/com/cloud/ha/HighAvailabilityManagerImpl.java

@@ -37,7 +37,7 @@ import org.apache.cloudstack.context.ServerContexts;
 import com.cloud.agent.AgentManager;
 import com.cloud.alert.AlertManager;
 import com.cloud.cluster.ClusterManagerListener;
-import com.cloud.cluster.ManagementServerHostVO;
+import com.cloud.cluster.ManagementServerHost;
 import com.cloud.configuration.Config;
 import com.cloud.configuration.dao.ConfigurationDao;
 import com.cloud.dc.ClusterDetailsDao;
@@ -866,12 +866,12 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai
     }
 
     @Override
-    public void onManagementNodeJoined(List<ManagementServerHostVO> nodeList, long selfNodeId) {
+    public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
     }
 
     @Override
-    public void onManagementNodeLeft(List<ManagementServerHostVO> nodeList, long selfNodeId) {
-        for (ManagementServerHostVO node : nodeList) {
+    public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
+        for (ManagementServerHost node : nodeList) {
             _haDao.releaseWorkItems(node.getMsid());
         }
     }

server/src/com/cloud/server/LockMasterListener.java

@@ -19,7 +19,7 @@ package com.cloud.server;
 import java.util.List;
 
 import com.cloud.cluster.ClusterManagerListener;
-import com.cloud.cluster.ManagementServerHostVO;
+import com.cloud.cluster.ManagementServerHost;
 import com.cloud.utils.db.Merovingian2;
 
 /**
@@ -34,12 +34,12 @@ public class LockMasterListener implements ClusterManagerListener {
     }
 
     @Override
-    public void onManagementNodeJoined(List<ManagementServerHostVO> nodeList, long selfNodeId) {
+    public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
     }
 
     @Override
-    public void onManagementNodeLeft(List<ManagementServerHostVO> nodeList, long selfNodeId) {
-        for (ManagementServerHostVO node : nodeList) {
+    public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
+        for (ManagementServerHost node : nodeList) {
             _lockMaster.cleanupForServer(node.getMsid());
         }
     }

server/src/com/cloud/storage/StorageManagerImpl.java

@@ -48,9 +48,9 @@ import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd;
 import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd;
 import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd;
 import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd;
-import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd;
 import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd;
 import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd;
+import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd;
 import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
@@ -100,7 +100,7 @@ import com.cloud.capacity.CapacityState;
 import com.cloud.capacity.CapacityVO;
 import com.cloud.capacity.dao.CapacityDao;
 import com.cloud.cluster.ClusterManagerListener;
-import com.cloud.cluster.ManagementServerHostVO;
+import com.cloud.cluster.ManagementServerHost;
 import com.cloud.configuration.Config;
 import com.cloud.configuration.ConfigurationManager;
 import com.cloud.configuration.dao.ConfigurationDao;
@@ -1271,14 +1271,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
     }
 
     @Override
-    public void onManagementNodeJoined(List<ManagementServerHostVO> nodeList, long selfNodeId) {
+    public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
         // TODO Auto-generated method stub
 
     }
 
     @Override
-    public void onManagementNodeLeft(List<ManagementServerHostVO> nodeList, long selfNodeId) {
-        for (ManagementServerHostVO vo : nodeList) {
+    public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
+        for (ManagementServerHost vo : nodeList) {
             if (vo.getMsid() == _serverId) {
                 s_logger.info("Cleaning up storage maintenance jobs associated with Management server: " + vo.getMsid());
                 List<Long> poolIds = _storagePoolWorkDao.searchForPoolIdsForPendingWorkJobs(vo.getMsid());
@@ -1886,7 +1886,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
             throw new InvalidParameterValueException("Cannot delete cache store with staging volumes currently in use!");
         }
 
-        List<TemplateDataStoreVO> templates = this._templateStoreDao.listActiveOnCache(storeId);
+        List<TemplateDataStoreVO> templates = _templateStoreDao.listActiveOnCache(storeId);
         if (templates != null && templates.size() > 0) {
             throw new InvalidParameterValueException("Cannot delete cache store with staging templates currently in use!");
         }

server/src/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java

@@ -25,7 +25,7 @@ import javax.naming.ConfigurationException;
 
 import com.cloud.cluster.ClusterManager;
 import com.cloud.cluster.ClusterManagerListener;
-import com.cloud.cluster.ManagementServerHostVO;
+import com.cloud.cluster.ManagementServerHost;
 
 @Local(value=VirtualMachineManager.class)
 public class ClusteredVirtualMachineManagerImpl extends VirtualMachineManagerImpl implements ClusterManagerListener {
@@ -37,13 +37,13 @@ public class ClusteredVirtualMachineManagerImpl extends VirtualMachineManagerImp
     }
     
     @Override
-    public void onManagementNodeJoined(List<ManagementServerHostVO> nodeList, long selfNodeId) {
+    public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
         
     }
     
     @Override
-    public void onManagementNodeLeft(List<ManagementServerHostVO> nodeList, long selfNodeId) {
-        for (ManagementServerHostVO node : nodeList) {
+    public void onManagementNodeLeft(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
+        for (ManagementServerHost node : nodeList) {
             cancelWorkItems(node.getMsid());
         }
     }

utils/src/org/apache/cloudstack/config/Configurable.java

@@ -0,0 +1,21 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.config;
+
+public interface Configurable {
+    ConfigKey<?>[] getConfigKeys();
+}