diff --git a/api/src/org/apache/cloudstack/jobs/JobInfo.java b/api/src/org/apache/cloudstack/jobs/JobInfo.java new file mode 100644 index 00000000000..ac8ffc3183b --- /dev/null +++ b/api/src/org/apache/cloudstack/jobs/JobInfo.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.jobs; + +import java.util.Date; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface JobInfo extends Identity, InternalIdentity { + public enum Status { + IN_PROGRESS(false), + SUCCEEDED(true), + FAILED(true), + CANCELLED(true); + + private final boolean done; + + private Status(boolean done) { + this.done = done; + } + + public boolean done() { + return done; + } + } + + String getType(); + + String getDispatcher(); + + int getPendingSignals(); + + long getUserId(); + + long getAccountId(); + + String getCmd(); + + int getCmdVersion(); + + String getCmdInfo(); + + Status getStatus(); + + int getProcessStatus(); + + int getResultCode(); + + String getResult(); + + Long getInitMsid(); + + Long getExecutingMsid(); + + Long getCompleteMsid(); + + Date getCreated(); + + Date getLastUpdated(); + + Date getLastPolled(); + + String getInstanceType(); + + Long getInstanceId(); +} diff --git a/framework/cluster/src/com/cloud/cluster/ClusterFenceManagerImpl.java b/framework/cluster/src/com/cloud/cluster/ClusterFenceManagerImpl.java index 7e4922e7967..5125a07b15e 100644 --- a/framework/cluster/src/com/cloud/cluster/ClusterFenceManagerImpl.java +++ b/framework/cluster/src/com/cloud/cluster/ClusterFenceManagerImpl.java @@ -43,11 +43,11 @@ public class ClusterFenceManagerImpl extends ManagerBase implements ClusterFence } @Override - public void onManagementNodeJoined(List nodeList, long selfNodeId) { + public void onManagementNodeJoined(List nodeList, long selfNodeId) { } @Override - public void onManagementNodeLeft(List nodeList, long selfNodeId) { + public void onManagementNodeLeft(List nodeList, long selfNodeId) { } @Override diff --git a/framework/cluster/src/com/cloud/cluster/ClusterManagerListener.java b/framework/cluster/src/com/cloud/cluster/ClusterManagerListener.java index bcb1736800e..12314348074 100644 --- a/framework/cluster/src/com/cloud/cluster/ClusterManagerListener.java +++ b/framework/cluster/src/com/cloud/cluster/ClusterManagerListener.java @@ -19,7 +19,8 @@ package com.cloud.cluster; import java.util.List; public interface ClusterManagerListener { - void onManagementNodeJoined(List nodeList, long selfNodeId); - void onManagementNodeLeft(List nodeList, long selfNodeId); + void onManagementNodeJoined(List nodeList, long selfNodeId); + + void onManagementNodeLeft(List nodeList, long selfNodeId); void onManagementNodeIsolated(); } diff --git a/framework/ipc/src/org/apache/cloudstack/framework/async/AsyncCallbackDispatcher.java b/framework/ipc/src/org/apache/cloudstack/framework/async/AsyncCallbackDispatcher.java index acbc5b60541..42cd8c5c726 100644 --- a/framework/ipc/src/org/apache/cloudstack/framework/async/AsyncCallbackDispatcher.java +++ b/framework/ipc/src/org/apache/cloudstack/framework/async/AsyncCallbackDispatcher.java @@ -22,20 +22,20 @@ package org.apache.cloudstack.framework.async; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import org.apache.log4j.Logger; - -import net.sf.cglib.proxy.CallbackFilter; import net.sf.cglib.proxy.Callback; +import net.sf.cglib.proxy.CallbackFilter; import net.sf.cglib.proxy.Enhancer; import net.sf.cglib.proxy.MethodInterceptor; import net.sf.cglib.proxy.MethodProxy; +import org.apache.log4j.Logger; + @SuppressWarnings("rawtypes") public class AsyncCallbackDispatcher implements AsyncCompletionCallback { private static final Logger s_logger = Logger.getLogger(AsyncCallbackDispatcher.class); private Method _callbackMethod; - private T _targetObject; + private final T _targetObject; private Object _contextObject; private Object _resultObject; private AsyncCallbackDriver _driver = new InplaceAsyncCallbackDriver(); @@ -84,6 +84,7 @@ public class AsyncCallbackDispatcher implements AsyncCompletionCallback { } }); en.setCallbackFilter(new CallbackFilter() { + @Override public int accept(Method method) { if (method.getParameterTypes().length == 0 && method.getName().equals("finalize")) { return 1; @@ -115,6 +116,7 @@ public class AsyncCallbackDispatcher implements AsyncCompletionCallback { return (P)_contextObject; } + @Override public void complete(Object resultObject) { _resultObject = resultObject; _driver.performCompletionCallback(this); diff --git a/framework/ipc/src/org/apache/cloudstack/framework/client/ClientEventBus.java b/framework/ipc/src/org/apache/cloudstack/framework/client/ClientMessageBus.java similarity index 93% rename from framework/ipc/src/org/apache/cloudstack/framework/client/ClientEventBus.java rename to framework/ipc/src/org/apache/cloudstack/framework/client/ClientMessageBus.java index d876b01981a..6a510ddfe54 100644 --- a/framework/ipc/src/org/apache/cloudstack/framework/client/ClientEventBus.java +++ b/framework/ipc/src/org/apache/cloudstack/framework/client/ClientMessageBus.java @@ -21,7 +21,7 @@ package org.apache.cloudstack.framework.client; import org.apache.cloudstack.framework.messagebus.MessageBusBase; import org.apache.cloudstack.framework.transport.TransportMultiplexier; -public class ClientEventBus extends MessageBusBase implements TransportMultiplexier { +public class ClientMessageBus extends MessageBusBase implements TransportMultiplexier { @Override public void onTransportMessage(String senderEndpointAddress, diff --git a/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageBusBase.java b/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageBusBase.java index 9cf5e77ce6e..a42a60416d7 100644 --- a/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageBusBase.java +++ b/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageBusBase.java @@ -30,10 +30,10 @@ import org.apache.cloudstack.framework.serializer.MessageSerializer; public class MessageBusBase implements MessageBus { - private Gate _gate; - private List _pendingActions; + private final Gate _gate; + private final List _pendingActions; - private SubscriptionNode _subscriberRoot; + private final SubscriptionNode _subscriberRoot; private MessageSerializer _messageSerializer; public MessageBusBase() { @@ -77,7 +77,7 @@ public class MessageBusBase implements MessageBus { if(current != null) current.removeSubscriber(subscriber, false); } else { - this._subscriberRoot.removeSubscriber(subscriber, true); + _subscriberRoot.removeSubscriber(subscriber, true); } _gate.leave(); } else { @@ -151,11 +151,10 @@ public class MessageBusBase implements MessageBus { private void onGateOpen() { synchronized(_pendingActions) { ActionRecord record = null; - if(_pendingActions.size() > 0) { - while((record = _pendingActions.remove(0)) != null) { + while (_pendingActions.size() > 0) { + record = _pendingActions.remove(0); switch(record.getType()) { - case Subscribe : - { + case Subscribe: { SubscriptionNode current = locate(record.getSubject(), null, true); assert(current != null); current.addSubscriber(record.getSubscriber()); @@ -168,7 +167,7 @@ public class MessageBusBase implements MessageBus { if(current != null) current.removeSubscriber(record.getSubscriber(), false); } else { - this._subscriberRoot.removeSubscriber(record.getSubscriber(), true); + _subscriberRoot.removeSubscriber(record.getSubscriber(), true); } break; @@ -188,7 +187,6 @@ public class MessageBusBase implements MessageBus { } } } - } private SubscriptionNode locate(String subject, List chainFromTop, boolean createPath) { @@ -223,7 +221,7 @@ public class MessageBusBase implements MessageBus { } if(subjectPathTokens.length > 1) { - return locate((String[])Arrays.copyOfRange(subjectPathTokens, 1, subjectPathTokens.length), + return locate(Arrays.copyOfRange(subjectPathTokens, 1, subjectPathTokens.length), next, chainFromTop, createPath); } else { return next; @@ -242,9 +240,9 @@ public class MessageBusBase implements MessageBus { } private static class ActionRecord { - private ActionType _type; - private String _subject; - private MessageSubscriber _subscriber; + private final ActionType _type; + private final String _subject; + private final MessageSubscriber _subscriber; public ActionRecord(ActionType type, String subject, MessageSubscriber subscriber) { _type = type; @@ -320,10 +318,10 @@ public class MessageBusBase implements MessageBus { } private static class SubscriptionNode { - private String _nodeKey; - private List _subscribers; - private Map _children; - private SubscriptionNode _parent; + private final String _nodeKey; + private final List _subscribers; + private final Map _children; + private final SubscriptionNode _parent; public SubscriptionNode(SubscriptionNode parent, String nodeKey, MessageSubscriber subscriber) { assert(nodeKey != null); diff --git a/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java b/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java new file mode 100644 index 00000000000..7a7a34aa008 --- /dev/null +++ b/framework/ipc/src/org/apache/cloudstack/framework/messagebus/MessageDetector.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.framework.messagebus; + +public class MessageDetector implements MessageSubscriber { + + private MessageBus _messageBus; + private String[] _subjects; + + private volatile boolean _signalled = false; + + public MessageDetector() { + _messageBus = null; + _subjects = null; + } + + public boolean waitAny(long timeoutInMiliseconds) { + _signalled = false; + synchronized(this) { + try { + wait(timeoutInMiliseconds); + } catch (InterruptedException e) { + } + } + return _signalled; + } + + public void open(MessageBus messageBus, String[] subjects) { + assert(messageBus != null); + assert(subjects != null); + + _messageBus = messageBus; + _subjects = subjects; + + if(subjects != null) { + for(String subject : subjects) { + messageBus.subscribe(subject, this); + } + } + } + + public void close() { + if(_subjects != null) { + assert(_messageBus != null); + + for(String subject : _subjects) { + _messageBus.unsubscribe(subject, this); + } + } + } + + @Override + public void onPublishMessage(String senderAddress, String subject, Object args) { + synchronized(this) { + _signalled = true; + notifyAll(); + } + } +} diff --git a/framework/ipc/src/org/apache/cloudstack/framework/server/ServerEventBus.java b/framework/ipc/src/org/apache/cloudstack/framework/server/ServerMessageBus.java similarity index 93% rename from framework/ipc/src/org/apache/cloudstack/framework/server/ServerEventBus.java rename to framework/ipc/src/org/apache/cloudstack/framework/server/ServerMessageBus.java index f3b782d6d35..a01146882ad 100644 --- a/framework/ipc/src/org/apache/cloudstack/framework/server/ServerEventBus.java +++ b/framework/ipc/src/org/apache/cloudstack/framework/server/ServerMessageBus.java @@ -21,7 +21,7 @@ package org.apache.cloudstack.framework.server; import org.apache.cloudstack.framework.messagebus.MessageBusBase; import org.apache.cloudstack.framework.transport.TransportMultiplexier; -public class ServerEventBus extends MessageBusBase implements TransportMultiplexier { +public class ServerMessageBus extends MessageBusBase implements TransportMultiplexier { @Override public void onTransportMessage(String senderEndpointAddress, diff --git a/framework/ipc/test/org/apache/cloudstack/framework/sampleserver/SampleManagementServer.java b/framework/ipc/test/org/apache/cloudstack/framework/sampleserver/SampleManagementServer.java index 2a168ac7cd7..28eb4abbb6b 100644 --- a/framework/ipc/test/org/apache/cloudstack/framework/sampleserver/SampleManagementServer.java +++ b/framework/ipc/test/org/apache/cloudstack/framework/sampleserver/SampleManagementServer.java @@ -18,9 +18,6 @@ */ package org.apache.cloudstack.framework.sampleserver; -import org.springframework.stereotype.Component; - -@Component public class SampleManagementServer { public void mainLoop() { diff --git a/framework/ipc/test/org/apache/cloudstack/messagebus/TestMessageBus.java b/framework/ipc/test/org/apache/cloudstack/messagebus/TestMessageBus.java index dabfdd3b102..33c5ce5d6f1 100644 --- a/framework/ipc/test/org/apache/cloudstack/messagebus/TestMessageBus.java +++ b/framework/ipc/test/org/apache/cloudstack/messagebus/TestMessageBus.java @@ -23,6 +23,7 @@ import javax.inject.Inject; import junit.framework.TestCase; import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.framework.messagebus.MessageDetector; import org.apache.cloudstack.framework.messagebus.MessageSubscriber; import org.apache.cloudstack.framework.messagebus.PublishScope; import org.junit.Assert; @@ -113,4 +114,42 @@ public class TestMessageBus extends TestCase { _messageBus.clearAll(); } + + public void testMessageDetector() { + MessageDetector detector = new MessageDetector(); + detector.open(_messageBus, new String[] {"VM", "Host"}); + + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + for(int i = 0; i < 2; i++) { + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + } + _messageBus.publish(null, "Host", PublishScope.GLOBAL, null); + } + } + }); + thread.start(); + + try { + int count = 0; + while(count < 2) { + if(detector.waitAny(1000)) { + System.out.println("Detected signal on bus"); + count++; + } else { + System.out.println("Waiting timed out"); + } + } + } finally { + detector.close(); + } + + try { + thread.join(); + } catch (InterruptedException e) { + } + } } diff --git a/framework/jobs/pom.xml b/framework/jobs/pom.xml index cf1fdd585a6..d5c16e88920 100644 --- a/framework/jobs/pom.xml +++ b/framework/jobs/pom.xml @@ -27,11 +27,6 @@ ../pom.xml - - org.quartz-scheduler - quartz - 2.1.6 - org.apache.cloudstack cloud-utils @@ -42,8 +37,20 @@ cloud-api ${project.version} + + org.apache.cloudstack + cloud-framework-ipc + ${project.version} + + + org.apache.cloudstack + cloud-framework-db + ${project.version} + + + org.apache.cloudstack + cloud-framework-cluster + ${project.version} + - - install - diff --git a/framework/jobs/src/org/apache/cloudstack/framework/job/Job.java b/framework/jobs/src/org/apache/cloudstack/framework/job/Job.java deleted file mode 100755 index 62ed72fe8d7..00000000000 --- a/framework/jobs/src/org/apache/cloudstack/framework/job/Job.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cloudstack.framework.job; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.METHOD) -public @interface Job { - - - -} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/job/JobInterceptor.java b/framework/jobs/src/org/apache/cloudstack/framework/job/JobInterceptor.java deleted file mode 100755 index d81077d6d7a..00000000000 --- a/framework/jobs/src/org/apache/cloudstack/framework/job/JobInterceptor.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cloudstack.framework.job; - -public class JobInterceptor { - -} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJob.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJob.java new file mode 100644 index 00000000000..61fb3962c37 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJob.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs; + +import java.util.Date; + +import org.apache.cloudstack.framework.jobs.impl.SyncQueueItem; +import org.apache.cloudstack.jobs.JobInfo; + +public interface AsyncJob extends JobInfo { + + public enum JournalType { + SUCCESS, FAILURE + }; + + public static interface Topics { + public static final String JOB_HEARTBEAT = "job.heartbeat"; + public static final String JOB_STATE = "job.state"; + } + + public static interface Constants { + + // Although we may have detailed masks for each individual wakeup event, i.e. + // periodical timer, matched topic from message bus, it seems that we don't + // need to distinguish them to such level. Therefore, only one wakeup signal + // is defined + public static final int SIGNAL_MASK_WAKEUP = 1; + + public static final String SYNC_LOCK_NAME = "SyncLock"; + } + + @Override + String getType(); + + @Override + String getDispatcher(); + + @Override + int getPendingSignals(); + + @Override + long getUserId(); + + @Override + long getAccountId(); + + @Override + String getCmd(); + + @Override + int getCmdVersion(); + + @Override + String getCmdInfo(); + + @Override + Status getStatus(); + + @Override + int getProcessStatus(); + + @Override + int getResultCode(); + + @Override + String getResult(); + + @Override + Long getInitMsid(); + + void setInitMsid(Long msid); + + @Override + Long getExecutingMsid(); + + @Override + Long getCompleteMsid(); + + void setCompleteMsid(Long msid); + + @Override + Date getCreated(); + + @Override + Date getLastUpdated(); + + @Override + Date getLastPolled(); + + @Override + String getInstanceType(); + + @Override + Long getInstanceId(); + + String getShortUuid(); + + SyncQueueItem getSyncSource(); + + void setSyncSource(SyncQueueItem item); + + String getRelated(); +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobDispatcher.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobDispatcher.java new file mode 100644 index 00000000000..5b0d15df41f --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobDispatcher.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs; + +import com.cloud.utils.component.Adapter; + +// +// We extend it from Adapter interface for +// 1) getName()/setName() +// 2) Confirming to general adapter pattern used across CloudStack +// +public interface AsyncJobDispatcher extends Adapter { + void runJob(AsyncJob job); +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java new file mode 100644 index 00000000000..01365939127 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobExecutionContext.java @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs; + +import org.apache.log4j.Logger; + +import org.apache.cloudstack.framework.jobs.dao.AsyncJobJoinMapDao; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobJoinMapVO; +import org.apache.cloudstack.framework.jobs.impl.JobSerializerHelper; +import org.apache.cloudstack.framework.jobs.impl.SyncQueueItem; +import org.apache.cloudstack.jobs.JobInfo; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ResourceUnavailableException; + +public class AsyncJobExecutionContext { + private AsyncJob _job; + + static private AsyncJobManager _jobMgr; + static private AsyncJobJoinMapDao _joinMapDao; + + public static void init(AsyncJobManager jobMgr, AsyncJobJoinMapDao joinMapDao) { + _jobMgr = jobMgr; + _joinMapDao = joinMapDao; + } + + private static ThreadLocal s_currentExectionContext = new ThreadLocal(); + + public AsyncJobExecutionContext() { + } + + public AsyncJobExecutionContext(AsyncJob job) { + _job = job; + } + + public SyncQueueItem getSyncSource() { + return _job.getSyncSource(); + } + + public void resetSyncSource() { + _job.setSyncSource(null); + } + + public AsyncJob getJob() { + return _job; + } + + public void setJob(AsyncJob job) { + _job = job; + } + + public void completeAsyncJob(JobInfo.Status jobStatus, int resultCode, String resultObject) { + assert(_job != null); + _jobMgr.completeAsyncJob(_job.getId(), jobStatus, resultCode, resultObject); + } + + public void updateAsyncJobStatus(int processStatus, String resultObject) { + assert(_job != null); + _jobMgr.updateAsyncJobStatus(_job.getId(), processStatus, resultObject); + } + + public void updateAsyncJobAttachment(String instanceType, Long instanceId) { + assert(_job != null); + _jobMgr.updateAsyncJobAttachment(_job.getId(), instanceType, instanceId); + } + + public void logJobJournal(AsyncJob.JournalType journalType, String journalText, String journalObjJson) { + assert(_job != null); + _jobMgr.logJobJournal(_job.getId(), journalType, journalText, journalObjJson); + } + + public void log(Logger logger, String journalText) { + _jobMgr.logJobJournal(_job.getId(), AsyncJob.JournalType.SUCCESS, journalText, null); + logger.debug(journalText); + } + + public void joinJob(long joinJobId) { + assert(_job != null); + _jobMgr.joinJob(_job.getId(), joinJobId); + } + + public void joinJob(long joinJobId, String wakeupHandler, String wakeupDispatcher, + String[] wakeupTopcisOnMessageBus, long wakeupIntervalInMilliSeconds, long timeoutInMilliSeconds) { + assert(_job != null); + _jobMgr.joinJob(_job.getId(), joinJobId, wakeupHandler, wakeupDispatcher, wakeupTopcisOnMessageBus, + wakeupIntervalInMilliSeconds, timeoutInMilliSeconds); + } + + // + // check failure exception before we disjoin the worker job + // TODO : it is ugly and this will become unnecessary after we switch to full-async mode + // + public void disjoinJob(long joinedJobId) throws InsufficientCapacityException, + ConcurrentOperationException, ResourceUnavailableException { + assert(_job != null); + + AsyncJobJoinMapVO record = _joinMapDao.getJoinRecord(_job.getId(), joinedJobId); + if(record.getJoinStatus() == JobInfo.Status.FAILED && record.getJoinResult() != null) { + Object exception = JobSerializerHelper.fromObjectSerializedString(record.getJoinResult()); + if(exception != null && exception instanceof Exception) { + if(exception instanceof InsufficientCapacityException) + throw (InsufficientCapacityException)exception; + else if(exception instanceof ConcurrentOperationException) + throw (ConcurrentOperationException)exception; + else if(exception instanceof ResourceUnavailableException) + throw (ResourceUnavailableException)exception; + else + throw new RuntimeException((Exception)exception); + } + } + + _jobMgr.disjoinJob(_job.getId(), joinedJobId); + } + + public void completeJoin(JobInfo.Status joinStatus, String joinResult) { + assert(_job != null); + _jobMgr.completeJoin(_job.getId(), joinStatus, joinResult); + } + + public void completeJobAndJoin(JobInfo.Status joinStatus, String joinResult) { + assert(_job != null); + _jobMgr.completeJoin(_job.getId(), joinStatus, joinResult); + _jobMgr.completeAsyncJob(_job.getId(), joinStatus, 0, null); + } + + public static AsyncJobExecutionContext getCurrentExecutionContext() { + AsyncJobExecutionContext context = s_currentExectionContext.get(); + return context; + } + + public static AsyncJobExecutionContext registerPseudoExecutionContext(long accountId, long userId) { + AsyncJobExecutionContext context = s_currentExectionContext.get(); + if (context == null) { + context = new AsyncJobExecutionContext(); + context.setJob(_jobMgr.getPseudoJob(accountId, userId)); + setCurrentExecutionContext(context); + } + + return context; + } + + public static AsyncJobExecutionContext unregister() { + AsyncJobExecutionContext context = s_currentExectionContext.get(); + setCurrentExecutionContext(null); + return context; + } + + // This is intended to be package level access for AsyncJobManagerImpl only. + public static void setCurrentExecutionContext(AsyncJobExecutionContext currentContext) { + s_currentExectionContext.set(currentContext); + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobMBean.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobMBean.java new file mode 100644 index 00000000000..8ba3c8733cc --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobMBean.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs; + +public interface AsyncJobMBean { + public long getAccountId(); + public long getUserId(); + public String getCmd(); + public String getCmdInfo(); + public String getStatus(); + public int getProcessStatus(); + public int getResultCode(); + public String getResult(); + public String getInstanceType(); + public String getInstanceId(); + public String getInitMsid(); + public String getCreateTime(); + public String getLastUpdateTime(); + public String getLastPollTime(); + public String getSyncQueueId(); + public String getSyncQueueContentType(); + public String getSyncQueueContentId(); +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobManager.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobManager.java new file mode 100644 index 00000000000..bc061018957 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobManager.java @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under ones +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs; + +import java.util.List; + +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; +import org.apache.cloudstack.jobs.JobInfo; + +import com.cloud.utils.Predicate; +import com.cloud.utils.component.Manager; + +public interface AsyncJobManager extends Manager { + + public static final String JOB_POOL_THREAD_PREFIX = "Job-Executor"; + + AsyncJobVO getAsyncJob(long jobId); + + List findInstancePendingAsyncJobs(String instanceType, Long accountId); + + long submitAsyncJob(AsyncJob job); + long submitAsyncJob(AsyncJob job, String syncObjType, long syncObjId); + + void completeAsyncJob(long jobId, JobInfo.Status jobStatus, int resultCode, String result); + + void updateAsyncJobStatus(long jobId, int processStatus, String resultObject); + void updateAsyncJobAttachment(long jobId, String instanceType, Long instanceId); + void logJobJournal(long jobId, AsyncJob.JournalType journalType, String + journalText, String journalObjJson); + + /** + * A running thread inside management server can have a 1:1 linked pseudo job. + * This is to help make some legacy code work without too dramatic changes. + * + * All pseudo jobs should be expunged upon management start event + * + * @return pseudo job for the thread + */ + AsyncJob getPseudoJob(long accountId, long userId); + + /** + * Used by upper level job to wait for completion of a down-level job (usually VmWork jobs) + * in synchronous way. Caller needs to use waitAndCheck() to check the completion status + * of the down-level job + * + * Due to the amount of legacy code that relies on synchronous-call semantics, this form of joinJob + * is used mostly + * + * + * @param jobId upper job that is going to wait the completion of a down-level job + * @param joinJobId down-level job + */ + void joinJob(long jobId, long joinJobId); + + /** + * Used by upper level job to wait for completion of a down-level job (usually VmWork jobs) + * in asynchronous way, it will cause upper job to cease current execution, upper job will be + * rescheduled to execute periodically or on wakeup events detected from message bus + * + * @param jobId upper job that is going to wait the completion of a down-level job + * @param joinJobId down-level job + * @Param wakeupHandler wake-up handler + * @Param wakeupDispatcher wake-up dispatcher + * @param wakeupTopicsOnMessageBus + * @param wakeupIntervalInMilliSeconds + * @param timeoutInMilliSeconds + */ + void joinJob(long jobId, long joinJobId, String wakeupHandler, String wakupDispatcher, + String[] wakeupTopicsOnMessageBus, long wakeupIntervalInMilliSeconds, long timeoutInMilliSeconds); + + /** + * Dis-join two related jobs + * + * @param jobId + * @param joinedJobId + */ + void disjoinJob(long jobId, long joinedJobId); + + /** + * Used by down-level job to notify its completion to upper level jobs + * + * @param joinJobId down-level job for upper level job to join with + * @param joinStatus AsyncJobConstants status code to indicate success or failure of the + * down-level job + * @param joinResult object-stream serialized result object + * this is primarily used by down-level job to pass error exception objects + * for legacy code to work. To help pass exception object easier, we use + * object-stream based serialization instead of GSON + */ + void completeJoin(long joinJobId, JobInfo.Status joinStatus, String joinResult); + + void releaseSyncSource(); + void syncAsyncJobExecution(AsyncJob job, String syncObjType, long syncObjId, long queueSizeLimit); + + /** + * This method will be deprecated after all code has been migrated to fully-asynchronous mode + * that uses async-feature of joinJob/disjoinJob + * + * @param wakupTopicsOnMessageBus topic on message bus to wakeup the wait + * @param checkIntervalInMilliSeconds time to break out wait for checking predicate condition + * @param timeoutInMiliseconds time out to break out the whole wait process + * @param predicate + * @return true, predicate condition is satisfied + * false, wait is timed out + */ + boolean waitAndCheck(AsyncJob job, String[] wakupTopicsOnMessageBus, long checkIntervalInMilliSeconds, + long timeoutInMiliseconds, Predicate predicate); + + AsyncJob queryJob(long jobId, boolean updatePollTime); + +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/JobCancellationException.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/JobCancellationException.java new file mode 100644 index 00000000000..28c1e5b1bc2 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/JobCancellationException.java @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs; + +import java.util.concurrent.CancellationException; + +import com.cloud.utils.SerialVersionUID; + + +/** + * This exception is fired when the job has been cancelled + * + */ +public class JobCancellationException extends CancellationException { + + private static final long serialVersionUID = SerialVersionUID.AffinityConflictException; + + public enum Reason { + RequestedByUser, + RequestedByCaller, + TimedOut; + } + + Reason reason; + + public JobCancellationException(Reason reason) { + super("The job was cancelled due to " + reason.toString()); + this.reason = reason; + } + + public Reason getReason() { + return reason; + } + +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/Outcome.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/Outcome.java new file mode 100644 index 00000000000..b400b71fd20 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/Outcome.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs; + +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +/** + * Outcome is returned by clients of jobs framework as a way to wait for the + * outcome of a job. It fully complies with how Future interface is designed. + * In addition, it allows the callee to file a task to be scheduled when the + * job completes. + * + * Note that the callee should schedule a job when using the Task interface. + * It shouldn't try to complete the job in the schedule code as that will take + * up threads in the jobs framework. + * + * For the client of the jobs framework, you can either use the OutcomeImpl + * class to implement this interface or you can add to this interface to + * allow for your specific exceptions to be thrown. + * + * @param Object returned to the callee when the job completes + */ +public interface Outcome extends Future { + AsyncJob getJob(); + + /** + * In addition to the normal Future methods, Outcome allows the ability + * to register a schedule task to be performed when the job is completed. + * + * @param listener + */ + void execute(Task task); + + void execute(Task task, long wait, TimeUnit unit); + + /** + * Listener is used by Outcome to schedule a task to run when a job + * completes. + * + * @param T result returned + */ + public interface Task extends Runnable { + void schedule(AsyncJobExecutionContext context, T result); + + void scheduleOnError(AsyncJobExecutionContext context, Throwable e); + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java new file mode 100644 index 00000000000..cfcd173a780 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.dao; + +import java.util.Date; +import java.util.List; + +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; + +import com.cloud.utils.db.GenericDao; + +public interface AsyncJobDao extends GenericDao { + AsyncJobVO findInstancePendingAsyncJob(String instanceType, long instanceId); + List findInstancePendingAsyncJobs(String instanceType, Long accountId); + + AsyncJobVO findPseudoJob(long threadId, long msid); + void cleanupPseduoJobs(long msid); + + List getExpiredJobs(Date cutTime, int limit); + List getExpiredUnfinishedJobs(Date cutTime, int limit); + void resetJobProcess(long msid, int jobResultCode, String jobResultMessage); + List getExpiredCompletedJobs(Date cutTime, int limit); +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java new file mode 100644 index 00000000000..fb3845caa31 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.dao; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Date; +import java.util.List; + +import org.apache.log4j.Logger; + +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; +import org.apache.cloudstack.jobs.JobInfo; + +import com.cloud.utils.db.DB; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.Transaction; + +public class AsyncJobDaoImpl extends GenericDaoBase implements AsyncJobDao { + private static final Logger s_logger = Logger.getLogger(AsyncJobDaoImpl.class.getName()); + + private final SearchBuilder pendingAsyncJobSearch; + private final SearchBuilder pendingAsyncJobsSearch; + private final SearchBuilder expiringAsyncJobSearch; + private final SearchBuilder pseudoJobSearch; + private final SearchBuilder pseudoJobCleanupSearch; + private final SearchBuilder expiringUnfinishedAsyncJobSearch; + private final SearchBuilder expiringCompletedAsyncJobSearch; + + + public AsyncJobDaoImpl() { + pendingAsyncJobSearch = createSearchBuilder(); + pendingAsyncJobSearch.and("instanceType", pendingAsyncJobSearch.entity().getInstanceType(), + SearchCriteria.Op.EQ); + pendingAsyncJobSearch.and("instanceId", pendingAsyncJobSearch.entity().getInstanceId(), + SearchCriteria.Op.EQ); + pendingAsyncJobSearch.and("status", pendingAsyncJobSearch.entity().getStatus(), + SearchCriteria.Op.EQ); + pendingAsyncJobSearch.done(); + + expiringAsyncJobSearch = createSearchBuilder(); + expiringAsyncJobSearch.and("created", expiringAsyncJobSearch.entity().getCreated(), SearchCriteria.Op.LTEQ); + expiringAsyncJobSearch.done(); + + pendingAsyncJobsSearch = createSearchBuilder(); + pendingAsyncJobsSearch.and("instanceType", pendingAsyncJobsSearch.entity().getInstanceType(), + SearchCriteria.Op.EQ); + pendingAsyncJobsSearch.and("accountId", pendingAsyncJobsSearch.entity().getAccountId(), + SearchCriteria.Op.EQ); + pendingAsyncJobsSearch.and("status", pendingAsyncJobsSearch.entity().getStatus(), + SearchCriteria.Op.EQ); + pendingAsyncJobsSearch.done(); + + expiringUnfinishedAsyncJobSearch = createSearchBuilder(); + expiringUnfinishedAsyncJobSearch.and("created", expiringUnfinishedAsyncJobSearch.entity().getCreated(), + SearchCriteria.Op.LTEQ); + expiringUnfinishedAsyncJobSearch.and("completeMsId", expiringUnfinishedAsyncJobSearch.entity().getCompleteMsid(), SearchCriteria.Op.NULL); + expiringUnfinishedAsyncJobSearch.and("jobStatus", expiringUnfinishedAsyncJobSearch.entity().getStatus(), SearchCriteria.Op.EQ); + expiringUnfinishedAsyncJobSearch.done(); + + expiringCompletedAsyncJobSearch = createSearchBuilder(); + expiringCompletedAsyncJobSearch.and("created", expiringCompletedAsyncJobSearch.entity().getCreated(), + SearchCriteria.Op.LTEQ); + expiringCompletedAsyncJobSearch.and("completeMsId", expiringCompletedAsyncJobSearch.entity().getCompleteMsid(), SearchCriteria.Op.NNULL); + expiringCompletedAsyncJobSearch.and("jobStatus", expiringCompletedAsyncJobSearch.entity().getStatus(), SearchCriteria.Op.NEQ); + expiringCompletedAsyncJobSearch.done(); + + pseudoJobSearch = createSearchBuilder(); + pseudoJobSearch.and("jobDispatcher", pseudoJobSearch.entity().getDispatcher(), Op.EQ); + pseudoJobSearch.and("instanceType", pseudoJobSearch.entity().getInstanceType(), Op.EQ); + pseudoJobSearch.and("instanceId", pseudoJobSearch.entity().getInstanceId(), Op.EQ); + pseudoJobSearch.done(); + + pseudoJobCleanupSearch = createSearchBuilder(); + pseudoJobCleanupSearch.and("initMsid", pseudoJobCleanupSearch.entity().getInitMsid(), Op.EQ); + pseudoJobCleanupSearch.done(); + + } + + @Override + public AsyncJobVO findInstancePendingAsyncJob(String instanceType, long instanceId) { + SearchCriteria sc = pendingAsyncJobSearch.create(); + sc.setParameters("instanceType", instanceType); + sc.setParameters("instanceId", instanceId); + sc.setParameters("status", JobInfo.Status.IN_PROGRESS); + + List l = listIncludingRemovedBy(sc); + if(l != null && l.size() > 0) { + if(l.size() > 1) { + s_logger.warn("Instance " + instanceType + "-" + instanceId + " has multiple pending async-job"); + } + + return l.get(0); + } + return null; + } + + @Override + public List findInstancePendingAsyncJobs(String instanceType, Long accountId) { + SearchCriteria sc = pendingAsyncJobsSearch.create(); + sc.setParameters("instanceType", instanceType); + + if (accountId != null) { + sc.setParameters("accountId", accountId); + } + sc.setParameters("status", JobInfo.Status.IN_PROGRESS); + + return listBy(sc); + } + + @Override + public AsyncJobVO findPseudoJob(long threadId, long msid) { + SearchCriteria sc = pseudoJobSearch.create(); + sc.setParameters("jobDispatcher", AsyncJobVO.JOB_DISPATCHER_PSEUDO); + sc.setParameters("instanceType", AsyncJobVO.PSEUDO_JOB_INSTANCE_TYPE); + sc.setParameters("instanceId", threadId); + + List result = listBy(sc); + if(result != null && result.size() > 0) { + assert(result.size() == 1); + return result.get(0); + } + + return null; + } + + @Override + public void cleanupPseduoJobs(long msid) { + SearchCriteria sc = pseudoJobCleanupSearch.create(); + sc.setParameters("initMsid", msid); + this.expunge(sc); + } + + @Override + public List getExpiredJobs(Date cutTime, int limit) { + SearchCriteria sc = expiringAsyncJobSearch.create(); + sc.setParameters("created", cutTime); + Filter filter = new Filter(AsyncJobVO.class, "created", true, 0L, (long)limit); + return listIncludingRemovedBy(sc, filter); + } + + @Override + public List getExpiredUnfinishedJobs(Date cutTime, int limit) { + SearchCriteria sc = expiringUnfinishedAsyncJobSearch.create(); + sc.setParameters("created", cutTime); + sc.setParameters("jobStatus", JobInfo.Status.IN_PROGRESS); + Filter filter = new Filter(AsyncJobVO.class, "created", true, 0L, (long)limit); + return listIncludingRemovedBy(sc, filter); + } + + @Override + public List getExpiredCompletedJobs(Date cutTime, int limit) { + SearchCriteria sc = expiringCompletedAsyncJobSearch.create(); + sc.setParameters("created", cutTime); + sc.setParameters("jobStatus", JobInfo.Status.IN_PROGRESS); + Filter filter = new Filter(AsyncJobVO.class, "created", true, 0L, (long)limit); + return listIncludingRemovedBy(sc, filter); + } + + @Override + @DB + public void resetJobProcess(long msid, int jobResultCode, String jobResultMessage) { + String sql = "UPDATE async_job SET job_status=" + JobInfo.Status.FAILED.ordinal() + ", job_result_code=" + jobResultCode + + ", job_result='" + jobResultMessage + "' where job_status=" + JobInfo.Status.IN_PROGRESS.ordinal() + + " AND (job_executing_msid=? OR (job_executing_msid IS NULL AND job_init_msid=?))"; + + Transaction txn = Transaction.currentTxn(); + PreparedStatement pstmt = null; + try { + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setLong(1, msid); + pstmt.setLong(2, msid); + pstmt.execute(); + } catch (SQLException e) { + s_logger.warn("Unable to reset job status for management server " + msid, e); + } catch (Throwable e) { + s_logger.warn("Unable to reset job status for management server " + msid, e); + } + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDao.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDao.java new file mode 100644 index 00000000000..577ed1057bb --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDao.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.dao; + +import java.util.Date; +import java.util.List; + +import org.apache.cloudstack.framework.jobs.impl.AsyncJobJoinMapVO; +import org.apache.cloudstack.jobs.JobInfo; + +import com.cloud.utils.db.GenericDao; + +public interface AsyncJobJoinMapDao extends GenericDao { + + Long joinJob(long jobId, long joinJobId, long joinMsid, + long wakeupIntervalMs, long expirationMs, + Long syncSourceId, String wakeupHandler, String wakeupDispatcher); + void disjoinJob(long jobId, long joinedJobId); + void disjoinAllJobs(long jobId); + + AsyncJobJoinMapVO getJoinRecord(long jobId, long joinJobId); + List listJoinRecords(long jobId); + + void completeJoin(long joinJobId, JobInfo.Status joinStatus, String joinResult, long completeMsid); + +// List wakeupScan(); + + List findJobsToWake(long joinedJobId); + + List findJobsToWakeBetween(Date cutDate); +// List wakeupByJoinedJobCompletion(long joinedJobId); +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDaoImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDaoImpl.java new file mode 100644 index 00000000000..20d8ba69fdc --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJoinMapDaoImpl.java @@ -0,0 +1,303 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.dao; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + +import org.apache.log4j.Logger; + +import org.apache.cloudstack.framework.jobs.impl.AsyncJobJoinMapVO; +import org.apache.cloudstack.jobs.JobInfo; + +import com.cloud.utils.DateUtil; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.UpdateBuilder; +import com.cloud.utils.exception.CloudRuntimeException; + +public class AsyncJobJoinMapDaoImpl extends GenericDaoBase implements AsyncJobJoinMapDao { + public static final Logger s_logger = Logger.getLogger(AsyncJobJoinMapDaoImpl.class); + + private final SearchBuilder RecordSearch; + private final SearchBuilder RecordSearchByOwner; + private final SearchBuilder CompleteJoinSearch; + private final SearchBuilder WakeupSearch; + +// private final GenericSearchBuilder JoinJobSearch; + + protected AsyncJobJoinMapDaoImpl() { + RecordSearch = createSearchBuilder(); + RecordSearch.and("jobId", RecordSearch.entity().getJobId(), Op.EQ); + RecordSearch.and("joinJobId", RecordSearch.entity().getJoinJobId(), Op.EQ); + RecordSearch.done(); + + RecordSearchByOwner = createSearchBuilder(); + RecordSearchByOwner.and("jobId", RecordSearchByOwner.entity().getJobId(), Op.EQ); + RecordSearchByOwner.done(); + + CompleteJoinSearch = createSearchBuilder(); + CompleteJoinSearch.and("joinJobId", CompleteJoinSearch.entity().getJoinJobId(), Op.EQ); + CompleteJoinSearch.done(); + + WakeupSearch = createSearchBuilder(); + WakeupSearch.and("nextWakeupTime", WakeupSearch.entity().getNextWakeupTime(), Op.LT); + WakeupSearch.and("expiration", WakeupSearch.entity().getExpiration(), Op.GT); + WakeupSearch.and("joinStatus", WakeupSearch.entity().getJoinStatus(), Op.EQ); + WakeupSearch.done(); + +// JoinJobSearch = createSearchBuilder(Long.class); +// JoinJobSearch.and(JoinJobSearch.entity().getJoinJobId(), Op.SC, "joinJobId"); +// JoinJobSearch.done(); + } + + @Override + public Long joinJob(long jobId, long joinJobId, long joinMsid, + long wakeupIntervalMs, long expirationMs, + Long syncSourceId, String wakeupHandler, String wakeupDispatcher) { + + AsyncJobJoinMapVO record = new AsyncJobJoinMapVO(); + record.setJobId(jobId); + record.setJoinJobId(joinJobId); + record.setJoinMsid(joinMsid); + record.setJoinStatus(JobInfo.Status.IN_PROGRESS); + record.setSyncSourceId(syncSourceId); + record.setWakeupInterval(wakeupIntervalMs / 1000); // convert millisecond to second + record.setWakeupHandler(wakeupHandler); + record.setWakeupDispatcher(wakeupDispatcher); + if(wakeupHandler != null) { + record.setNextWakeupTime(new Date(DateUtil.currentGMTTime().getTime() + wakeupIntervalMs)); + record.setExpiration(new Date(DateUtil.currentGMTTime().getTime() + expirationMs)); + } + + persist(record); + return record.getId(); + } + + @Override + public void disjoinJob(long jobId, long joinedJobId) { + SearchCriteria sc = RecordSearch.create(); + sc.setParameters("jobId", jobId); + sc.setParameters("joinJobId", joinedJobId); + + this.expunge(sc); + } + + @Override + public void disjoinAllJobs(long jobId) { + SearchCriteria sc = RecordSearchByOwner.create(); + sc.setParameters("jobId", jobId); + + this.expunge(sc); + } + + @Override + public AsyncJobJoinMapVO getJoinRecord(long jobId, long joinJobId) { + SearchCriteria sc = RecordSearch.create(); + sc.setParameters("jobId", jobId); + sc.setParameters("joinJobId", joinJobId); + + List result = this.listBy(sc); + if(result != null && result.size() > 0) { + assert(result.size() == 1); + return result.get(0); + } + + return null; + } + + @Override + public List listJoinRecords(long jobId) { + SearchCriteria sc = RecordSearchByOwner.create(); + sc.setParameters("jobId", jobId); + + return this.listBy(sc); + } + + @Override + public void completeJoin(long joinJobId, JobInfo.Status joinStatus, String joinResult, long completeMsid) { + AsyncJobJoinMapVO record = createForUpdate(); + record.setJoinStatus(joinStatus); + record.setJoinResult(joinResult); + record.setCompleteMsid(completeMsid); + record.setLastUpdated(DateUtil.currentGMTTime()); + + UpdateBuilder ub = getUpdateBuilder(record); + + SearchCriteria sc = CompleteJoinSearch.create(); + sc.setParameters("joinJobId", joinJobId); + update(ub, sc, null); + } + +// @Override +// public List wakeupScan() { +// List standaloneList = new ArrayList(); +// +// Date cutDate = DateUtil.currentGMTTime(); +// +// Transaction txn = Transaction.currentTxn(); +// PreparedStatement pstmt = null; +// try { +// txn.start(); +// +// // +// // performance sensitive processing, do it in plain SQL +// // +// String sql = "UPDATE async_job SET job_pending_signals=? WHERE id IN " + +// "(SELECT job_id FROM async_job_join_map WHERE next_wakeup < ? AND expiration > ?)"; +// pstmt = txn.prepareStatement(sql); +// pstmt.setInt(1, AsyncJob.Constants.SIGNAL_MASK_WAKEUP); +// pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate)); +// pstmt.setString(3, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate)); +// pstmt.executeUpdate(); +// pstmt.close(); +// +// sql = "UPDATE sync_queue_item SET queue_proc_msid=NULL, queue_proc_number=NULL WHERE content_id IN " + +// "(SELECT job_id FROM async_job_join_map WHERE next_wakeup < ? AND expiration > ?)"; +// pstmt = txn.prepareStatement(sql); +// pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate)); +// pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate)); +// pstmt.executeUpdate(); +// pstmt.close(); +// +// sql = "SELECT job_id FROM async_job_join_map WHERE next_wakeup < ? AND expiration > ? AND job_id NOT IN (SELECT content_id FROM sync_queue_item)"; +// pstmt = txn.prepareStatement(sql); +// pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate)); +// pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate)); +// ResultSet rs = pstmt.executeQuery(); +// while(rs.next()) { +// standaloneList.add(rs.getLong(1)); +// } +// rs.close(); +// pstmt.close(); +// +// // update for next wake-up +// sql = "UPDATE async_job_join_map SET next_wakeup=DATE_ADD(next_wakeup, INTERVAL wakeup_interval SECOND) WHERE next_wakeup < ? AND expiration > ?"; +// pstmt = txn.prepareStatement(sql); +// pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate)); +// pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate)); +// pstmt.executeUpdate(); +// pstmt.close(); +// +// txn.commit(); +// } catch (SQLException e) { +// s_logger.error("Unexpected exception", e); +// } +// +// return standaloneList; +// } + + @Override + public List findJobsToWake(long joinedJobId) { + // TODO: We should fix this. We shouldn't be crossing daos in a dao code. + List standaloneList = new ArrayList(); + Transaction txn = Transaction.currentTxn(); + String sql = "SELECT job_id FROM async_job_join_map WHERE join_job_id = ? AND job_id NOT IN (SELECT content_id FROM sync_queue_item)"; + try { + PreparedStatement pstmt = txn.prepareStatement(sql); + pstmt.setLong(1, joinedJobId); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + standaloneList.add(rs.getLong(1)); + } + } catch (SQLException e) { + throw new CloudRuntimeException("Unable to execute " + sql, e); + } + return standaloneList; + } + + @Override + public List findJobsToWakeBetween(Date cutDate) { + List standaloneList = new ArrayList(); + Transaction txn = Transaction.currentTxn(); + try { + String sql = "SELECT job_id FROM async_job_join_map WHERE next_wakeup < ? AND expiration > ? AND job_id NOT IN (SELECT content_id FROM sync_queue_item)"; + PreparedStatement pstmt = txn.prepareStatement(sql); + pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate)); + pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate)); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + standaloneList.add(rs.getLong(1)); + } + + // update for next wake-up + sql = "UPDATE async_job_join_map SET next_wakeup=DATE_ADD(next_wakeup, INTERVAL wakeup_interval SECOND) WHERE next_wakeup < ? AND expiration > ?"; + pstmt = txn.prepareStatement(sql); + pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate)); + pstmt.setString(2, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutDate)); + pstmt.executeUpdate(); + + return standaloneList; + } catch (SQLException e) { + throw new CloudRuntimeException("Unable to handle SQL exception", e); + } + + } + +// @Override +// public List wakeupByJoinedJobCompletion(long joinedJobId) { +// List standaloneList = new ArrayList(); +// +// Transaction txn = Transaction.currentTxn(); +// PreparedStatement pstmt = null; +// try { +// txn.start(); +// +// // +// // performance sensitive processing, do it in plain SQL +// // +// String sql = "UPDATE async_job SET job_pending_signals=? WHERE id IN " + +// "(SELECT job_id FROM async_job_join_map WHERE join_job_id = ?)"; +// pstmt = txn.prepareStatement(sql); +// pstmt.setInt(1, AsyncJob.Contants.SIGNAL_MASK_WAKEUP); +// pstmt.setLong(2, joinedJobId); +// pstmt.executeUpdate(); +// pstmt.close(); +// +// sql = "UPDATE sync_queue_item SET queue_proc_msid=NULL, queue_proc_number=NULL WHERE content_id IN " + +// "(SELECT job_id FROM async_job_join_map WHERE join_job_id = ?)"; +// pstmt = txn.prepareStatement(sql); +// pstmt.setLong(1, joinedJobId); +// pstmt.executeUpdate(); +// pstmt.close(); +// +// sql = "SELECT job_id FROM async_job_join_map WHERE join_job_id = ? AND job_id NOT IN (SELECT content_id FROM sync_queue_item)"; +// pstmt = txn.prepareStatement(sql); +// pstmt.setLong(1, joinedJobId); +// ResultSet rs = pstmt.executeQuery(); +// while(rs.next()) { +// standaloneList.add(rs.getLong(1)); +// } +// rs.close(); +// pstmt.close(); +// +// txn.commit(); +// } catch (SQLException e) { +// s_logger.error("Unexpected exception", e); +// } +// +// return standaloneList; +// } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJournalDao.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJournalDao.java new file mode 100644 index 00000000000..fb6a2421ba4 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJournalDao.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.dao; + +import java.util.List; + +import org.apache.cloudstack.framework.jobs.impl.AsyncJobJournalVO; + +import com.cloud.utils.db.GenericDao; + +public interface AsyncJobJournalDao extends GenericDao { + List getJobJournal(long jobId); +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJournalDaoImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJournalDaoImpl.java new file mode 100644 index 00000000000..d26e6ed63ed --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/AsyncJobJournalDaoImpl.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.dao; + +import java.util.List; + +import org.apache.cloudstack.framework.jobs.impl.AsyncJobJournalVO; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Op; + +public class AsyncJobJournalDaoImpl extends GenericDaoBase implements AsyncJobJournalDao { + + private final SearchBuilder JobJournalSearch; + + public AsyncJobJournalDaoImpl() { + JobJournalSearch = createSearchBuilder(); + JobJournalSearch.and("jobId", JobJournalSearch.entity().getJobId(), Op.EQ); + JobJournalSearch.done(); + } + + @Override + public List getJobJournal(long jobId) { + SearchCriteria sc = JobJournalSearch.create(); + sc.setParameters("jobId", jobId); + + return this.listBy(sc); + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueDao.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueDao.java new file mode 100644 index 00000000000..fa617adbf62 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueDao.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.dao; + +import org.apache.cloudstack.framework.jobs.impl.SyncQueueVO; + +import com.cloud.utils.db.GenericDao; + +public interface SyncQueueDao extends GenericDao{ + public void ensureQueue(String syncObjType, long syncObjId); + public SyncQueueVO find(String syncObjType, long syncObjId); +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueDaoImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueDaoImpl.java new file mode 100644 index 00000000000..f7d9d72dc0b --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueDaoImpl.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.jobs.dao; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Date; +import java.util.TimeZone; + +import org.apache.log4j.Logger; + +import org.apache.cloudstack.framework.jobs.impl.SyncQueueVO; + +import com.cloud.utils.DateUtil; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; + +public class SyncQueueDaoImpl extends GenericDaoBase implements SyncQueueDao { + private static final Logger s_logger = Logger.getLogger(SyncQueueDaoImpl.class.getName()); + + SearchBuilder TypeIdSearch = createSearchBuilder(); + + public SyncQueueDaoImpl() { + super(); + TypeIdSearch = createSearchBuilder(); + TypeIdSearch.and("syncObjType", TypeIdSearch.entity().getSyncObjType(), SearchCriteria.Op.EQ); + TypeIdSearch.and("syncObjId", TypeIdSearch.entity().getSyncObjId(), SearchCriteria.Op.EQ); + TypeIdSearch.done(); + } + + @Override + public void ensureQueue(String syncObjType, long syncObjId) { + Date dt = DateUtil.currentGMTTime(); + String sql = "INSERT IGNORE INTO sync_queue(sync_objtype, sync_objid, created, last_updated)" + + " values(?, ?, ?, ?)"; + + Transaction txn = Transaction.currentTxn(); + PreparedStatement pstmt = null; + try { + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setString(1, syncObjType); + pstmt.setLong(2, syncObjId); + pstmt.setString(3, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), dt)); + pstmt.setString(4, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), dt)); + pstmt.execute(); + } catch (SQLException e) { + s_logger.warn("Unable to create sync queue " + syncObjType + "-" + syncObjId + ":" + e.getMessage(), e); + } catch (Throwable e) { + s_logger.warn("Unable to create sync queue " + syncObjType + "-" + syncObjId + ":" + e.getMessage(), e); + } + } + + @Override + public SyncQueueVO find(String syncObjType, long syncObjId) { + SearchCriteria sc = TypeIdSearch.create(); + sc.setParameters("syncObjType", syncObjType); + sc.setParameters("syncObjId", syncObjId); + return findOneBy(sc); + } + +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java new file mode 100644 index 00000000000..61670bf7043 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.dao; + +import java.util.List; + +import org.apache.cloudstack.framework.jobs.impl.SyncQueueItemVO; + +import com.cloud.utils.db.GenericDao; + +public interface SyncQueueItemDao extends GenericDao { + public SyncQueueItemVO getNextQueueItem(long queueId); + public List getNextQueueItems(int maxItems); + public List getActiveQueueItems(Long msid, boolean exclusive); + public List getBlockedQueueItems(long thresholdMs, boolean exclusive); + public Long getQueueItemIdByContentIdAndType(long contentId, String contentType); +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java new file mode 100644 index 00000000000..ccb7f103742 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.jobs.dao; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + +import org.apache.log4j.Logger; + +import org.apache.cloudstack.framework.jobs.impl.SyncQueueItemVO; + +import com.cloud.utils.DateUtil; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.Transaction; + +@DB +public class SyncQueueItemDaoImpl extends GenericDaoBase implements SyncQueueItemDao { + private static final Logger s_logger = Logger.getLogger(SyncQueueItemDaoImpl.class); + final GenericSearchBuilder queueIdSearch; + + public SyncQueueItemDaoImpl() { + super(); + queueIdSearch = createSearchBuilder(Long.class); + queueIdSearch.and("contentId", queueIdSearch.entity().getContentId(), Op.EQ); + queueIdSearch.and("contentType", queueIdSearch.entity().getContentType(), Op.EQ); + queueIdSearch.selectField(queueIdSearch.entity().getId()); + queueIdSearch.done(); + } + + @Override + public SyncQueueItemVO getNextQueueItem(long queueId) { + + SearchBuilder sb = createSearchBuilder(); + sb.and("queueId", sb.entity().getQueueId(), SearchCriteria.Op.EQ); + sb.and("lastProcessNumber", sb.entity().getLastProcessNumber(), SearchCriteria.Op.NULL); + sb.done(); + + SearchCriteria sc = sb.create(); + sc.setParameters("queueId", queueId); + + Filter filter = new Filter(SyncQueueItemVO.class, "created", true, 0L, 1L); + List l = listBy(sc, filter); + if(l != null && l.size() > 0) + return l.get(0); + + return null; + } + + @Override + public List getNextQueueItems(int maxItems) { + List l = new ArrayList(); + + String sql = "SELECT i.id, i.queue_id, i.content_type, i.content_id, i.created " + + " FROM sync_queue AS q JOIN sync_queue_item AS i ON q.id = i.queue_id " + + " WHERE i.queue_proc_number IS NULL " + + " GROUP BY q.id " + + " ORDER BY i.id " + + " LIMIT 0, ?"; + + Transaction txn = Transaction.currentTxn(); + PreparedStatement pstmt = null; + try { + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setInt(1, maxItems); + ResultSet rs = pstmt.executeQuery(); + while(rs.next()) { + SyncQueueItemVO item = new SyncQueueItemVO(); + item.setId(rs.getLong(1)); + item.setQueueId(rs.getLong(2)); + item.setContentType(rs.getString(3)); + item.setContentId(rs.getLong(4)); + item.setCreated(DateUtil.parseDateString(TimeZone.getTimeZone("GMT"), rs.getString(5))); + l.add(item); + } + } catch (SQLException e) { + s_logger.error("Unexpected sql excetpion, ", e); + } catch (Throwable e) { + s_logger.error("Unexpected excetpion, ", e); + } + return l; + } + + @Override + public List getActiveQueueItems(Long msid, boolean exclusive) { + SearchBuilder sb = createSearchBuilder(); + sb.and("lastProcessMsid", sb.entity().getLastProcessMsid(), + SearchCriteria.Op.EQ); + sb.done(); + + SearchCriteria sc = sb.create(); + sc.setParameters("lastProcessMsid", msid); + + Filter filter = new Filter(SyncQueueItemVO.class, "created", true, null, null); + + if(exclusive) + return lockRows(sc, filter, true); + return listBy(sc, filter); + } + + @Override + public List getBlockedQueueItems(long thresholdMs, boolean exclusive) { + Date cutTime = DateUtil.currentGMTTime(); + + SearchBuilder sbItem = createSearchBuilder(); + sbItem.and("lastProcessMsid", sbItem.entity().getLastProcessMsid(), SearchCriteria.Op.NNULL); + sbItem.and("lastProcessNumber", sbItem.entity().getLastProcessNumber(), SearchCriteria.Op.NNULL); + sbItem.and("lastProcessNumber", sbItem.entity().getLastProcessTime(), SearchCriteria.Op.NNULL); + sbItem.and("lastProcessTime2", sbItem.entity().getLastProcessTime(), SearchCriteria.Op.LT); + + sbItem.done(); + + SearchCriteria sc = sbItem.create(); + sc.setParameters("lastProcessTime2", new Date(cutTime.getTime() - thresholdMs)); + + if(exclusive) + return lockRows(sc, null, true); + return listBy(sc, null); + } + + @Override + public Long getQueueItemIdByContentIdAndType(long contentId, String contentType) { + SearchCriteria sc = queueIdSearch.create(); + sc.setParameters("contentId", contentId); + sc.setParameters("contentType", contentType); + List id = customSearch(sc, null); + + return id.size() == 0 ? null : id.get(0); + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobJoinMapVO.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobJoinMapVO.java new file mode 100644 index 00000000000..287121f1c64 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobJoinMapVO.java @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.impl; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import org.apache.cloudstack.jobs.JobInfo; + +import com.cloud.utils.DateUtil; +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name="async_job_join_map") +public class AsyncJobJoinMapVO { + @Id + @GeneratedValue(strategy=GenerationType.IDENTITY) + @Column(name="id") + private Long id = null; + + @Column(name="job_id") + private long jobId; + + @Column(name="join_job_id") + private long joinJobId; + + @Column(name="join_status") + @Enumerated(EnumType.ORDINAL) + private JobInfo.Status joinStatus; + + @Column(name="join_result", length=1024) + private String joinResult; + + @Column(name="join_msid") + private long joinMsid; + + @Column(name="complete_msid") + private Long completeMsid; + + @Column(name="sync_source_id") + private Long syncSourceId; + + @Column(name="wakeup_handler") + private String wakeupHandler; + + @Column(name="wakeup_dispatcher") + private String wakeupDispatcher; + + @Column(name="wakeup_interval") + private long wakeupInterval; + + @Column(name=GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name="last_updated") + @Temporal(TemporalType.TIMESTAMP) + private Date lastUpdated; + + @Column(name="next_wakeup") + @Temporal(TemporalType.TIMESTAMP) + private Date nextWakeupTime; + + @Column(name="expiration") + @Temporal(TemporalType.TIMESTAMP) + private Date expiration; + + public AsyncJobJoinMapVO() { + created = DateUtil.currentGMTTime(); + lastUpdated = DateUtil.currentGMTTime(); + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public long getJobId() { + return jobId; + } + + public void setJobId(long jobId) { + this.jobId = jobId; + } + + public long getJoinJobId() { + return joinJobId; + } + + public void setJoinJobId(long joinJobId) { + this.joinJobId = joinJobId; + } + + public JobInfo.Status getJoinStatus() { + return joinStatus; + } + + public void setJoinStatus(JobInfo.Status joinStatus) { + this.joinStatus = joinStatus; + } + + public String getJoinResult() { + return joinResult; + } + + public void setJoinResult(String joinResult) { + this.joinResult = joinResult; + } + + public long getJoinMsid() { + return joinMsid; + } + + public void setJoinMsid(long joinMsid) { + this.joinMsid = joinMsid; + } + + public Long getCompleteMsid() { + return completeMsid; + } + + public void setCompleteMsid(Long completeMsid) { + this.completeMsid = completeMsid; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Date getLastUpdated() { + return lastUpdated; + } + + public void setLastUpdated(Date lastUpdated) { + this.lastUpdated = lastUpdated; + } + + public Long getSyncSourceId() { + return syncSourceId; + } + + public void setSyncSourceId(Long syncSourceId) { + this.syncSourceId = syncSourceId; + } + + public String getWakeupHandler() { + return wakeupHandler; + } + + public void setWakeupHandler(String wakeupHandler) { + this.wakeupHandler = wakeupHandler; + } + + public String getWakeupDispatcher() { + return wakeupDispatcher; + } + + public void setWakeupDispatcher(String wakeupDispatcher) { + this.wakeupDispatcher = wakeupDispatcher; + } + + public long getWakeupInterval() { + return wakeupInterval; + } + + public void setWakeupInterval(long wakeupInterval) { + this.wakeupInterval = wakeupInterval; + } + + public Date getNextWakeupTime() { + return nextWakeupTime; + } + + public void setNextWakeupTime(Date nextWakeupTime) { + this.nextWakeupTime = nextWakeupTime; + } + + public Date getExpiration() { + return expiration; + } + + public void setExpiration(Date expiration) { + this.expiration = expiration; + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobJournalVO.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobJournalVO.java new file mode 100644 index 00000000000..b78a7e02254 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobJournalVO.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.impl; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.framework.jobs.AsyncJob.JournalType; + +import com.cloud.utils.DateUtil; +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name="async_job_journal") +public class AsyncJobJournalVO { + @Id + @GeneratedValue(strategy=GenerationType.IDENTITY) + @Column(name="id") + private Long id = null; + + @Column(name="job_id") + private long jobId; + + @Column(name="journal_type", updatable=false, nullable=false, length=32) + @Enumerated(value=EnumType.STRING) + private AsyncJob.JournalType journalType; + + @Column(name="journal_text", length=1024) + private String journalText; + + @Column(name="journal_obj", length=1024) + private String journalObjJsonString; + + @Column(name=GenericDao.CREATED_COLUMN) + protected Date created; + + public AsyncJobJournalVO() { + created = DateUtil.currentGMTTime(); + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public long getJobId() { + return jobId; + } + + public void setJobId(long jobId) { + this.jobId = jobId; + } + + public AsyncJob.JournalType getJournalType() { + return journalType; + } + + public void setJournalType(AsyncJob.JournalType journalType) { + this.journalType = journalType; + } + + public String getJournalText() { + return journalText; + } + + public void setJournalText(String journalText) { + this.journalText = journalText; + } + + public String getJournalObjJsonString() { + return journalObjJsonString; + } + + public void setJournalObjJsonString(String journalObjJsonString) { + this.journalObjJsonString = journalObjJsonString; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMBeanImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMBeanImpl.java new file mode 100644 index 00000000000..0a48da38aea --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMBeanImpl.java @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.impl; + +import java.util.Date; +import java.util.TimeZone; + +import javax.management.StandardMBean; + +import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.framework.jobs.AsyncJobMBean; + +import com.cloud.utils.DateUtil; + +public class AsyncJobMBeanImpl extends StandardMBean implements AsyncJobMBean { + private final AsyncJob _job; + + public AsyncJobMBeanImpl(AsyncJob job) { + super(AsyncJobMBean.class, false); + + _job = job; + } + + @Override + public long getAccountId() { + return _job.getAccountId(); + } + + @Override + public long getUserId() { + return _job.getUserId(); + } + + @Override + public String getCmd() { + return _job.getCmd(); + } + + @Override + public String getCmdInfo() { + return _job.getCmdInfo(); + } + + @Override + public String getStatus() { + switch (_job.getStatus()) { + case SUCCEEDED: + return "Completed"; + + case IN_PROGRESS: + return "In preogress"; + + case FAILED: + return "failed"; + + case CANCELLED: + return "cancelled"; + } + + return "Unknow"; + } + + @Override + public int getProcessStatus() { + return _job.getProcessStatus(); + } + + @Override + public int getResultCode() { + return _job.getResultCode(); + } + + @Override + public String getResult() { + return _job.getResult(); + } + + @Override + public String getInstanceType() { + if(_job.getInstanceType() != null) + return _job.getInstanceType().toString(); + return "N/A"; + } + + @Override + public String getInstanceId() { + if(_job.getInstanceId() != null) + return String.valueOf(_job.getInstanceId()); + return "N/A"; + } + + @Override + public String getInitMsid() { + if(_job.getInitMsid() != null) { + return String.valueOf(_job.getInitMsid()); + } + return "N/A"; + } + + @Override + public String getCreateTime() { + Date time = _job.getCreated(); + if(time != null) + return DateUtil.getDateDisplayString(TimeZone.getDefault(), time); + return "N/A"; + } + + @Override + public String getLastUpdateTime() { + Date time = _job.getLastUpdated(); + if(time != null) + return DateUtil.getDateDisplayString(TimeZone.getDefault(), time); + return "N/A"; + } + + @Override + public String getLastPollTime() { + Date time = _job.getLastPolled(); + + if(time != null) + return DateUtil.getDateDisplayString(TimeZone.getDefault(), time); + return "N/A"; + } + + @Override + public String getSyncQueueId() { + SyncQueueItem item = _job.getSyncSource(); + if(item != null && item.getQueueId() != null) { + return String.valueOf(item.getQueueId()); + } + return "N/A"; + } + + @Override + public String getSyncQueueContentType() { + SyncQueueItem item = _job.getSyncSource(); + if(item != null) { + return item.getContentType(); + } + return "N/A"; + } + + @Override + public String getSyncQueueContentId() { + SyncQueueItem item = _job.getSyncSource(); + if(item != null && item.getContentId() != null) { + return String.valueOf(item.getContentId()); + } + return "N/A"; + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java new file mode 100644 index 00000000000..9b12a57787d --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -0,0 +1,988 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.jobs.impl; + +import java.io.File; +import java.io.FileInputStream; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.log4j.Logger; + +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.config.ConfigDepot; +import org.apache.cloudstack.config.ConfigKey; +import org.apache.cloudstack.config.ConfigValue; +import org.apache.cloudstack.config.Configurable; +import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.framework.jobs.AsyncJobDispatcher; +import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobDao; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobJoinMapDao; +import org.apache.cloudstack.framework.jobs.dao.AsyncJobJournalDao; +import org.apache.cloudstack.framework.jobs.dao.SyncQueueItemDao; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.framework.messagebus.MessageDetector; +import org.apache.cloudstack.framework.messagebus.PublishScope; +import org.apache.cloudstack.jobs.JobInfo; +import org.apache.cloudstack.jobs.JobInfo.Status; +import org.apache.cloudstack.utils.identity.ManagementServerNode; + +import com.cloud.cluster.ClusterManagerListener; +import com.cloud.cluster.ManagementServerHost; +import com.cloud.utils.DateUtil; +import com.cloud.utils.Predicate; +import com.cloud.utils.PropertiesUtil; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.GenericDao; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.exception.ExceptionUtil; +import com.cloud.utils.mgmt.JmxUtil; + +import edu.emory.mathcs.backport.java.util.Collections; + +public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, ClusterManagerListener, Configurable { + // Advanced + private static final ConfigKey JobExpireMinutes = new ConfigKey(Long.class, "job.expire.minutes", "Advanced", AsyncJobManager.class, "1440", + "Time (in minutes) for async-jobs to be kept in system", true, null); + private static final ConfigKey JobCancelThresholdMinutes = new ConfigKey(Long.class, "job.cancel.threshold.minutes", "Advanced", AsyncJobManager.class, + "60", "Time (in minutes) for async-jobs to be forcely cancelled if it has been in process for long", true, null); + + private static final Logger s_logger = Logger.getLogger(AsyncJobManagerImpl.class); + + private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 3; // 3 seconds + private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC = 60; // 60 seconds + + private static final int MAX_ONETIME_SCHEDULE_SIZE = 50; + private static final int HEARTBEAT_INTERVAL = 2000; + private static final int GC_INTERVAL = 10000; // 10 seconds + + @Inject + private SyncQueueItemDao _queueItemDao; + @Inject private SyncQueueManager _queueMgr; + @Inject private AsyncJobDao _jobDao; + @Inject private AsyncJobJournalDao _journalDao; + @Inject private AsyncJobJoinMapDao _joinMapDao; + @Inject private List _jobDispatchers; + @Inject private MessageBus _messageBus; + @Inject private AsyncJobMonitor _jobMonitor; + @Inject + private ConfigDepot _configDepot; + + private ConfigValue _jobExpireSeconds; // 1 day + private ConfigValue _jobCancelThresholdSeconds; // 1 hour (for cancelling the jobs blocking other jobs) + + private volatile long _executionRunNumber = 1; + + private final ScheduledExecutorService _heartbeatScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("AsyncJobMgr-Heartbeat")); + private ExecutorService _executor; + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {JobExpireMinutes, JobCancelThresholdMinutes}; + } + + @Override + public AsyncJobVO getAsyncJob(long jobId) { + return _jobDao.findById(jobId); + } + + @Override + public List findInstancePendingAsyncJobs(String instanceType, Long accountId) { + return _jobDao.findInstancePendingAsyncJobs(instanceType, accountId); + } + + @Override @DB + public AsyncJob getPseudoJob(long accountId, long userId) { + AsyncJobVO job = _jobDao.findPseudoJob(Thread.currentThread().getId(), getMsid()); + if(job == null) { + job = new AsyncJobVO(); + job.setAccountId(accountId); + job.setUserId(userId); + job.setInitMsid(getMsid()); + job.setDispatcher(AsyncJobVO.JOB_DISPATCHER_PSEUDO); + job.setInstanceType(AsyncJobVO.PSEUDO_JOB_INSTANCE_TYPE); + job.setInstanceId(Thread.currentThread().getId()); + _jobDao.persist(job); + } + return job; + } + + @Override + public long submitAsyncJob(AsyncJob job) { + return submitAsyncJob(job, false); + } + + @SuppressWarnings("unchecked") + @DB + public long submitAsyncJob(AsyncJob job, boolean scheduleJobExecutionInContext) { + @SuppressWarnings("rawtypes") + GenericDao dao = GenericDaoBase.getDao(job.getClass()); + job.setInitMsid(getMsid()); + job.setSyncSource(null); // no sync source originally + dao.persist(job); + + scheduleExecution(job, scheduleJobExecutionInContext); + if (s_logger.isDebugEnabled()) { + s_logger.debug("submit async job-" + job.getId() + ", details: " + job.toString()); + } + return job.getId(); + } + + @SuppressWarnings("unchecked") + @Override @DB + public long submitAsyncJob(AsyncJob job, String syncObjType, long syncObjId) { + Transaction txt = Transaction.currentTxn(); + try { + @SuppressWarnings("rawtypes") + GenericDao dao = GenericDaoBase.getDao(job.getClass()); + + txt.start(); + job.setInitMsid(getMsid()); + dao.persist(job); + + syncAsyncJobExecution(job, syncObjType, syncObjId, 1); + txt.commit(); + return job.getId(); + } catch(Exception e) { + String errMsg = "Unable to schedule async job for command " + job.getCmd() + ", unexpected exception."; + s_logger.warn(errMsg, e); + throw new CloudRuntimeException(errMsg); + } + } + + @Override @DB + public void completeAsyncJob(long jobId, Status jobStatus, int resultCode, String resultObject) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Complete async job-" + jobId + ", jobStatus: " + jobStatus + + ", resultCode: " + resultCode + ", result: " + resultObject); + } + + Transaction txn = Transaction.currentTxn(); + try { + txn.start(); + AsyncJobVO job = _jobDao.findById(jobId); + if(job == null) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("job-" + jobId + " no longer exists, we just log completion info here. " + jobStatus + + ", resultCode: " + resultCode + ", result: " + resultObject); + } + + txn.rollback(); + return; + } + + if(job.getStatus() != JobInfo.Status.IN_PROGRESS) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("job-" + jobId + " is already completed."); + } + + txn.rollback(); + return; + } + + job.setCompleteMsid(getMsid()); + job.setStatus(jobStatus); + job.setResultCode(resultCode); + + // reset attached object + job.setInstanceType(null); + job.setInstanceId(null); + + if (resultObject != null) { + job.setResult(resultObject); + } + + job.setLastUpdated(DateUtil.currentGMTTime()); + _jobDao.update(jobId, job); + + List wakeupList = wakeupByJoinedJobCompletion(jobId); + _joinMapDao.disjoinAllJobs(jobId); + + txn.commit(); + + for(Long id : wakeupList) { + // TODO, we assume that all jobs in this category is API job only + AsyncJobVO jobToWakeup = _jobDao.findById(id); + if (jobToWakeup != null && (jobToWakeup.getPendingSignals() & AsyncJob.Constants.SIGNAL_MASK_WAKEUP) != 0) + scheduleExecution(jobToWakeup, false); + } + + _messageBus.publish(null, AsyncJob.Topics.JOB_STATE, PublishScope.GLOBAL, jobId); + } catch(Exception e) { + s_logger.error("Unexpected exception while completing async job-" + jobId, e); + txn.rollback(); + } + } + + @Override @DB + public void updateAsyncJobStatus(long jobId, int processStatus, String resultObject) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Update async-job progress, job-" + jobId + ", processStatus: " + processStatus + + ", result: " + resultObject); + } + + Transaction txt = Transaction.currentTxn(); + try { + txt.start(); + AsyncJobVO job = _jobDao.findById(jobId); + if(job == null) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("job-" + jobId + " no longer exists, we just log progress info here. progress status: " + processStatus); + } + + txt.rollback(); + return; + } + + job.setProcessStatus(processStatus); + if(resultObject != null) { + job.setResult(resultObject); + } + job.setLastUpdated(DateUtil.currentGMTTime()); + _jobDao.update(jobId, job); + txt.commit(); + } catch(Exception e) { + s_logger.error("Unexpected exception while updating async job-" + jobId + " status: ", e); + txt.rollback(); + } + } + + @Override @DB + public void updateAsyncJobAttachment(long jobId, String instanceType, Long instanceId) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Update async-job attachment, job-" + jobId + ", instanceType: " + instanceType + + ", instanceId: " + instanceId); + } + + Transaction txt = Transaction.currentTxn(); + try { + txt.start(); + + AsyncJobVO job = _jobDao.createForUpdate(); + job.setInstanceType(instanceType); + job.setInstanceId(instanceId); + job.setLastUpdated(DateUtil.currentGMTTime()); + _jobDao.update(jobId, job); + + txt.commit(); + } catch(Exception e) { + s_logger.error("Unexpected exception while updating async job-" + jobId + " attachment: ", e); + txt.rollback(); + } + } + + @Override @DB + public void logJobJournal(long jobId, AsyncJob.JournalType journalType, String + journalText, String journalObjJson) { + AsyncJobJournalVO journal = new AsyncJobJournalVO(); + journal.setJobId(jobId); + journal.setJournalType(journalType); + journal.setJournalText(journalText); + journal.setJournalObjJsonString(journalObjJson); + + _journalDao.persist(journal); + } + + @Override @DB + public void joinJob(long jobId, long joinJobId) { + _joinMapDao.joinJob(jobId, joinJobId, getMsid(), 0, 0, null, null, null); + } + + @Override @DB + public void joinJob(long jobId, long joinJobId, String wakeupHandler, String wakeupDispatcher, + String[] wakeupTopcisOnMessageBus, long wakeupIntervalInMilliSeconds, long timeoutInMilliSeconds) { + + Long syncSourceId = null; + AsyncJobExecutionContext context = AsyncJobExecutionContext.getCurrentExecutionContext(); + assert(context.getJob() != null); + if(context.getJob().getSyncSource() != null) { + syncSourceId = context.getJob().getSyncSource().getQueueId(); + } + + _joinMapDao.joinJob(jobId, joinJobId, getMsid(), + wakeupIntervalInMilliSeconds, timeoutInMilliSeconds, + syncSourceId, wakeupHandler, wakeupDispatcher); + } + + @Override @DB + public void disjoinJob(long jobId, long joinedJobId) { + _joinMapDao.disjoinJob(jobId, joinedJobId); + } + + @Override @DB + public void completeJoin(long joinJobId, JobInfo.Status joinStatus, String joinResult) { + // + // TODO + // this is a temporary solution to solve strange MySQL deadlock issue, + // completeJoin() causes deadlock happens at async_job table + // I removed the temporary solution already. I think my changes should fix the deadlock. + +/* + ------------------------ + LATEST DETECTED DEADLOCK + ------------------------ + 130625 20:03:10 + *** (1) TRANSACTION: + TRANSACTION 0 98087127, ACTIVE 0 sec, process no 1489, OS thread id 139837829175040 fetching rows, thread declared inside InnoDB 494 + mysql tables in use 2, locked 1 + LOCK WAIT 3 lock struct(s), heap size 368, 2 row lock(s), undo log entries 1 + MySQL thread id 28408, query id 368571321 localhost 127.0.0.1 cloud preparing + UPDATE async_job SET job_pending_signals=1 WHERE id IN (SELECT job_id FROM async_job_join_map WHERE join_job_id = 9) + *** (1) WAITING FOR THIS LOCK TO BE GRANTED: + RECORD LOCKS space id 0 page no 1275 n bits 80 index `PRIMARY` of table `cloud`.`async_job` trx id 0 98087127 lock_mode X locks rec but not gap waiting + Record lock, heap no 9 PHYSICAL RECORD: n_fields 26; compact format; info bits 0 + 0: len 8; hex 0000000000000008; asc ;; 1: len 6; hex 000005d8b0d8; asc ;; 2: len 7; hex 00000009270110; asc ' ;; 3: len 8; hex 0000000000000002; asc ;; 4: len 8; hex 0000000000000002; asc ;; 5: SQL NULL; 6: SQL NULL; 7: len 30; hex 6f72672e6170616368652e636c6f7564737461636b2e6170692e636f6d6d; asc org.apache.cloudstack.api.comm;...(truncated); 8: len 30; hex 7b226964223a2232222c22706879736963616c6e6574776f726b6964223a; asc {"id":"2","physicalnetworkid":;...(truncated); 9: len 4; hex 80000000; asc ;; 10: len 4; hex 80000001; asc ;; 11: len 4; hex 80000000; asc ;; 12: len 4; hex 80000000; asc ;; 13: len 30; hex 6f72672e6170616368652e636c6f7564737461636b2e6170692e72657370; asc org.apache.cloudstack.api.resp;...(truncated); 14: len 8; hex 80001a6f7bb0d0a8; asc o{ ;; 15: len 8; hex 80001a6f7bb0d0a8; asc o{ ;; 16: len 8; hex 8000124f06cfd5b6; asc O ;; 17: len 8; hex 8000124f06cfd5b6; asc O ;; 18: SQL NULL; 19: SQL NULL; 20: len 30; hex 66376466396532362d323139622d346338652d393231332d393766653636; asc f7df9e26-219b-4c8e-9213-97fe66;...(truncated); 21: len 30; hex 36623238306364362d663436652d343563322d383833642d333863616439; asc 6b280cd6-f46e-45c2-883d-38cad9;...(truncated); 22: SQL NULL; 23: len 21; hex 4170694173796e634a6f6244697370617463686572; asc ApiAsyncJobDispatcher;; 24: SQL NULL; 25: len 4; hex 80000000; asc ;; + + *** (2) TRANSACTION: + TRANSACTION 0 98087128, ACTIVE 0 sec, process no 1489, OS thread id 139837671909120 fetching rows, thread declared inside InnoDB 492 + mysql tables in use 2, locked 1 + 3 lock struct(s), heap size 368, 2 row lock(s), undo log entries 1 + MySQL thread id 28406, query id 368571323 localhost 127.0.0.1 cloud preparing + UPDATE async_job SET job_pending_signals=1 WHERE id IN (SELECT job_id FROM async_job_join_map WHERE join_job_id = 8) + *** (2) HOLDS THE LOCK(S): + RECORD LOCKS space id 0 page no 1275 n bits 80 index `PRIMARY` of table `cloud`.`async_job` trx id 0 98087128 lock_mode X locks rec but not gap + Record lock, heap no 9 PHYSICAL RECORD: n_fields 26; compact format; info bits 0 + 0: len 8; hex 0000000000000008; asc ;; 1: len 6; hex 000005d8b0d8; asc ;; 2: len 7; hex 00000009270110; asc ' ;; 3: len 8; hex 0000000000000002; asc ;; 4: len 8; hex 0000000000000002; asc ;; 5: SQL NULL; 6: SQL NULL; 7: len 30; hex 6f72672e6170616368652e636c6f7564737461636b2e6170692e636f6d6d; asc org.apache.cloudstack.api.comm;...(truncated); 8: len 30; hex 7b226964223a2232222c22706879736963616c6e6574776f726b6964223a; asc {"id":"2","physicalnetworkid":;...(truncated); 9: len 4; hex 80000000; asc ;; 10: len 4; hex 80000001; asc ;; 11: len 4; hex 80000000; asc ;; 12: len 4; hex 80000000; asc ;; 13: len 30; hex 6f72672e6170616368652e636c6f7564737461636b2e6170692e72657370; asc org.apache.cloudstack.api.resp;...(truncated); 14: len 8; hex 80001a6f7bb0d0a8; asc o{ ;; 15: len 8; hex 80001a6f7bb0d0a8; asc o{ ;; 16: len 8; hex 8000124f06cfd5b6; asc O ;; 17: len 8; hex 8000124f06cfd5b6; asc O ;; 18: SQL NULL; 19: SQL NULL; 20: len 30; hex 66376466396532362d323139622d346338652d393231332d393766653636; asc f7df9e26-219b-4c8e-9213-97fe66;...(truncated); 21: len 30; hex 36623238306364362d663436652d343563322d383833642d333863616439; asc 6b280cd6-f46e-45c2-883d-38cad9;...(truncated); 22: SQL NULL; 23: len 21; hex 4170694173796e634a6f6244697370617463686572; asc ApiAsyncJobDispatcher;; 24: SQL NULL; 25: len 4; hex 80000000; asc ;; + + *** (2) WAITING FOR THIS LOCK TO BE GRANTED: + RECORD LOCKS space id 0 page no 1275 n bits 80 index `PRIMARY` of table `cloud`.`async_job` trx id 0 98087128 lock_mode X locks rec but not gap waiting + Record lock, heap no 10 PHYSICAL RECORD: n_fields 26; compact format; info bits 0 + 0: len 8; hex 0000000000000009; asc ;; 1: len 6; hex 000005d8b0d7; asc ;; 2: len 7; hex 00000009280110; asc ( ;; 3: len 8; hex 0000000000000002; asc ;; 4: len 8; hex 0000000000000002; asc ;; 5: SQL NULL; 6: SQL NULL; 7: len 30; hex 6f72672e6170616368652e636c6f7564737461636b2e6170692e636f6d6d; asc org.apache.cloudstack.api.comm;...(truncated); 8: len 30; hex 7b226964223a2233222c22706879736963616c6e6574776f726b6964223a; asc {"id":"3","physicalnetworkid":;...(truncated); 9: len 4; hex 80000000; asc ;; 10: len 4; hex 80000001; asc ;; 11: len 4; hex 80000000; asc ;; 12: len 4; hex 80000000; asc ;; 13: len 30; hex 6f72672e6170616368652e636c6f7564737461636b2e6170692e72657370; asc org.apache.cloudstack.api.resp;...(truncated); 14: len 8; hex 80001a6f7bb0d0a8; asc o{ ;; 15: len 8; hex 80001a6f7bb0d0a8; asc o{ ;; 16: len 8; hex 8000124f06cfd5b6; asc O ;; 17: len 8; hex 8000124f06cfd5b6; asc O ;; 18: SQL NULL; 19: SQL NULL; 20: len 30; hex 62313065306432342d336233352d343663622d386361622d623933623562; asc b10e0d24-3b35-46cb-8cab-b93b5b;...(truncated); 21: len 30; hex 39353664383563632d383336622d346663612d623738622d646238343739; asc 956d85cc-836b-4fca-b78b-db8479;...(truncated); 22: SQL NULL; 23: len 21; hex 4170694173796e634a6f6244697370617463686572; asc ApiAsyncJobDispatcher;; 24: SQL NULL; 25: len 4; hex 80000000; asc ;; + + *** WE ROLL BACK TRANSACTION (2) +*/ + + _joinMapDao.completeJoin(joinJobId, joinStatus, joinResult, getMsid()); + } + + @Override + public void syncAsyncJobExecution(AsyncJob job, String syncObjType, long syncObjId, long queueSizeLimit) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Sync job-" + job.getId() + " execution on object " + syncObjType + "." + syncObjId); + } + + SyncQueueVO queue = null; + + // to deal with temporary DB exceptions like DB deadlock/Lock-wait time out cased rollbacks + // we retry five times until we throw an exception + Random random = new Random(); + + for(int i = 0; i < 5; i++) { + queue = _queueMgr.queue(syncObjType, syncObjId, SyncQueueItem.AsyncJobContentType, job.getId(), queueSizeLimit); + if(queue != null) { + break; + } + + try { + Thread.sleep(1000 + random.nextInt(5000)); + } catch (InterruptedException e) { + } + } + + if (queue == null) + throw new CloudRuntimeException("Unable to insert queue item into database, DB is full?"); + } + + @Override + public AsyncJob queryJob(long jobId, boolean updatePollTime) { + AsyncJobVO job = _jobDao.findById(jobId); + + if (updatePollTime) { + job.setLastPolled(DateUtil.currentGMTTime()); + _jobDao.update(jobId, job); + } + return job; + } + + + private void scheduleExecution(final AsyncJobVO job) { + scheduleExecution(job, false); + } + + private void scheduleExecution(final AsyncJob job, boolean executeInContext) { + Runnable runnable = getExecutorRunnable(job); + if (executeInContext) { + runnable.run(); + } else { + _executor.submit(runnable); + } + } + + private AsyncJobDispatcher getDispatcher(String dispatcherName) { + assert (dispatcherName != null && !dispatcherName.isEmpty()) : "Who's not setting the dispatcher when submitting a job? Who am I suppose to call if you do that!"; + + for (AsyncJobDispatcher dispatcher : _jobDispatchers) { + if (dispatcherName.equals(dispatcher.getName())) + return dispatcher; + } + + throw new CloudRuntimeException("Unable to find dispatcher name: " + dispatcherName); + } + + private AsyncJobDispatcher getWakeupDispatcher(AsyncJob job) { + if(_jobDispatchers != null) { + List joinRecords = _joinMapDao.listJoinRecords(job.getId()); + if(joinRecords.size() > 0) { + AsyncJobJoinMapVO joinRecord = joinRecords.get(0); + for(AsyncJobDispatcher dispatcher : _jobDispatchers) { + if(dispatcher.getName().equals(joinRecord.getWakeupDispatcher())) + return dispatcher; + } + } else { + s_logger.warn("job-" + job.getId() + " is scheduled for wakeup run, but there is no joining info anymore"); + } + } + return null; + } + + private long getJobRunNumber() { + synchronized(this) { + return _executionRunNumber++; + } + } + + private Runnable getExecutorRunnable(final AsyncJob job) { + return new Runnable() { + @Override + public void run() { + Transaction txn = null; + long runNumber = getJobRunNumber(); + + try { + // + // setup execution environment + // + txn = Transaction.open(Transaction.CLOUD_DB); + + try { + JmxUtil.registerMBean("AsyncJobManager", "Active Job " + job.getId(), new AsyncJobMBeanImpl(job)); + } catch(Exception e) { + // Due to co-existence of normal-dispatched-job/wakeup-dispatched-job, MBean register() call + // is expected to fail under situations + if(s_logger.isTraceEnabled()) + s_logger.trace("Unable to register active job " + job.getId() + " to JMX monitoring due to exception " + ExceptionUtil.toString(e)); + } + + _jobMonitor.registerActiveTask(runNumber, job.getId()); + AsyncJobExecutionContext.setCurrentExecutionContext(new AsyncJobExecutionContext(job)); + + // execute the job + if(s_logger.isDebugEnabled()) { + s_logger.debug("Executing " + job); + } + + if ((getAndResetPendingSignals(job) & AsyncJob.Constants.SIGNAL_MASK_WAKEUP) != 0) { + AsyncJobDispatcher jobDispatcher = getWakeupDispatcher(job); + if(jobDispatcher != null) { + jobDispatcher.runJob(job); + } else { + s_logger.error("Unable to find a wakeup dispatcher from the joined job: " + job); + } + } else { + AsyncJobDispatcher jobDispatcher = getDispatcher(job.getDispatcher()); + if(jobDispatcher != null) { + jobDispatcher.runJob(job); + } else { + s_logger.error("Unable to find job dispatcher, job will be cancelled"); + completeAsyncJob(job.getId(), JobInfo.Status.FAILED, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), null); + } + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Done executing " + job.getCmd() + " for job-" + job.getId()); + } + + } catch (Throwable e) { + s_logger.error("Unexpected exception", e); + completeAsyncJob(job.getId(), JobInfo.Status.FAILED, ApiErrorCode.INTERNAL_ERROR.getHttpCode(), null); + } finally { + // guard final clause as well + try { + AsyncJobVO jobToUpdate = _jobDao.findById(job.getId()); + jobToUpdate.setExecutingMsid(null); + _jobDao.update(job.getId(), jobToUpdate); + + if (job.getSyncSource() != null) { + _queueMgr.purgeItem(job.getSyncSource().getId()); + checkQueue(job.getSyncSource().getQueueId()); + } + + try { + JmxUtil.unregisterMBean("AsyncJobManager", "Active Job " + job.getId()); + } catch(Exception e) { + // Due to co-existence of normal-dispatched-job/wakeup-dispatched-job, MBean unregister() call + // is expected to fail under situations + if(s_logger.isTraceEnabled()) + s_logger.trace("Unable to unregister job " + job.getId() + " to JMX monitoring due to exception " + ExceptionUtil.toString(e)); + } + + if(txn != null) + txn.close(); + + // + // clean execution environment + // + AsyncJobExecutionContext.unregister(); + _jobMonitor.unregisterActiveTask(runNumber); + + } catch(Throwable e) { + s_logger.error("Double exception", e); + } + } + } + }; + } + + private int getAndResetPendingSignals(AsyncJob job) { + int signals = job.getPendingSignals(); + if(signals != 0) { + AsyncJobVO jobRecord = _jobDao.findById(job.getId()); + jobRecord.setPendingSignals(0); + _jobDao.update(job.getId(), jobRecord); + } + return signals; + } + + private void executeQueueItem(SyncQueueItemVO item, boolean fromPreviousSession) { + AsyncJobVO job = _jobDao.findById(item.getContentId()); + if (job != null) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Schedule queued job-" + job.getId()); + } + + job.setSyncSource(item); + + job.setExecutingMsid(getMsid()); + _jobDao.update(job.getId(), job); + + try { + scheduleExecution(job); + } catch(RejectedExecutionException e) { + s_logger.warn("Execution for job-" + job.getId() + " is rejected, return it to the queue for next turn"); + _queueMgr.returnItem(item.getId()); + + job.setExecutingMsid(null); + _jobDao.update(job.getId(), job); + } + + } else { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Unable to find related job for queue item: " + item.toString()); + } + + _queueMgr.purgeItem(item.getId()); + } + } + + @Override + public void releaseSyncSource() { + AsyncJobExecutionContext executionContext = AsyncJobExecutionContext.getCurrentExecutionContext(); + assert(executionContext != null); + + if(executionContext.getSyncSource() != null) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Release sync source for job-" + executionContext.getJob().getId() + " sync source: " + + executionContext.getSyncSource().getContentType() + "-" + + executionContext.getSyncSource().getContentId()); + } + + _queueMgr.purgeItem(executionContext.getSyncSource().getId()); + checkQueue(executionContext.getSyncSource().getQueueId()); + } + } + + @Override + public boolean waitAndCheck(AsyncJob job, String[] wakeupTopicsOnMessageBus, long checkIntervalInMilliSeconds, + long timeoutInMiliseconds, Predicate predicate) { + + MessageDetector msgDetector = new MessageDetector(); + String[] topics = Arrays.copyOf(wakeupTopicsOnMessageBus, wakeupTopicsOnMessageBus.length + 1); + topics[topics.length - 1] = AsyncJob.Topics.JOB_STATE; + + msgDetector.open(_messageBus, topics); + try { + long startTick = System.currentTimeMillis(); + while(System.currentTimeMillis() - startTick < timeoutInMiliseconds) { + msgDetector.waitAny(checkIntervalInMilliSeconds); + job = _jobDao.findById(job.getId()); + if (job.getStatus().done()) { + return true; + } + + if (predicate.checkCondition()) { + return true; + } + } + } finally { + msgDetector.close(); + } + + return false; + } + + private void checkQueue(long queueId) { + while(true) { + try { + SyncQueueItemVO item = _queueMgr.dequeueFromOne(queueId, getMsid()); + if(item != null) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Executing sync queue item: " + item.toString()); + } + + executeQueueItem(item, false); + } else { + break; + } + } catch(Throwable e) { + s_logger.error("Unexpected exception when kicking sync queue-" + queueId, e); + break; + } + } + } + + private Runnable getHeartbeatTask() { + return new Runnable() { + @Override + public void run() { + Transaction txn = Transaction.open("AsyncJobManagerImpl.getHeartbeatTask"); + try { + List l = _queueMgr.dequeueFromAny(getMsid(), MAX_ONETIME_SCHEDULE_SIZE); + if(l != null && l.size() > 0) { + for(SyncQueueItemVO item: l) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Execute sync-queue item: " + item.toString()); + } + executeQueueItem(item, false); + } + } + + List standaloneWakeupJobs = wakeupScan(); + for(Long jobId : standaloneWakeupJobs) { + // TODO, we assume that all jobs in this category is API job only + AsyncJobVO job = _jobDao.findById(jobId); + if (job != null && (job.getPendingSignals() & AsyncJob.Constants.SIGNAL_MASK_WAKEUP) != 0) + scheduleExecution(job, false); + } + } catch(Throwable e) { + s_logger.error("Unexpected exception when trying to execute queue item, ", e); + } finally { + try { + txn.close(); + } catch(Throwable e) { + s_logger.error("Unexpected exception", e); + } + } + } + }; + } + + @DB + private Runnable getGCTask() { + return new Runnable() { + @Override + public void run() { + GlobalLock scanLock = GlobalLock.getInternLock("AsyncJobManagerGC"); + try { + if(scanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { + try { + reallyRun(); + } finally { + scanLock.unlock(); + } + } + } finally { + scanLock.releaseRef(); + } + } + + public void reallyRun() { + try { + s_logger.trace("Begin cleanup expired async-jobs"); + + Date cutTime = new Date(DateUtil.currentGMTTime().getTime() - _jobExpireSeconds.value() * 1000); + + // limit to 100 jobs per turn, this gives cleanup throughput as 600 jobs per minute + // hopefully this will be fast enough to balance potential growth of job table + //1) Expire unfinished jobs that weren't processed yet + List l = _jobDao.getExpiredUnfinishedJobs(cutTime, 100); + for(AsyncJobVO job : l) { + s_logger.trace("Expunging unfinished job " + job); + expungeAsyncJob(job); + } + + //2) Expunge finished jobs + List completedJobs = _jobDao.getExpiredCompletedJobs(cutTime, 100); + for(AsyncJobVO job : completedJobs) { + s_logger.trace("Expunging completed job " + job); + expungeAsyncJob(job); + } + + // forcefully cancel blocking queue items if they've been staying there for too long + List blockItems = _queueMgr.getBlockedQueueItems(_jobCancelThresholdSeconds.value() + * 1000, false); + if(blockItems != null && blockItems.size() > 0) { + for(SyncQueueItemVO item : blockItems) { + if(item.getContentType().equalsIgnoreCase(SyncQueueItem.AsyncJobContentType)) { + completeAsyncJob(item.getContentId(), JobInfo.Status.FAILED, 0, "Job is cancelled as it has been blocking others for too long"); + } + + // purge the item and resume queue processing + _queueMgr.purgeItem(item.getId()); + } + } + + s_logger.trace("End cleanup expired async-jobs"); + } catch(Throwable e) { + s_logger.error("Unexpected exception when trying to execute queue item, ", e); + } + } + }; + } + + @DB + protected void expungeAsyncJob(AsyncJobVO job) { + Transaction txn = Transaction.currentTxn(); + txn.start(); + _jobDao.expunge(job.getId()); + //purge corresponding sync queue item + _queueMgr.purgeAsyncJobQueueItemId(job.getId()); + txn.commit(); + } + + private long getMsid() { + return ManagementServerNode.getManagementServerId(); + } + + private void cleanupPendingJobs(List l) { + for (SyncQueueItemVO item : l) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Discard left-over queue item: " + item.toString()); + } + + String contentType = item.getContentType(); + if (contentType != null && contentType.equalsIgnoreCase(SyncQueueItem.AsyncJobContentType)) { + Long jobId = item.getContentId(); + if (jobId != null) { + s_logger.warn("Mark job as failed as its correspoding queue-item has been discarded. job id: " + jobId); + completeAsyncJob(jobId, JobInfo.Status.FAILED, 0, "Execution was cancelled because of server shutdown"); + } + } + _queueMgr.purgeItem(item.getId()); + } + } + + @DB + protected List wakeupByJoinedJobCompletion(long joinedJobId) { + SearchCriteria joinJobSC = JoinJobSearch.create("joinJobId", joinedJobId); + + List result = _joinMapDao.customSearch(joinJobSC, null); + if (result.size() > 0) { + Collections.sort(result); + Long[] ids = result.toArray(new Long[result.size()]); + + SearchCriteria jobsSC = JobIdsSearch.create("ids", ids); + SearchCriteria queueItemsSC = QueueJobIdsSearch.create("contentIds", ids); + + Transaction txn = Transaction.currentTxn(); + txn.start(); + AsyncJobVO job = _jobDao.createForUpdate(); + job.setPendingSignals(AsyncJob.Constants.SIGNAL_MASK_WAKEUP); + _jobDao.update(job, jobsSC); + + SyncQueueItemVO item = _queueItemDao.createForUpdate(); + item.setLastProcessNumber(null); + item.setLastProcessMsid(null); + _queueItemDao.update(item, queueItemsSC); + txn.commit(); + } + return _joinMapDao.findJobsToWake(joinedJobId); + } + + @DB + protected List wakeupScan() { + Date cutDate = DateUtil.currentGMTTime(); + Transaction txn = Transaction.currentTxn(); + + SearchCriteria sc = JoinJobTimeSearch.create(); + sc.setParameters("beginTime", cutDate); + sc.setParameters("endTime", cutDate); + + List result = _joinMapDao.customSearch(sc, null); + + txn.start(); + if (result.size() > 0) { + Collections.sort(result); + Long[] ids = result.toArray(new Long[result.size()]); + + AsyncJobVO job = _jobDao.createForUpdate(); + job.setPendingSignals(AsyncJob.Constants.SIGNAL_MASK_WAKEUP); + + SearchCriteria sc2 = JobIdsSearch.create("ids", ids); + SearchCriteria queueItemsSC = QueueJobIdsSearch.create("contentIds", ids); + + _jobDao.update(job, sc2); + + SyncQueueItemVO item = _queueItemDao.createForUpdate(); + item.setLastProcessNumber(null); + item.setLastProcessMsid(null); + _queueItemDao.update(item, queueItemsSC); + } + + List wakupIds = _joinMapDao.findJobsToWakeBetween(cutDate); + txn.commit(); + + return wakupIds; + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + _jobExpireSeconds = _configDepot.get(JobExpireMinutes).setMultiplier(60); + _jobCancelThresholdSeconds = _configDepot.get(JobCancelThresholdMinutes).setMultiplier(60); + + try { + final File dbPropsFile = PropertiesUtil.findConfigFile("db.properties"); + final Properties dbProps = new Properties(); + dbProps.load(new FileInputStream(dbPropsFile)); + + final int cloudMaxActive = Integer.parseInt(dbProps.getProperty("db.cloud.maxActive")); + + int poolSize = (cloudMaxActive * 2) / 3; + + s_logger.info("Start AsyncJobManager thread pool in size " + poolSize); + _executor = Executors.newFixedThreadPool(poolSize, new NamedThreadFactory(AsyncJobManager.JOB_POOL_THREAD_PREFIX)); + } catch (final Exception e) { + throw new ConfigurationException("Unable to load db.properties to configure AsyncJobManagerImpl"); + } + + JoinJobSearch = _joinMapDao.createSearchBuilder(Long.class); + JoinJobSearch.and(JoinJobSearch.entity().getJoinJobId(), Op.EQ, "joinJobId"); + JoinJobSearch.selectField(JoinJobSearch.entity().getJobId()); + JoinJobSearch.done(); + + JoinJobTimeSearch = _joinMapDao.createSearchBuilder(Long.class); + JoinJobTimeSearch.and(JoinJobTimeSearch.entity().getNextWakeupTime(), Op.LT, "beginTime"); + JoinJobTimeSearch.and(JoinJobTimeSearch.entity().getExpiration(), Op.GT, "endTime"); + JoinJobTimeSearch.selectField(JoinJobTimeSearch.entity().getJobId()).done(); + + JobIdsSearch = _jobDao.createSearchBuilder(); + JobIdsSearch.and(JobIdsSearch.entity().getId(), Op.IN, "ids").done(); + + QueueJobIdsSearch = _queueItemDao.createSearchBuilder(); + QueueJobIdsSearch.and(QueueJobIdsSearch.entity().getContentId(), Op.IN, "contentIds").done(); + + JoinJobIdsSearch = _joinMapDao.createSearchBuilder(Long.class); + JoinJobIdsSearch.selectField(JoinJobIdsSearch.entity().getJobId()); + JoinJobIdsSearch.and(JoinJobIdsSearch.entity().getJoinJobId(), Op.EQ, "joinJobId"); + JoinJobIdsSearch.and(JoinJobIdsSearch.entity().getJobId(), Op.NIN, "jobIds"); + JoinJobIdsSearch.done(); + + ContentIdsSearch = _queueItemDao.createSearchBuilder(Long.class); + ContentIdsSearch.selectField(ContentIdsSearch.entity().getContentId()).done(); + + AsyncJobExecutionContext.init(this, _joinMapDao); + OutcomeImpl.init(this); + + return true; + } + + @Override + public void onManagementNodeJoined(List nodeList, long selfNodeId) { + } + + @Override + public void onManagementNodeLeft(List nodeList, long selfNodeId) { + for (ManagementServerHost msHost : nodeList) { + Transaction txn = Transaction.open(Transaction.CLOUD_DB); + try { + txn.start(); + List items = _queueMgr.getActiveQueueItems(msHost.getId(), true); + cleanupPendingJobs(items); + _jobDao.resetJobProcess(msHost.getId(), ApiErrorCode.INTERNAL_ERROR.getHttpCode(), "job cancelled because of management server restart"); + txn.commit(); + } catch(Throwable e) { + s_logger.warn("Unexpected exception ", e); + } finally { + txn.close(); + } + } + } + + @Override + public void onManagementNodeIsolated() { + } + + @Override + public boolean start() { + try { + _jobDao.cleanupPseduoJobs(getMsid()); + + List l = _queueMgr.getActiveQueueItems(getMsid(), false); + cleanupPendingJobs(l); + _jobDao.resetJobProcess(getMsid(), ApiErrorCode.INTERNAL_ERROR.getHttpCode(), "job cancelled because of management server restart"); + } catch(Throwable e) { + s_logger.error("Unexpected exception " + e.getMessage(), e); + } + + _heartbeatScheduler.scheduleAtFixedRate(getHeartbeatTask(), HEARTBEAT_INTERVAL, HEARTBEAT_INTERVAL, TimeUnit.MILLISECONDS); + _heartbeatScheduler.scheduleAtFixedRate(getGCTask(), GC_INTERVAL, GC_INTERVAL, TimeUnit.MILLISECONDS); + + return true; + } + + @Override + public boolean stop() { + _heartbeatScheduler.shutdown(); + _executor.shutdown(); + return true; + } + + private GenericSearchBuilder ContentIdsSearch; + private GenericSearchBuilder JoinJobSearch; + private SearchBuilder JobIdsSearch; + private SearchBuilder QueueJobIdsSearch; + private GenericSearchBuilder JoinJobIdsSearch; + private GenericSearchBuilder JoinJobTimeSearch; + + protected AsyncJobManagerImpl() { + + } + +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java new file mode 100644 index 00000000000..3bf362251fc --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java @@ -0,0 +1,185 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.impl; + +import java.util.HashMap; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.log4j.Logger; + +import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.framework.messagebus.MessageDispatcher; +import org.apache.cloudstack.framework.messagebus.MessageHandler; + +import com.cloud.utils.component.ManagerBase; + +public class AsyncJobMonitor extends ManagerBase { + public static final Logger s_logger = Logger.getLogger(AsyncJobMonitor.class); + + @Inject private MessageBus _messageBus; + + private final Map _activeTasks = new HashMap(); + private final Timer _timer = new Timer(); + + private volatile int _activePoolThreads = 0; + private volatile int _activeInplaceThreads = 0; + + // configuration + private long _inactivityCheckIntervalMs = 60000; + private long _inactivityWarningThresholdMs = 90000; + + public AsyncJobMonitor() { + } + + public long getInactivityCheckIntervalMs() { + return _inactivityCheckIntervalMs; + } + + public void setInactivityCheckIntervalMs(long intervalMs) { + _inactivityCheckIntervalMs = intervalMs; + } + + public long getInactivityWarningThresholdMs() { + return _inactivityWarningThresholdMs; + } + + public void setInactivityWarningThresholdMs(long thresholdMs) { + _inactivityWarningThresholdMs = thresholdMs; + } + + @MessageHandler(topic = AsyncJob.Topics.JOB_HEARTBEAT) + public void onJobHeartbeatNotify(String subject, String senderAddress, Object args) { + if(args != null && args instanceof Long) { + synchronized(this) { + ActiveTaskRecord record = _activeTasks.get(args); + if(record != null) { + record.updateJobHeartbeatTick(); + } + } + } + } + + private void heartbeat() { + synchronized(this) { + for(Map.Entry entry : _activeTasks.entrySet()) { + if(entry.getValue().millisSinceLastJobHeartbeat() > _inactivityWarningThresholdMs) { + s_logger.warn("Task (job-" + entry.getValue().getJobId() + ") has been pending for " + + entry.getValue().millisSinceLastJobHeartbeat()/1000 + " seconds"); + } + } + } + } + + @Override + public boolean configure(String name, Map params) + throws ConfigurationException { + + _messageBus.subscribe(AsyncJob.Topics.JOB_HEARTBEAT, MessageDispatcher.getDispatcher(this)); + _timer.scheduleAtFixedRate(new TimerTask() { + + @Override + public void run() { + heartbeat(); + } + + }, _inactivityCheckIntervalMs, _inactivityCheckIntervalMs); + return true; + } + + public void registerActiveTask(long runNumber, long jobId) { + synchronized(this) { + s_logger.info("Add job-" + jobId + " into job monitoring"); + + assert(_activeTasks.get(runNumber) == null); + + long threadId = Thread.currentThread().getId(); + boolean fromPoolThread = Thread.currentThread().getName().contains(AsyncJobManager.JOB_POOL_THREAD_PREFIX); + ActiveTaskRecord record = new ActiveTaskRecord(jobId, threadId, fromPoolThread); + _activeTasks.put(runNumber, record); + if(fromPoolThread) + _activePoolThreads++; + else + _activeInplaceThreads++; + } + } + + public void unregisterActiveTask(long runNumber) { + synchronized(this) { + ActiveTaskRecord record = _activeTasks.get(runNumber); + assert(record != null); + if(record != null) { + s_logger.info("Remove job-" + record.getJobId() + " from job monitoring"); + + if(record.isPoolThread()) + _activePoolThreads--; + else + _activeInplaceThreads--; + + _activeTasks.remove(runNumber); + } + } + } + + public int getActivePoolThreads() { + return _activePoolThreads; + } + + public int getActiveInplaceThread() { + return _activeInplaceThreads; + } + + private static class ActiveTaskRecord { + long _jobId; + long _threadId; + boolean _fromPoolThread; + long _jobLastHeartbeatTick; + + public ActiveTaskRecord(long jobId, long threadId, boolean fromPoolThread) { + _threadId = threadId; + _jobId = jobId; + _fromPoolThread = fromPoolThread; + _jobLastHeartbeatTick = System.currentTimeMillis(); + } + + public long getThreadId() { + return _threadId; + } + + public long getJobId() { + return _jobId; + } + + public boolean isPoolThread() { + return _fromPoolThread; + } + + public void updateJobHeartbeatTick() { + _jobLastHeartbeatTick = System.currentTimeMillis(); + } + + public long millisSinceLastJobHeartbeat() { + return System.currentTimeMillis() - _jobLastHeartbeatTick; + } + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobVO.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobVO.java new file mode 100644 index 00000000000..89bbd8668f6 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobVO.java @@ -0,0 +1,398 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.impl; + +import java.util.Date; +import java.util.UUID; + +import javax.persistence.Column; +import javax.persistence.DiscriminatorColumn; +import javax.persistence.DiscriminatorType; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Inheritance; +import javax.persistence.InheritanceType; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import javax.persistence.Transient; + +import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.jobs.JobInfo; + +import com.cloud.utils.UuidUtils; +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name="async_job") +@Inheritance(strategy=InheritanceType.JOINED) +@DiscriminatorColumn(name="job_type", discriminatorType=DiscriminatorType.STRING, length=32) +public class AsyncJobVO implements AsyncJob, JobInfo { + + public static final String JOB_DISPATCHER_PSEUDO = "pseudoJobDispatcher"; + public static final String PSEUDO_JOB_INSTANCE_TYPE = "Thread"; + + @Id + @GeneratedValue(strategy=GenerationType.IDENTITY) + @Column(name="id") + private long id; + + @Column(name="job_type", length=32) + protected String type; + + @Column(name="job_dispatcher", length=64) + protected String dispatcher; + + @Column(name="job_pending_signals") + protected int pendingSignals; + + @Column(name="user_id") + private long userId; + + @Column(name="account_id") + private long accountId; + + @Column(name="job_cmd") + private String cmd; + + @Column(name="job_cmd_ver") + private int cmdVersion; + + @Column(name = "related") + private String related; + + @Column(name="job_cmd_info", length=65535) + private String cmdInfo; + + @Column(name="job_status") + @Enumerated(value = EnumType.ORDINAL) + private Status status; + + @Column(name="job_process_status") + private int processStatus; + + @Column(name="job_result_code") + private int resultCode; + + @Column(name="job_result", length=65535) + private String result; + + @Column(name="instance_type", length=64) + private String instanceType; + + @Column(name="instance_id", length=64) + private Long instanceId; + + @Column(name="job_init_msid") + private Long initMsid; + + @Column(name="job_complete_msid") + private Long completeMsid; + + @Column(name="job_executing_msid") + private Long executingMsid; + + @Column(name=GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name="last_updated") + @Temporal(TemporalType.TIMESTAMP) + private Date lastUpdated; + + @Column(name="last_polled") + @Temporal(TemporalType.TIMESTAMP) + private Date lastPolled; + + @Column(name=GenericDao.REMOVED_COLUMN) + private Date removed; + + @Column(name="uuid") + private String uuid; + + @Transient + private SyncQueueItem syncSource = null; + + public AsyncJobVO() { + uuid = UUID.randomUUID().toString(); + related = UUID.randomUUID().toString(); + status = Status.IN_PROGRESS; + } + + public AsyncJobVO(String related, long userId, long accountId, String cmd, String cmdInfo, Long instanceId, String instanceType) { + this.userId = userId; + this.accountId = accountId; + this.cmd = cmd; + this.cmdInfo = cmdInfo; + uuid = UUID.randomUUID().toString(); + this.related = related; + this.instanceId = instanceId; + this.instanceType = instanceType; + status = Status.IN_PROGRESS; + } + + @Override + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + @Override + public String getShortUuid() { + return UuidUtils.first(uuid); + } + + public void setRelated(String related) { + this.related = related; + } + + @Override + public String getRelated() { + return related; + } + + @Override + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + @Override + public String getDispatcher() { + return dispatcher; + } + + public void setDispatcher(String dispatcher) { + this.dispatcher = dispatcher; + } + + @Override + public int getPendingSignals() { + return pendingSignals; + } + + public void setPendingSignals(int signals) { + pendingSignals = signals; + } + + @Override + public long getUserId() { + return userId; + } + + public void setUserId(long userId) { + this.userId = userId; + } + + @Override + public long getAccountId() { + return accountId; + } + + public void setAccountId(long accountId) { + this.accountId = accountId; + } + + @Override + public String getCmd() { + return cmd; + } + + public void setCmd(String cmd) { + this.cmd = cmd; + } + + @Override + public int getCmdVersion() { + return cmdVersion; + } + + public void setCmdVersion(int version) { + cmdVersion = version; + } + + @Override + public String getCmdInfo() { + return cmdInfo; + } + + public void setCmdInfo(String cmdInfo) { + this.cmdInfo = cmdInfo; + } + + @Override + public Status getStatus() { + return status; + } + + public void setStatus(Status status) { + this.status = status; + } + + @Override + public int getProcessStatus() { + return processStatus; + } + + public void setProcessStatus(int status) { + processStatus = status; + } + + @Override + public int getResultCode() { + return resultCode; + } + + public void setResultCode(int resultCode) { + this.resultCode = resultCode; + } + + @Override + public String getResult() { + return result; + } + + public void setResult(String result) { + this.result = result; + } + + @Override + public Long getInitMsid() { + return initMsid; + } + + @Override + public void setInitMsid(Long initMsid) { + this.initMsid = initMsid; + } + + @Override + public Long getExecutingMsid() { + return executingMsid; + } + + public void setExecutingMsid(Long executingMsid) { + this.executingMsid = executingMsid; + } + + @Override + public Long getCompleteMsid() { + return completeMsid; + } + + @Override + public void setCompleteMsid(Long completeMsid) { + this.completeMsid = completeMsid; + } + + @Override + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + @Override + public Date getLastUpdated() { + return lastUpdated; + } + + public void setLastUpdated(Date lastUpdated) { + this.lastUpdated = lastUpdated; + } + + @Override + public Date getLastPolled() { + return lastPolled; + } + + public void setLastPolled(Date lastPolled) { + this.lastPolled = lastPolled; + } + + @Override + public String getInstanceType() { + return instanceType; + } + + public void setInstanceType(String instanceType) { + this.instanceType = instanceType; + } + + @Override + public Long getInstanceId() { + return instanceId; + } + + public void setInstanceId(Long instanceId) { + this.instanceId = instanceId; + } + + @Override + public SyncQueueItem getSyncSource() { + return syncSource; + } + + @Override + public void setSyncSource(SyncQueueItem syncSource) { + this.syncSource = syncSource; + } + + @Override + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("AsyncJobVO {id:").append(getId()); + sb.append(", userId: ").append(getUserId()); + sb.append(", accountId: ").append(getAccountId()); + sb.append(", instanceType: ").append(getInstanceType()); + sb.append(", instanceId: ").append(getInstanceId()); + sb.append(", cmd: ").append(getCmd()); + sb.append(", cmdInfo: ").append(getCmdInfo()); + sb.append(", cmdVersion: ").append(getCmdVersion()); + sb.append(", status: ").append(getStatus()); + sb.append(", processStatus: ").append(getProcessStatus()); + sb.append(", resultCode: ").append(getResultCode()); + sb.append(", result: ").append(getResult()); + sb.append(", initMsid: ").append(getInitMsid()); + sb.append(", completeMsid: ").append(getCompleteMsid()); + sb.append(", lastUpdated: ").append(getLastUpdated()); + sb.append(", lastPolled: ").append(getLastPolled()); + sb.append(", created: ").append(getCreated()); + sb.append("}"); + return sb.toString(); + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/JobSerializerHelper.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/JobSerializerHelper.java new file mode 100644 index 00000000000..6acc93387c1 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/JobSerializerHelper.java @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.impl; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Type; + +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +import com.cloud.utils.exception.CloudRuntimeException; + +/** + * Note: toPairList and appendPairList only support simple POJO objects currently + */ +public class JobSerializerHelper { + private static final Logger s_logger = Logger.getLogger(JobSerializerHelper.class); + public static String token = "/"; + + private static Gson s_gson; + static { + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.setVersion(1.5); + s_logger.debug("Job GSON Builder initialized."); + gsonBuilder.registerTypeAdapter(Class.class, new ClassTypeAdapter()); + gsonBuilder.registerTypeAdapter(Throwable.class, new ThrowableTypeAdapter()); + s_gson = gsonBuilder.create(); + } + + public static String toSerializedString(Object result) { + if(result != null) { + Class clz = result.getClass(); + return clz.getName() + token + s_gson.toJson(result); + } + return null; + } + + public static Object fromSerializedString(String result) { + try { + if(result != null && !result.isEmpty()) { + + String[] serializedParts = result.split(token); + + if (serializedParts.length < 2) { + return null; + } + String clzName = serializedParts[0]; + String nameField = null; + String content = null; + if (serializedParts.length == 2) { + content = serializedParts[1]; + } else { + nameField = serializedParts[1]; + int index = result.indexOf(token + nameField + token); + content = result.substring(index + nameField.length() + 2); + } + + Class clz; + try { + clz = Class.forName(clzName); + } catch (ClassNotFoundException e) { + return null; + } + + Object obj = s_gson.fromJson(content, clz); + return obj; + } + return null; + } catch(RuntimeException e) { + throw new CloudRuntimeException("Unable to deserialize: " + result, e); + } + } + + public static String toObjectSerializedString(Serializable object) { + assert(object != null); + + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + try { + ObjectOutputStream os = new ObjectOutputStream(bs); + os.writeObject(object); + os.close(); + bs.close(); + + return Base64.encodeBase64URLSafeString(bs.toByteArray()); + } catch(IOException e) { + throw new CloudRuntimeException("Unable to serialize: " + object, e); + } + } + + public static Object fromObjectSerializedString(String base64EncodedString) { + if(base64EncodedString == null) + return null; + + byte[] content = Base64.decodeBase64(base64EncodedString); + ByteArrayInputStream bs = new ByteArrayInputStream(content); + try { + ObjectInputStream is = new ObjectInputStream(bs); + Object obj = is.readObject(); + is.close(); + bs.close(); + return obj; + } catch(IOException e) { + throw new CloudRuntimeException("Unable to serialize: " + base64EncodedString, e); + } catch (ClassNotFoundException e) { + throw new CloudRuntimeException("Unable to serialize: " + base64EncodedString, e); + } + } + + public static class ClassTypeAdapter implements JsonSerializer>, JsonDeserializer> { + @Override + public JsonElement serialize(Class clazz, Type typeOfResponseObj, JsonSerializationContext ctx) { + return new JsonPrimitive(clazz.getName()); + } + + @Override + public Class deserialize(JsonElement arg0, Type arg1, JsonDeserializationContext arg2) throws JsonParseException { + String str = arg0.getAsString(); + try { + return Class.forName(str); + } catch (ClassNotFoundException e) { + throw new CloudRuntimeException("Unable to find class " + str); + } + } + } + + public static class ThrowableTypeAdapter implements JsonSerializer, JsonDeserializer { + + @Override + public Throwable deserialize(JsonElement json, Type type, JsonDeserializationContext ctx) throws JsonParseException { + JsonObject obj = (JsonObject)json; + + String className = obj.get("class").getAsString(); + try { + Class clazz = (Class)Class.forName(className); + Throwable cause = s_gson.fromJson(obj.get("cause"), Throwable.class); + String msg = obj.get("msg").getAsString(); + Constructor constructor = clazz.getConstructor(String.class, Throwable.class); + Throwable th = constructor.newInstance(msg, cause); + return th; + } catch (ClassNotFoundException e) { + throw new JsonParseException("Unable to find " + className); + } catch (NoSuchMethodException e) { + throw new JsonParseException("Unable to find constructor for " + className); + } catch (SecurityException e) { + throw new JsonParseException("Unable to get over security " + className); + } catch (InstantiationException e) { + throw new JsonParseException("Unable to instantiate " + className); + } catch (IllegalAccessException e) { + throw new JsonParseException("Illegal access to " + className, e); + } catch (IllegalArgumentException e) { + throw new JsonParseException("Illegal argument to " + className, e); + } catch (InvocationTargetException e) { + throw new JsonParseException("Cannot invoke " + className, e); + } + } + + @Override + public JsonElement serialize(Throwable th, Type type, JsonSerializationContext ctx) { + JsonObject json = new JsonObject(); + + json.add("class", new JsonPrimitive(th.getClass().getName())); + json.add("cause", s_gson.toJsonTree(th.getCause())); + json.add("msg", new JsonPrimitive(th.getMessage())); +// json.add("stack", s_gson.toJsonTree(th.getStackTrace())); + + return json; + } + + } + +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/OutcomeImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/OutcomeImpl.java new file mode 100644 index 00000000000..03c652c388a --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/OutcomeImpl.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.impl; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; +import org.apache.cloudstack.framework.jobs.Outcome; + +import com.cloud.utils.Predicate; +public class OutcomeImpl implements Outcome { + protected AsyncJob _job; + protected Class _clazz; + protected String[] _topics; + protected Predicate _predicate; + protected long _checkIntervalInMs; + + protected T _result; + + private static AsyncJobManagerImpl s_jobMgr; + + public static void init(AsyncJobManagerImpl jobMgr) { + s_jobMgr = jobMgr; + } + + public OutcomeImpl(Class clazz, AsyncJob job, long checkIntervalInMs, Predicate predicate, String... topics) { + _clazz = clazz; + _job = job; + _topics = topics; + _predicate = predicate; + _checkIntervalInMs = checkIntervalInMs; + } + + @Override + public AsyncJob getJob() { + return _job; + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public T get() throws InterruptedException, ExecutionException { + s_jobMgr.waitAndCheck(getJob(), _topics, _checkIntervalInMs, -1, _predicate); + try { + AsyncJobExecutionContext.getCurrentExecutionContext().disjoinJob(_job.getId()); + } catch (Throwable e) { + throw new ExecutionException("Job task has trouble executing", e); + } + + return retrieve(); + } + + @Override + public T get(long timeToWait, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + s_jobMgr.waitAndCheck(getJob(), _topics, _checkIntervalInMs, unit.toMillis(timeToWait), _predicate); + try { + AsyncJobExecutionContext.getCurrentExecutionContext().disjoinJob(_job.getId()); + } catch (Throwable e) { + throw new ExecutionException("Job task has trouble executing", e); + } + return retrieve(); + } + + /** + * This method can be overridden by children classes to retrieve the + * actual object. + */ + protected T retrieve() { + return _result; + } + + protected Outcome set(T result) { + _result = result; + return this; + } + + @Override + public boolean isCancelled() { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean isDone() { + // TODO Auto-generated method stub + return false; + } + + @Override + public void execute(Task task) { + // TODO Auto-generated method stub + + } + + @Override + public void execute(Task task, long wait, TimeUnit unit) { + // TODO Auto-generated method stub + + } + + public Predicate getPredicate() { + return _predicate; + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueItem.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueItem.java new file mode 100644 index 00000000000..04519e70a5d --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueItem.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.impl; + +public interface SyncQueueItem { + public final String AsyncJobContentType = "AsyncJob"; + + /** + * @return queue item id + */ + long getId(); + + /** + * @return queue id + */ + Long getQueueId(); + + /** + * @return subject object type pointed by the queue item + */ + String getContentType(); + + /** + * @return subject object id pointed by the queue item + */ + Long getContentId(); +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueItemVO.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueItemVO.java new file mode 100644 index 00000000000..f8bba0262ff --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueItemVO.java @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.impl; + +import org.apache.cloudstack.api.InternalIdentity; + + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +@Entity +@Table(name="sync_queue_item") +public class SyncQueueItemVO implements SyncQueueItem, InternalIdentity { + + @Id + @GeneratedValue(strategy=GenerationType.IDENTITY) + @Column(name="id") + private Long id = null; + + @Column(name="queue_id") + private Long queueId; + + @Column(name="content_type") + private String contentType; + + @Column(name="content_id") + private Long contentId; + + @Column(name="queue_proc_msid") + private Long lastProcessMsid; + + @Column(name="queue_proc_number") + private Long lastProcessNumber; + + @Column(name="queue_proc_time") + @Temporal(TemporalType.TIMESTAMP) + private Date lastProcessTime; + + @Column(name="created") + private Date created; + + public long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + @Override + public Long getQueueId() { + return queueId; + } + + public void setQueueId(Long queueId) { + this.queueId = queueId; + } + + @Override + public String getContentType() { + return contentType; + } + + public void setContentType(String contentType) { + this.contentType = contentType; + } + + @Override + public Long getContentId() { + return contentId; + } + + public void setContentId(Long contentId) { + this.contentId = contentId; + } + + public Long getLastProcessMsid() { + return lastProcessMsid; + } + + public void setLastProcessMsid(Long lastProcessMsid) { + this.lastProcessMsid = lastProcessMsid; + } + + public Long getLastProcessNumber() { + return lastProcessNumber; + } + + public void setLastProcessNumber(Long lastProcessNumber) { + this.lastProcessNumber = lastProcessNumber; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("SyncQueueItemVO {id:").append(getId()).append(", queueId: ").append(getQueueId()); + sb.append(", contentType: ").append(getContentType()); + sb.append(", contentId: ").append(getContentId()); + sb.append(", lastProcessMsid: ").append(getLastProcessMsid()); + sb.append(", lastprocessNumber: ").append(getLastProcessNumber()); + sb.append(", lastProcessTime: ").append(getLastProcessTime()); + sb.append(", created: ").append(getCreated()); + sb.append("}"); + return sb.toString(); + } + + public Date getLastProcessTime() { + return lastProcessTime; + } + + public void setLastProcessTime(Date lastProcessTime) { + this.lastProcessTime = lastProcessTime; + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java new file mode 100644 index 00000000000..202a704ee36 --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.impl; + +import java.util.List; + +import com.cloud.utils.component.Manager; + +public interface SyncQueueManager extends Manager { + public SyncQueueVO queue(String syncObjType, long syncObjId, String itemType, long itemId, long queueSizeLimit); + public SyncQueueItemVO dequeueFromOne(long queueId, Long msid); + public List dequeueFromAny(Long msid, int maxItems); + public void purgeItem(long queueItemId); + public void returnItem(long queueItemId); + + public List getActiveQueueItems(Long msid, boolean exclusive); + public List getBlockedQueueItems(long thresholdMs, boolean exclusive); + + void purgeAsyncJobQueueItemId(long asyncJobId); +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java new file mode 100644 index 00000000000..b9b5d6bdabd --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java @@ -0,0 +1,258 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.framework.jobs.impl; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import javax.inject.Inject; +import org.apache.log4j.Logger; + +import org.apache.cloudstack.framework.jobs.dao.SyncQueueDao; +import org.apache.cloudstack.framework.jobs.dao.SyncQueueItemDao; + +import com.cloud.utils.DateUtil; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.exception.CloudRuntimeException; + +public class SyncQueueManagerImpl extends ManagerBase implements SyncQueueManager { + public static final Logger s_logger = Logger.getLogger(SyncQueueManagerImpl.class.getName()); + + @Inject private SyncQueueDao _syncQueueDao; + @Inject private SyncQueueItemDao _syncQueueItemDao; + + @Override + @DB + public SyncQueueVO queue(String syncObjType, long syncObjId, String itemType, long itemId, long queueSizeLimit) { + Transaction txn = Transaction.currentTxn(); + try { + txn.start(); + + _syncQueueDao.ensureQueue(syncObjType, syncObjId); + SyncQueueVO queueVO = _syncQueueDao.find(syncObjType, syncObjId); + if(queueVO == null) + throw new CloudRuntimeException("Unable to queue item into DB, DB is full?"); + + queueVO.setQueueSizeLimit(queueSizeLimit); + _syncQueueDao.update(queueVO.getId(), queueVO); + + Date dt = DateUtil.currentGMTTime(); + SyncQueueItemVO item = new SyncQueueItemVO(); + item.setQueueId(queueVO.getId()); + item.setContentType(itemType); + item.setContentId(itemId); + item.setCreated(dt); + + _syncQueueItemDao.persist(item); + txn.commit(); + + return queueVO; + } catch(Exception e) { + s_logger.error("Unexpected exception: ", e); + txn.rollback(); + } + return null; + } + + @Override + @DB + public SyncQueueItemVO dequeueFromOne(long queueId, Long msid) { + Transaction txt = Transaction.currentTxn(); + try { + txt.start(); + + SyncQueueVO queueVO = _syncQueueDao.lockRow(queueId, true); + if(queueVO == null) { + s_logger.error("Sync queue(id: " + queueId + ") does not exist"); + txt.commit(); + return null; + } + + if(queueReadyToProcess(queueVO)) { + SyncQueueItemVO itemVO = _syncQueueItemDao.getNextQueueItem(queueVO.getId()); + if(itemVO != null) { + Long processNumber = queueVO.getLastProcessNumber(); + if(processNumber == null) + processNumber = new Long(1); + else + processNumber = processNumber + 1; + Date dt = DateUtil.currentGMTTime(); + queueVO.setLastProcessNumber(processNumber); + queueVO.setLastUpdated(dt); + queueVO.setQueueSize(queueVO.getQueueSize() + 1); + _syncQueueDao.update(queueVO.getId(), queueVO); + + itemVO.setLastProcessMsid(msid); + itemVO.setLastProcessNumber(processNumber); + itemVO.setLastProcessTime(dt); + _syncQueueItemDao.update(itemVO.getId(), itemVO); + + txt.commit(); + return itemVO; + } else { + if(s_logger.isDebugEnabled()) + s_logger.debug("Sync queue (" + queueId + ") is currently empty"); + } + } else { + if(s_logger.isDebugEnabled()) + s_logger.debug("There is a pending process in sync queue(id: " + queueId + ")"); + } + txt.commit(); + } catch(Exception e) { + s_logger.error("Unexpected exception: ", e); + txt.rollback(); + } + + return null; + } + + @Override + @DB + public List dequeueFromAny(Long msid, int maxItems) { + + List resultList = new ArrayList(); + Transaction txt = Transaction.currentTxn(); + try { + txt.start(); + + List l = _syncQueueItemDao.getNextQueueItems(maxItems); + if(l != null && l.size() > 0) { + for(SyncQueueItemVO item : l) { + SyncQueueVO queueVO = _syncQueueDao.lockRow(item.getQueueId(), true); + SyncQueueItemVO itemVO = _syncQueueItemDao.lockRow(item.getId(), true); + if(queueReadyToProcess(queueVO) && itemVO.getLastProcessNumber() == null) { + Long processNumber = queueVO.getLastProcessNumber(); + if(processNumber == null) + processNumber = new Long(1); + else + processNumber = processNumber + 1; + + Date dt = DateUtil.currentGMTTime(); + queueVO.setLastProcessNumber(processNumber); + queueVO.setLastUpdated(dt); + queueVO.setQueueSize(queueVO.getQueueSize() + 1); + _syncQueueDao.update(queueVO.getId(), queueVO); + + itemVO.setLastProcessMsid(msid); + itemVO.setLastProcessNumber(processNumber); + itemVO.setLastProcessTime(dt); + _syncQueueItemDao.update(item.getId(), itemVO); + + resultList.add(item); + } + } + } + txt.commit(); + return resultList; + } catch(Exception e) { + s_logger.error("Unexpected exception: ", e); + txt.rollback(); + } + return null; + } + + @Override + @DB + public void purgeItem(long queueItemId) { + Transaction txt = Transaction.currentTxn(); + try { + txt.start(); + + SyncQueueItemVO itemVO = _syncQueueItemDao.findById(queueItemId); + if(itemVO != null) { + SyncQueueVO queueVO = _syncQueueDao.lockRow(itemVO.getQueueId(), true); + + _syncQueueItemDao.expunge(itemVO.getId()); + + // if item is active, reset queue information + if (itemVO.getLastProcessMsid() != null) { + queueVO.setLastUpdated(DateUtil.currentGMTTime()); + // decrement the count + assert (queueVO.getQueueSize() > 0) : "Count reduce happens when it's already <= 0!"; + queueVO.setQueueSize(queueVO.getQueueSize() - 1); + _syncQueueDao.update(queueVO.getId(), queueVO); + } + } + txt.commit(); + } catch(Exception e) { + s_logger.error("Unexpected exception: ", e); + txt.rollback(); + } + } + + @Override + @DB + public void returnItem(long queueItemId) { + Transaction txt = Transaction.currentTxn(); + try { + txt.start(); + + SyncQueueItemVO itemVO = _syncQueueItemDao.findById(queueItemId); + if(itemVO != null) { + SyncQueueVO queueVO = _syncQueueDao.lockRow(itemVO.getQueueId(), true); + + itemVO.setLastProcessMsid(null); + itemVO.setLastProcessNumber(null); + itemVO.setLastProcessTime(null); + _syncQueueItemDao.update(queueItemId, itemVO); + + queueVO.setLastUpdated(DateUtil.currentGMTTime()); + _syncQueueDao.update(queueVO.getId(), queueVO); + } + txt.commit(); + } catch(Exception e) { + s_logger.error("Unexpected exception: ", e); + txt.rollback(); + } + } + + @Override + public List getActiveQueueItems(Long msid, boolean exclusive) { + return _syncQueueItemDao.getActiveQueueItems(msid, exclusive); + } + + @Override + public List getBlockedQueueItems(long thresholdMs, boolean exclusive) { + return _syncQueueItemDao.getBlockedQueueItems(thresholdMs, exclusive); + } + + private boolean queueReadyToProcess(SyncQueueVO queueVO) { + return true; + + // + // TODO + // + // Need to disable concurrency disable at queue level due to the need to support + // job wake-up dispatching task + // + // Concurrency control is better done at higher level and leave the job scheduling/serializing simpler + // + + // return queueVO.getQueueSize() < queueVO.getQueueSizeLimit(); + } + + @Override + public void purgeAsyncJobQueueItemId(long asyncJobId) { + Long itemId = _syncQueueItemDao.getQueueItemIdByContentIdAndType(asyncJobId, SyncQueueItem.AsyncJobContentType); + if (itemId != null) { + purgeItem(itemId); + } + } +} diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueVO.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueVO.java new file mode 100644 index 00000000000..4fd4740c8aa --- /dev/null +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueVO.java @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.jobs.impl; + +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +@Entity +@Table(name="sync_queue") +public class SyncQueueVO implements InternalIdentity { + + @Id + @GeneratedValue(strategy=GenerationType.IDENTITY) + @Column(name="id") + private Long id; + + @Column(name="sync_objtype") + + private String syncObjType; + + @Column(name="sync_objid") + private Long syncObjId; + + @Column(name="queue_proc_number") + private Long lastProcessNumber; + + @Column(name="created") + @Temporal(TemporalType.TIMESTAMP) + private Date created; + + @Column(name="last_updated") + @Temporal(TemporalType.TIMESTAMP) + private Date lastUpdated; + + @Column(name="queue_size") + private long queueSize = 0; + + @Column(name="queue_size_limit") + private long queueSizeLimit = 0; + + public long getId() { + return id; + } + + public String getSyncObjType() { + return syncObjType; + } + + public void setSyncObjType(String syncObjType) { + this.syncObjType = syncObjType; + } + + public Long getSyncObjId() { + return syncObjId; + } + + public void setSyncObjId(Long syncObjId) { + this.syncObjId = syncObjId; + } + + public Long getLastProcessNumber() { + return lastProcessNumber; + } + + public void setLastProcessNumber(Long number) { + lastProcessNumber = number; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Date getLastUpdated() { + return lastUpdated; + } + + public void setLastUpdated(Date lastUpdated) { + this.lastUpdated = lastUpdated; + } + + public String toString() { + StringBuffer sb = new StringBuffer(); + sb.append("SyncQueueVO {id:").append(getId()); + sb.append(", syncObjType: ").append(getSyncObjType()); + sb.append(", syncObjId: ").append(getSyncObjId()); + sb.append(", lastProcessNumber: ").append(getLastProcessNumber()); + sb.append(", lastUpdated: ").append(getLastUpdated()); + sb.append(", created: ").append(getCreated()); + sb.append(", count: ").append(getQueueSize()); + sb.append("}"); + return sb.toString(); + } + + public long getQueueSize() { + return queueSize; + } + + public void setQueueSize(long queueSize) { + this.queueSize = queueSize; + } + + public long getQueueSizeLimit() { + return queueSizeLimit; + } + + public void setQueueSizeLimit(long queueSizeLimit) { + this.queueSizeLimit = queueSizeLimit; + } +} diff --git a/server/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/server/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index 91b0343cef8..bc72aff054d 100755 --- a/server/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/server/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -713,12 +713,12 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust } @Override - public void onManagementNodeJoined(List nodeList, long selfNodeId) { + public void onManagementNodeJoined(List nodeList, long selfNodeId) { } @Override - public void onManagementNodeLeft(List nodeList, long selfNodeId) { - for (ManagementServerHostVO vo : nodeList) { + public void onManagementNodeLeft(List nodeList, long selfNodeId) { + for (ManagementServerHost vo : nodeList) { s_logger.info("Marking hosts as disconnected on Management server" + vo.getMsid()); long lastPing = (System.currentTimeMillis() >> 10) - getTimeout(); _hostDao.markHostsAsDisconnected(vo.getMsid(), lastPing); diff --git a/server/src/com/cloud/async/AsyncJobManagerImpl.java b/server/src/com/cloud/async/AsyncJobManagerImpl.java index faf3e719b6e..3aceec4a9f6 100644 --- a/server/src/com/cloud/async/AsyncJobManagerImpl.java +++ b/server/src/com/cloud/async/AsyncJobManagerImpl.java @@ -61,7 +61,7 @@ import com.cloud.api.ApiGsonHelper; import com.cloud.api.ApiSerializerHelper; import com.cloud.async.dao.AsyncJobDao; import com.cloud.cluster.ClusterManagerListener; -import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.ManagementServerHost; import com.cloud.configuration.Config; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.domain.DomainVO; @@ -823,12 +823,12 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, } @Override - public void onManagementNodeJoined(List nodeList, long selfNodeId) { + public void onManagementNodeJoined(List nodeList, long selfNodeId) { } @Override - public void onManagementNodeLeft(List nodeList, long selfNodeId) { - for(ManagementServerHostVO msHost : nodeList) { + public void onManagementNodeLeft(List nodeList, long selfNodeId) { + for (ManagementServerHost msHost : nodeList) { Transaction txn = Transaction.open(Transaction.CLOUD_DB); try { txn.start(); diff --git a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java index 71c1a4ddc04..93de351ac2b 100755 --- a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -37,7 +37,7 @@ import org.apache.cloudstack.context.ServerContexts; import com.cloud.agent.AgentManager; import com.cloud.alert.AlertManager; import com.cloud.cluster.ClusterManagerListener; -import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.ManagementServerHost; import com.cloud.configuration.Config; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.dc.ClusterDetailsDao; @@ -866,12 +866,12 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai } @Override - public void onManagementNodeJoined(List nodeList, long selfNodeId) { + public void onManagementNodeJoined(List nodeList, long selfNodeId) { } @Override - public void onManagementNodeLeft(List nodeList, long selfNodeId) { - for (ManagementServerHostVO node : nodeList) { + public void onManagementNodeLeft(List nodeList, long selfNodeId) { + for (ManagementServerHost node : nodeList) { _haDao.releaseWorkItems(node.getMsid()); } } diff --git a/server/src/com/cloud/server/LockMasterListener.java b/server/src/com/cloud/server/LockMasterListener.java index ee9c9a9d50c..8bd64bb0327 100644 --- a/server/src/com/cloud/server/LockMasterListener.java +++ b/server/src/com/cloud/server/LockMasterListener.java @@ -19,7 +19,7 @@ package com.cloud.server; import java.util.List; import com.cloud.cluster.ClusterManagerListener; -import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.ManagementServerHost; import com.cloud.utils.db.Merovingian2; /** @@ -34,12 +34,12 @@ public class LockMasterListener implements ClusterManagerListener { } @Override - public void onManagementNodeJoined(List nodeList, long selfNodeId) { + public void onManagementNodeJoined(List nodeList, long selfNodeId) { } @Override - public void onManagementNodeLeft(List nodeList, long selfNodeId) { - for (ManagementServerHostVO node : nodeList) { + public void onManagementNodeLeft(List nodeList, long selfNodeId) { + for (ManagementServerHost node : nodeList) { _lockMaster.cleanupForServer(node.getMsid()); } } diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index 20d5a625095..a8cfcc0d19c 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -48,9 +48,9 @@ import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd; import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; -import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; @@ -100,7 +100,7 @@ import com.cloud.capacity.CapacityState; import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDao; import com.cloud.cluster.ClusterManagerListener; -import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.ManagementServerHost; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; import com.cloud.configuration.dao.ConfigurationDao; @@ -1271,14 +1271,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } @Override - public void onManagementNodeJoined(List nodeList, long selfNodeId) { + public void onManagementNodeJoined(List nodeList, long selfNodeId) { // TODO Auto-generated method stub } @Override - public void onManagementNodeLeft(List nodeList, long selfNodeId) { - for (ManagementServerHostVO vo : nodeList) { + public void onManagementNodeLeft(List nodeList, long selfNodeId) { + for (ManagementServerHost vo : nodeList) { if (vo.getMsid() == _serverId) { s_logger.info("Cleaning up storage maintenance jobs associated with Management server: " + vo.getMsid()); List poolIds = _storagePoolWorkDao.searchForPoolIdsForPendingWorkJobs(vo.getMsid()); @@ -1886,7 +1886,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C throw new InvalidParameterValueException("Cannot delete cache store with staging volumes currently in use!"); } - List templates = this._templateStoreDao.listActiveOnCache(storeId); + List templates = _templateStoreDao.listActiveOnCache(storeId); if (templates != null && templates.size() > 0) { throw new InvalidParameterValueException("Cannot delete cache store with staging templates currently in use!"); } diff --git a/server/src/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java b/server/src/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java index 2ee2d564fdd..8f0e00ef57a 100644 --- a/server/src/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java +++ b/server/src/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java @@ -25,7 +25,7 @@ import javax.naming.ConfigurationException; import com.cloud.cluster.ClusterManager; import com.cloud.cluster.ClusterManagerListener; -import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.ManagementServerHost; @Local(value=VirtualMachineManager.class) public class ClusteredVirtualMachineManagerImpl extends VirtualMachineManagerImpl implements ClusterManagerListener { @@ -37,13 +37,13 @@ public class ClusteredVirtualMachineManagerImpl extends VirtualMachineManagerImp } @Override - public void onManagementNodeJoined(List nodeList, long selfNodeId) { + public void onManagementNodeJoined(List nodeList, long selfNodeId) { } @Override - public void onManagementNodeLeft(List nodeList, long selfNodeId) { - for (ManagementServerHostVO node : nodeList) { + public void onManagementNodeLeft(List nodeList, long selfNodeId) { + for (ManagementServerHost node : nodeList) { cancelWorkItems(node.getMsid()); } } diff --git a/utils/src/org/apache/cloudstack/config/Configurable.java b/utils/src/org/apache/cloudstack/config/Configurable.java new file mode 100644 index 00000000000..3c50ebac083 --- /dev/null +++ b/utils/src/org/apache/cloudstack/config/Configurable.java @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.config; + +public interface Configurable { + ConfigKey[] getConfigKeys(); +}