CLOUDSTACK-5765: cleanup internal serialization and exception propagation issues

api/src/com/cloud/deploy/DeployDestination.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.deploy;
 
+import java.io.Serializable;
 import java.util.Map;
 
 import com.cloud.dc.DataCenter;
@@ -26,7 +27,9 @@ import com.cloud.storage.StoragePool;
 import com.cloud.storage.Volume;
 import com.cloud.utils.NumbersUtil;
 
-public class DeployDestination {
+public class DeployDestination implements Serializable {
+    private static final long serialVersionUID = 7113840781939014695L;
+
     DataCenter _dc;
     Pod _pod;
     Cluster _cluster;
@@ -76,28 +79,28 @@ public class DeployDestination {
     @Override
     public boolean equals(Object obj) {
         DeployDestination that = (DeployDestination)obj;
-        if (this._dc == null || that._dc == null) {
+        if (_dc == null || that._dc == null) {
             return false;
         }
-        if (this._dc.getId() != that._dc.getId()) {
+        if (_dc.getId() != that._dc.getId()) {
             return false;
         }
-        if (this._pod == null || that._pod == null) {
+        if (_pod == null || that._pod == null) {
             return false;
         }
-        if (this._pod.getId() != that._pod.getId()) {
+        if (_pod.getId() != that._pod.getId()) {
             return false;
         }
-        if (this._cluster == null || that._cluster == null) {
+        if (_cluster == null || that._cluster == null) {
             return false;
         }
-        if (this._cluster.getId() != that._cluster.getId()) {
+        if (_cluster.getId() != that._cluster.getId()) {
             return false;
         }
-        if (this._host == null || that._host == null) {
+        if (_host == null || that._host == null) {
             return false;
         }
-        return this._host.getId() == that._host.getId();
+        return _host.getId() == that._host.getId();
     }
 
     @Override

api/src/com/cloud/deploy/DeploymentPlanner.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.deploy;
 
+import java.io.Serializable;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Set;
@@ -89,7 +90,9 @@ public interface DeploymentPlanner extends Adapter {
         Shared, Dedicated;
     }
 
-    public static class ExcludeList {
+    public static class ExcludeList implements Serializable {
+        private static final long serialVersionUID = -482175549460148301L;
+
         private Set<Long> _dcIds;
         private Set<Long> _podIds;
         private Set<Long> _clusterIds;

api/src/com/cloud/exception/CloudException.java

@@ -27,6 +27,7 @@ import com.cloud.utils.exception.CSExceptionErrorCode;
  */
 
 public class CloudException extends Exception {
+    private static final long serialVersionUID = 8784427323859682503L;
 
     // This holds a list of uuids and their names. Add uuid:fieldname pairs
     protected ArrayList<String> idList = new ArrayList<String>();
@@ -58,10 +59,10 @@ public class CloudException extends Exception {
     }
 
     public void setCSErrorCode(int cserrcode) {
-        this.csErrorCode = cserrcode;
+        csErrorCode = cserrcode;
     }
 
     public int getCSErrorCode() {
-        return this.csErrorCode;
+        return csErrorCode;
     }
 }

api/src/com/cloud/exception/OperationTimedoutException.java

@@ -28,7 +28,15 @@ public class OperationTimedoutException extends CloudException {
     long _agentId;
     long _seqId;
     int _time;
-    Command[] _cmds;
+
+    // TODO
+    // I did a reference search on usage of getCommands() and found none
+    //
+    // to prevent serialization problems across boundaries, I'm disabling serialization of _cmds here
+    // getCommands() will still be available within the same serialization boundary, but it will be lost
+    // when exception is propagated across job boundaries.
+    //
+    transient Command[] _cmds;
     boolean _isActive;
 
     public OperationTimedoutException(Command[] cmds, long agentId, long seqId, int time, boolean isActive) {

engine/api/src/com/cloud/vm/VirtualMachineManager.java

@@ -190,7 +190,9 @@ public interface VirtualMachineManager extends Manager {
      */
     VirtualMachineTO toVmTO(VirtualMachineProfile profile);
 
-    VirtualMachine reConfigureVm(String vmUuid, ServiceOffering newServiceOffering, boolean sameHost) throws ResourceUnavailableException, ConcurrentOperationException;
+    VirtualMachine reConfigureVm(String vmUuid, ServiceOffering newServiceOffering, boolean sameHost) throws ResourceUnavailableException, ConcurrentOperationException,
+            InsufficientServerCapacityException;
+
     void findHostAndMigrate(String vmUuid, Long newSvcOfferingId, DeploymentPlanner.ExcludeList excludeHostList) throws InsufficientCapacityException,
         ConcurrentOperationException, ResourceUnavailableException;
 

engine/components-api/src/com/cloud/vm/VmWorkJobHandlerProxy.java

@@ -116,8 +116,10 @@ public class VmWorkJobHandlerProxy implements VmWorkJobHandler {
 
                 // legacy CloudStack code relies on checked exception for error handling
                 // we need to re-throw the real exception here
-                if (e.getCause() != null && e.getCause() instanceof Exception)
+                if (e.getCause() != null && e.getCause() instanceof Exception) {
+                    s_logger.info("Rethrow exception " + e.getCause());
                     throw (Exception)e.getCause();
+                }
 
                 throw e;
             }

engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -3670,7 +3670,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     @Override
     public VMInstanceVO reConfigureVm(String vmUuid, ServiceOffering oldServiceOffering,
             boolean reconfiguringOnExistingHost)
-            throws ResourceUnavailableException, ConcurrentOperationException {
+            throws ResourceUnavailableException, InsufficientServerCapacityException, ConcurrentOperationException {
 
         AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext();
         if (!VmJobEnabled.value() || jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) {
@@ -3688,20 +3688,21 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                 throw new RuntimeException("Execution excetion", e);
             }
 
-            AsyncJobVO jobVo = _entityMgr.findById(AsyncJobVO.class, outcome.getJob().getId());
+            Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob());
-            if (jobVo.getResultCode() == JobInfo.Status.SUCCEEDED.ordinal()) {
+            if (jobResult != null) {
-                return _entityMgr.findById(VMInstanceVO.class, vm.getId());
+                if (jobResult instanceof ResourceUnavailableException)
-            } else {
+                    throw (ResourceUnavailableException)jobResult;
-                Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob());
+                else if (jobResult instanceof ConcurrentOperationException)
-                if (jobResult != null) {
+                    throw (ConcurrentOperationException)jobResult;
-                    if (jobResult instanceof ResourceUnavailableException)
+                else if (jobResult instanceof InsufficientServerCapacityException)
-                        throw (ResourceUnavailableException)jobResult;
+                    throw (InsufficientServerCapacityException)jobResult;
-                    else if (jobResult instanceof ConcurrentOperationException)
+                else if (jobResult instanceof Throwable) {
-                        throw (ConcurrentOperationException)jobResult;
+                    s_logger.error("Unhandled exception", (Throwable)jobResult);
+                    throw new RuntimeException("Unhandled exception", (Throwable)jobResult);
                 }
-
-                throw new RuntimeException("Failed with un-handled exception");
             }
+
+            return (VMInstanceVO)vm;
         }
     }
 
@@ -4633,7 +4634,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     }
 
     public Outcome<VirtualMachine> reconfigureVmThroughJobQueue(
-            final String vmUuid, final ServiceOffering oldServiceOffering, final boolean reconfiguringOnExistingHost) {
+            final String vmUuid, final ServiceOffering newServiceOffering, final boolean reconfiguringOnExistingHost) {
 
         final CallContext context = CallContext.current();
         final User user = context.getCallingUser();
@@ -4668,7 +4669,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
 
                     // save work context info (there are some duplications)
                     VmWorkReconfigure workInfo = new VmWorkReconfigure(user.getId(), account.getId(), vm.getId(),
-                            VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, oldServiceOffering, reconfiguringOnExistingHost);
+                            VirtualMachineManagerImpl.VM_WORK_JOB_HANDLER, newServiceOffering.getId(), reconfiguringOnExistingHost);
                     workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo));
 
                     _jobMgr.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId());
@@ -4796,7 +4797,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
             s_logger.info("Unable to find vm " + work.getVmId());
         }
         assert (vm != null);
-        reConfigureVm(vm.getUuid(), work.getNewServiceOffering(),
+
+        ServiceOffering newServiceOffering = _offeringDao.findById(vm.getId(), work.getNewServiceOfferingId());
+
+        reConfigureVm(vm.getUuid(), newServiceOffering,
                 work.isSameHost());
         return new Pair<JobInfo.Status, String>(JobInfo.Status.SUCCEEDED, null);
     }

engine/orchestration/src/com/cloud/vm/VmWorkJobDispatcher.java

@@ -26,7 +26,6 @@ import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.framework.jobs.AsyncJob;
 import org.apache.cloudstack.framework.jobs.AsyncJobDispatcher;
 import org.apache.cloudstack.framework.jobs.AsyncJobManager;
-import org.apache.cloudstack.framework.jobs.impl.JobSerializerHelper;
 import org.apache.cloudstack.jobs.JobInfo;
 
 import com.cloud.utils.Pair;
@@ -105,9 +104,8 @@ public class VmWorkJobDispatcher extends AdapterBase implements AsyncJobDispatch
         } catch(Throwable e) {
             s_logger.error("Unable to complete " + job + ", job origin:" + job.getRelated(), e);
 
-            String exceptionJson = JobSerializerHelper.toSerializedString(e);
+            RuntimeException ex = new RuntimeException("Job failed due to exception " + e.getMessage());
-            s_logger.info("Serialize exception object into json: " + exceptionJson + ", job origin: " + job.getRelated());
+            _asyncJobMgr.completeAsyncJob(job.getId(), JobInfo.Status.FAILED, 0, _asyncJobMgr.marshallResultObject(ex));
-            _asyncJobMgr.completeAsyncJob(job.getId(), JobInfo.Status.FAILED, 0, exceptionJson);
         } finally {
             CallContext.unregister();
         }

engine/orchestration/src/com/cloud/vm/VmWorkReconfigure.java

@@ -16,25 +16,24 @@
 // under the License.
 package com.cloud.vm;
 
-import com.cloud.offering.ServiceOffering;
 
 public class VmWorkReconfigure extends VmWork {
     private static final long serialVersionUID = -4517030323758086615L;
 
-    ServiceOffering newServiceOffering;
+    Long newServiceOfferingId;
     boolean sameHost;
 
     public VmWorkReconfigure(long userId, long accountId, long vmId, String handlerName,
-            ServiceOffering newServiceOffering, boolean sameHost) {
+            Long newServiceOfferingId, boolean sameHost) {
 
         super(userId, accountId, vmId, handlerName);
 
-        this.newServiceOffering = newServiceOffering;
+        this.newServiceOfferingId = newServiceOfferingId;
         this.sameHost = sameHost;
     }
 
-    public ServiceOffering getNewServiceOffering() {
+    public Long getNewServiceOfferingId() {
-        return newServiceOffering;
+        return newServiceOfferingId;
     }
 
     public boolean isSameHost() {

engine/schema/src/com/cloud/service/ServiceOfferingVO.java

@@ -97,8 +97,8 @@ public class ServiceOfferingVO extends DiskOfferingVO implements ServiceOffering
         this.rateMbps = rateMbps;
         this.multicastRateMbps = multicastRateMbps;
         this.offerHA = offerHA;
-        this.limitCpuUse = false;
+        limitCpuUse = false;
-        this.volatileVm = false;
+        volatileVm = false;
         this.defaultUse = defaultUse;
         this.vmType = vmType == null ? null : vmType.toString().toLowerCase();
     }
@@ -174,16 +174,16 @@ public class ServiceOfferingVO extends DiskOfferingVO implements ServiceOffering
             offering.getSystemUse(),
             true,
             offering.getDomainId());
-        this.cpu = offering.getCpu();
+        cpu = offering.getCpu();
-        this.ramSize = offering.getRamSize();
+        ramSize = offering.getRamSize();
-        this.speed = offering.getSpeed();
+        speed = offering.getSpeed();
-        this.rateMbps = offering.getRateMbps();
+        rateMbps = offering.getRateMbps();
-        this.multicastRateMbps = offering.getMulticastRateMbps();
+        multicastRateMbps = offering.getMulticastRateMbps();
-        this.offerHA = offering.getOfferHA();
+        offerHA = offering.getOfferHA();
-        this.limitCpuUse = offering.getLimitCpuUse();
+        limitCpuUse = offering.getLimitCpuUse();
-        this.volatileVm = offering.getVolatileVm();
+        volatileVm = offering.getVolatileVm();
-        this.hostTag = offering.getHostTag();
+        hostTag = offering.getHostTag();
-        this.vmType = offering.getSystemVmType();
+        vmType = offering.getSystemVmType();
     }
 
     @Override
@@ -325,6 +325,6 @@ public class ServiceOfferingVO extends DiskOfferingVO implements ServiceOffering
     }
 
     public void setDynamicFlag(boolean isdynamic) {
-        this.isDynamic = isdynamic;
+        isDynamic = isdynamic;
     }
 }

server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java

@@ -131,7 +131,7 @@ public class VMSnapshotManagerImpl extends ManagerBase implements VMSnapshotMana
 
     // TODO
     static final ConfigKey<Boolean> VmJobEnabled = new ConfigKey<Boolean>("Advanced",
-            Boolean.class, "vm.job.enabled", "true",
+            Boolean.class, "vm.job.enabled", "false",
             "True to enable new VM sync model. false to use the old way", false);
     static final ConfigKey<Long> VmJobCheckInterval = new ConfigKey<Long>("Advanced",
             Long.class, "vm.job.check.interval", "3000",

utils/src/com/cloud/utils/exception/CloudRuntimeException.java

@@ -16,6 +16,9 @@
 // under the License.
 package com.cloud.utils.exception;
 
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -30,9 +33,9 @@ public class CloudRuntimeException extends RuntimeException implements ErrorCont
     private static final long serialVersionUID = SerialVersionUID.CloudRuntimeException;
 
     // This holds a list of uuids and their descriptive names.
-    protected ArrayList<ExceptionProxyObject> idList = new ArrayList<ExceptionProxyObject>();
+    transient protected ArrayList<ExceptionProxyObject> idList = new ArrayList<ExceptionProxyObject>();
 
-    protected ArrayList<Pair<Class<?>, String>> uuidList = new ArrayList<Pair<Class<?>, String>>();
+    transient protected ArrayList<Pair<Class<?>, String>> uuidList = new ArrayList<Pair<Class<?>, String>>();
 
     protected int csErrorCode;
 
@@ -48,6 +51,7 @@ public class CloudRuntimeException extends RuntimeException implements ErrorCont
 
     protected CloudRuntimeException() {
         super();
+
         setCSErrorCode(CSExceptionErrorCode.getCSErrCode(this.getClass().getName()));
     }
 
@@ -90,4 +94,45 @@ public class CloudRuntimeException extends RuntimeException implements ErrorCont
     public List<Pair<Class<?>, String>> getEntitiesInError() {
         return uuidList;
     }
+
+    private void writeObject(ObjectOutputStream out) throws IOException {
+        out.defaultWriteObject();
+
+        int idListSize = idList.size();
+        out.writeInt(idListSize);
+        for (ExceptionProxyObject proxy : idList) {
+            out.writeObject(proxy);
+        }
+
+        int uuidListSize = uuidList.size();
+        out.writeInt(uuidListSize);
+        for (Pair<Class<?>, String> entry : uuidList) {
+            out.writeObject(entry.first().getCanonicalName());
+            out.writeObject(entry.second());
+        }
+    }
+
+    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
+        in.defaultReadObject();
+
+        int idListSize = in.readInt();
+        if (idList == null)
+            idList = new ArrayList<ExceptionProxyObject>();
+        if (uuidList == null)
+            uuidList = new ArrayList<Pair<Class<?>, String>>();
+
+        for (int i = 0; i < idListSize; i++) {
+            ExceptionProxyObject proxy = (ExceptionProxyObject)in.readObject();
+
+            idList.add(proxy);
+        }
+
+        int uuidListSize = in.readInt();
+        for (int i = 0; i < uuidListSize; i++) {
+            String clzName = (String)in.readObject();
+            String val = (String)in.readObject();
+
+            uuidList.add(new Pair<Class<?>, String>(Class.forName(clzName), val));
+        }
+    }
 }

utils/src/com/cloud/utils/exception/ExceptionProxyObject.java

@@ -16,17 +16,22 @@
 // under the License.
 package com.cloud.utils.exception;
 
-public class ExceptionProxyObject {
+import java.io.Serializable;
+
+
+public class ExceptionProxyObject implements Serializable {
+    private static final long serialVersionUID = -7514266713085362352L;
+
     private String uuid;
     private String description;
 
-    public ExceptionProxyObject() {
+    public ExceptionProxyObject(){
 
     }
 
-    public ExceptionProxyObject(String uuid, String desc) {
+    public ExceptionProxyObject(String uuid, String desc){
         this.uuid = uuid;
-        this.description = desc;
+        description = desc;
     }
 
     public String getUuid() {
@@ -44,5 +49,4 @@ public class ExceptionProxyObject {
     public void setDescription(String description) {
         this.description = description;
     }
-
 }