Clean up entries in template_store_ref, volume_store_ref and

snapshot_store_ref in case of operation failure.

engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/StorageCacheManager.java

@@ -28,5 +28,5 @@ public interface StorageCacheManager {
      * @return
      */
     DataObject getCacheObject(DataObject data, Scope scope);
-    DataObject deleteCacheObject(DataObject data);
+    boolean deleteCacheObject(DataObject data);
 }

engine/api/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java

@@ -81,8 +81,6 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
 	@Column (name="install_path")
     private String installPath;
 
-    @Column(name = GenericDao.REMOVED_COLUMN)
-    Date removed;
 
     @Column(name="update_count", updatable = true, nullable=false)
     protected long updatedCount;

engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java

@@ -368,6 +368,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
         int _backupsnapshotwait = NumbersUtil.parseInt(value, Integer.parseInt(Config.BackupSnapshotWait.getDefaultValue()));
 
         DataObject cacheData = null;
+        Answer answer = null;
         try {
             if (needCacheStorage(srcData, destData)) {
                 cacheData = cacheMgr.getCacheObject(srcData, destData.getDataStore().getScope());
@@ -375,14 +376,19 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
                 CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait);
                 cmd.setCacheTO(cacheData.getTO());
                 EndPoint ep = selector.select(srcData, destData);
-            Answer answer = ep.sendMessage(cmd);
+                answer = ep.sendMessage(cmd);
-            return answer;
             } else {
                 CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait);
                 EndPoint ep = selector.select(srcData, destData);
-            Answer answer = ep.sendMessage(cmd);
+                answer = ep.sendMessage(cmd);
-            return answer;
             }
+            // clean up cache entry in case of failure
+            if (answer == null || !answer.getResult()) {
+                if (cacheData != null) {
+                    cacheMgr.deleteCacheObject(cacheData);
+                }
+            }
+            return answer;
         } catch (Exception e) {
             s_logger.debug("copy snasphot failed: " + e.toString());
             if (cacheData != null) {

engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java

@@ -63,7 +63,7 @@ public class TemplateObject implements TemplateInfo {
     @Inject
     VMTemplateDao imageDao;
     @Inject
-    ObjectInDataStoreManager ojbectInStoreMgr;
+    ObjectInDataStoreManager objectInStoreMgr;
     @Inject VMTemplatePoolDao templatePoolDao;
     @Inject TemplateDataStoreDao templateStoreDao;
 
@@ -191,10 +191,15 @@ public class TemplateObject implements TemplateInfo {
                 }
             }
 
-            ojbectInStoreMgr.update(this, event);
+            objectInStoreMgr.update(this, event);
         } catch (NoTransitionException e) {
             s_logger.debug("failed to update state", e);
             throw new CloudRuntimeException("Failed to update state" + e.toString());
+        } finally{
+            // in case of OperationFailed, expunge the entry
+            if ( event == ObjectInDataStoreStateMachine.Event.OperationFailed){
+                objectInStoreMgr.delete(this);
+            }
         }
     }
 
@@ -247,10 +252,15 @@ public class TemplateObject implements TemplateInfo {
         		    this.stateTransit(templEvent);
         		}
         	}
-            ojbectInStoreMgr.update(this, event);
+            objectInStoreMgr.update(this, event);
         } catch (NoTransitionException e) {
             s_logger.debug("failed to update state", e);
             throw new CloudRuntimeException("Failed to update state" + e.toString());
+        } finally{
+            // in case of OperationFailed, expunge the entry
+            if ( event == ObjectInDataStoreStateMachine.Event.OperationFailed){
+                objectInStoreMgr.delete(this);
+            }
         }
     }
 
@@ -266,7 +276,7 @@ public class TemplateObject implements TemplateInfo {
 
 	@Override
 	public String getInstallPath() {
-		 DataObjectInStore obj = ojbectInStoreMgr.findObject(this, this.dataStore);
+		 DataObjectInStore obj = objectInStoreMgr.findObject(this, this.dataStore);
          return obj.getInstallPath();
 	}
 

engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java

@@ -59,14 +59,17 @@ public class SnapshotObject implements SnapshotInfo {
     protected SnapshotDao snapshotDao;
     @Inject
     protected VolumeDao volumeDao;
-    @Inject protected VolumeDataFactory volFactory;
+    @Inject
-    @Inject protected SnapshotStateMachineManager stateMachineMgr;
+    protected VolumeDataFactory volFactory;
+    @Inject
+    protected SnapshotStateMachineManager stateMachineMgr;
     @Inject
     SnapshotDataFactory snapshotFactory;
     @Inject
-    ObjectInDataStoreManager ojbectInStoreMgr;
+    ObjectInDataStoreManager objectInStoreMgr;
     @Inject
     SnapshotDataStoreDao snapshotStoreDao;
+
     public SnapshotObject() {
 
     }
@@ -144,20 +147,22 @@ public class SnapshotObject implements SnapshotInfo {
         return DataObjectType.SNAPSHOT;
     }
 
-
     @Override
     public String getUuid() {
         return this.snapshot.getUuid();
     }
 
     @Override
-	public void processEvent(
+    public void processEvent(ObjectInDataStoreStateMachine.Event event) {
-			ObjectInDataStoreStateMachine.Event event) {
         try {
-			ojbectInStoreMgr.update(this, event);
+            objectInStoreMgr.update(this, event);
         } catch (Exception e) {
             s_logger.debug("Failed to update state:" + e.toString());
             throw new CloudRuntimeException("Failed to update state: " + e.toString());
+        } finally {
+            if (event == ObjectInDataStoreStateMachine.Event.OperationFailed) {
+                objectInStoreMgr.delete(this);
+            }
         }
     }
 
@@ -173,7 +178,7 @@ public class SnapshotObject implements SnapshotInfo {
 
     @Override
     public String getPath() {
-		return this.ojbectInStoreMgr.findObject(this, getDataStore()).getInstallPath();
+        return this.objectInStoreMgr.findObject(this, getDataStore()).getInstallPath();
     }
 
     @Override
@@ -216,18 +221,16 @@ public class SnapshotObject implements SnapshotInfo {
         return this.snapshot.getDomainId();
     }
 
-
     @Override
     public Long getDataCenterId() {
         return this.snapshot.getDataCenterId();
     }
 
-	public void processEvent(Snapshot.Event event)
+    public void processEvent(Snapshot.Event event) throws NoTransitionException {
-			throws NoTransitionException {
         stateMachineMgr.processEvent(this.snapshot, event);
     }
 
-	public SnapshotVO getSnapshotVO(){
+    public SnapshotVO getSnapshotVO() {
         return this.snapshot;
     }
 
@@ -242,14 +245,15 @@ public class SnapshotObject implements SnapshotInfo {
 
     @Override
     public void processEvent(ObjectInDataStoreStateMachine.Event event, Answer answer) {
-    	SnapshotDataStoreVO snapshotStore = this.snapshotStoreDao.findByStoreSnapshot(this.getDataStore().getRole(),
+        try {
-    		   this.getDataStore().getId(), this.getId());
+            SnapshotDataStoreVO snapshotStore = this.snapshotStoreDao.findByStoreSnapshot(this.getDataStore().getRole(), this.getDataStore().getId(),
+                    this.getId());
             if (answer instanceof CreateObjectAnswer) {
-    		SnapshotObjectTO snapshotTO = (SnapshotObjectTO)((CreateObjectAnswer) answer).getData();
+                SnapshotObjectTO snapshotTO = (SnapshotObjectTO) ((CreateObjectAnswer) answer).getData();
                 snapshotStore.setInstallPath(snapshotTO.getPath());
                 this.snapshotStoreDao.update(snapshotStore.getId(), snapshotStore);
             } else if (answer instanceof CopyCmdAnswer) {
-    	    SnapshotObjectTO snapshotTO = (SnapshotObjectTO)((CopyCmdAnswer) answer).getNewData();
+                SnapshotObjectTO snapshotTO = (SnapshotObjectTO) ((CopyCmdAnswer) answer).getNewData();
                 snapshotStore.setInstallPath(snapshotTO.getPath());
                 if (snapshotTO.getParentSnapshotPath() == null) {
                     snapshotStore.setParentSnapshotId(0L);
@@ -258,12 +262,18 @@ public class SnapshotObject implements SnapshotInfo {
             } else {
                 throw new CloudRuntimeException("Unknown answer: " + answer.getClass());
             }
+        } catch (RuntimeException ex) {
+            if (event == ObjectInDataStoreStateMachine.Event.OperationFailed) {
+                objectInStoreMgr.delete(this);
+            }
+            throw ex;
+        }
         this.processEvent(event);
     }
 
     @Override
     public ObjectInDataStoreStateMachine.State getStatus() {
-       return this.ojbectInStoreMgr.findObject(this, store).getObjectInStoreState();
+        return this.objectInStoreMgr.findObject(this, store).getObjectInStoreState();
     }
 
     @Override

engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeObject.java

@@ -59,7 +59,7 @@ public class VolumeObject implements VolumeInfo {
     @Inject
     VolumeDataStoreDao volumeStoreDao;
     @Inject
-    ObjectInDataStoreManager ojbectInStoreMgr;
+    ObjectInDataStoreManager objectInStoreMgr;
     @Inject
     VMInstanceDao vmInstanceDao;
     private Object payload;
@@ -159,7 +159,7 @@ public class VolumeObject implements VolumeInfo {
         if (this.dataStore == null) {
             throw new CloudRuntimeException("datastore must be set before using this object");
         }
-        DataObjectInStore obj = ojbectInStoreMgr.findObject(this.volumeVO.getId(), DataObjectType.VOLUME, this.dataStore.getId(), this.dataStore.getRole());
+        DataObjectInStore obj = objectInStoreMgr.findObject(this.volumeVO.getId(), DataObjectType.VOLUME, this.dataStore.getId(), this.dataStore.getRole());
         if (obj.getState() != ObjectInDataStoreStateMachine.State.Ready) {
             return this.dataStore.getUri() +
                     "&" + EncodingType.OBJTYPE + "=" + DataObjectType.VOLUME +
@@ -187,11 +187,11 @@ public class VolumeObject implements VolumeInfo {
         try {
             Volume.Event volEvent = null;
             if ( this.dataStore.getRole() == DataStoreRole.ImageCache){
-                ojbectInStoreMgr.update(this, event);
+                objectInStoreMgr.update(this, event);
                 return;
             }
             if (this.dataStore.getRole() == DataStoreRole.Image) {
-                ojbectInStoreMgr.update(this, event);
+                objectInStoreMgr.update(this, event);
                 if (event == ObjectInDataStoreStateMachine.Event.CreateRequested) {
                     volEvent = Volume.Event.UploadRequested;
                 } else if (event == ObjectInDataStoreStateMachine.Event.OperationSuccessed) {
@@ -225,6 +225,11 @@ public class VolumeObject implements VolumeInfo {
         } catch (Exception e) {
             s_logger.debug("Failed to update state", e);
             throw new CloudRuntimeException("Failed to update state:" + e.toString());
+        } finally{
+            // in case of OperationFailed, expunge the entry
+            if ( event == ObjectInDataStoreStateMachine.Event.OperationFailed){
+                objectInStoreMgr.delete(this);
+            }
         }
 
     }
@@ -249,7 +254,7 @@ public class VolumeObject implements VolumeInfo {
         if (this.dataStore.getRole() == DataStoreRole.Primary) {
             return this.volumeVO.getPath();
         } else {
-            DataObjectInStore objInStore = this.ojbectInStoreMgr.findObject(this, dataStore);
+            DataObjectInStore objInStore = this.objectInStoreMgr.findObject(this, dataStore);
             return objInStore.getInstallPath();
         }
     }
@@ -379,18 +384,19 @@ public class VolumeObject implements VolumeInfo {
 
     @Override
     public void processEvent(ObjectInDataStoreStateMachine.Event event, Answer answer) {
+        try {
             if (this.dataStore.getRole() == DataStoreRole.Primary) {
                 if (answer instanceof CopyCmdAnswer) {
-               CopyCmdAnswer cpyAnswer = (CopyCmdAnswer)answer;
+                    CopyCmdAnswer cpyAnswer = (CopyCmdAnswer) answer;
                     VolumeVO vol = this.volumeDao.findById(this.getId());
-               VolumeObjectTO newVol = (VolumeObjectTO)cpyAnswer.getNewData();
+                    VolumeObjectTO newVol = (VolumeObjectTO) cpyAnswer.getNewData();
                     vol.setPath(newVol.getPath());
                     vol.setSize(newVol.getSize());
                     vol.setPoolId(this.getDataStore().getId());
                     volumeDao.update(vol.getId(), vol);
                 } else if (answer instanceof CreateObjectAnswer) {
-               CreateObjectAnswer createAnswer =(CreateObjectAnswer)answer;
+                    CreateObjectAnswer createAnswer = (CreateObjectAnswer) answer;
-               VolumeObjectTO newVol = (VolumeObjectTO)createAnswer.getData();
+                    VolumeObjectTO newVol = (VolumeObjectTO) createAnswer.getData();
                     VolumeVO vol = this.volumeDao.findById(this.getId());
                     vol.setPath(newVol.getPath());
                     vol.setSize(newVol.getSize());
@@ -399,14 +405,20 @@ public class VolumeObject implements VolumeInfo {
                 }
             } else if (this.dataStore.getRole() == DataStoreRole.Image) {
                 if (answer instanceof DownloadAnswer) {
-               DownloadAnswer dwdAnswer = (DownloadAnswer)answer;
+                    DownloadAnswer dwdAnswer = (DownloadAnswer) answer;
                     VolumeDataStoreVO volStore = this.volumeStoreDao.findByStoreVolume(this.dataStore.getId(), this.getId());
                     volStore.setInstallPath(dwdAnswer.getInstallPath());
                     volStore.setChecksum(dwdAnswer.getCheckSum());
                     this.volumeStoreDao.update(volStore.getId(), volStore);
                 }
             }
-
+        } catch (RuntimeException ex) {
+            if (event == ObjectInDataStoreStateMachine.Event.OperationFailed) {
+                objectInStoreMgr.delete(this);
+            }
+            throw ex;
+        }
         this.processEvent(event);
+
     }
 }