diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index 4b83e312c7b..20cff5671b7 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -63,4 +63,5 @@ public interface SnapshotDataStoreDao extends GenericDao listByState(ObjectInDataStoreStateMachine.State... states); } diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index c7e9aa1234f..b47f0cf251b 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -191,7 +191,9 @@ public class SnapshotObject implements SnapshotInfo { s_logger.debug("Failed to update state:" + e.toString()); throw new CloudRuntimeException("Failed to update state: " + e.toString()); } finally { - if (event == ObjectInDataStoreStateMachine.Event.OperationFailed) { + DataObjectInStore obj = objectInStoreMgr.findObject(this, this.getDataStore()); + if (event == ObjectInDataStoreStateMachine.Event.OperationFailed && !obj.getState().equals(ObjectInDataStoreStateMachine.State.Destroying)) { + // Don't delete db entry if snapshot is successfully removed. objectInStoreMgr.deleteIfNotReady(this); } } diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotStateMachineManagerImpl.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotStateMachineManagerImpl.java index 1cec2dc3c1d..9c6e3796887 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotStateMachineManagerImpl.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotStateMachineManagerImpl.java @@ -52,6 +52,7 @@ public class SnapshotStateMachineManagerImpl implements SnapshotStateMachineMana stateMachine.addTransition(Snapshot.State.Copying, Event.OperationFailed, Snapshot.State.BackedUp); stateMachine.addTransition(Snapshot.State.Destroying, Event.OperationSucceeded, Snapshot.State.Destroyed); stateMachine.addTransition(Snapshot.State.Destroying, Event.OperationFailed, State.BackedUp); + stateMachine.addTransition(Snapshot.State.Destroying, Event.DestroyRequested, Snapshot.State.Destroying); stateMachine.registerListener(new SnapshotStateListener()); } diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java index 5b410237506..165d3564b75 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java @@ -194,18 +194,23 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { } } if (!deleted) { - boolean r = snapshotSvr.deleteSnapshot(snapshot); - if (r) { - // delete snapshot in cache if there is - List cacheSnaps = snapshotDataFactory.listSnapshotOnCache(snapshot.getId()); - for (SnapshotInfo cacheSnap : cacheSnaps) { - s_logger.debug("Delete snapshot " + snapshot.getId() + " from image cache store: " + cacheSnap.getDataStore().getName()); - cacheSnap.delete(); + try { + boolean r = snapshotSvr.deleteSnapshot(snapshot); + if (r) { + // delete snapshot in cache if there is + List cacheSnaps = snapshotDataFactory.listSnapshotOnCache(snapshot.getId()); + for (SnapshotInfo cacheSnap : cacheSnaps) { + s_logger.debug("Delete snapshot " + snapshot.getId() + " from image cache store: " + cacheSnap.getDataStore().getName()); + cacheSnap.delete(); + } } - } - if (!resultIsSet) { - result = r; - resultIsSet = true; + if (!resultIsSet) { + result = r; + resultIsSet = true; + } + } catch (Exception e) { + // Snapshots which are not successfully deleted will be retried again. + s_logger.debug("Failed to delete snapshot on storage. ", e); } } snapshot = parent; @@ -245,7 +250,8 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { return true; } - if (!Snapshot.State.BackedUp.equals(snapshotVO.getState()) && !Snapshot.State.Error.equals(snapshotVO.getState())) { + if (!Snapshot.State.BackedUp.equals(snapshotVO.getState()) && !Snapshot.State.Error.equals(snapshotVO.getState()) && + !Snapshot.State.Destroying.equals(snapshotVO.getState())) { throw new InvalidParameterValueException("Can't delete snapshotshot " + snapshotId + " due to it is in " + snapshotVO.getState() + " Status"); } diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java index 24bb542ec04..9ffeae06e0b 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java @@ -53,6 +53,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase snapshotIdSearch; private SearchBuilder volumeIdSearch; private SearchBuilder volumeSearch; + private SearchBuilder stateSearch; private final String parentSearch = "select store_id, store_role, snapshot_id from cloud.snapshot_store_ref where store_id = ? " + " and store_role = ? and volume_id = ? and state = 'Ready'" + " order by created DESC " + " limit 1"; @@ -123,6 +124,10 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase listByState(ObjectInDataStoreStateMachine.State... states) { + SearchCriteria sc = stateSearch.create(); + sc.setParameters("state", (Object[])states); + return listBy(sc, null); + } } diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index 8f5b053d384..a816a355d80 100644 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -41,6 +41,8 @@ import javax.naming.ConfigurationException; import com.cloud.hypervisor.Hypervisor; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd; @@ -291,6 +293,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C @Inject EntityManager _entityMgr; @Inject + SnapshotService _snapshotService; + @Inject StoragePoolTagsDao _storagePoolTagsDao; protected List _discoverers; @@ -1078,6 +1082,16 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } } + //destroy snapshots in destroying state in snapshot_store_ref + List ssSnapshots = _snapshotStoreDao.listByState(ObjectInDataStoreStateMachine.State.Destroying); + for(SnapshotDataStoreVO ssSnapshotVO : ssSnapshots){ + try { + _snapshotService.deleteSnapshot(snapshotFactory.getSnapshot(ssSnapshotVO.getSnapshotId(), DataStoreRole.Image)); + } catch (Exception e) { + s_logger.debug("Failed to delete snapshot: " + ssSnapshotVO.getId() + " from storage"); + } + } + cleanupSecondaryStorage(recurring); List vols = _volsDao.listVolumesToBeDestroyed(new Date(System.currentTimeMillis() - ((long) StorageCleanupDelay.value() << 10))); diff --git a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 2ffc8acc618..686ca6aa4cf 100755 --- a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -1192,6 +1192,17 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement @Override public boolean start() { + //destroy snapshots in destroying state + List snapshots = _snapshotDao.listAllByStatus(Snapshot.State.Destroying); + for (SnapshotVO snapshotVO : snapshots) { + try { + if (!deleteSnapshot(snapshotVO.getId())) { + s_logger.debug("Failed to delete snapshot in destroying state with id " + snapshotVO.getUuid()); + } + } catch (Exception e) { + s_logger.debug("Failed to delete snapshot in destroying state with id " + snapshotVO.getUuid()); + } + } return true; }