bug CS-10789: Adding more code changes for usage, delete volume .

api/src/com/cloud/api/commands/UploadVolumeCmd.java

@@ -15,22 +15,22 @@ package com.cloud.api.commands;
 import org.apache.log4j.Logger;
 
 import com.cloud.api.ApiConstants;
-import com.cloud.api.BaseAsyncCreateCmd;
+import com.cloud.api.BaseCmd;
 import com.cloud.api.IdentityMapper;
 import com.cloud.api.Implementation;
 import com.cloud.api.Parameter;
 import com.cloud.api.ServerApiException;
 import com.cloud.api.response.VolumeResponse;
-import com.cloud.event.EventTypes;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.NetworkRuleConflictException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.storage.Volume;
 import com.cloud.user.UserContext;
 
 @Implementation(description="Uploads a data disk.", responseObject=VolumeResponse.class)
-public class UploadVolumeCmd extends BaseAsyncCreateCmd {
+public class UploadVolumeCmd extends BaseCmd {
 	public static final Logger s_logger = Logger.getLogger(UploadVolumeCmd.class.getName());
     private static final String s_name = "uploadvolumeresponse";
 	
@@ -38,10 +38,7 @@ public class UploadVolumeCmd extends BaseAsyncCreateCmd {
     //////////////// API parameters /////////////////////
     /////////////////////////////////////////////////////
 
-    @Parameter(name=ApiConstants.DISPLAY_TEXT, type=CommandType.STRING, required=true, description="the display text of the volume. This is usually used for display purposes.", length=4096)
+    @Parameter(name=ApiConstants.FORMAT, type=CommandType.STRING, required=true, description="the format for the volume. Possible values include QCOW2, OVA, and VHD.")
-    private String displayText;
-
-    @Parameter(name=ApiConstants.FORMAT, type=CommandType.STRING, required=true, description="the format for the volume. Possible values include QCOW2, RAW, and VHD.")
     private String format;
 
     @Parameter(name=ApiConstants.HYPERVISOR, type=CommandType.STRING, required=true, description="the target hypervisor for the volume")
@@ -71,10 +68,6 @@ public class UploadVolumeCmd extends BaseAsyncCreateCmd {
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
 
-    public String getDisplayText() {
-        return displayText;
-    }
-
     public String getFormat() {
         return format;
     }
@@ -110,33 +103,21 @@ public class UploadVolumeCmd extends BaseAsyncCreateCmd {
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
-    @Override
-	public void create() throws ResourceAllocationException {
-
-	}
-
-	@Override
-	public String getEntityTable() {
-		return "volumes";
-	}
-
-	@Override
-	public String getEventDescription() {
-		return  "creating volume: " + getVolumeName();
-	}
-
-	@Override
-	public String getEventType() {
-		return EventTypes.EVENT_VOLUME_CREATE;		
-	}
 
 	@Override
 	public void execute() throws ResourceUnavailableException,
 			InsufficientCapacityException, ServerApiException,
 			ConcurrentOperationException, ResourceAllocationException,
 			NetworkRuleConflictException {
-		// TODO Auto-generated method stub
 
+	        Volume volume = _storageService.uploadVolume(this);
+	        if (volume != null){	            
+	            VolumeResponse response = _responseGenerator.createVolumeResponse(volume);
+	            response.setResponseName(getCommandName());              
+	            this.setResponseObject(response);
+	        } else {
+	            throw new ServerApiException(BaseCmd.INTERNAL_ERROR, "Failed to upload volume");
+	        }
 	}
 
 	@Override

api/src/com/cloud/storage/Volume.java

@@ -62,7 +62,9 @@ public interface Volume extends ControlledEntity, BasedOn, StateObject<Volume.St
             s_fsm.addTransition(Allocated, Event.UploadRequested, Uploading);
             s_fsm.addTransition(Uploading, Event.UploadSucceeded, Uploaded);
             s_fsm.addTransition(Uploading, Event.OperationFailed, UploadError);
+            s_fsm.addTransition(Uploaded, Event.UploadSucceeded, Uploaded);
             s_fsm.addTransition(Uploaded, Event.CopyRequested, Creating);
+            s_fsm.addTransition(Uploaded, Event.DestroyRequested, Destroy);
             s_fsm.addTransition(Creating, Event.CopySucceeded, Ready);
             s_fsm.addTransition(Creating, Event.CopyFailed, Uploaded);
             s_fsm.addTransition(Ready, Event.DestroyRequested, Destroy);

core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java

@@ -72,6 +72,7 @@ import com.cloud.agent.api.storage.ListVolumeCommand;
 import com.cloud.agent.api.storage.UploadCommand;
 import com.cloud.agent.api.storage.ssCommand;
 import com.cloud.agent.api.to.SwiftTO;
+import com.cloud.api.commands.DeleteVolumeCmd;
 import com.cloud.exception.InternalErrorException;
 import com.cloud.host.Host;
 import com.cloud.host.Host.Type;
@@ -144,6 +145,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
             return new CheckHealthAnswer((CheckHealthCommand)cmd, true);
         } else if (cmd instanceof DeleteTemplateCommand) {
         	return execute((DeleteTemplateCommand) cmd);
+        } else if (cmd instanceof DeleteVolumeCommand) {
+        	return execute((DeleteVolumeCommand) cmd);
         }else if (cmd instanceof ReadyCommand) {
             return new ReadyAnswer((ReadyCommand)cmd);
         } else if (cmd instanceof SecStorageFirewallCfgCommand){

server/src/com/cloud/configuration/Config.java

@@ -65,7 +65,7 @@ public enum Config {
 	StorageOverprovisioningFactor("Storage", StoragePoolAllocator.class, String.class, "storage.overprovisioning.factor", "2", "Used for storage overprovisioning calculation; available storage will be (actualStorageSize * storage.overprovisioning.factor)", null),
 	StorageStatsInterval("Storage", ManagementServer.class, String.class, "storage.stats.interval", "60000", "The interval (in milliseconds) when storage stats (per host) are retrieved from agents.", null),
 	MaxVolumeSize("Storage", ManagementServer.class, Integer.class, "storage.max.volume.size", "2000", "The maximum size for a volume (in GB).", null),
-	MaxUploadVolumeSize("Storage",  ManagementServer.class, Integer.class, "storage.max.volume.upload.size", "50", "The maximum size for a uploaded volume(in GB).", null),
+	MaxUploadVolumeSize("Storage",  ManagementServer.class, Integer.class, "storage.max.volume.upload.size", "500", "The maximum size for a uploaded volume(in GB).", null),
 	TotalRetries("Storage", AgentManager.class, Integer.class, "total.retries", "4", "The number of times each command sent to a host should be retried in case of failure.", null),
 	StoragePoolMaxWaitSeconds("Storage", ManagementServer.class, Integer.class, "storage.pool.max.waitseconds", "3600", "Timeout (in seconds) to synchronize storage pool operations.", null),
 	StorageTemplateCleanupEnabled("Storage", ManagementServer.class, Boolean.class, "storage.template.cleanup.enabled", "true", "Enable/disable template cleanup activity, only take effect when overall storage cleanup is enabled", null),

server/src/com/cloud/storage/StorageManagerImpl.java

@@ -60,6 +60,7 @@ import com.cloud.agent.api.storage.CopyVolumeCommand;
 import com.cloud.agent.api.storage.CreateAnswer;
 import com.cloud.agent.api.storage.CreateCommand;
 import com.cloud.agent.api.storage.DeleteTemplateCommand;
+import com.cloud.agent.api.storage.DeleteVolumeCommand;
 import com.cloud.agent.api.storage.DestroyCommand;
 import com.cloud.agent.api.to.StorageFilerTO;
 import com.cloud.agent.api.to.VolumeTO;
@@ -751,6 +752,8 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
         volume.setPoolId(destPool.getId());        
         volume.setPodId(destPool.getPodId());
         stateTransitTo(volume, Event.CopySucceeded); 
+        UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), volume.getDiskOfferingId(), null, volume.getSize());
+        _usageEventDao.persist(usageEvent);
         _volumeHostDao.remove(volumeHostVO.getId());
     	txn.commit();
 		return volume;
@@ -1691,7 +1694,7 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
      */
     @Override
     @DB
-    @ActionEvent(eventType = EventTypes.EVENT_VOLUME_CREATE, eventDescription = "creating volume", create = true)
+    @ActionEvent(eventType = EventTypes.EVENT_VOLUME_UPLOAD, eventDescription = "creating volume", create = true)
     public VolumeVO uploadVolume(UploadVolumeCmd cmd) throws ResourceAllocationException{
     	Account caller = UserContext.current().getCaller();
         long ownerId = cmd.getEntityOwnerId();
@@ -1806,8 +1809,6 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
         volume.setDomainId((caller == null) ? Domain.ROOT_DOMAIN : caller.getDomainId());
 
         volume = _volsDao.persist(volume);
-        UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), diskOfferingId, null, 0l);
-        _usageEventDao.persist(usageEvent);
         try {
 			stateTransitTo(volume, Event.UploadRequested);
 		} catch (NoTransitionException e) {
@@ -2354,6 +2355,38 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
                 }
             }
             
+            //CleanUp volumes on Secondary Storage.
+            for (HostVO secondaryStorageHost : secondaryStorageHosts) {
+                try {
+                    long hostId = secondaryStorageHost.getId();
+                    List<VolumeHostVO> destroyedVolumeHostVOs = _volumeHostDao.listDestroyed(hostId);
+                    s_logger.debug("Secondary storage garbage collector found " + destroyedVolumeHostVOs.size() + " templates to cleanup on secondary storage host: "
+                            + secondaryStorageHost.getName());
+                    for (VolumeHostVO destroyedVolumeHostVO : destroyedVolumeHostVOs) {
+                        if (s_logger.isDebugEnabled()) {
+                            s_logger.debug("Deleting volume host: " + destroyedVolumeHostVO);
+                        }
+
+                        String installPath = destroyedVolumeHostVO.getInstallPath();
+
+                        if (installPath != null) {
+                            Answer answer = _agentMgr.sendToSecStorage(secondaryStorageHost, new DeleteVolumeCommand(secondaryStorageHost.getStorageUrl(), destroyedVolumeHostVO.getInstallPath()));
+
+                            if (answer == null || !answer.getResult()) {
+                                s_logger.debug("Failed to delete " + destroyedVolumeHostVO + " due to " + ((answer == null) ? "answer is null" : answer.getDetails()));
+                            } else {
+                                _volumeHostDao.remove(destroyedVolumeHostVO.getId());
+                                s_logger.debug("Deleted volume at: " + destroyedVolumeHostVO.getInstallPath());
+                            }
+                        } else {
+                            _volumeHostDao.remove(destroyedVolumeHostVO.getId());
+                        }
+                    }
+                
+                }catch (Exception e2) {
+                    s_logger.warn("problem cleaning up volumes in secondary storage " + secondaryStorageHost, e2);
+                }
+            }
         } catch (Exception e3) {
             s_logger.warn("problem cleaning up secondary storage ", e3);
         }
@@ -3330,6 +3363,22 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
         if (s_logger.isDebugEnabled()) {
             s_logger.debug("Expunging " + vol);
         }
+        
+        //Find out if the volume is present on secondary storage
+        VolumeHostVO volumeHost = _volumeHostDao.findByVolumeId(vol.getId());
+        if(volumeHost != null){
+        	HostVO ssHost = _hostDao.findById(volumeHost.getHostId());
+        	DeleteVolumeCommand dtCommand = new DeleteVolumeCommand(ssHost.getStorageUrl(), volumeHost.getInstallPath());            
+        	Answer answer = _agentMgr.sendToSecStorage(ssHost, dtCommand);
+             if (answer == null || !answer.getResult()) {
+                 s_logger.debug("Failed to delete " + volumeHost + " due to " + ((answer == null) ? "answer is null" : answer.getDetails()));
+                 return;
+             }
+            _volumeHostDao.remove(volumeHost.getId());
+            _volumeDao.remove(vol.getId());
+            return;             
+        }
+        
         String vmName = null;
         if (vol.getVolumeType() == Type.ROOT && vol.getInstanceId() != null) {
             VirtualMachine vm = _vmInstanceDao.findByIdIncludingRemoved(vol.getInstanceId());

server/src/com/cloud/storage/dao/VolumeHostDao.java

@@ -14,4 +14,6 @@ public interface VolumeHostDao extends GenericDao<VolumeHostVO, Long> {
 
 	List<VolumeHostVO> listBySecStorage(long sserverId);
 
+	List<VolumeHostVO> listDestroyed(long hostId);
+
 }

server/src/com/cloud/storage/dao/VolumeHostDaoImpl.java

@@ -6,6 +6,7 @@ import java.util.List;
 import javax.ejb.Local;
 
 import com.cloud.host.HostVO;
+import com.cloud.storage.VMTemplateHostVO;
 import com.cloud.storage.VolumeHostVO;
 import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.SearchBuilder;
@@ -16,6 +17,7 @@ public class VolumeHostDaoImpl extends GenericDaoBase<VolumeHostVO, Long> implem
 	protected final SearchBuilder<VolumeHostVO> HostVolumeSearch;
 	protected final SearchBuilder<VolumeHostVO> VolumeSearch;
 	protected final SearchBuilder<VolumeHostVO> HostSearch;
+	protected final SearchBuilder<VolumeHostVO> HostDestroyedSearch;
 	
 	VolumeHostDaoImpl(){
 		HostVolumeSearch = createSearchBuilder();
@@ -33,6 +35,11 @@ public class VolumeHostDaoImpl extends GenericDaoBase<VolumeHostVO, Long> implem
 		VolumeSearch.and("volume_id", VolumeSearch.entity().getVolumeId(), SearchCriteria.Op.EQ);
 		VolumeSearch.and("destroyed", VolumeSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
 		VolumeSearch.done();
+		
+		HostDestroyedSearch = createSearchBuilder();
+		HostDestroyedSearch.and("host_id", HostDestroyedSearch.entity().getHostId(), SearchCriteria.Op.EQ);
+		HostDestroyedSearch.and("destroyed", HostDestroyedSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
+		HostDestroyedSearch.done();	
 	}
     
     
@@ -64,4 +71,12 @@ public class VolumeHostDaoImpl extends GenericDaoBase<VolumeHostVO, Long> implem
 	    return listAll();
 	}
 	
+	@Override
+	public List<VolumeHostVO> listDestroyed(long hostId){
+		SearchCriteria<VolumeHostVO> sc = HostDestroyedSearch.create();
+		sc.setParameters("host_id", hostId);
+		sc.setParameters("destroyed", true);
+		return listIncludingRemovedBy(sc);
+	}
+
 }

server/src/com/cloud/storage/download/DownloadMonitorImpl.java

@@ -439,12 +439,7 @@ public class DownloadMonitorImpl implements  DownloadMonitor {
             downloadJobExists = true;
         }
         
-        try {
+
-			_storageMgr.stateTransitTo(volume, Event.UploadRequested);
-		} catch (NoTransitionException e1) {
-			// TODO Auto-generated catch block
-			e1.printStackTrace();
-		}
         Long maxVolumeSizeInBytes = getMaxVolumeSizeInBytes();
         String secUrl = sserver.getStorageUrl();
 		if(volumeHost != null) {
@@ -478,7 +473,7 @@ public class DownloadMonitorImpl implements  DownloadMonitor {
 			try {
 	            send(ssvm.getId(), dcmd, dl);
             } catch (AgentUnavailableException e) {
-				s_logger.warn("Unable to start /resume download of template " + volume.getName() + " to " + sserver.getName(), e);
+				s_logger.warn("Unable to start /resume download of volume " + volume.getName() + " to " + sserver.getName(), e);
 				dl.setDisconnected();
 				dl.scheduleStatusCheck(RequestType.GET_OR_RESTART);
             }
@@ -551,7 +546,7 @@ public class DownloadMonitorImpl implements  DownloadMonitor {
             else{
                 s_logger.warn("Failed to get size for volume" + volume.getName());
             }
-            String eventType = EventTypes.EVENT_VOLUME_CREATE;            
+            String eventType = EventTypes.EVENT_VOLUME_UPLOAD;            
             if(volume.getAccountId() != Account.ACCOUNT_ID_SYSTEM){
                UsageEventVO usageEvent = new UsageEventVO(eventType, volume.getAccountId(), host.getDataCenterId(), volume.getId(), volume.getName(), null, 0l , size);
                _usageEventDao.persist(usageEvent);