Code clean up for Extract functionality and making it more robust.

core/src/com/cloud/storage/UploadVO.java

@@ -82,6 +82,9 @@ public class UploadVO implements Upload {
 	@Column (name="url")
 	private String uploadUrl;
 
+	@Column (name="install_path")
+	private String installPath;
+	   
 	public long getHostId() {
 		return hostId;
 	}
@@ -113,17 +116,15 @@ public class UploadVO implements Upload {
 	}
 
 	public UploadVO(long hostId, long typeId, Date lastUpdated,
-			Status uploadState, int uploadPercent, Type type,
+			Status uploadState, Type type,
-			String errorString, String jobId, String uploadUrl) {
+			String uploadUrl, Mode mode) {
 		super();
 		this.hostId = hostId;
 		this.typeId = typeId;
 		this.lastUpdated = lastUpdated;
 		this.uploadState = uploadState;
-		this.uploadPercent = uploadPercent;
+		this.mode = mode;
 		this.type = type;
-		this.errorString = errorString;
-		this.jobId = jobId;
 		this.uploadUrl = uploadUrl;
 	}
 	
@@ -241,4 +242,12 @@ public class UploadVO implements Upload {
 		this.created = created;
 	}
 
+    public String getInstallPath() {
+        return installPath;
+    }
+
+    public void setInstallPath(String installPath) {
+        this.installPath = installPath;
+    }
+
 }

core/src/com/cloud/storage/template/UploadManagerImpl.java

@@ -358,11 +358,13 @@ public class UploadManagerImpl implements UploadManager {
     @Override
     public DeleteEntityDownloadURLAnswer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd){
 
-        //Delete the soft link
+        //Delete the soft link. Example path = volumes/8/74eeb2c6-8ab1-4357-841f-2e9d06d1f360.vhd
-        s_logger.debug("handleDeleteEntityDownloadURLCommand "+cmd.getPath());
+        s_logger.warn("handleDeleteEntityDownloadURLCommand Path:"+cmd.getPath() + " Type:" +cmd.getType().toString());
+        String path = cmd.getPath();
         Script command = new Script("/bin/bash", s_logger);
         command.add("-c");
-        command.add("unlink /var/www/html/"+cmd.getPath());
+        //We just need to remove the UUID.vhd
+        command.add("unlink /var/www/html/" +path.substring(path.lastIndexOf(File.separator) + 1));
         String result = command.execute();
         if (result != null) {
             String errorString = "Error in deleting =" + result;
@@ -370,11 +372,12 @@ public class UploadManagerImpl implements UploadManager {
             return new DeleteEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
         }
         
-        // If its a volume also delete the Hard link
+        // If its a volume also delete the Hard link since it was created only for the purpose of download.
         if(cmd.getType() == Upload.Type.VOLUME){
             command = new Script("/bin/bash", s_logger);
             command.add("-c");
-            command.add("rm -f " + publicTemplateRepo + cmd.getPath());
+            command.add("rm -f " + parentDir +File.separator+ path);
+            s_logger.warn(" " +parentDir +File.separator+ path);
             result = command.execute();
             if (result != null) {
                 String errorString = "Error in linking  err=" + result;

server/src/com/cloud/server/ManagementServerImpl.java

@@ -61,6 +61,7 @@ import com.cloud.agent.api.storage.CopyVolumeCommand;
 import com.cloud.alert.AlertManager;
 import com.cloud.alert.AlertVO;
 import com.cloud.alert.dao.AlertDao;
+import com.cloud.api.ApiDBUtils;
 import com.cloud.api.BaseCmd;
 import com.cloud.api.ServerApiException;
 import com.cloud.api.commands.AssignPortForwardingServiceCmd;
@@ -6696,7 +6697,12 @@ public class ManagementServerImpl implements ManagementServer {
         if (_dcDao.findById(zoneId) == null) {
             throw new ServerApiException(BaseCmd.PARAM_ERROR, "Please specify a valid zone.");          
         }
-
+        //Extract activity only for detached volumes or for volumes whose instance is stopped
+        if(volume.getInstanceId() != null && ApiDBUtils.findVMInstanceById(volume.getInstanceId()).getState() != State.Stopped ){
+            s_logger.debug("Invalid state of the volume with ID: " + volumeId + ". It should be either detached or the VM should be in stopped state.");
+            throw new PermissionDeniedException("Invalid state of the volume with ID: " + volumeId + ". It should be either detached or the VM should be in stopped state.");
+        }
+        
         Upload.Mode extractMode;
         if( mode == null || (!mode.equals(Upload.Mode.FTP_UPLOAD.toString()) && !mode.equals(Upload.Mode.HTTP_DOWNLOAD.toString())) ){
             throw new ServerApiException(BaseCmd.PARAM_ERROR, "Please specify a valid extract Mode ");
@@ -6704,7 +6710,7 @@ public class ManagementServerImpl implements ManagementServer {
             extractMode = mode.equals(Upload.Mode.FTP_UPLOAD.toString()) ? Upload.Mode.FTP_UPLOAD : Upload.Mode.HTTP_DOWNLOAD;
         }
         
-        if (account != null) {                  
+        if (account != null) {    
             if(!isAdmin(account.getType())){
                 if (volume.getAccountId() != account.getId()){
                     throw new PermissionDeniedException("Unable to find volume with ID: " + volumeId + " for account: " + account.getAccountName());
@@ -6751,17 +6757,19 @@ public class ManagementServerImpl implements ManagementServer {
         List<HostVO> storageServers = _hostDao.listByTypeDataCenter(Host.Type.SecondaryStorage, zoneId);
         HostVO sserver = storageServers.get(0);
 
-        EventUtils.saveStartedEvent(userId, accountId, EventTypes.EVENT_VOLUME_UPLOAD, "Starting extraction of " +volume.getName()+ " mode:"+mode, cmd.getStartEventId());
+        EventUtils.saveStartedEvent(userId, accountId, cmd.getEventType(), "Starting extraction of " +volume.getName()+ " mode:"+mode, cmd.getStartEventId());
         List<UploadVO> extractURLList = _uploadDao.listByTypeUploadStatus(volumeId, Upload.Type.VOLUME, UploadVO.Status.DOWNLOAD_URL_CREATED);
         
-        if (extractMode == Upload.Mode.HTTP_DOWNLOAD && extractURLList.size() > 0){                       
+        if (extractMode == Upload.Mode.HTTP_DOWNLOAD && extractURLList.size() > 0){   
             return extractURLList.get(0).getId(); // If download url already exists then return 
         }else {
-            UploadVO uploadJob = _uploadMonitor.createNewUploadEntry(sserver.getId(), volumeId, UploadVO.Status.COPY_IN_PROGRESS, 0, Type.VOLUME, null, null, url);
+            UploadVO uploadJob = _uploadMonitor.createNewUploadEntry(sserver.getId(), volumeId, UploadVO.Status.COPY_IN_PROGRESS, Type.VOLUME, url, extractMode);
+            s_logger.debug("Extract Mode - " +uploadJob.getMode());
             uploadJob = _uploadDao.createForUpdate(uploadJob.getId());
             
             // Update the async Job
             ExtractResponse resultObj = new ExtractResponse(volumeId, volume.getName(), accountId, UploadVO.Status.COPY_IN_PROGRESS.toString(), uploadJob.getId());
+            resultObj.setResponseName(cmd.getName());
             _asyncMgr.updateAsyncJobAttachment(job.getId(), Type.VOLUME.toString(), volumeId);
             _asyncMgr.updateAsyncJobStatus(job.getId(), AsyncJobResult.STATUS_IN_PROGRESS, resultObj);
     
@@ -6789,16 +6797,16 @@ public class ManagementServerImpl implements ManagementServer {
             }
             
             String volumeLocalPath = "volumes/"+volume.getId()+"/"+cvAnswer.getVolumePath()+".vhd";
-            //Update the DB that volume is copied
+            //Update the DB that volume is copied and volumePath
             uploadJob.setUploadState(UploadVO.Status.COPY_COMPLETE);
             uploadJob.setLastUpdated(new Date());
+            uploadJob.setInstallPath(volumeLocalPath);
             _uploadDao.update(uploadJob.getId(), uploadJob);
             
             if (extractMode == Mode.FTP_UPLOAD){ // Now that the volume is copied perform the actual uploading
                 _uploadMonitor.extractVolume(uploadJob, sserver, volume, url, zoneId, volumeLocalPath, cmd.getStartEventId(), job.getId(), _asyncMgr);
                 return uploadJob.getId();
             }else{ // Volume is copied now make it visible under apache and create a URL.
-                s_logger.debug("volumepath " +volumeLocalPath);
                 _uploadMonitor.createVolumeDownloadURL(volumeId, volumeLocalPath, Type.VOLUME, zoneId, uploadJob.getId());                
                 EventUtils.saveEvent(userId, accountId, EventVO.LEVEL_INFO, cmd.getEventType(), "Completed extraction of "+volume.getName()+ " in mode:" +mode, null, cmd.getStartEventId());
                 return uploadJob.getId();

server/src/com/cloud/storage/upload/UploadMonitor.java

@@ -19,7 +19,9 @@
 package com.cloud.storage.upload;
 
 import com.cloud.async.AsyncJobManager;
+import com.cloud.exception.InternalErrorException;
 import com.cloud.host.HostVO;
+import com.cloud.storage.Upload.Mode;
 import com.cloud.storage.Upload.Status;
 import com.cloud.storage.Upload.Type;
 import com.cloud.storage.UploadVO;
@@ -46,8 +48,7 @@ public interface UploadMonitor extends Manager{
     void handleUploadSync(long sserverId);
 
     UploadVO createNewUploadEntry(Long hostId, Long typeId, Status uploadState,
-            int uploadPercent, Type type, String errorString, String jobId,
+            Type type, String errorString, Mode extractMode);
-            String uploadUrl);
 
     void extractVolume(UploadVO uploadVolumeObj, HostVO sserver, VolumeVO volume, String url,
             Long dataCenterId, String installPath, long eventId,
@@ -57,6 +58,6 @@ public interface UploadMonitor extends Manager{
             VMTemplateHostVO vmTemplateHost, Long dataCenterId, long eventId);
 
     void createVolumeDownloadURL(Long entityId, String path, Type type,
-            Long dataCenterId, Long uploadId) throws CloudRuntimeException;
+            Long dataCenterId, Long uploadId) throws InternalErrorException;
 
 }

server/src/com/cloud/storage/upload/UploadMonitorImpl.java

@@ -30,6 +30,7 @@ import com.cloud.configuration.dao.ConfigurationDao;
 import com.cloud.event.EventTypes;
 import com.cloud.event.EventVO;
 import com.cloud.event.dao.EventDao;
+import com.cloud.exception.InternalErrorException;
 import com.cloud.host.Host;
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
@@ -53,6 +54,7 @@ import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.SecondaryStorageVmVO;
 import com.cloud.vm.State;
 import com.cloud.vm.dao.SecondaryStorageVmDao;
+import com.sun.corba.se.impl.logging.InterceptorsSystemException;
 
 /**
  * @author nitin
@@ -115,11 +117,10 @@ public class UploadMonitorImpl implements UploadMonitor {
 	
 	@Override
 	public UploadVO createNewUploadEntry(Long hostId, Long typeId, UploadVO.Status  uploadState,
-	                                        int uploadPercent, Type  type, 
+	                                        Type  type, String uploadUrl, Upload.Mode mode){
-	                                        String  errorString, String  jobId, String  uploadUrl){
 	       
         UploadVO uploadObj = new UploadVO(hostId, typeId, new Date(), 
-                            uploadState, 0, type, null, null, uploadUrl);
+                                          uploadState, type, uploadUrl, mode);
         _uploadDao.persist(uploadObj);
         
         return uploadObj;
@@ -156,8 +157,7 @@ public class UploadMonitorImpl implements UploadMonitor {
 		HostVO sserver = storageServers.get(0);			
 		
 		UploadVO uploadTemplateObj = new UploadVO(sserver.getId(), template.getId(), new Date(), 
-													Upload.Status.NOT_UPLOADED, 0, type, 
+													Upload.Status.NOT_UPLOADED, type, url, Mode.FTP_UPLOAD);
-													null, "jobid0000", url);
 		_uploadDao.persist(uploadTemplateObj);        		               
         		
 		if(vmTemplateHost != null) {
@@ -197,7 +197,7 @@ public class UploadMonitorImpl implements UploadMonitor {
 	    HostVO sserver = storageServers.get(0);
 	    UploadVO uploadTemplateObj = new UploadVO(sserver.getId(), template.getId(), new Date(), 
 	                                                Status.DOWNLOAD_URL_NOT_CREATED, 0, type, Mode.HTTP_DOWNLOAD); 
-	                                                
+	    uploadTemplateObj.setInstallPath(vmTemplateHost.getInstallPath());	                                                
 	    _uploadDao.persist(uploadTemplateObj);
 	    
 	    // Create Symlink at ssvm
@@ -234,50 +234,61 @@ public class UploadMonitorImpl implements UploadMonitor {
 	}
 	
 	@Override
-    public void createVolumeDownloadURL(Long entityId, String path, Type type, Long dataCenterId, Long uploadId) throws CloudRuntimeException{
+    public void createVolumeDownloadURL(Long entityId, String path, Type type, Long dataCenterId, Long uploadId) throws InternalErrorException{
         
-        List<HostVO> storageServers = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorage, dataCenterId);
+	    String errorString = "";
-        if(storageServers == null ) 
+	    boolean success = false;
-            throw new CloudRuntimeException("No Storage Server found at the datacenter - " +dataCenterId);        
+	    try{
-        
+            List<HostVO> storageServers = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorage, dataCenterId);
-        // Update DB for state = DOWNLOAD_URL_NOT_CREATED.        
+            if(storageServers == null ){
-        UploadVO uploadJob = _uploadDao.createForUpdate(uploadId);
+                errorString = "No Storage Server found at the datacenter - " +dataCenterId;
-        uploadJob.setUploadState(Status.DOWNLOAD_URL_NOT_CREATED);
+                throw new CloudRuntimeException(errorString);   
-        uploadJob.setLastUpdated(new Date());
+            }                    
-        _uploadDao.update(uploadJob.getId(), uploadJob);
-        
-        // Create Symlink at ssvm
-        CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(path);
-        long result = send(ApiDBUtils.findUploadById(uploadId).getHostId(), cmd, null);
-        if (result == -1){
-            String errorString = "Unable to create a link for " +type+ " id:"+entityId;
-            s_logger.warn(errorString);
-            throw new CloudRuntimeException(errorString);
-        }
-        
-        //Construct actual URL locally now that the symlink exists at SSVM
-        List<SecondaryStorageVmVO> ssVms = _secStorageVmDao.getSecStorageVmListInStates(dataCenterId, State.Running);
-        if (ssVms.size() > 0) {
-            SecondaryStorageVmVO ssVm = ssVms.get(0);
-            if (ssVm.getPublicIpAddress() == null) {
-                s_logger.warn("A running secondary storage vm has a null public ip?");
-                throw new CloudRuntimeException("SSVM has null public IP - couldnt create the URL");
-            }
-            String extractURL = generateCopyUrl(ssVm.getPublicIpAddress(), path);
-            UploadVO vo = _uploadDao.createForUpdate();
-            vo.setLastUpdated(new Date());
-            vo.setUploadUrl(extractURL);
-            vo.setUploadState(Status.DOWNLOAD_URL_CREATED);
             
-            if(extractURL == null){
+            // Update DB for state = DOWNLOAD_URL_NOT_CREATED.        
-                vo.setUploadState(Status.ERROR);
+            UploadVO uploadJob = _uploadDao.createForUpdate(uploadId);
-                vo.setErrorString("Could not create the download URL");
+            uploadJob.setUploadState(Status.DOWNLOAD_URL_NOT_CREATED);
+            uploadJob.setLastUpdated(new Date());
+            _uploadDao.update(uploadJob.getId(), uploadJob);
+            
+            // Create Symlink at ssvm
+            CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(path);
+            long result = send(ApiDBUtils.findUploadById(uploadId).getHostId(), cmd, null);
+            if (result == -1){
+                errorString = "Unable to create a link for " +type+ " id:"+entityId;
+                s_logger.warn(errorString);
+                throw new InternalErrorException(errorString);
             }
-            _uploadDao.update(uploadId, vo);
+            
-            return;
+            //Construct actual URL locally now that the symlink exists at SSVM
-        }
+            List<SecondaryStorageVmVO> ssVms = _secStorageVmDao.getSecStorageVmListInStates(dataCenterId, State.Running);
-        throw new CloudRuntimeException("Couldnt find a running SSVM in the zone" + dataCenterId+ ". Couldnt create the extraction URL.");
+            if (ssVms.size() > 0) {
-        
+                SecondaryStorageVmVO ssVm = ssVms.get(0);
+                if (ssVm.getPublicIpAddress() == null) {
+                    errorString = "A running secondary storage vm has a null public ip?";
+                    s_logger.warn(errorString);
+                    throw new InternalErrorException(errorString);
+                }
+                String extractURL = generateCopyUrl(ssVm.getPublicIpAddress(), path);
+                UploadVO vo = _uploadDao.createForUpdate();
+                vo.setLastUpdated(new Date());
+                vo.setUploadUrl(extractURL);
+                vo.setUploadState(Status.DOWNLOAD_URL_CREATED);
+                _uploadDao.update(uploadId, vo);
+                success = true;
+                return;
+            }
+            errorString = "Couldnt find a running SSVM in the zone" + dataCenterId+ ". Couldnt create the extraction URL.";
+            throw new InternalErrorException(errorString);
+	    }finally{
+	        if(!success){
+	            UploadVO uploadJob = _uploadDao.createForUpdate(uploadId);
+	            uploadJob.setLastUpdated(new Date());
+	            uploadJob.setErrorString(errorString);
+	            uploadJob.setUploadState(Status.ERROR);
+	            _uploadDao.update(uploadId, uploadJob);
+	        }
+	    }
     }
 	
 	   private String generateCopyUrl(String ipAddress, String path){
@@ -328,7 +339,7 @@ public class UploadMonitorImpl implements UploadMonitor {
 
 	@Override
 	public boolean start() {
-	    //FIX ME - Make the timings configurable.
+	    //FIX ME - Make the timings configurable. // Keep them to 86400 for now.
 	    _executor.scheduleWithFixedDelay(new StorageGarbageCollector(), 86400, 86400, TimeUnit.SECONDS);
 		_timer = new Timer();
 		return true;
@@ -437,27 +448,30 @@ public class UploadMonitorImpl implements UploadMonitor {
     
     
     private long getTimeDiff(Date date){
-        Calendar currentCalendar = Calendar.getInstance();
+        Calendar currentDateCalendar = Calendar.getInstance();
         Calendar givenDateCalendar = Calendar.getInstance();
         givenDateCalendar.setTime(date);
         
-        return (currentCalendar.getTimeInMillis() - givenDateCalendar.getTimeInMillis() )/1000;  
+        return (currentDateCalendar.getTimeInMillis() - givenDateCalendar.getTimeInMillis() )/1000;  
     }
     
     public void cleanupStorage() {
 
-        final int EXTRACT_URL_TIME_LIMIT = 40;
+        final int EXTRACT_URL_LIFE_LIMIT_IN_SECONDS = 86400;//FIX ME make it configurable.
-        List<UploadVO> extractURLs= _uploadDao.listByModeAndStatus(Mode.HTTP_DOWNLOAD, Status.DOWNLOAD_URL_CREATED);
+        List<UploadVO> extractJobs= _uploadDao.listByModeAndStatus(Mode.HTTP_DOWNLOAD, Status.DOWNLOAD_URL_CREATED);
         
-        for (UploadVO extractURL : extractURLs){
+        for (UploadVO extractJob : extractJobs){
-            if( getTimeDiff(extractURL.getLastUpdated()) < EXTRACT_URL_TIME_LIMIT) continue;
+            if( getTimeDiff(extractJob.getLastUpdated()) > EXTRACT_URL_LIFE_LIMIT_IN_SECONDS ){                           
-            String path = extractURL.getUploadUrl().substring( (extractURL.getUploadUrl().lastIndexOf("/")) +1 );
+                String path = extractJob.getInstallPath();
-            DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(path, extractURL.getType());
+                s_logger.debug("Sending deletion of extract URL "+extractJob.getUploadUrl());
-            long result = send(extractURL.getHostId(), cmd, null);
+                // Would delete the symlink for the Type and if Type == VOLUME then also the volume
-            if (result == -1){
+                DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(path, extractJob.getType());            
-                s_logger.warn("Unable to delete the link for " +extractURL.getType()+ " id=" +extractURL.getTypeId()+ " url="+extractURL.getUploadUrl());
+                long result = send(extractJob.getHostId(), cmd, null);
-            }else{
+                if (result == -1){
-                _uploadDao.remove(extractURL.getId());
+                    s_logger.warn("Unable to delete the link for " +extractJob.getType()+ " id=" +extractJob.getTypeId()+ " url="+extractJob.getUploadUrl());
+                }else{
+                    _uploadDao.remove(extractJob.getId());
+                }
             }
         }
                 

setup/db/create-schema.sql

@@ -648,6 +648,7 @@ CREATE TABLE  `cloud`.`upload` (
   `upload_state` varchar(255),
   `error_str` varchar(255),
   `url` varchar(255),
+  `install_path` varchar(255),
   PRIMARY KEY  (`id`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;