diff --git a/core/src/com/cloud/storage/UploadVO.java b/core/src/com/cloud/storage/UploadVO.java index 80164708cc7..92db23d0192 100755 --- a/core/src/com/cloud/storage/UploadVO.java +++ b/core/src/com/cloud/storage/UploadVO.java @@ -82,6 +82,9 @@ public class UploadVO implements Upload { @Column (name="url") private String uploadUrl; + @Column (name="install_path") + private String installPath; + public long getHostId() { return hostId; } @@ -113,17 +116,15 @@ public class UploadVO implements Upload { } public UploadVO(long hostId, long typeId, Date lastUpdated, - Status uploadState, int uploadPercent, Type type, - String errorString, String jobId, String uploadUrl) { + Status uploadState, Type type, + String uploadUrl, Mode mode) { super(); this.hostId = hostId; this.typeId = typeId; this.lastUpdated = lastUpdated; this.uploadState = uploadState; - this.uploadPercent = uploadPercent; + this.mode = mode; this.type = type; - this.errorString = errorString; - this.jobId = jobId; this.uploadUrl = uploadUrl; } @@ -241,4 +242,12 @@ public class UploadVO implements Upload { this.created = created; } + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } + } diff --git a/core/src/com/cloud/storage/template/UploadManagerImpl.java b/core/src/com/cloud/storage/template/UploadManagerImpl.java index ad3ea86ecb1..c6609e70da1 100755 --- a/core/src/com/cloud/storage/template/UploadManagerImpl.java +++ b/core/src/com/cloud/storage/template/UploadManagerImpl.java @@ -358,11 +358,13 @@ public class UploadManagerImpl implements UploadManager { @Override public DeleteEntityDownloadURLAnswer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd){ - //Delete the soft link - s_logger.debug("handleDeleteEntityDownloadURLCommand "+cmd.getPath()); + //Delete the soft link. Example path = volumes/8/74eeb2c6-8ab1-4357-841f-2e9d06d1f360.vhd + s_logger.warn("handleDeleteEntityDownloadURLCommand Path:"+cmd.getPath() + " Type:" +cmd.getType().toString()); + String path = cmd.getPath(); Script command = new Script("/bin/bash", s_logger); command.add("-c"); - command.add("unlink /var/www/html/"+cmd.getPath()); + //We just need to remove the UUID.vhd + command.add("unlink /var/www/html/" +path.substring(path.lastIndexOf(File.separator) + 1)); String result = command.execute(); if (result != null) { String errorString = "Error in deleting =" + result; @@ -370,11 +372,12 @@ public class UploadManagerImpl implements UploadManager { return new DeleteEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE); } - // If its a volume also delete the Hard link + // If its a volume also delete the Hard link since it was created only for the purpose of download. if(cmd.getType() == Upload.Type.VOLUME){ command = new Script("/bin/bash", s_logger); command.add("-c"); - command.add("rm -f " + publicTemplateRepo + cmd.getPath()); + command.add("rm -f " + parentDir +File.separator+ path); + s_logger.warn(" " +parentDir +File.separator+ path); result = command.execute(); if (result != null) { String errorString = "Error in linking err=" + result; diff --git a/server/src/com/cloud/server/ManagementServerImpl.java b/server/src/com/cloud/server/ManagementServerImpl.java index b346e2fa274..b4ae499ddab 100755 --- a/server/src/com/cloud/server/ManagementServerImpl.java +++ b/server/src/com/cloud/server/ManagementServerImpl.java @@ -61,6 +61,7 @@ import com.cloud.agent.api.storage.CopyVolumeCommand; import com.cloud.alert.AlertManager; import com.cloud.alert.AlertVO; import com.cloud.alert.dao.AlertDao; +import com.cloud.api.ApiDBUtils; import com.cloud.api.BaseCmd; import com.cloud.api.ServerApiException; import com.cloud.api.commands.AssignPortForwardingServiceCmd; @@ -6696,7 +6697,12 @@ public class ManagementServerImpl implements ManagementServer { if (_dcDao.findById(zoneId) == null) { throw new ServerApiException(BaseCmd.PARAM_ERROR, "Please specify a valid zone."); } - + //Extract activity only for detached volumes or for volumes whose instance is stopped + if(volume.getInstanceId() != null && ApiDBUtils.findVMInstanceById(volume.getInstanceId()).getState() != State.Stopped ){ + s_logger.debug("Invalid state of the volume with ID: " + volumeId + ". It should be either detached or the VM should be in stopped state."); + throw new PermissionDeniedException("Invalid state of the volume with ID: " + volumeId + ". It should be either detached or the VM should be in stopped state."); + } + Upload.Mode extractMode; if( mode == null || (!mode.equals(Upload.Mode.FTP_UPLOAD.toString()) && !mode.equals(Upload.Mode.HTTP_DOWNLOAD.toString())) ){ throw new ServerApiException(BaseCmd.PARAM_ERROR, "Please specify a valid extract Mode "); @@ -6704,7 +6710,7 @@ public class ManagementServerImpl implements ManagementServer { extractMode = mode.equals(Upload.Mode.FTP_UPLOAD.toString()) ? Upload.Mode.FTP_UPLOAD : Upload.Mode.HTTP_DOWNLOAD; } - if (account != null) { + if (account != null) { if(!isAdmin(account.getType())){ if (volume.getAccountId() != account.getId()){ throw new PermissionDeniedException("Unable to find volume with ID: " + volumeId + " for account: " + account.getAccountName()); @@ -6751,17 +6757,19 @@ public class ManagementServerImpl implements ManagementServer { List storageServers = _hostDao.listByTypeDataCenter(Host.Type.SecondaryStorage, zoneId); HostVO sserver = storageServers.get(0); - EventUtils.saveStartedEvent(userId, accountId, EventTypes.EVENT_VOLUME_UPLOAD, "Starting extraction of " +volume.getName()+ " mode:"+mode, cmd.getStartEventId()); + EventUtils.saveStartedEvent(userId, accountId, cmd.getEventType(), "Starting extraction of " +volume.getName()+ " mode:"+mode, cmd.getStartEventId()); List extractURLList = _uploadDao.listByTypeUploadStatus(volumeId, Upload.Type.VOLUME, UploadVO.Status.DOWNLOAD_URL_CREATED); - if (extractMode == Upload.Mode.HTTP_DOWNLOAD && extractURLList.size() > 0){ + if (extractMode == Upload.Mode.HTTP_DOWNLOAD && extractURLList.size() > 0){ return extractURLList.get(0).getId(); // If download url already exists then return }else { - UploadVO uploadJob = _uploadMonitor.createNewUploadEntry(sserver.getId(), volumeId, UploadVO.Status.COPY_IN_PROGRESS, 0, Type.VOLUME, null, null, url); + UploadVO uploadJob = _uploadMonitor.createNewUploadEntry(sserver.getId(), volumeId, UploadVO.Status.COPY_IN_PROGRESS, Type.VOLUME, url, extractMode); + s_logger.debug("Extract Mode - " +uploadJob.getMode()); uploadJob = _uploadDao.createForUpdate(uploadJob.getId()); // Update the async Job ExtractResponse resultObj = new ExtractResponse(volumeId, volume.getName(), accountId, UploadVO.Status.COPY_IN_PROGRESS.toString(), uploadJob.getId()); + resultObj.setResponseName(cmd.getName()); _asyncMgr.updateAsyncJobAttachment(job.getId(), Type.VOLUME.toString(), volumeId); _asyncMgr.updateAsyncJobStatus(job.getId(), AsyncJobResult.STATUS_IN_PROGRESS, resultObj); @@ -6789,16 +6797,16 @@ public class ManagementServerImpl implements ManagementServer { } String volumeLocalPath = "volumes/"+volume.getId()+"/"+cvAnswer.getVolumePath()+".vhd"; - //Update the DB that volume is copied + //Update the DB that volume is copied and volumePath uploadJob.setUploadState(UploadVO.Status.COPY_COMPLETE); uploadJob.setLastUpdated(new Date()); + uploadJob.setInstallPath(volumeLocalPath); _uploadDao.update(uploadJob.getId(), uploadJob); if (extractMode == Mode.FTP_UPLOAD){ // Now that the volume is copied perform the actual uploading _uploadMonitor.extractVolume(uploadJob, sserver, volume, url, zoneId, volumeLocalPath, cmd.getStartEventId(), job.getId(), _asyncMgr); return uploadJob.getId(); }else{ // Volume is copied now make it visible under apache and create a URL. - s_logger.debug("volumepath " +volumeLocalPath); _uploadMonitor.createVolumeDownloadURL(volumeId, volumeLocalPath, Type.VOLUME, zoneId, uploadJob.getId()); EventUtils.saveEvent(userId, accountId, EventVO.LEVEL_INFO, cmd.getEventType(), "Completed extraction of "+volume.getName()+ " in mode:" +mode, null, cmd.getStartEventId()); return uploadJob.getId(); diff --git a/server/src/com/cloud/storage/upload/UploadMonitor.java b/server/src/com/cloud/storage/upload/UploadMonitor.java index df0f68b7920..c5100f4dd4e 100755 --- a/server/src/com/cloud/storage/upload/UploadMonitor.java +++ b/server/src/com/cloud/storage/upload/UploadMonitor.java @@ -19,7 +19,9 @@ package com.cloud.storage.upload; import com.cloud.async.AsyncJobManager; +import com.cloud.exception.InternalErrorException; import com.cloud.host.HostVO; +import com.cloud.storage.Upload.Mode; import com.cloud.storage.Upload.Status; import com.cloud.storage.Upload.Type; import com.cloud.storage.UploadVO; @@ -46,8 +48,7 @@ public interface UploadMonitor extends Manager{ void handleUploadSync(long sserverId); UploadVO createNewUploadEntry(Long hostId, Long typeId, Status uploadState, - int uploadPercent, Type type, String errorString, String jobId, - String uploadUrl); + Type type, String errorString, Mode extractMode); void extractVolume(UploadVO uploadVolumeObj, HostVO sserver, VolumeVO volume, String url, Long dataCenterId, String installPath, long eventId, @@ -57,6 +58,6 @@ public interface UploadMonitor extends Manager{ VMTemplateHostVO vmTemplateHost, Long dataCenterId, long eventId); void createVolumeDownloadURL(Long entityId, String path, Type type, - Long dataCenterId, Long uploadId) throws CloudRuntimeException; + Long dataCenterId, Long uploadId) throws InternalErrorException; } \ No newline at end of file diff --git a/server/src/com/cloud/storage/upload/UploadMonitorImpl.java b/server/src/com/cloud/storage/upload/UploadMonitorImpl.java index 6b718d560dc..b11ef7416bb 100755 --- a/server/src/com/cloud/storage/upload/UploadMonitorImpl.java +++ b/server/src/com/cloud/storage/upload/UploadMonitorImpl.java @@ -30,6 +30,7 @@ import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.event.EventTypes; import com.cloud.event.EventVO; import com.cloud.event.dao.EventDao; +import com.cloud.exception.InternalErrorException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; @@ -53,6 +54,7 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.SecondaryStorageVmVO; import com.cloud.vm.State; import com.cloud.vm.dao.SecondaryStorageVmDao; +import com.sun.corba.se.impl.logging.InterceptorsSystemException; /** * @author nitin @@ -115,11 +117,10 @@ public class UploadMonitorImpl implements UploadMonitor { @Override public UploadVO createNewUploadEntry(Long hostId, Long typeId, UploadVO.Status uploadState, - int uploadPercent, Type type, - String errorString, String jobId, String uploadUrl){ + Type type, String uploadUrl, Upload.Mode mode){ UploadVO uploadObj = new UploadVO(hostId, typeId, new Date(), - uploadState, 0, type, null, null, uploadUrl); + uploadState, type, uploadUrl, mode); _uploadDao.persist(uploadObj); return uploadObj; @@ -156,8 +157,7 @@ public class UploadMonitorImpl implements UploadMonitor { HostVO sserver = storageServers.get(0); UploadVO uploadTemplateObj = new UploadVO(sserver.getId(), template.getId(), new Date(), - Upload.Status.NOT_UPLOADED, 0, type, - null, "jobid0000", url); + Upload.Status.NOT_UPLOADED, type, url, Mode.FTP_UPLOAD); _uploadDao.persist(uploadTemplateObj); if(vmTemplateHost != null) { @@ -197,7 +197,7 @@ public class UploadMonitorImpl implements UploadMonitor { HostVO sserver = storageServers.get(0); UploadVO uploadTemplateObj = new UploadVO(sserver.getId(), template.getId(), new Date(), Status.DOWNLOAD_URL_NOT_CREATED, 0, type, Mode.HTTP_DOWNLOAD); - + uploadTemplateObj.setInstallPath(vmTemplateHost.getInstallPath()); _uploadDao.persist(uploadTemplateObj); // Create Symlink at ssvm @@ -234,50 +234,61 @@ public class UploadMonitorImpl implements UploadMonitor { } @Override - public void createVolumeDownloadURL(Long entityId, String path, Type type, Long dataCenterId, Long uploadId) throws CloudRuntimeException{ + public void createVolumeDownloadURL(Long entityId, String path, Type type, Long dataCenterId, Long uploadId) throws InternalErrorException{ - List storageServers = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorage, dataCenterId); - if(storageServers == null ) - throw new CloudRuntimeException("No Storage Server found at the datacenter - " +dataCenterId); - - // Update DB for state = DOWNLOAD_URL_NOT_CREATED. - UploadVO uploadJob = _uploadDao.createForUpdate(uploadId); - uploadJob.setUploadState(Status.DOWNLOAD_URL_NOT_CREATED); - uploadJob.setLastUpdated(new Date()); - _uploadDao.update(uploadJob.getId(), uploadJob); - - // Create Symlink at ssvm - CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(path); - long result = send(ApiDBUtils.findUploadById(uploadId).getHostId(), cmd, null); - if (result == -1){ - String errorString = "Unable to create a link for " +type+ " id:"+entityId; - s_logger.warn(errorString); - throw new CloudRuntimeException(errorString); - } - - //Construct actual URL locally now that the symlink exists at SSVM - List ssVms = _secStorageVmDao.getSecStorageVmListInStates(dataCenterId, State.Running); - if (ssVms.size() > 0) { - SecondaryStorageVmVO ssVm = ssVms.get(0); - if (ssVm.getPublicIpAddress() == null) { - s_logger.warn("A running secondary storage vm has a null public ip?"); - throw new CloudRuntimeException("SSVM has null public IP - couldnt create the URL"); - } - String extractURL = generateCopyUrl(ssVm.getPublicIpAddress(), path); - UploadVO vo = _uploadDao.createForUpdate(); - vo.setLastUpdated(new Date()); - vo.setUploadUrl(extractURL); - vo.setUploadState(Status.DOWNLOAD_URL_CREATED); + String errorString = ""; + boolean success = false; + try{ + List storageServers = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorage, dataCenterId); + if(storageServers == null ){ + errorString = "No Storage Server found at the datacenter - " +dataCenterId; + throw new CloudRuntimeException(errorString); + } - if(extractURL == null){ - vo.setUploadState(Status.ERROR); - vo.setErrorString("Could not create the download URL"); + // Update DB for state = DOWNLOAD_URL_NOT_CREATED. + UploadVO uploadJob = _uploadDao.createForUpdate(uploadId); + uploadJob.setUploadState(Status.DOWNLOAD_URL_NOT_CREATED); + uploadJob.setLastUpdated(new Date()); + _uploadDao.update(uploadJob.getId(), uploadJob); + + // Create Symlink at ssvm + CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(path); + long result = send(ApiDBUtils.findUploadById(uploadId).getHostId(), cmd, null); + if (result == -1){ + errorString = "Unable to create a link for " +type+ " id:"+entityId; + s_logger.warn(errorString); + throw new InternalErrorException(errorString); } - _uploadDao.update(uploadId, vo); - return; - } - throw new CloudRuntimeException("Couldnt find a running SSVM in the zone" + dataCenterId+ ". Couldnt create the extraction URL."); - + + //Construct actual URL locally now that the symlink exists at SSVM + List ssVms = _secStorageVmDao.getSecStorageVmListInStates(dataCenterId, State.Running); + if (ssVms.size() > 0) { + SecondaryStorageVmVO ssVm = ssVms.get(0); + if (ssVm.getPublicIpAddress() == null) { + errorString = "A running secondary storage vm has a null public ip?"; + s_logger.warn(errorString); + throw new InternalErrorException(errorString); + } + String extractURL = generateCopyUrl(ssVm.getPublicIpAddress(), path); + UploadVO vo = _uploadDao.createForUpdate(); + vo.setLastUpdated(new Date()); + vo.setUploadUrl(extractURL); + vo.setUploadState(Status.DOWNLOAD_URL_CREATED); + _uploadDao.update(uploadId, vo); + success = true; + return; + } + errorString = "Couldnt find a running SSVM in the zone" + dataCenterId+ ". Couldnt create the extraction URL."; + throw new InternalErrorException(errorString); + }finally{ + if(!success){ + UploadVO uploadJob = _uploadDao.createForUpdate(uploadId); + uploadJob.setLastUpdated(new Date()); + uploadJob.setErrorString(errorString); + uploadJob.setUploadState(Status.ERROR); + _uploadDao.update(uploadId, uploadJob); + } + } } private String generateCopyUrl(String ipAddress, String path){ @@ -328,7 +339,7 @@ public class UploadMonitorImpl implements UploadMonitor { @Override public boolean start() { - //FIX ME - Make the timings configurable. + //FIX ME - Make the timings configurable. // Keep them to 86400 for now. _executor.scheduleWithFixedDelay(new StorageGarbageCollector(), 86400, 86400, TimeUnit.SECONDS); _timer = new Timer(); return true; @@ -437,27 +448,30 @@ public class UploadMonitorImpl implements UploadMonitor { private long getTimeDiff(Date date){ - Calendar currentCalendar = Calendar.getInstance(); + Calendar currentDateCalendar = Calendar.getInstance(); Calendar givenDateCalendar = Calendar.getInstance(); givenDateCalendar.setTime(date); - return (currentCalendar.getTimeInMillis() - givenDateCalendar.getTimeInMillis() )/1000; + return (currentDateCalendar.getTimeInMillis() - givenDateCalendar.getTimeInMillis() )/1000; } public void cleanupStorage() { - final int EXTRACT_URL_TIME_LIMIT = 40; - List extractURLs= _uploadDao.listByModeAndStatus(Mode.HTTP_DOWNLOAD, Status.DOWNLOAD_URL_CREATED); + final int EXTRACT_URL_LIFE_LIMIT_IN_SECONDS = 86400;//FIX ME make it configurable. + List extractJobs= _uploadDao.listByModeAndStatus(Mode.HTTP_DOWNLOAD, Status.DOWNLOAD_URL_CREATED); - for (UploadVO extractURL : extractURLs){ - if( getTimeDiff(extractURL.getLastUpdated()) < EXTRACT_URL_TIME_LIMIT) continue; - String path = extractURL.getUploadUrl().substring( (extractURL.getUploadUrl().lastIndexOf("/")) +1 ); - DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(path, extractURL.getType()); - long result = send(extractURL.getHostId(), cmd, null); - if (result == -1){ - s_logger.warn("Unable to delete the link for " +extractURL.getType()+ " id=" +extractURL.getTypeId()+ " url="+extractURL.getUploadUrl()); - }else{ - _uploadDao.remove(extractURL.getId()); + for (UploadVO extractJob : extractJobs){ + if( getTimeDiff(extractJob.getLastUpdated()) > EXTRACT_URL_LIFE_LIMIT_IN_SECONDS ){ + String path = extractJob.getInstallPath(); + s_logger.debug("Sending deletion of extract URL "+extractJob.getUploadUrl()); + // Would delete the symlink for the Type and if Type == VOLUME then also the volume + DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(path, extractJob.getType()); + long result = send(extractJob.getHostId(), cmd, null); + if (result == -1){ + s_logger.warn("Unable to delete the link for " +extractJob.getType()+ " id=" +extractJob.getTypeId()+ " url="+extractJob.getUploadUrl()); + }else{ + _uploadDao.remove(extractJob.getId()); + } } } diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index e91c6994445..836a32d8894 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -648,6 +648,7 @@ CREATE TABLE `cloud`.`upload` ( `upload_state` varchar(255), `error_str` varchar(255), `url` varchar(255), + `install_path` varchar(255), PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;