apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Code clean up for Extract functionality and making it more robust.

Browse Source
This commit is contained in:
nit 2010-10-28 18:42:40 +05:30
parent 471dafd1f7
commit c9069303ce
6 changed files with 117 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
core/src/com/cloud/storage/UploadVO.java
View File

@ -82,6 +82,9 @@ public class UploadVO implements Upload {
@Column (name="url")
private String uploadUrl;
@Column (name="install_path")
private String installPath;
public long getHostId() {
return hostId;
}
@ -113,17 +116,15 @@ public class UploadVO implements Upload {
}
public UploadVO(long hostId, long typeId, Date lastUpdated,
Status uploadState, int uploadPercent, Type type,
String errorString, String jobId, String uploadUrl) {
Status uploadState, Type type,
String uploadUrl, Mode mode) {
super();
this.hostId = hostId;
this.typeId = typeId;
this.lastUpdated = lastUpdated;
this.uploadState = uploadState;
this.uploadPercent = uploadPercent;
this.mode = mode;
this.type = type;
this.errorString = errorString;
this.jobId = jobId;
this.uploadUrl = uploadUrl;
}
@ -241,4 +242,12 @@ public class UploadVO implements Upload {
this.created = created;
}
public String getInstallPath() {
return installPath;
}
public void setInstallPath(String installPath) {
this.installPath = installPath;
}
}

13
core/src/com/cloud/storage/template/UploadManagerImpl.java
View File

@ -358,11 +358,13 @@ public class UploadManagerImpl implements UploadManager {
@Override
public DeleteEntityDownloadURLAnswer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd){
//Delete the soft link
s_logger.debug("handleDeleteEntityDownloadURLCommand "+cmd.getPath());
//Delete the soft link. Example path = volumes/8/74eeb2c6-8ab1-4357-841f-2e9d06d1f360.vhd
s_logger.warn("handleDeleteEntityDownloadURLCommand Path:"+cmd.getPath() + " Type:" +cmd.getType().toString());
String path = cmd.getPath();
Script command = new Script("/bin/bash", s_logger);
command.add("-c");
command.add("unlink /var/www/html/"+cmd.getPath());
//We just need to remove the UUID.vhd
command.add("unlink /var/www/html/" +path.substring(path.lastIndexOf(File.separator) + 1));
String result = command.execute();
if (result != null) {
String errorString = "Error in deleting =" + result;
@ -370,11 +372,12 @@ public class UploadManagerImpl implements UploadManager {
return new DeleteEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
}
// If its a volume also delete the Hard link
// If its a volume also delete the Hard link since it was created only for the purpose of download.
if(cmd.getType() == Upload.Type.VOLUME){
command = new Script("/bin/bash", s_logger);
command.add("-c");
command.add("rm -f " + publicTemplateRepo + cmd.getPath());
command.add("rm -f " + parentDir +File.separator+ path);
s_logger.warn(" " +parentDir +File.separator+ path);
result = command.execute();
if (result != null) {
String errorString = "Error in linking err=" + result;

22
server/src/com/cloud/server/ManagementServerImpl.java
View File

@ -61,6 +61,7 @@ import com.cloud.agent.api.storage.CopyVolumeCommand;
import com.cloud.alert.AlertManager;
import com.cloud.alert.AlertVO;
import com.cloud.alert.dao.AlertDao;
import com.cloud.api.ApiDBUtils;
import com.cloud.api.BaseCmd;
import com.cloud.api.ServerApiException;
import com.cloud.api.commands.AssignPortForwardingServiceCmd;
@ -6696,7 +6697,12 @@ public class ManagementServerImpl implements ManagementServer {
if (_dcDao.findById(zoneId) == null) {
throw new ServerApiException(BaseCmd.PARAM_ERROR, "Please specify a valid zone.");
}
//Extract activity only for detached volumes or for volumes whose instance is stopped
if(volume.getInstanceId() != null && ApiDBUtils.findVMInstanceById(volume.getInstanceId()).getState() != State.Stopped ){
s_logger.debug("Invalid state of the volume with ID: " + volumeId + ". It should be either detached or the VM should be in stopped state.");
throw new PermissionDeniedException("Invalid state of the volume with ID: " + volumeId + ". It should be either detached or the VM should be in stopped state.");
}
Upload.Mode extractMode;
if( mode == null || (!mode.equals(Upload.Mode.FTP_UPLOAD.toString()) && !mode.equals(Upload.Mode.HTTP_DOWNLOAD.toString())) ){
throw new ServerApiException(BaseCmd.PARAM_ERROR, "Please specify a valid extract Mode ");
@ -6704,7 +6710,7 @@ public class ManagementServerImpl implements ManagementServer {
extractMode = mode.equals(Upload.Mode.FTP_UPLOAD.toString()) ? Upload.Mode.FTP_UPLOAD : Upload.Mode.HTTP_DOWNLOAD;
}
if (account != null) {
if (account != null) {
if(!isAdmin(account.getType())){
if (volume.getAccountId() != account.getId()){
throw new PermissionDeniedException("Unable to find volume with ID: " + volumeId + " for account: " + account.getAccountName());
@ -6751,17 +6757,19 @@ public class ManagementServerImpl implements ManagementServer {
List<HostVO> storageServers = _hostDao.listByTypeDataCenter(Host.Type.SecondaryStorage, zoneId);
HostVO sserver = storageServers.get(0);
EventUtils.saveStartedEvent(userId, accountId, EventTypes.EVENT_VOLUME_UPLOAD, "Starting extraction of " +volume.getName()+ " mode:"+mode, cmd.getStartEventId());
EventUtils.saveStartedEvent(userId, accountId, cmd.getEventType(), "Starting extraction of " +volume.getName()+ " mode:"+mode, cmd.getStartEventId());
List<UploadVO> extractURLList = _uploadDao.listByTypeUploadStatus(volumeId, Upload.Type.VOLUME, UploadVO.Status.DOWNLOAD_URL_CREATED);
if (extractMode == Upload.Mode.HTTP_DOWNLOAD && extractURLList.size() > 0){
if (extractMode == Upload.Mode.HTTP_DOWNLOAD && extractURLList.size() > 0){
return extractURLList.get(0).getId(); // If download url already exists then return
}else {
UploadVO uploadJob = _uploadMonitor.createNewUploadEntry(sserver.getId(), volumeId, UploadVO.Status.COPY_IN_PROGRESS, 0, Type.VOLUME, null, null, url);
UploadVO uploadJob = _uploadMonitor.createNewUploadEntry(sserver.getId(), volumeId, UploadVO.Status.COPY_IN_PROGRESS, Type.VOLUME, url, extractMode);
s_logger.debug("Extract Mode - " +uploadJob.getMode());
uploadJob = _uploadDao.createForUpdate(uploadJob.getId());
// Update the async Job
ExtractResponse resultObj = new ExtractResponse(volumeId, volume.getName(), accountId, UploadVO.Status.COPY_IN_PROGRESS.toString(), uploadJob.getId());
resultObj.setResponseName(cmd.getName());
_asyncMgr.updateAsyncJobAttachment(job.getId(), Type.VOLUME.toString(), volumeId);
_asyncMgr.updateAsyncJobStatus(job.getId(), AsyncJobResult.STATUS_IN_PROGRESS, resultObj);
@ -6789,16 +6797,16 @@ public class ManagementServerImpl implements ManagementServer {
}
String volumeLocalPath = "volumes/"+volume.getId()+"/"+cvAnswer.getVolumePath()+".vhd";
//Update the DB that volume is copied
//Update the DB that volume is copied and volumePath
uploadJob.setUploadState(UploadVO.Status.COPY_COMPLETE);
uploadJob.setLastUpdated(new Date());
uploadJob.setInstallPath(volumeLocalPath);
_uploadDao.update(uploadJob.getId(), uploadJob);
if (extractMode == Mode.FTP_UPLOAD){ // Now that the volume is copied perform the actual uploading
_uploadMonitor.extractVolume(uploadJob, sserver, volume, url, zoneId, volumeLocalPath, cmd.getStartEventId(), job.getId(), _asyncMgr);
return uploadJob.getId();
}else{ // Volume is copied now make it visible under apache and create a URL.
s_logger.debug("volumepath " +volumeLocalPath);
_uploadMonitor.createVolumeDownloadURL(volumeId, volumeLocalPath, Type.VOLUME, zoneId, uploadJob.getId());
EventUtils.saveEvent(userId, accountId, EventVO.LEVEL_INFO, cmd.getEventType(), "Completed extraction of "+volume.getName()+ " in mode:" +mode, null, cmd.getStartEventId());
return uploadJob.getId();

7
server/src/com/cloud/storage/upload/UploadMonitor.java
View File

@ -19,7 +19,9 @@
package com.cloud.storage.upload;
import com.cloud.async.AsyncJobManager;
import com.cloud.exception.InternalErrorException;
import com.cloud.host.HostVO;
import com.cloud.storage.Upload.Mode;
import com.cloud.storage.Upload.Status;
import com.cloud.storage.Upload.Type;
import com.cloud.storage.UploadVO;
@ -46,8 +48,7 @@ public interface UploadMonitor extends Manager{
void handleUploadSync(long sserverId);
UploadVO createNewUploadEntry(Long hostId, Long typeId, Status uploadState,
int uploadPercent, Type type, String errorString, String jobId,
String uploadUrl);
Type type, String errorString, Mode extractMode);
void extractVolume(UploadVO uploadVolumeObj, HostVO sserver, VolumeVO volume, String url,
Long dataCenterId, String installPath, long eventId,
@ -57,6 +58,6 @@ public interface UploadMonitor extends Manager{
VMTemplateHostVO vmTemplateHost, Long dataCenterId, long eventId);
void createVolumeDownloadURL(Long entityId, String path, Type type,
Long dataCenterId, Long uploadId) throws CloudRuntimeException;
Long dataCenterId, Long uploadId) throws InternalErrorException;
}

136
server/src/com/cloud/storage/upload/UploadMonitorImpl.java
View File

@ -30,6 +30,7 @@ import com.cloud.configuration.dao.ConfigurationDao;
import com.cloud.event.EventTypes;
import com.cloud.event.EventVO;
import com.cloud.event.dao.EventDao;
import com.cloud.exception.InternalErrorException;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
@ -53,6 +54,7 @@ import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.SecondaryStorageVmVO;
import com.cloud.vm.State;
import com.cloud.vm.dao.SecondaryStorageVmDao;
import com.sun.corba.se.impl.logging.InterceptorsSystemException;
/**
* @author nitin
@ -115,11 +117,10 @@ public class UploadMonitorImpl implements UploadMonitor {
@Override
public UploadVO createNewUploadEntry(Long hostId, Long typeId, UploadVO.Status uploadState,
int uploadPercent, Type type,
String errorString, String jobId, String uploadUrl){
Type type, String uploadUrl, Upload.Mode mode){
UploadVO uploadObj = new UploadVO(hostId, typeId, new Date(),
uploadState, 0, type, null, null, uploadUrl);
uploadState, type, uploadUrl, mode);
_uploadDao.persist(uploadObj);
return uploadObj;
@ -156,8 +157,7 @@ public class UploadMonitorImpl implements UploadMonitor {
HostVO sserver = storageServers.get(0);
UploadVO uploadTemplateObj = new UploadVO(sserver.getId(), template.getId(), new Date(),
Upload.Status.NOT_UPLOADED, 0, type,
null, "jobid0000", url);
Upload.Status.NOT_UPLOADED, type, url, Mode.FTP_UPLOAD);
_uploadDao.persist(uploadTemplateObj);
if(vmTemplateHost != null) {
@ -197,7 +197,7 @@ public class UploadMonitorImpl implements UploadMonitor {
HostVO sserver = storageServers.get(0);
UploadVO uploadTemplateObj = new UploadVO(sserver.getId(), template.getId(), new Date(),
Status.DOWNLOAD_URL_NOT_CREATED, 0, type, Mode.HTTP_DOWNLOAD);
uploadTemplateObj.setInstallPath(vmTemplateHost.getInstallPath());
_uploadDao.persist(uploadTemplateObj);
// Create Symlink at ssvm
@ -234,50 +234,61 @@ public class UploadMonitorImpl implements UploadMonitor {
}
@Override
public void createVolumeDownloadURL(Long entityId, String path, Type type, Long dataCenterId, Long uploadId) throws CloudRuntimeException{
public void createVolumeDownloadURL(Long entityId, String path, Type type, Long dataCenterId, Long uploadId) throws InternalErrorException{
List<HostVO> storageServers = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorage, dataCenterId);
if(storageServers == null )
throw new CloudRuntimeException("No Storage Server found at the datacenter - " +dataCenterId);
// Update DB for state = DOWNLOAD_URL_NOT_CREATED.
UploadVO uploadJob = _uploadDao.createForUpdate(uploadId);
uploadJob.setUploadState(Status.DOWNLOAD_URL_NOT_CREATED);
uploadJob.setLastUpdated(new Date());
_uploadDao.update(uploadJob.getId(), uploadJob);
// Create Symlink at ssvm
CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(path);
long result = send(ApiDBUtils.findUploadById(uploadId).getHostId(), cmd, null);
if (result == -1){
String errorString = "Unable to create a link for " +type+ " id:"+entityId;
s_logger.warn(errorString);
throw new CloudRuntimeException(errorString);
}
//Construct actual URL locally now that the symlink exists at SSVM
List<SecondaryStorageVmVO> ssVms = _secStorageVmDao.getSecStorageVmListInStates(dataCenterId, State.Running);
if (ssVms.size() > 0) {
SecondaryStorageVmVO ssVm = ssVms.get(0);
if (ssVm.getPublicIpAddress() == null) {
s_logger.warn("A running secondary storage vm has a null public ip?");
throw new CloudRuntimeException("SSVM has null public IP - couldnt create the URL");
}
String extractURL = generateCopyUrl(ssVm.getPublicIpAddress(), path);
UploadVO vo = _uploadDao.createForUpdate();
vo.setLastUpdated(new Date());
vo.setUploadUrl(extractURL);
vo.setUploadState(Status.DOWNLOAD_URL_CREATED);
String errorString = "";
boolean success = false;
try{
List<HostVO> storageServers = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorage, dataCenterId);
if(storageServers == null ){
errorString = "No Storage Server found at the datacenter - " +dataCenterId;
throw new CloudRuntimeException(errorString);
}
if(extractURL == null){
vo.setUploadState(Status.ERROR);
vo.setErrorString("Could not create the download URL");
// Update DB for state = DOWNLOAD_URL_NOT_CREATED.
UploadVO uploadJob = _uploadDao.createForUpdate(uploadId);
uploadJob.setUploadState(Status.DOWNLOAD_URL_NOT_CREATED);
uploadJob.setLastUpdated(new Date());
_uploadDao.update(uploadJob.getId(), uploadJob);
// Create Symlink at ssvm
CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(path);
long result = send(ApiDBUtils.findUploadById(uploadId).getHostId(), cmd, null);
if (result == -1){
errorString = "Unable to create a link for " +type+ " id:"+entityId;
s_logger.warn(errorString);
throw new InternalErrorException(errorString);
}
_uploadDao.update(uploadId, vo);
return;
}
throw new CloudRuntimeException("Couldnt find a running SSVM in the zone" + dataCenterId+ ". Couldnt create the extraction URL.");
//Construct actual URL locally now that the symlink exists at SSVM
List<SecondaryStorageVmVO> ssVms = _secStorageVmDao.getSecStorageVmListInStates(dataCenterId, State.Running);
if (ssVms.size() > 0) {
SecondaryStorageVmVO ssVm = ssVms.get(0);
if (ssVm.getPublicIpAddress() == null) {
errorString = "A running secondary storage vm has a null public ip?";
s_logger.warn(errorString);
throw new InternalErrorException(errorString);
}
String extractURL = generateCopyUrl(ssVm.getPublicIpAddress(), path);
UploadVO vo = _uploadDao.createForUpdate();
vo.setLastUpdated(new Date());
vo.setUploadUrl(extractURL);
vo.setUploadState(Status.DOWNLOAD_URL_CREATED);
_uploadDao.update(uploadId, vo);
success = true;
return;
}
errorString = "Couldnt find a running SSVM in the zone" + dataCenterId+ ". Couldnt create the extraction URL.";
throw new InternalErrorException(errorString);
}finally{
if(!success){
UploadVO uploadJob = _uploadDao.createForUpdate(uploadId);
uploadJob.setLastUpdated(new Date());
uploadJob.setErrorString(errorString);
uploadJob.setUploadState(Status.ERROR);
_uploadDao.update(uploadId, uploadJob);
}
}
}
private String generateCopyUrl(String ipAddress, String path){
@ -328,7 +339,7 @@ public class UploadMonitorImpl implements UploadMonitor {
@Override
public boolean start() {
//FIX ME - Make the timings configurable.
//FIX ME - Make the timings configurable. // Keep them to 86400 for now.
_executor.scheduleWithFixedDelay(new StorageGarbageCollector(), 86400, 86400, TimeUnit.SECONDS);
_timer = new Timer();
return true;
@ -437,27 +448,30 @@ public class UploadMonitorImpl implements UploadMonitor {
private long getTimeDiff(Date date){
Calendar currentCalendar = Calendar.getInstance();
Calendar currentDateCalendar = Calendar.getInstance();
Calendar givenDateCalendar = Calendar.getInstance();
givenDateCalendar.setTime(date);
return (currentCalendar.getTimeInMillis() - givenDateCalendar.getTimeInMillis() )/1000;
return (currentDateCalendar.getTimeInMillis() - givenDateCalendar.getTimeInMillis() )/1000;
}
public void cleanupStorage() {
final int EXTRACT_URL_TIME_LIMIT = 40;
List<UploadVO> extractURLs= _uploadDao.listByModeAndStatus(Mode.HTTP_DOWNLOAD, Status.DOWNLOAD_URL_CREATED);
final int EXTRACT_URL_LIFE_LIMIT_IN_SECONDS = 86400;//FIX ME make it configurable.
List<UploadVO> extractJobs= _uploadDao.listByModeAndStatus(Mode.HTTP_DOWNLOAD, Status.DOWNLOAD_URL_CREATED);
for (UploadVO extractURL : extractURLs){
if( getTimeDiff(extractURL.getLastUpdated()) < EXTRACT_URL_TIME_LIMIT) continue;
String path = extractURL.getUploadUrl().substring( (extractURL.getUploadUrl().lastIndexOf("/")) +1 );
DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(path, extractURL.getType());
long result = send(extractURL.getHostId(), cmd, null);
if (result == -1){
s_logger.warn("Unable to delete the link for " +extractURL.getType()+ " id=" +extractURL.getTypeId()+ " url="+extractURL.getUploadUrl());
}else{
_uploadDao.remove(extractURL.getId());
for (UploadVO extractJob : extractJobs){
if( getTimeDiff(extractJob.getLastUpdated()) > EXTRACT_URL_LIFE_LIMIT_IN_SECONDS ){
String path = extractJob.getInstallPath();
s_logger.debug("Sending deletion of extract URL "+extractJob.getUploadUrl());
// Would delete the symlink for the Type and if Type == VOLUME then also the volume
DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(path, extractJob.getType());
long result = send(extractJob.getHostId(), cmd, null);
if (result == -1){
s_logger.warn("Unable to delete the link for " +extractJob.getType()+ " id=" +extractJob.getTypeId()+ " url="+extractJob.getUploadUrl());
}else{
_uploadDao.remove(extractJob.getId());
}
}
}

1
setup/db/create-schema.sql
View File

@ -648,6 +648,7 @@ CREATE TABLE `cloud`.`upload` (
`upload_state` varchar(255),
`error_str` varchar(255),
`url` varchar(255),
`install_path` varchar(255),
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;