apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

[KVM] Direct download agnostic of the storage provider (#3828)

Browse Source 
* Remove constraint for NFS storage

* Add new property on agent.properties

* Add free disk space on the host prior template download

* Add unit tests for the free space check

* Fix free space check - retrieve avaiable size in bytes

* Update default location for direct download

* Improve the method to retrieve hosts to retry on depending on the destination pool type and scope

* Verify location for temporary download exists before checking free space

* In progress - refactor and extension

* Refactor and fix

* Last fixes and marvin tests

* Remove unused test file

* Improve logging

* Change default path for direct download

* Fix upload certificate

* Fix ISO failure after retry

* Fix metalink filename mismatch error

* Fix iso direct download

* Fix for direct download ISOs on local storage and shared mount point

* Last fix iso

* Fix VM migration with ISO

* Refactor volume migration to remove secondary storage intermediate

* Fix simulator issue
This commit is contained in:
Nicolas Vazquez 2020-03-06 15:56:54 -03:00 committed by GitHub
parent dc225de811
commit 73122fd0a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 621 additions and 284 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
agent/conf/agent.properties
View File

@ -115,6 +115,9 @@ domr.scripts.dir=scripts/network/domr/kvm
# set the hypervisor type, values are: kvm, lxc
hypervisor.type=kvm
# This parameter specifies a directory on the host local storage for temporary storing direct download templates
#direct.download.temporary.download.location=/var/lib/libvirt/images
# set the hypervisor URI. Usually there is no need for changing this
# For KVM: qemu:///system
# For LXC: lxc:///

42
agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java
View File

@ -19,49 +19,15 @@
package com.cloud.agent.direct.download;
import com.cloud.utils.Pair;
public interface DirectTemplateDownloader {
class DirectTemplateInformation {
private String installPath;
private Long size;
private String checksum;
public DirectTemplateInformation(String installPath, Long size, String checksum) {
this.installPath = installPath;
this.size = size;
this.checksum = checksum;
}
public String getInstallPath() {
return installPath;
}
public Long getSize() {
return size;
}
public String getChecksum() {
return checksum;
}
}
/**
* Perform template download to pool specified on downloader creation
* @return true if successful, false if not
* @return (true if successful, false if not, download file path)
*/
boolean downloadTemplate();
/**
* Perform extraction (if necessary) and installation of previously downloaded template
* @return true if successful, false if not
*/
boolean extractAndInstallDownloadedTemplate();
/**
* Get template information after it is properly installed on pool
* @return template information
*/
DirectTemplateInformation getTemplateInformation();
Pair<Boolean, String> downloadTemplate();
/**
* Perform checksum validation of previously downloadeed template

73
agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java
View File

@ -19,7 +19,6 @@
package com.cloud.agent.direct.download;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import org.apache.cloudstack.utils.security.DigestHelper;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
@ -28,7 +27,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
import java.util.UUID;
public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader {
@ -36,16 +34,19 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
private String destPoolPath;
private Long templateId;
private String downloadedFilePath;
private String installPath;
private String checksum;
private boolean redownload = false;
protected String temporaryDownloadPath;
public static final Logger s_logger = Logger.getLogger(DirectTemplateDownloaderImpl.class.getName());
protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, final String checksum) {
protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId,
final String checksum, final String temporaryDownloadPath) {
this.url = url;
this.destPoolPath = destPoolPath;
this.templateId = templateId;
this.checksum = checksum;
this.temporaryDownloadPath = temporaryDownloadPath;
}
private static String directDownloadDir = "template";
@ -53,10 +54,10 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
/**
* Return direct download temporary path to download template
*/
protected static String getDirectDownloadTempPath(Long templateId) {
protected String getDirectDownloadTempPath(Long templateId) {
String templateIdAsString = String.valueOf(templateId);
return directDownloadDir + File.separator + templateIdAsString.substring(0,1) +
File.separator + templateIdAsString;
return this.temporaryDownloadPath + File.separator + directDownloadDir + File.separator +
templateIdAsString.substring(0,1) + File.separator + templateIdAsString;
}
/**
@ -113,64 +114,6 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown
return urlParts[urlParts.length - 1];
}
/**
* Checks if downloaded template is extractable
* @return true if it should be extracted, false if not
*/
private boolean isTemplateExtractable() {
String type = Script.runSimpleBashScript("file " + downloadedFilePath + " | awk -F' ' '{print $2}'");
return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
}
@Override
public boolean extractAndInstallDownloadedTemplate() {
installPath = UUID.randomUUID().toString();
if (isTemplateExtractable()) {
extractDownloadedTemplate();
} else {
Script.runSimpleBashScript("mv " + downloadedFilePath + " " + getInstallFullPath());
}
return true;
}
/**
* Return install full path
*/
private String getInstallFullPath() {
return destPoolPath + File.separator + installPath;
}
/**
* Return extract command to execute given downloaded file
*/
private String getExtractCommandForDownloadedFile() {
if (downloadedFilePath.endsWith(".zip")) {
return "unzip -p " + downloadedFilePath + " | cat > " + getInstallFullPath();
} else if (downloadedFilePath.endsWith(".bz2")) {
return "bunzip2 -c " + downloadedFilePath + " > " + getInstallFullPath();
} else if (downloadedFilePath.endsWith(".gz")) {
return "gunzip -c " + downloadedFilePath + " > " + getInstallFullPath();
} else {
throw new CloudRuntimeException("Unable to extract template " + templateId + " on " + downloadedFilePath);
}
}
/**
* Extract downloaded template into installPath, remove compressed file
*/
private void extractDownloadedTemplate() {
String extractCommand = getExtractCommandForDownloadedFile();
Script.runSimpleBashScript(extractCommand);
Script.runSimpleBashScript("rm -f " + downloadedFilePath);
}
@Override
public DirectTemplateInformation getTemplateInformation() {
String sizeResult = Script.runSimpleBashScript("ls -als " + getInstallFullPath() + " | awk '{print $1}'");
long size = Long.parseLong(sizeResult);
return new DirectTemplateInformation(installPath, size, checksum);
}
@Override
public boolean validateChecksum() {
if (StringUtils.isNotBlank(checksum)) {

33
agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java
View File

@ -27,6 +27,8 @@ import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
@ -35,8 +37,6 @@ import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.io.IOUtils;
import org.apache.log4j.Logger;
import com.cloud.utils.exception.CloudRuntimeException;
public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected HttpClient client;
@ -45,20 +45,25 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected GetMethod request;
protected Map<String, String> reqHeaders = new HashMap<>();
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers, Integer connectTimeout, Integer soTimeout) {
super(url, destPoolPath, templateId, checksum);
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
super(url, destPoolPath, templateId, checksum, downloadPath);
s_httpClientManager.getParams().setConnectionTimeout(connectTimeout == null ? 5000 : connectTimeout);
s_httpClientManager.getParams().setSoTimeout(soTimeout == null ? 5000 : soTimeout);
client = new HttpClient(s_httpClientManager);
request = createRequest(url, headers);
String downloadDir = getDirectDownloadTempPath(templateId);
createTemporaryDirectoryAndFile(downloadDir);
File tempFile = createTemporaryDirectoryAndFile(downloadDir);
setDownloadedFilePath(tempFile.getAbsolutePath());
}
protected void createTemporaryDirectoryAndFile(String downloadDir) {
createFolder(getDestPoolPath() + File.separator + downloadDir);
File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl());
setDownloadedFilePath(f.getAbsolutePath());
/**
* Create download directory (if it does not exist) and set the download file
* @return
*/
protected File createTemporaryDirectoryAndFile(String downloadDir) {
createFolder(downloadDir);
return new File(downloadDir + File.separator + getFileNameFromUrl());
}
protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
@ -74,12 +79,12 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
}
@Override
public boolean downloadTemplate() {
public Pair<Boolean, String> downloadTemplate() {
try {
int status = client.executeMethod(request);
if (status != HttpStatus.SC_OK) {
s_logger.warn("Not able to download template, status code: " + status);
return false;
return new Pair<>(false, null);
}
return performDownload();
} catch (IOException e) {
@ -89,7 +94,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
}
}
protected boolean performDownload() {
protected Pair<Boolean, String> performDownload() {
s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath());
try (
InputStream in = request.getResponseBodyAsStream();
@ -98,8 +103,8 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
IOUtils.copy(in, out);
} catch (IOException e) {
s_logger.error("Error downloading template " + getTemplateId() + " due to: " + e.getMessage());
return false;
return new Pair<>(false, null);
}
return true;
return new Pair<>(true, getDownloadedFilePath());
}
}

48
agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java
View File

@ -19,6 +19,23 @@
package com.cloud.agent.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.commons.collections.MapUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.ssl.SSLContexts;
import javax.net.ssl.SSLContext;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@ -32,31 +49,14 @@ import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.util.Map;
import javax.net.ssl.SSLContext;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.ssl.SSLContexts;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader {
private CloseableHttpClient httpsClient;
private HttpUriRequest req;
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers, Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout) {
super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout);
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) {
super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, temporaryDownloadPath);
SSLContext sslcontext = null;
try {
sslcontext = getSSLContext();
@ -98,7 +98,7 @@ public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader
}
@Override
public boolean downloadTemplate() {
public Pair<Boolean, String> downloadTemplate() {
CloseableHttpResponse response;
try {
response = httpsClient.execute(req);
@ -111,7 +111,7 @@ public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader
/**
* Consume response and persist it on getDownloadedFilePath() file
*/
protected boolean consumeResponse(CloseableHttpResponse response) {
protected Pair<Boolean, String> consumeResponse(CloseableHttpResponse response) {
s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath());
if (response.getStatusLine().getStatusCode() != 200) {
throw new CloudRuntimeException("Error on HTTPS response");
@ -123,9 +123,9 @@ public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader
IOUtils.copy(in, out);
} catch (Exception e) {
s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage());
return false;
return new Pair<>(false, null);
}
return true;
return new Pair<>(true, getDownloadedFilePath());
}
}

32
agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java
View File

@ -18,17 +18,17 @@
//
package com.cloud.agent.direct.download;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.Random;
import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader {
@ -38,8 +38,9 @@ public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownload
private Random random = new Random();
private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName());
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, Map<String, String> headers, Integer connectTimeout, Integer soTimeout) {
super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout);
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum,
Map<String, String> headers, Integer connectTimeout, Integer soTimeout, String downloadPath) {
super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, downloadPath);
metalinkUrl = url;
metalinkUrls = UriUtils.getMetalinkUrls(metalinkUrl);
metalinkChecksums = UriUtils.getMetalinkChecksums(metalinkUrl);
@ -53,27 +54,28 @@ public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownload
}
@Override
public boolean downloadTemplate() {
public Pair<Boolean, String> downloadTemplate() {
if (StringUtils.isBlank(getUrl())) {
throw new CloudRuntimeException("Download url has not been set, aborting");
}
String downloadDir = getDirectDownloadTempPath(getTemplateId());
boolean downloaded = false;
int i = 0;
String downloadDir = getDirectDownloadTempPath(getTemplateId());
do {
if (!isRedownload()) {
setUrl(metalinkUrls.get(i));
}
s_logger.info("Trying to download template from url: " + getUrl());
try {
File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl());
setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl());
File f = new File(getDownloadedFilePath());
if (f.exists()) {
f.delete();
f.createNewFile();
}
setDownloadedFilePath(f.getAbsolutePath());
request = createRequest(getUrl(), reqHeaders);
downloaded = super.downloadTemplate();
Pair<Boolean, String> downloadResult = super.downloadTemplate();
downloaded = downloadResult.first();
if (downloaded) {
s_logger.info("Successfully downloaded template from url: " + getUrl());
}
@ -84,7 +86,7 @@ public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownload
i++;
}
while (!downloaded && !isRedownload() && i < metalinkUrls.size());
return downloaded;
return new Pair<>(downloaded, getDownloadedFilePath());
}
@Override

9
agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java
View File

@ -18,6 +18,7 @@
//
package com.cloud.agent.direct.download;
import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
@ -51,13 +52,13 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
}
}
public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum) {
super(url, destPool, templateId, checksum);
public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) {
super(url, destPool, templateId, checksum, downloadPath);
parseUrl();
}
@Override
public boolean downloadTemplate() {
public Pair<Boolean, String> downloadTemplate() {
String mountSrcUuid = UUID.randomUUID().toString();
String mount = String.format(mountCommand, srcHost + ":" + srcPath, "/mnt/" + mountSrcUuid);
Script.runSimpleBashScript(mount);
@ -65,6 +66,6 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl());
Script.runSimpleBashScript("cp /mnt/" + mountSrcUuid + srcPath + " " + getDownloadedFilePath());
Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid);
return true;
return new Pair<>(true, getDownloadedFilePath());
}
}

36
agent/src/test/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImplTest.java
View File

@ -1,36 +0,0 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.runners.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class DirectTemplateDownloaderImplTest {
private static final Long templateId = 202l;
@Test
public void testGetDirectDownloadTempPath() {
String path = DirectTemplateDownloaderImpl.getDirectDownloadTempPath(templateId);
Assert.assertEquals("template/2/202", path);
}
}

2
core/src/main/java/com/cloud/storage/resource/StorageProcessor.java
View File

@ -74,4 +74,6 @@ public interface StorageProcessor {
public Answer resignature(ResignatureCommand cmd);
public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd);
Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd);
}

5
core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java
View File

@ -20,6 +20,7 @@
package com.cloud.storage.resource;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.log4j.Logger;
import org.apache.cloudstack.storage.command.AttachCommand;
@ -95,7 +96,9 @@ public class StorageSubsystemCommandHandlerBase implements StorageSubsystemComma
//copy volume from image cache to primary
return processor.copyVolumeFromImageCacheToPrimary(cmd);
} else if (srcData.getObjectType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.Primary) {
if (destData.getObjectType() == DataObjectType.VOLUME) {
if (destData.getObjectType() == DataObjectType.VOLUME && srcData instanceof VolumeObjectTO && ((VolumeObjectTO)srcData).isDirectDownload()) {
return processor.copyVolumeFromPrimaryToPrimary(cmd);
} else if (destData.getObjectType() == DataObjectType.VOLUME) {
return processor.copyVolumeFromPrimaryToSecondary(cmd);
} else if (destData.getObjectType() == DataObjectType.TEMPLATE) {
return processor.createTemplateFromVolume(cmd);

18
core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
View File

@ -38,6 +38,8 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
private Integer connectTimeout;
private Integer soTimeout;
private Integer connectionRequestTimeout;
private Long templateSize;
private boolean iso;
protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map<String, String> headers, final Integer connectTimeout, final Integer soTimeout, final Integer connectionRequestTimeout) {
this.url = url;
@ -94,6 +96,22 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
this.connectionRequestTimeout = connectionRequestTimeout;
}
public Long getTemplateSize() {
return templateSize;
}
public void setTemplateSize(Long templateSize) {
this.templateSize = templateSize;
}
public boolean isIso() {
return iso;
}
public void setIso(boolean iso) {
this.iso = iso;
}
@Override
public void setExecuteInSequence(boolean inSeq) {
}

6
core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
View File

@ -61,6 +61,7 @@ public class VolumeObjectTO implements DataTO {
private DiskCacheMode cacheMode;
private Hypervisor.HypervisorType hypervisorType;
private MigrationOptions migrationOptions;
private boolean directDownload;
public VolumeObjectTO() {
@ -100,6 +101,7 @@ public class VolumeObjectTO implements DataTO {
hypervisorType = volume.getHypervisorType();
setDeviceId(volume.getDeviceId());
this.migrationOptions = volume.getMigrationOptions();
this.directDownload = volume.isDirectDownload();
}
public String getUuid() {
@ -307,4 +309,8 @@ public class VolumeObjectTO implements DataTO {
public MigrationOptions getMigrationOptions() {
return migrationOptions;
}
public boolean isDirectDownload() {
return directDownload;
}
}

4
engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java
View File

@ -79,4 +79,8 @@ public interface VolumeInfo extends DataObject, Volume {
MigrationOptions getMigrationOptions();
void setMigrationOptions(MigrationOptions migrationOptions);
boolean isDirectDownload();
void setDirectDownload(boolean directDownload);
}

18
engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
View File

@ -325,7 +325,25 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
Scope destScope = getZoneScope(destData.getDataStore().getScope());
DataStore cacheStore = cacheMgr.getCacheStorage(destScope);
boolean bypassSecondaryStorage = false;
if (srcData instanceof VolumeInfo && ((VolumeInfo)srcData).isDirectDownload()) {
bypassSecondaryStorage = true;
}
if (cacheStore == null) {
if (bypassSecondaryStorage) {
CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _copyvolumewait, VirtualMachineManager.ExecuteInSequence.value());
EndPoint ep = selector.select(srcData, destData);
Answer answer = null;
if (ep == null) {
String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
s_logger.error(errMsg);
answer = new Answer(cmd, false, errMsg);
} else {
answer = ep.sendMessage(cmd);
}
return answer;
}
// need to find a nfs or cifs image store, assuming that can't copy volume
// directly to s3
ImageStoreEntity imageStore = (ImageStoreEntity)dataStoreMgr.getImageStoreWithFreeCapacity(destScope.getScopeId());

8
engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
View File

@ -84,6 +84,12 @@ public class DefaultEndPointSelector implements EndPointSelector {
}
}
private boolean moveBetweenPrimaryDirectDownload(DataStore srcStore, DataStore destStore) {
DataStoreRole srcRole = srcStore.getRole();
DataStoreRole destRole = destStore.getRole();
return srcRole == DataStoreRole.Primary && destRole == DataStoreRole.Primary;
}
protected boolean moveBetweenCacheAndImage(DataStore srcStore, DataStore destStore) {
DataStoreRole srcRole = srcStore.getRole();
DataStoreRole destRole = destStore.getRole();
@ -182,6 +188,8 @@ public class DefaultEndPointSelector implements EndPointSelector {
DataStore destStore = destData.getDataStore();
if (moveBetweenPrimaryImage(srcStore, destStore)) {
return findEndPointForImageMove(srcStore, destStore);
} else if (moveBetweenPrimaryDirectDownload(srcStore, destStore)) {
return findEndPointForImageMove(srcStore, destStore);
} else if (moveBetweenCacheAndImage(srcStore, destStore)) {
// pick ssvm based on image cache dc
DataStore selectedStore = null;

10
engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java
View File

@ -23,6 +23,8 @@ import java.util.List;
import javax.inject.Inject;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.VMTemplateDao;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@ -42,6 +44,8 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory {
VolumeDataStoreDao volumeStoreDao;
@Inject
DataStoreManager storeMgr;
@Inject
VMTemplateDao templateDao;
@Override
public VolumeInfo getVolume(long volumeId, DataStore store) {
@ -90,6 +94,12 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory {
DataStore store = storeMgr.getDataStore(volumeVO.getPoolId(), DataStoreRole.Primary);
vol = VolumeObject.getVolumeObject(store, volumeVO);
}
if (vol.getTemplateId() != null) {
VMTemplateVO template = templateDao.findById(vol.getTemplateId());
if (template != null) {
vol.setDirectDownload(template.isDirectDownload());
}
}
return vol;
}

11
engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
View File

@ -74,6 +74,7 @@ public class VolumeObject implements VolumeInfo {
DiskOfferingDao diskOfferingDao;
private Object payload;
private MigrationOptions migrationOptions;
private boolean directDownload;
public VolumeObject() {
_volStateMachine = Volume.State.getStateMachine();
@ -327,6 +328,16 @@ public class VolumeObject implements VolumeInfo {
this.migrationOptions = migrationOptions;
}
@Override
public boolean isDirectDownload() {
return directDownload;
}
@Override
public void setDirectDownload(boolean directDownload) {
this.directDownload = directDownload;
}
public void update() {
volumeDao.update(volumeVO.getId(), volumeVO);
volumeVO = volumeDao.findById(volumeVO.getId());

50
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View File

@ -478,6 +478,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
return _ovsPvlanVmPath;
}
public String getDirectDownloadTemporaryDownloadPath() {
return directDownloadTemporaryDownloadPath;
}
public String getResizeVolumePath() {
return _resizeVolumePath;
}
@ -530,6 +534,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
protected boolean dpdkSupport = false;
protected String dpdkOvsPath;
protected String directDownloadTemporaryDownloadPath;
private String getEndIpFromStartIp(final String startIp, final int numIps) {
final String[] tokens = startIp.split("[.]");
@ -577,6 +582,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
}
}
private String getDefaultDirectDownloadTemporaryPath() {
return "/var/lib/libvirt/images";
}
protected String getDefaultNetworkScriptsDir() {
return "scripts/vm/network/vnet";
}
@ -656,6 +665,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
}
}
directDownloadTemporaryDownloadPath = (String) params.get("direct.download.temporary.download.location");
if (org.apache.commons.lang.StringUtils.isBlank(directDownloadTemporaryDownloadPath)) {
directDownloadTemporaryDownloadPath = getDefaultDirectDownloadTemporaryPath();
}
params.put("domr.scripts.dir", domrScriptsDir);
_virtRouterResource = new VirtualRoutingResource(this);
@ -2320,18 +2334,20 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
if (dataStore instanceof NfsTO) {
NfsTO nfsStore = (NfsTO)data.getDataStore();
dataStoreUrl = nfsStore.getUrl();
} else if (dataStore instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO) dataStore).getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
physicalDisk = getPhysicalDiskFromNfsStore(dataStoreUrl, data);
} else if (dataStore instanceof PrimaryDataStoreTO) {
//In order to support directly downloaded ISOs
String psHost = ((PrimaryDataStoreTO) dataStore).getHost();
String psPath = ((PrimaryDataStoreTO) dataStore).getPath();
dataStoreUrl = "nfs://" + psHost + File.separator + psPath;
PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) dataStore;
if (primaryDataStoreTO.getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
String psHost = primaryDataStoreTO.getHost();
String psPath = primaryDataStoreTO.getPath();
dataStoreUrl = "nfs://" + psHost + File.separator + psPath;
physicalDisk = getPhysicalDiskFromNfsStore(dataStoreUrl, data);
} else if (primaryDataStoreTO.getPoolType().equals(StoragePoolType.SharedMountPoint) ||
primaryDataStoreTO.getPoolType().equals(StoragePoolType.Filesystem)) {
physicalDisk = getPhysicalDiskPrimaryStore(primaryDataStoreTO, data);
}
}
final String volPath = dataStoreUrl + File.separator + data.getPath();
final int index = volPath.lastIndexOf("/");
final String volDir = volPath.substring(0, index);
final String volName = volPath.substring(index + 1);
final KVMStoragePool secondaryStorage = _storagePoolMgr.getStoragePoolByURI(volDir);
physicalDisk = secondaryStorage.getPhysicalDisk(volName);
} else if (volume.getType() != Volume.Type.ISO) {
final PrimaryDataStoreTO store = (PrimaryDataStoreTO)data.getDataStore();
physicalDisk = _storagePoolMgr.getPhysicalDisk(store.getPoolType(), store.getUuid(), data.getPath());
@ -2472,6 +2488,20 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
}
private KVMPhysicalDisk getPhysicalDiskPrimaryStore(PrimaryDataStoreTO primaryDataStoreTO, DataTO data) {
KVMStoragePool storagePool = _storagePoolMgr.getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid());
return storagePool.getPhysicalDisk(data.getPath());
}
private KVMPhysicalDisk getPhysicalDiskFromNfsStore(String dataStoreUrl, DataTO data) {
final String volPath = dataStoreUrl + File.separator + data.getPath();
final int index = volPath.lastIndexOf("/");
final String volDir = volPath.substring(0, index);
final String volName = volPath.substring(index + 1);
final KVMStoragePool storage = _storagePoolMgr.getStoragePoolByURI(volDir);
return storage.getPhysicalDisk(volName);
}
private void setBurstProperties(final VolumeObjectTO volumeObjectTO, final DiskDef disk ) {
if (volumeObjectTO.getBytesReadRate() != null && volumeObjectTO.getBytesReadRate() > 0) {
disk.setBytesReadRate(volumeObjectTO.getBytesReadRate());

5
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java
View File

@ -445,4 +445,9 @@ public class IscsiAdmStorageAdaptor implements StorageAdaptor {
public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout) {
return null;
}
@Override
public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, KVMStoragePool destPool, boolean isIso) {
return null;
}
}

5
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java
View File

@ -405,4 +405,9 @@ public class KVMStoragePoolManager {
return adaptor.createDiskFromTemplateBacking(template, name, format, size, destPool, timeout);
}
public KVMPhysicalDisk createPhysicalDiskFromDirectDownloadTemplate(String templateFilePath, KVMStoragePool destPool, boolean isIso) {
StorageAdaptor adaptor = getStorageAdaptor(destPool.getType());
return adaptor.createTemplateFromDirectDownloadFile(templateFilePath, destPool, isIso);
}
}

125
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View File

@ -36,6 +36,7 @@ import java.util.UUID;
import javax.naming.ConfigurationException;
import com.cloud.utils.Pair;
import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
@ -89,7 +90,6 @@ import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.direct.download.DirectTemplateDownloader;
import com.cloud.agent.direct.download.DirectTemplateDownloader.DirectTemplateInformation;
import com.cloud.agent.direct.download.HttpDirectTemplateDownloader;
import com.cloud.agent.direct.download.HttpsDirectTemplateDownloader;
import com.cloud.agent.direct.download.MetalinkDirectTemplateDownloader;
@ -1676,15 +1676,20 @@ public class KVMStorageProcessor implements StorageProcessor {
/**
* Get direct template downloader from direct download command and destination pool
*/
private DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd, KVMStoragePool destPool) {
private DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd,
KVMStoragePool destPool,
String temporaryDownloadPath) {
if (cmd instanceof HttpDirectDownloadCommand) {
return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout());
return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
} else if (cmd instanceof HttpsDirectDownloadCommand) {
return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout());
return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath);
} else if (cmd instanceof NfsDirectDownloadCommand) {
return new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum());
return new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath);
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout());
return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(),
cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath);
} else {
throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink");
}
@ -1693,38 +1698,112 @@ public class KVMStorageProcessor implements StorageProcessor {
@Override
public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
final PrimaryDataStoreTO pool = cmd.getDestPool();
if (!pool.getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
return new DirectDownloadAnswer(false, "Unsupported pool type " + pool.getPoolType().toString(), true);
}
KVMStoragePool destPool = storagePoolMgr.getStoragePool(pool.getPoolType(), pool.getUuid());
DirectTemplateDownloader downloader;
KVMPhysicalDisk template;
try {
downloader = getDirectTemplateDownloaderFromCommand(cmd, destPool);
} catch (IllegalArgumentException e) {
return new DirectDownloadAnswer(false, "Unable to create direct downloader: " + e.getMessage(), true);
}
s_logger.debug("Verifying temporary location for downloading the template exists on the host");
String temporaryDownloadPath = resource.getDirectDownloadTemporaryDownloadPath();
if (!isLocationAccessible(temporaryDownloadPath)) {
String msg = "The temporary location path for downloading templates does not exist: " +
temporaryDownloadPath + " on this host";
s_logger.error(msg);
return new DirectDownloadAnswer(false, msg, true);
}
try {
s_logger.info("Trying to download template");
if (!downloader.downloadTemplate()) {
s_logger.debug("Checking for free space on the host for downloading the template");
if (!isEnoughSpaceForDownloadTemplateOnTemporaryLocation(cmd.getTemplateSize())) {
String msg = "Not enough space on the defined temporary location to download the template " + cmd.getTemplateId();
s_logger.error(msg);
return new DirectDownloadAnswer(false, msg, true);
}
KVMStoragePool destPool = storagePoolMgr.getStoragePool(pool.getPoolType(), pool.getUuid());
downloader = getDirectTemplateDownloaderFromCommand(cmd, destPool, temporaryDownloadPath);
s_logger.debug("Trying to download template");
Pair<Boolean, String> result = downloader.downloadTemplate();
if (!result.first()) {
s_logger.warn("Couldn't download template");
return new DirectDownloadAnswer(false, "Unable to download template", true);
}
String tempFilePath = result.second();
if (!downloader.validateChecksum()) {
s_logger.warn("Couldn't validate template checksum");
return new DirectDownloadAnswer(false, "Checksum validation failed", false);
}
if (!downloader.extractAndInstallDownloadedTemplate()) {
s_logger.warn("Couldn't extract and install template");
return new DirectDownloadAnswer(false, "Extraction and installation failed", false);
}
template = storagePoolMgr.createPhysicalDiskFromDirectDownloadTemplate(tempFilePath, destPool, cmd.isIso());
} catch (CloudRuntimeException e) {
s_logger.warn("Error downloading template " + cmd.getTemplateId() + " due to: " + e.getMessage());
return new DirectDownloadAnswer(false, "Unable to download template: " + e.getMessage(), true);
} catch (IllegalArgumentException e) {
return new DirectDownloadAnswer(false, "Unable to create direct downloader: " + e.getMessage(), true);
}
DirectTemplateInformation info = downloader.getTemplateInformation();
return new DirectDownloadAnswer(true, info.getSize(), info.getInstallPath());
return new DirectDownloadAnswer(true, template.getSize(), template.getName());
}
@Override
public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) {
final DataTO srcData = cmd.getSrcTO();
final DataTO destData = cmd.getDestTO();
final VolumeObjectTO srcVol = (VolumeObjectTO)srcData;
final VolumeObjectTO destVol = (VolumeObjectTO)destData;
final ImageFormat srcFormat = srcVol.getFormat();
final ImageFormat destFormat = destVol.getFormat();
final DataStoreTO srcStore = srcData.getDataStore();
final DataStoreTO destStore = destData.getDataStore();
final PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)srcStore;
final PrimaryDataStoreTO primaryStoreDest = (PrimaryDataStoreTO)destStore;
final String srcVolumePath = srcData.getPath();
final String destVolumePath = destData.getPath();
KVMStoragePool destPool = null;
try {
final String volumeName = UUID.randomUUID().toString();
final String destVolumeName = volumeName + "." + destFormat.getFileExtension();
final KVMPhysicalDisk volume = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), srcVolumePath);
volume.setFormat(PhysicalDiskFormat.valueOf(srcFormat.toString()));
destPool = storagePoolMgr.getStoragePool(primaryStoreDest.getPoolType(), primaryStoreDest.getUuid());
storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds());
final VolumeObjectTO newVol = new VolumeObjectTO();
newVol.setPath(destVolumePath + File.separator + destVolumeName);
newVol.setFormat(destFormat);
return new CopyCmdAnswer(newVol);
} catch (final CloudRuntimeException e) {
s_logger.debug("Failed to copyVolumeFromPrimaryToPrimary: ", e);
return new CopyCmdAnswer(e.toString());
}
}
/**
* True if location exists
*/
private boolean isLocationAccessible(String temporaryDownloadPath) {
File dir = new File(temporaryDownloadPath);
return dir.exists();
}
/**
* Perform a free space check on the host for downloading the direct download templates
* @param templateSize template size obtained from remote server when registering the template (in bytes)
*/
protected boolean isEnoughSpaceForDownloadTemplateOnTemporaryLocation(Long templateSize) {
if (templateSize == null || templateSize == 0L) {
s_logger.info("The server did not provide the template size, assuming there is enough space to download it");
return true;
}
String cmd = String.format("df --output=avail %s -B 1 | tail -1", resource.getDirectDownloadTemporaryDownloadPath());
String resultInBytes = Script.runSimpleBashScript(cmd);
Long availableBytes;
try {
availableBytes = Long.parseLong(resultInBytes);
} catch (NumberFormatException e) {
String msg = "Could not parse the output " + resultInBytes + " as a number, therefore not able to check for free space";
s_logger.error(msg, e);
return false;
}
return availableBytes >= templateSize;
}
}

60
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java
View File

@ -122,6 +122,64 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
return disk;
}
/**
* Checks if downloaded template is extractable
* @return true if it should be extracted, false if not
*/
private boolean isTemplateExtractable(String templatePath) {
String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'");
return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
}
/**
* Return extract command to execute given downloaded file
* @param downloadedTemplateFile
* @param templateUuid
*/
private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateUuid) {
if (downloadedTemplateFile.endsWith(".zip")) {
return "unzip -p " + downloadedTemplateFile + " | cat > " + templateUuid;
} else if (downloadedTemplateFile.endsWith(".bz2")) {
return "bunzip2 -c " + downloadedTemplateFile + " > " + templateUuid;
} else if (downloadedTemplateFile.endsWith(".gz")) {
return "gunzip -c " + downloadedTemplateFile + " > " + templateUuid;
} else {
throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile);
}
}
/**
* Extract downloaded template into installPath, remove compressed file
*/
private void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, String destinationFile) {
String extractCommand = getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile);
Script.runSimpleBashScript(extractCommand);
Script.runSimpleBashScript("rm -f " + downloadedTemplateFile);
}
@Override
public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, KVMStoragePool destPool, boolean isIso) {
File sourceFile = new File(templateFilePath);
if (!sourceFile.exists()) {
throw new CloudRuntimeException("Direct download template file " + sourceFile + " does not exist on this host");
}
String templateUuid = UUID.randomUUID().toString();
if (isIso) {
templateUuid += ".iso";
}
String destinationFile = destPool.getLocalPath() + File.separator + templateUuid;
if (destPool.getType() == StoragePoolType.NetworkFilesystem || destPool.getType() == StoragePoolType.Filesystem
|| destPool.getType() == StoragePoolType.SharedMountPoint) {
if (!isIso && isTemplateExtractable(templateFilePath)) {
extractDownloadedTemplate(templateFilePath, destPool, destinationFile);
} else {
Script.runSimpleBashScript("mv " + templateFilePath + " " + destinationFile);
}
}
return destPool.getPhysicalDisk(templateUuid);
}
public StorageVol getVolume(StoragePool pool, String volName) {
StorageVol vol = null;
@ -1198,7 +1256,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
if (disk.getFormat() == PhysicalDiskFormat.TAR) {
newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, Storage.ProvisioningType.THIN, disk.getVirtualSize());
} else {
newDisk = destPool.createPhysicalDisk(name, Storage.ProvisioningType.THIN, disk.getVirtualSize());
newDisk = destPool.createPhysicalDisk(name, Storage.ProvisioningType.THIN, disk.getVirtualSize());
}
} else {
newDisk = new KVMPhysicalDisk(destPool.getSourceDir() + "/" + name, name, destPool);

5
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java
View File

@ -318,6 +318,11 @@ public class ManagedNfsStorageAdaptor implements StorageAdaptor {
return null;
}
@Override
public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, KVMStoragePool destPool, boolean isIso) {
return null;
}
@Override
public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, PhysicalDiskFormat format, ProvisioningType provisioningType, long size) {
return null;

8
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java
View File

@ -81,4 +81,12 @@ public interface StorageAdaptor {
KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template,
String name, PhysicalDiskFormat format, long size,
KVMStoragePool destPool, int timeout);
/**
* Create physical disk on Primary Storage from direct download template on the host (in temporary location)
* @param templateFilePath
* @param destPool
* @param isIso
*/
KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, KVMStoragePool destPool, boolean isIso);
}

46
plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java
View File

@ -21,13 +21,22 @@ package com.cloud.hypervisor.kvm.storage;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import javax.naming.ConfigurationException;
import com.cloud.utils.script.Script;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Matchers;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.Spy;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@PrepareForTest({ Script.class })
@RunWith(PowerMockRunner.class)
public class KVMStorageProcessorTest {
@Mock
@ -35,26 +44,47 @@ public class KVMStorageProcessorTest {
@Mock
LibvirtComputingResource resource;
private static final Long TEMPLATE_ID = 202l;
private static final String EXPECTED_DIRECT_DOWNLOAD_DIR = "template/2/202";
@Spy
@InjectMocks
private KVMStorageProcessor storageProcessor;
private static final String directDownloadTemporaryPath = "/var/lib/libvirt/images/dd";
private static final long templateSize = 80000L;
@Before
public void setUp() throws ConfigurationException {
MockitoAnnotations.initMocks(this);
storageProcessor = new KVMStorageProcessor(storagePoolManager, resource);
PowerMockito.mockStatic(Script.class);
Mockito.when(resource.getDirectDownloadTemporaryDownloadPath()).thenReturn(directDownloadTemporaryPath);
}
@Test
public void testCloneVolumeFromBaseTemplate() throws Exception {
public void testIsEnoughSpaceForDownloadTemplateOnTemporaryLocationAssumeEnoughSpaceWhenNotProvided() {
boolean result = storageProcessor.isEnoughSpaceForDownloadTemplateOnTemporaryLocation(null);
Assert.assertTrue(result);
}
@Test
public void testCopyVolumeFromImageCacheToPrimary() throws Exception {
public void testIsEnoughSpaceForDownloadTemplateOnTemporaryLocationNotEnoughSpace() {
String output = String.valueOf(templateSize - 30000L);
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).thenReturn(output);
boolean result = storageProcessor.isEnoughSpaceForDownloadTemplateOnTemporaryLocation(templateSize);
Assert.assertFalse(result);
}
@Test
public void testIsEnoughSpaceForDownloadTemplateOnTemporaryLocationEnoughSpace() {
String output = String.valueOf(templateSize + 30000L);
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).thenReturn(output);
boolean result = storageProcessor.isEnoughSpaceForDownloadTemplateOnTemporaryLocation(templateSize);
Assert.assertTrue(result);
}
@Test
public void testIsEnoughSpaceForDownloadTemplateOnTemporaryLocationNotExistingLocation() {
String output = String.format("df: ‘%s’: No such file or directory", directDownloadTemporaryPath);
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).thenReturn(output);
boolean result = storageProcessor.isEnoughSpaceForDownloadTemplateOnTemporaryLocation(templateSize);
Assert.assertFalse(result);
}
}

5
plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java
View File

@ -826,6 +826,11 @@ public class Ovm3StorageProcessor implements StorageProcessor {
return null;
}
@Override
public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) {
return null;
}
/**
* Attach disks
* @param cmd

5
plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java
View File

@ -264,4 +264,9 @@ public class SimulatorStorageProcessor implements StorageProcessor {
// TODO Auto-generated method stub
return null;
}
@Override
public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) {
return null;
}
}

5
plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java
View File

@ -3554,4 +3554,9 @@ public class VmwareStorageProcessor implements StorageProcessor {
public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
return null;
}
@Override
public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) {
return null;
}
}

5
plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
View File

@ -209,6 +209,11 @@ public class XenServerStorageProcessor implements StorageProcessor {
return null;
}
@Override
public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) {
return null;
}
@Override
public AttachAnswer attachIso(final AttachCommand cmd) {
final DiskTO disk = cmd.getDisk();

70
server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
View File

@ -20,6 +20,32 @@ package org.apache.cloudstack.direct.download;
import static com.cloud.storage.Storage.ImageFormat;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.event.ActionEventUtils;
import com.cloud.event.EventTypes;
import com.cloud.event.EventVO;
import com.cloud.exception.AgentUnavailableException;
import com.cloud.exception.OperationTimedoutException;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage;
import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.exception.CloudRuntimeException;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.cert.Certificate;
@ -27,7 +53,6 @@ import java.security.cert.CertificateException;
import java.security.cert.CertificateExpiredException;
import java.security.cert.CertificateNotYetValidException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -67,29 +92,6 @@ import org.apache.log4j.Logger;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.event.ActionEventUtils;
import com.cloud.event.EventTypes;
import com.cloud.event.EventVO;
import com.cloud.exception.AgentUnavailableException;
import com.cloud.exception.OperationTimedoutException;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.security.CertificateHelper;
import sun.security.x509.X509CertImpl;
@ -202,7 +204,7 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
*/
protected Long[] createHostIdsList(List<Long> hostIds, long hostId) {
if (CollectionUtils.isEmpty(hostIds)) {
return Arrays.asList(hostId).toArray(new Long[1]);
return Collections.singletonList(hostId).toArray(new Long[1]);
}
Long[] ids = new Long[hostIds.size() + 1];
ids[0] = hostId;
@ -215,11 +217,15 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
}
/**
* Get hosts to retry download having hostId as the first element
* Get alternative hosts to retry downloading a template. The planner have previously selected a host and a storage pool
* @return array of host ids which can access the storage pool
*/
protected Long[] getHostsToRetryOn(Long clusterId, long dataCenterId, HypervisorType hypervisorType, long hostId) {
List<Long> hostIds = getRunningHostIdsInTheSameCluster(clusterId, dataCenterId, hypervisorType, hostId);
return createHostIdsList(hostIds, hostId);
protected Long[] getHostsToRetryOn(Host host, StoragePoolVO storagePool) {
List<Long> clusterHostIds = new ArrayList<>();
if (storagePool.getPoolType() != Storage.StoragePoolType.Filesystem || storagePool.getScope() != ScopeType.HOST) {
clusterHostIds = getRunningHostIdsInTheSameCluster(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType(), host.getId());
}
return createHostIdsList(clusterHostIds, host.getId());
}
@Override
@ -252,6 +258,8 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
DownloadProtocol protocol = getProtocolFromUrl(url);
DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers);
cmd.setTemplateSize(template.getSize());
cmd.setIso(template.getFormat() == ImageFormat.ISO);
Answer answer = sendDirectDownloadCommand(cmd, template, poolId, host);
@ -284,7 +292,9 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown
private Answer sendDirectDownloadCommand(DirectDownloadCommand cmd, VMTemplateVO template, long poolId, HostVO host) {
boolean downloaded = false;
int retry = 3;
Long[] hostsToRetry = getHostsToRetryOn(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType(), host.getId());
StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId);
Long[] hostsToRetry = getHostsToRetryOn(host, storagePoolVO);
int hostIndex = 0;
Answer answer = null;
Long hostToSendDownloadCmd = hostsToRetry[hostIndex];

153
test/integration/smoke/test_direct_download.py
View File

@ -23,12 +23,14 @@ from marvin.lib.base import (ServiceOffering,
NetworkOffering,
Network,
Template,
VirtualMachine)
VirtualMachine,
StoragePool)
from marvin.lib.common import (get_pod,
get_zone)
from nose.plugins.attrib import attr
from marvin.cloudstackAPI import (uploadTemplateDirectDownloadCertificate, revokeTemplateDirectDownloadCertificate)
from marvin.lib.decoratorGenerators import skipTestIf
import uuid
class TestUploadDirectDownloadCertificates(cloudstackTestCase):
@ -90,7 +92,7 @@ class TestUploadDirectDownloadCertificates(cloudstackTestCase):
cmd = uploadTemplateDirectDownloadCertificate.uploadTemplateDirectDownloadCertificateCmd()
cmd.hypervisor = self.hypervisor
cmd.name = "marvin-test-verify-certs"
cmd.name = "marvin-test-verify-certs" + str(uuid.uuid1())
cmd.certificate = self.certificates["invalid"]
cmd.zoneid = self.zone.id
@ -125,7 +127,7 @@ class TestUploadDirectDownloadCertificates(cloudstackTestCase):
cmd = uploadTemplateDirectDownloadCertificate.uploadTemplateDirectDownloadCertificateCmd()
cmd.hypervisor = self.hypervisor
cmd.name = "marvin-test-verify-certs"
cmd.name = "marvin-test-verify-certs" + str(uuid.uuid1())
cmd.certificate = self.certificates["valid"]
cmd.zoneid = self.zone.id
@ -160,11 +162,15 @@ class TestDirectDownloadTemplates(cloudstackTestCase):
cls.services = cls.testClient.getParsedTestDataConfig()
cls._cleanup = []
cls.hypervisorNotSupported = False
if cls.hypervisor.lower() not in ['kvm', 'lxc']:
cls.hypervisorNotSupported = True
cls.hypervisorSupported = False
cls.nfsStorageFound = False
cls.localStorageFound = False
cls.sharedMountPointFound = False
if not cls.hypervisorNotSupported:
if cls.hypervisor.lower() in ['kvm', 'lxc']:
cls.hypervisorSupported = True
if cls.hypervisorSupported:
cls.services["test_templates"]["kvm"]["directdownload"] = "true"
cls.template = Template.register(cls.apiclient, cls.services["test_templates"]["kvm"],
zoneid=cls.zone.id, hypervisor=cls.hypervisor)
@ -192,6 +198,25 @@ class TestDirectDownloadTemplates(cloudstackTestCase):
)
cls._cleanup.append(cls.l2_network)
cls._cleanup.append(cls.network_offering)
storage_pools = StoragePool.list(
cls.apiclient,
zoneid=cls.zone.id
)
for pool in storage_pools:
if not cls.nfsStorageFound and pool.type == "NetworkFilesystem":
cls.nfsStorageFound = True
cls.nfsPoolId = pool.id
elif not cls.localStorageFound and pool.type == "Filesystem":
cls.localStorageFound = True
cls.localPoolId = pool.id
elif not cls.sharedMountPointFound and pool.type == "SharedMountPoint":
cls.sharedMountPointFound = True
cls.sharedPoolId = pool.id
cls.nfsKvmNotAvailable = not cls.hypervisorSupported or not cls.nfsStorageFound
cls.localStorageKvmNotAvailable = not cls.hypervisorSupported or not cls.localStorageFound
cls.sharedMountPointKvmNotAvailable = not cls.hypervisorSupported or not cls.sharedMountPointFound
return
@classmethod
@ -215,26 +240,124 @@ class TestDirectDownloadTemplates(cloudstackTestCase):
raise Exception("Warning: Exception during cleanup : %s" % e)
return
@skipTestIf("hypervisorNotSupported")
def getCurrentStoragePoolTags(self, poolId):
local_pool = StoragePool.list(
self.apiclient,
id=poolId
)
return local_pool[0].tags
def updateStoragePoolTags(self, poolId, tags):
StoragePool.update(
self.apiclient,
id=poolId,
tags=tags
)
def createServiceOffering(self, name, type, tags):
services = {
"cpunumber": 1,
"cpuspeed": 512,
"memory": 256,
"displaytext": name,
"name": name,
"storagetype": type
}
return ServiceOffering.create(
self.apiclient,
services,
tags=tags
)
@skipTestIf("nfsKvmNotAvailable")
@attr(tags=["advanced", "basic", "eip", "advancedns", "sg"], required_hardware="false")
def test_01_deploy_vm_from_direct_download_template(self):
"""Test Deploy VM from direct download template
def test_01_deploy_vm_from_direct_download_template_nfs_storage(self):
"""Test Deploy VM from direct download template on NFS storage
"""
# Validate the following
# 1. Register direct download template
# 2. Deploy VM from direct download template
# Create service offering for local storage using storage tags
tags = self.getCurrentStoragePoolTags(self.nfsPoolId)
test_tag = "marvin_test_nfs_storage_direct_download"
self.updateStoragePoolTags(self.nfsPoolId, test_tag)
nfs_storage_offering = self.createServiceOffering("TestNFSStorageDirectDownload", "shared", test_tag)
vm = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
serviceofferingid=self.service_offering.id,
serviceofferingid=nfs_storage_offering.id,
networkids=self.l2_network.id
)
self.assertEqual(
vm.state,
"Running",
"Check VM deployed from direct download template is running"
"Check VM deployed from direct download template is running on NFS storage"
)
# Revert storage tags for the storage pool used in this test
self.updateStoragePoolTags(self.nfsPoolId, tags)
self.cleanup.append(vm)
self.cleanup.append(nfs_storage_offering)
return
@skipTestIf("localStorageKvmNotAvailable")
@attr(tags=["advanced", "basic", "eip", "advancedns", "sg"], required_hardware="false")
def test_02_deploy_vm_from_direct_download_template_local_storage(self):
"""Test Deploy VM from direct download template on local storage
"""
# Create service offering for local storage using storage tags
tags = self.getCurrentStoragePoolTags(self.localPoolId)
test_tag = "marvin_test_local_storage_direct_download"
self.updateStoragePoolTags(self.localPoolId, test_tag)
local_storage_offering = self.createServiceOffering("TestLocalStorageDirectDownload", "local", test_tag)
# Deploy VM
vm = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
serviceofferingid=local_storage_offering.id,
networkids=self.l2_network.id,
)
self.assertEqual(
vm.state,
"Running",
"Check VM deployed from direct download template is running on local storage"
)
# Revert storage tags for the storage pool used in this test
self.updateStoragePoolTags(self.localPoolId, tags)
self.cleanup.append(vm)
self.cleanup.append(local_storage_offering)
return
@skipTestIf("sharedMountPointKvmNotAvailable")
@attr(tags=["advanced", "basic", "eip", "advancedns", "sg"], required_hardware="false")
def test_03_deploy_vm_from_direct_download_template_shared_mount_point_storage(self):
"""Test Deploy VM from direct download template on shared mount point
"""
# Create service offering for local storage using storage tags
tags = self.getCurrentStoragePoolTags(self.sharedPoolId)
test_tag = "marvin_test_shared_mount_point_storage_direct_download"
self.updateStoragePoolTags(self.sharedPoolId, test_tag)
shared_offering = self.createServiceOffering("TestSharedMountPointStorageDirectDownload", "shared", test_tag)
# Deploy VM
vm = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
serviceofferingid=shared_offering.id,
networkids=self.l2_network.id,
)
self.assertEqual(
vm.state,
"Running",
"Check VM deployed from direct download template is running on shared mount point"
)
# Revert storage tags for the storage pool used in this test
self.updateStoragePoolTags(self.sharedPoolId, tags)
self.cleanup.append(vm)
self.cleanup.append(shared_offering)
return