diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 7472e392227..24592387b09 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -115,6 +115,9 @@ domr.scripts.dir=scripts/network/domr/kvm # set the hypervisor type, values are: kvm, lxc hypervisor.type=kvm +# This parameter specifies a directory on the host local storage for temporary storing direct download templates +#direct.download.temporary.download.location=/var/lib/libvirt/images + # set the hypervisor URI. Usually there is no need for changing this # For KVM: qemu:///system # For LXC: lxc:/// diff --git a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java index a88b4526e9d..32b84a34436 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java +++ b/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloader.java @@ -19,49 +19,15 @@ package com.cloud.agent.direct.download; +import com.cloud.utils.Pair; + public interface DirectTemplateDownloader { - class DirectTemplateInformation { - private String installPath; - private Long size; - private String checksum; - - public DirectTemplateInformation(String installPath, Long size, String checksum) { - this.installPath = installPath; - this.size = size; - this.checksum = checksum; - } - - public String getInstallPath() { - return installPath; - } - - public Long getSize() { - return size; - } - - public String getChecksum() { - return checksum; - } - } - /** * Perform template download to pool specified on downloader creation - * @return true if successful, false if not + * @return (true if successful, false if not, download file path) */ - boolean downloadTemplate(); - - /** - * Perform extraction (if necessary) and installation of previously downloaded template - * @return true if successful, false if not - */ - boolean extractAndInstallDownloadedTemplate(); - - /** - * Get template information after it is properly installed on pool - * @return template information - */ - DirectTemplateInformation getTemplateInformation(); + Pair downloadTemplate(); /** * Perform checksum validation of previously downloadeed template diff --git a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java b/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java index 419ab7d1bbd..9c150e92a98 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java +++ b/agent/src/main/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java @@ -19,7 +19,6 @@ package com.cloud.agent.direct.download; import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.Script; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; @@ -28,7 +27,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.security.NoSuchAlgorithmException; -import java.util.UUID; public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader { @@ -36,16 +34,19 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown private String destPoolPath; private Long templateId; private String downloadedFilePath; - private String installPath; private String checksum; private boolean redownload = false; + protected String temporaryDownloadPath; + public static final Logger s_logger = Logger.getLogger(DirectTemplateDownloaderImpl.class.getName()); - protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, final String checksum) { + protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, + final String checksum, final String temporaryDownloadPath) { this.url = url; this.destPoolPath = destPoolPath; this.templateId = templateId; this.checksum = checksum; + this.temporaryDownloadPath = temporaryDownloadPath; } private static String directDownloadDir = "template"; @@ -53,10 +54,10 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown /** * Return direct download temporary path to download template */ - protected static String getDirectDownloadTempPath(Long templateId) { + protected String getDirectDownloadTempPath(Long templateId) { String templateIdAsString = String.valueOf(templateId); - return directDownloadDir + File.separator + templateIdAsString.substring(0,1) + - File.separator + templateIdAsString; + return this.temporaryDownloadPath + File.separator + directDownloadDir + File.separator + + templateIdAsString.substring(0,1) + File.separator + templateIdAsString; } /** @@ -113,64 +114,6 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown return urlParts[urlParts.length - 1]; } - /** - * Checks if downloaded template is extractable - * @return true if it should be extracted, false if not - */ - private boolean isTemplateExtractable() { - String type = Script.runSimpleBashScript("file " + downloadedFilePath + " | awk -F' ' '{print $2}'"); - return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); - } - - @Override - public boolean extractAndInstallDownloadedTemplate() { - installPath = UUID.randomUUID().toString(); - if (isTemplateExtractable()) { - extractDownloadedTemplate(); - } else { - Script.runSimpleBashScript("mv " + downloadedFilePath + " " + getInstallFullPath()); - } - return true; - } - - /** - * Return install full path - */ - private String getInstallFullPath() { - return destPoolPath + File.separator + installPath; - } - - /** - * Return extract command to execute given downloaded file - */ - private String getExtractCommandForDownloadedFile() { - if (downloadedFilePath.endsWith(".zip")) { - return "unzip -p " + downloadedFilePath + " | cat > " + getInstallFullPath(); - } else if (downloadedFilePath.endsWith(".bz2")) { - return "bunzip2 -c " + downloadedFilePath + " > " + getInstallFullPath(); - } else if (downloadedFilePath.endsWith(".gz")) { - return "gunzip -c " + downloadedFilePath + " > " + getInstallFullPath(); - } else { - throw new CloudRuntimeException("Unable to extract template " + templateId + " on " + downloadedFilePath); - } - } - - /** - * Extract downloaded template into installPath, remove compressed file - */ - private void extractDownloadedTemplate() { - String extractCommand = getExtractCommandForDownloadedFile(); - Script.runSimpleBashScript(extractCommand); - Script.runSimpleBashScript("rm -f " + downloadedFilePath); - } - - @Override - public DirectTemplateInformation getTemplateInformation() { - String sizeResult = Script.runSimpleBashScript("ls -als " + getInstallFullPath() + " | awk '{print $1}'"); - long size = Long.parseLong(sizeResult); - return new DirectTemplateInformation(installPath, size, checksum); - } - @Override public boolean validateChecksum() { if (StringUtils.isNotBlank(checksum)) { diff --git a/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java index f91af40dd40..fc236034404 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java +++ b/agent/src/main/java/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java @@ -27,6 +27,8 @@ import java.io.OutputStream; import java.util.HashMap; import java.util.Map; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.collections.MapUtils; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpStatus; @@ -35,8 +37,6 @@ import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; -import com.cloud.utils.exception.CloudRuntimeException; - public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected HttpClient client; @@ -45,20 +45,25 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected GetMethod request; protected Map reqHeaders = new HashMap<>(); - public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, Integer connectTimeout, Integer soTimeout) { - super(url, destPoolPath, templateId, checksum); + public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, + Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { + super(url, destPoolPath, templateId, checksum, downloadPath); s_httpClientManager.getParams().setConnectionTimeout(connectTimeout == null ? 5000 : connectTimeout); s_httpClientManager.getParams().setSoTimeout(soTimeout == null ? 5000 : soTimeout); client = new HttpClient(s_httpClientManager); request = createRequest(url, headers); String downloadDir = getDirectDownloadTempPath(templateId); - createTemporaryDirectoryAndFile(downloadDir); + File tempFile = createTemporaryDirectoryAndFile(downloadDir); + setDownloadedFilePath(tempFile.getAbsolutePath()); } - protected void createTemporaryDirectoryAndFile(String downloadDir) { - createFolder(getDestPoolPath() + File.separator + downloadDir); - File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl()); - setDownloadedFilePath(f.getAbsolutePath()); + /** + * Create download directory (if it does not exist) and set the download file + * @return + */ + protected File createTemporaryDirectoryAndFile(String downloadDir) { + createFolder(downloadDir); + return new File(downloadDir + File.separator + getFileNameFromUrl()); } protected GetMethod createRequest(String downloadUrl, Map headers) { @@ -74,12 +79,12 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { } @Override - public boolean downloadTemplate() { + public Pair downloadTemplate() { try { int status = client.executeMethod(request); if (status != HttpStatus.SC_OK) { s_logger.warn("Not able to download template, status code: " + status); - return false; + return new Pair<>(false, null); } return performDownload(); } catch (IOException e) { @@ -89,7 +94,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { } } - protected boolean performDownload() { + protected Pair performDownload() { s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); try ( InputStream in = request.getResponseBodyAsStream(); @@ -98,8 +103,8 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { IOUtils.copy(in, out); } catch (IOException e) { s_logger.error("Error downloading template " + getTemplateId() + " due to: " + e.getMessage()); - return false; + return new Pair<>(false, null); } - return true; + return new Pair<>(true, getDownloadedFilePath()); } } \ No newline at end of file diff --git a/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java index f320846e75d..d788310f68e 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java +++ b/agent/src/main/java/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java @@ -19,6 +19,23 @@ package com.cloud.agent.direct.download; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.commons.collections.MapUtils; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.ssl.SSLContexts; + +import javax.net.ssl.SSLContext; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -32,31 +49,14 @@ import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.util.Map; -import javax.net.ssl.SSLContext; - -import org.apache.commons.collections.MapUtils; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.ssl.SSLContexts; - -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.Script; - public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader { private CloseableHttpClient httpsClient; private HttpUriRequest req; - public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout) { - super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout); + public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map headers, + Integer connectTimeout, Integer soTimeout, Integer connectionRequestTimeout, String temporaryDownloadPath) { + super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, temporaryDownloadPath); SSLContext sslcontext = null; try { sslcontext = getSSLContext(); @@ -98,7 +98,7 @@ public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader } @Override - public boolean downloadTemplate() { + public Pair downloadTemplate() { CloseableHttpResponse response; try { response = httpsClient.execute(req); @@ -111,7 +111,7 @@ public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader /** * Consume response and persist it on getDownloadedFilePath() file */ - protected boolean consumeResponse(CloseableHttpResponse response) { + protected Pair consumeResponse(CloseableHttpResponse response) { s_logger.info("Downloading template " + getTemplateId() + " from " + getUrl() + " to: " + getDownloadedFilePath()); if (response.getStatusLine().getStatusCode() != 200) { throw new CloudRuntimeException("Error on HTTPS response"); @@ -123,9 +123,9 @@ public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader IOUtils.copy(in, out); } catch (Exception e) { s_logger.error("Error parsing response for template " + getTemplateId() + " due to: " + e.getMessage()); - return false; + return new Pair<>(false, null); } - return true; + return new Pair<>(true, getDownloadedFilePath()); } } diff --git a/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java index 0a571015bee..c8e85277913 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java +++ b/agent/src/main/java/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java @@ -18,17 +18,17 @@ // package com.cloud.agent.direct.download; -import java.io.File; -import java.util.List; -import java.util.Map; -import java.util.Random; - +import com.cloud.utils.Pair; +import com.cloud.utils.UriUtils; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import com.cloud.utils.UriUtils; -import com.cloud.utils.exception.CloudRuntimeException; +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Random; public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownloader { @@ -38,8 +38,9 @@ public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownload private Random random = new Random(); private static final Logger s_logger = Logger.getLogger(MetalinkDirectTemplateDownloader.class.getName()); - public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, Map headers, Integer connectTimeout, Integer soTimeout) { - super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout); + public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum, + Map headers, Integer connectTimeout, Integer soTimeout, String downloadPath) { + super(url, templateId, destPoolPath, checksum, headers, connectTimeout, soTimeout, downloadPath); metalinkUrl = url; metalinkUrls = UriUtils.getMetalinkUrls(metalinkUrl); metalinkChecksums = UriUtils.getMetalinkChecksums(metalinkUrl); @@ -53,27 +54,28 @@ public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownload } @Override - public boolean downloadTemplate() { + public Pair downloadTemplate() { if (StringUtils.isBlank(getUrl())) { throw new CloudRuntimeException("Download url has not been set, aborting"); } - String downloadDir = getDirectDownloadTempPath(getTemplateId()); boolean downloaded = false; int i = 0; + String downloadDir = getDirectDownloadTempPath(getTemplateId()); do { if (!isRedownload()) { setUrl(metalinkUrls.get(i)); } s_logger.info("Trying to download template from url: " + getUrl()); try { - File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl()); + setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); + File f = new File(getDownloadedFilePath()); if (f.exists()) { f.delete(); f.createNewFile(); } - setDownloadedFilePath(f.getAbsolutePath()); request = createRequest(getUrl(), reqHeaders); - downloaded = super.downloadTemplate(); + Pair downloadResult = super.downloadTemplate(); + downloaded = downloadResult.first(); if (downloaded) { s_logger.info("Successfully downloaded template from url: " + getUrl()); } @@ -84,7 +86,7 @@ public class MetalinkDirectTemplateDownloader extends HttpDirectTemplateDownload i++; } while (!downloaded && !isRedownload() && i < metalinkUrls.size()); - return downloaded; + return new Pair<>(downloaded, getDownloadedFilePath()); } @Override diff --git a/agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java b/agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java index 16901afedf1..932477031d6 100644 --- a/agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java +++ b/agent/src/main/java/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java @@ -18,6 +18,7 @@ // package com.cloud.agent.direct.download; +import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; @@ -51,13 +52,13 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { } } - public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum) { - super(url, destPool, templateId, checksum); + public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum, String downloadPath) { + super(url, destPool, templateId, checksum, downloadPath); parseUrl(); } @Override - public boolean downloadTemplate() { + public Pair downloadTemplate() { String mountSrcUuid = UUID.randomUUID().toString(); String mount = String.format(mountCommand, srcHost + ":" + srcPath, "/mnt/" + mountSrcUuid); Script.runSimpleBashScript(mount); @@ -65,6 +66,6 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); Script.runSimpleBashScript("cp /mnt/" + mountSrcUuid + srcPath + " " + getDownloadedFilePath()); Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid); - return true; + return new Pair<>(true, getDownloadedFilePath()); } } diff --git a/agent/src/test/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImplTest.java b/agent/src/test/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImplTest.java deleted file mode 100644 index b244d02f499..00000000000 --- a/agent/src/test/java/com/cloud/agent/direct/download/DirectTemplateDownloaderImplTest.java +++ /dev/null @@ -1,36 +0,0 @@ -// -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -package com.cloud.agent.direct.download; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.runners.MockitoJUnitRunner; - -@RunWith(MockitoJUnitRunner.class) -public class DirectTemplateDownloaderImplTest { - - private static final Long templateId = 202l; - - @Test - public void testGetDirectDownloadTempPath() { - String path = DirectTemplateDownloaderImpl.getDirectDownloadTempPath(templateId); - Assert.assertEquals("template/2/202", path); - } -} diff --git a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java index 5d57616b7a4..f940e22f45a 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java @@ -74,4 +74,6 @@ public interface StorageProcessor { public Answer resignature(ResignatureCommand cmd); public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd); + + Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd); } diff --git a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java index 8c0399e9e19..17b9b700d6c 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java @@ -20,6 +20,7 @@ package com.cloud.storage.resource; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.log4j.Logger; import org.apache.cloudstack.storage.command.AttachCommand; @@ -95,7 +96,9 @@ public class StorageSubsystemCommandHandlerBase implements StorageSubsystemComma //copy volume from image cache to primary return processor.copyVolumeFromImageCacheToPrimary(cmd); } else if (srcData.getObjectType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.Primary) { - if (destData.getObjectType() == DataObjectType.VOLUME) { + if (destData.getObjectType() == DataObjectType.VOLUME && srcData instanceof VolumeObjectTO && ((VolumeObjectTO)srcData).isDirectDownload()) { + return processor.copyVolumeFromPrimaryToPrimary(cmd); + } else if (destData.getObjectType() == DataObjectType.VOLUME) { return processor.copyVolumeFromPrimaryToSecondary(cmd); } else if (destData.getObjectType() == DataObjectType.TEMPLATE) { return processor.createTemplateFromVolume(cmd); diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java index 506cd3140de..aafcb5370a5 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java @@ -38,6 +38,8 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand { private Integer connectTimeout; private Integer soTimeout; private Integer connectionRequestTimeout; + private Long templateSize; + private boolean iso; protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum, final Map headers, final Integer connectTimeout, final Integer soTimeout, final Integer connectionRequestTimeout) { this.url = url; @@ -94,6 +96,22 @@ public abstract class DirectDownloadCommand extends StorageSubSystemCommand { this.connectionRequestTimeout = connectionRequestTimeout; } + public Long getTemplateSize() { + return templateSize; + } + + public void setTemplateSize(Long templateSize) { + this.templateSize = templateSize; + } + + public boolean isIso() { + return iso; + } + + public void setIso(boolean iso) { + this.iso = iso; + } + @Override public void setExecuteInSequence(boolean inSeq) { } diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java index 5a9ff21cc12..e47d13ed669 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java @@ -61,6 +61,7 @@ public class VolumeObjectTO implements DataTO { private DiskCacheMode cacheMode; private Hypervisor.HypervisorType hypervisorType; private MigrationOptions migrationOptions; + private boolean directDownload; public VolumeObjectTO() { @@ -100,6 +101,7 @@ public class VolumeObjectTO implements DataTO { hypervisorType = volume.getHypervisorType(); setDeviceId(volume.getDeviceId()); this.migrationOptions = volume.getMigrationOptions(); + this.directDownload = volume.isDirectDownload(); } public String getUuid() { @@ -307,4 +309,8 @@ public class VolumeObjectTO implements DataTO { public MigrationOptions getMigrationOptions() { return migrationOptions; } + + public boolean isDirectDownload() { + return directDownload; + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java index 99e47df8149..f4a73810901 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java @@ -79,4 +79,8 @@ public interface VolumeInfo extends DataObject, Volume { MigrationOptions getMigrationOptions(); void setMigrationOptions(MigrationOptions migrationOptions); + + boolean isDirectDownload(); + + void setDirectDownload(boolean directDownload); } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index bc49a53db58..9c8f4df3b6e 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -325,7 +325,25 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { Scope destScope = getZoneScope(destData.getDataStore().getScope()); DataStore cacheStore = cacheMgr.getCacheStorage(destScope); + boolean bypassSecondaryStorage = false; + if (srcData instanceof VolumeInfo && ((VolumeInfo)srcData).isDirectDownload()) { + bypassSecondaryStorage = true; + } + if (cacheStore == null) { + if (bypassSecondaryStorage) { + CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _copyvolumewait, VirtualMachineManager.ExecuteInSequence.value()); + EndPoint ep = selector.select(srcData, destData); + Answer answer = null; + if (ep == null) { + String errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; + s_logger.error(errMsg); + answer = new Answer(cmd, false, errMsg); + } else { + answer = ep.sendMessage(cmd); + } + return answer; + } // need to find a nfs or cifs image store, assuming that can't copy volume // directly to s3 ImageStoreEntity imageStore = (ImageStoreEntity)dataStoreMgr.getImageStoreWithFreeCapacity(destScope.getScopeId()); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index 9879f1bcf6a..6e8bdaf4b8c 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -84,6 +84,12 @@ public class DefaultEndPointSelector implements EndPointSelector { } } + private boolean moveBetweenPrimaryDirectDownload(DataStore srcStore, DataStore destStore) { + DataStoreRole srcRole = srcStore.getRole(); + DataStoreRole destRole = destStore.getRole(); + return srcRole == DataStoreRole.Primary && destRole == DataStoreRole.Primary; + } + protected boolean moveBetweenCacheAndImage(DataStore srcStore, DataStore destStore) { DataStoreRole srcRole = srcStore.getRole(); DataStoreRole destRole = destStore.getRole(); @@ -182,6 +188,8 @@ public class DefaultEndPointSelector implements EndPointSelector { DataStore destStore = destData.getDataStore(); if (moveBetweenPrimaryImage(srcStore, destStore)) { return findEndPointForImageMove(srcStore, destStore); + } else if (moveBetweenPrimaryDirectDownload(srcStore, destStore)) { + return findEndPointForImageMove(srcStore, destStore); } else if (moveBetweenCacheAndImage(srcStore, destStore)) { // pick ssvm based on image cache dc DataStore selectedStore = null; diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java index 692f3cc9791..53fa21f3a79 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java @@ -23,6 +23,8 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -42,6 +44,8 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory { VolumeDataStoreDao volumeStoreDao; @Inject DataStoreManager storeMgr; + @Inject + VMTemplateDao templateDao; @Override public VolumeInfo getVolume(long volumeId, DataStore store) { @@ -90,6 +94,12 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory { DataStore store = storeMgr.getDataStore(volumeVO.getPoolId(), DataStoreRole.Primary); vol = VolumeObject.getVolumeObject(store, volumeVO); } + if (vol.getTemplateId() != null) { + VMTemplateVO template = templateDao.findById(vol.getTemplateId()); + if (template != null) { + vol.setDirectDownload(template.isDirectDownload()); + } + } return vol; } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index d62a0baa04a..690a1124402 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -74,6 +74,7 @@ public class VolumeObject implements VolumeInfo { DiskOfferingDao diskOfferingDao; private Object payload; private MigrationOptions migrationOptions; + private boolean directDownload; public VolumeObject() { _volStateMachine = Volume.State.getStateMachine(); @@ -327,6 +328,16 @@ public class VolumeObject implements VolumeInfo { this.migrationOptions = migrationOptions; } + @Override + public boolean isDirectDownload() { + return directDownload; + } + + @Override + public void setDirectDownload(boolean directDownload) { + this.directDownload = directDownload; + } + public void update() { volumeDao.update(volumeVO.getId(), volumeVO); volumeVO = volumeDao.findById(volumeVO.getId()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 0c1593b3cbc..9404be2bd71 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -478,6 +478,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return _ovsPvlanVmPath; } + public String getDirectDownloadTemporaryDownloadPath() { + return directDownloadTemporaryDownloadPath; + } + public String getResizeVolumePath() { return _resizeVolumePath; } @@ -530,6 +534,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected boolean dpdkSupport = false; protected String dpdkOvsPath; + protected String directDownloadTemporaryDownloadPath; private String getEndIpFromStartIp(final String startIp, final int numIps) { final String[] tokens = startIp.split("[.]"); @@ -577,6 +582,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } } + private String getDefaultDirectDownloadTemporaryPath() { + return "/var/lib/libvirt/images"; + } + protected String getDefaultNetworkScriptsDir() { return "scripts/vm/network/vnet"; } @@ -656,6 +665,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } } + directDownloadTemporaryDownloadPath = (String) params.get("direct.download.temporary.download.location"); + if (org.apache.commons.lang.StringUtils.isBlank(directDownloadTemporaryDownloadPath)) { + directDownloadTemporaryDownloadPath = getDefaultDirectDownloadTemporaryPath(); + } + params.put("domr.scripts.dir", domrScriptsDir); _virtRouterResource = new VirtualRoutingResource(this); @@ -2320,18 +2334,20 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv if (dataStore instanceof NfsTO) { NfsTO nfsStore = (NfsTO)data.getDataStore(); dataStoreUrl = nfsStore.getUrl(); - } else if (dataStore instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO) dataStore).getPoolType().equals(StoragePoolType.NetworkFilesystem)) { + physicalDisk = getPhysicalDiskFromNfsStore(dataStoreUrl, data); + } else if (dataStore instanceof PrimaryDataStoreTO) { //In order to support directly downloaded ISOs - String psHost = ((PrimaryDataStoreTO) dataStore).getHost(); - String psPath = ((PrimaryDataStoreTO) dataStore).getPath(); - dataStoreUrl = "nfs://" + psHost + File.separator + psPath; + PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) dataStore; + if (primaryDataStoreTO.getPoolType().equals(StoragePoolType.NetworkFilesystem)) { + String psHost = primaryDataStoreTO.getHost(); + String psPath = primaryDataStoreTO.getPath(); + dataStoreUrl = "nfs://" + psHost + File.separator + psPath; + physicalDisk = getPhysicalDiskFromNfsStore(dataStoreUrl, data); + } else if (primaryDataStoreTO.getPoolType().equals(StoragePoolType.SharedMountPoint) || + primaryDataStoreTO.getPoolType().equals(StoragePoolType.Filesystem)) { + physicalDisk = getPhysicalDiskPrimaryStore(primaryDataStoreTO, data); + } } - final String volPath = dataStoreUrl + File.separator + data.getPath(); - final int index = volPath.lastIndexOf("/"); - final String volDir = volPath.substring(0, index); - final String volName = volPath.substring(index + 1); - final KVMStoragePool secondaryStorage = _storagePoolMgr.getStoragePoolByURI(volDir); - physicalDisk = secondaryStorage.getPhysicalDisk(volName); } else if (volume.getType() != Volume.Type.ISO) { final PrimaryDataStoreTO store = (PrimaryDataStoreTO)data.getDataStore(); physicalDisk = _storagePoolMgr.getPhysicalDisk(store.getPoolType(), store.getUuid(), data.getPath()); @@ -2472,6 +2488,20 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } + private KVMPhysicalDisk getPhysicalDiskPrimaryStore(PrimaryDataStoreTO primaryDataStoreTO, DataTO data) { + KVMStoragePool storagePool = _storagePoolMgr.getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid()); + return storagePool.getPhysicalDisk(data.getPath()); + } + + private KVMPhysicalDisk getPhysicalDiskFromNfsStore(String dataStoreUrl, DataTO data) { + final String volPath = dataStoreUrl + File.separator + data.getPath(); + final int index = volPath.lastIndexOf("/"); + final String volDir = volPath.substring(0, index); + final String volName = volPath.substring(index + 1); + final KVMStoragePool storage = _storagePoolMgr.getStoragePoolByURI(volDir); + return storage.getPhysicalDisk(volName); + } + private void setBurstProperties(final VolumeObjectTO volumeObjectTO, final DiskDef disk ) { if (volumeObjectTO.getBytesReadRate() != null && volumeObjectTO.getBytesReadRate() > 0) { disk.setBytesReadRate(volumeObjectTO.getBytesReadRate()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java index bad0151ed8f..0418dbbb000 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java @@ -445,4 +445,9 @@ public class IscsiAdmStorageAdaptor implements StorageAdaptor { public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout) { return null; } + + @Override + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, KVMStoragePool destPool, boolean isIso) { + return null; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index c1f73d7a088..544c47f07e5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -405,4 +405,9 @@ public class KVMStoragePoolManager { return adaptor.createDiskFromTemplateBacking(template, name, format, size, destPool, timeout); } + public KVMPhysicalDisk createPhysicalDiskFromDirectDownloadTemplate(String templateFilePath, KVMStoragePool destPool, boolean isIso) { + StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); + return adaptor.createTemplateFromDirectDownloadFile(templateFilePath, destPool, isIso); + } + } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 979583b1399..17503ffd49b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -36,6 +36,7 @@ import java.util.UUID; import javax.naming.ConfigurationException; +import com.cloud.utils.Pair; import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand; @@ -89,7 +90,6 @@ import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.S3TO; import com.cloud.agent.direct.download.DirectTemplateDownloader; -import com.cloud.agent.direct.download.DirectTemplateDownloader.DirectTemplateInformation; import com.cloud.agent.direct.download.HttpDirectTemplateDownloader; import com.cloud.agent.direct.download.HttpsDirectTemplateDownloader; import com.cloud.agent.direct.download.MetalinkDirectTemplateDownloader; @@ -1676,15 +1676,20 @@ public class KVMStorageProcessor implements StorageProcessor { /** * Get direct template downloader from direct download command and destination pool */ - private DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd, KVMStoragePool destPool) { + private DirectTemplateDownloader getDirectTemplateDownloaderFromCommand(DirectDownloadCommand cmd, + KVMStoragePool destPool, + String temporaryDownloadPath) { if (cmd instanceof HttpDirectDownloadCommand) { - return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout()); + return new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(), + cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath); } else if (cmd instanceof HttpsDirectDownloadCommand) { - return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout()); + return new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), cmd.getHeaders(), + cmd.getConnectTimeout(), cmd.getSoTimeout(), cmd.getConnectionRequestTimeout(), temporaryDownloadPath); } else if (cmd instanceof NfsDirectDownloadCommand) { - return new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum()); + return new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), temporaryDownloadPath); } else if (cmd instanceof MetalinkDirectDownloadCommand) { - return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(), cmd.getConnectTimeout(), cmd.getSoTimeout()); + return new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum(), cmd.getHeaders(), + cmd.getConnectTimeout(), cmd.getSoTimeout(), temporaryDownloadPath); } else { throw new IllegalArgumentException("Unsupported protocol, please provide HTTP(S), NFS or a metalink"); } @@ -1693,38 +1698,112 @@ public class KVMStorageProcessor implements StorageProcessor { @Override public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) { final PrimaryDataStoreTO pool = cmd.getDestPool(); - if (!pool.getPoolType().equals(StoragePoolType.NetworkFilesystem)) { - return new DirectDownloadAnswer(false, "Unsupported pool type " + pool.getPoolType().toString(), true); - } - KVMStoragePool destPool = storagePoolMgr.getStoragePool(pool.getPoolType(), pool.getUuid()); DirectTemplateDownloader downloader; + KVMPhysicalDisk template; try { - downloader = getDirectTemplateDownloaderFromCommand(cmd, destPool); - } catch (IllegalArgumentException e) { - return new DirectDownloadAnswer(false, "Unable to create direct downloader: " + e.getMessage(), true); - } + s_logger.debug("Verifying temporary location for downloading the template exists on the host"); + String temporaryDownloadPath = resource.getDirectDownloadTemporaryDownloadPath(); + if (!isLocationAccessible(temporaryDownloadPath)) { + String msg = "The temporary location path for downloading templates does not exist: " + + temporaryDownloadPath + " on this host"; + s_logger.error(msg); + return new DirectDownloadAnswer(false, msg, true); + } - try { - s_logger.info("Trying to download template"); - if (!downloader.downloadTemplate()) { + s_logger.debug("Checking for free space on the host for downloading the template"); + if (!isEnoughSpaceForDownloadTemplateOnTemporaryLocation(cmd.getTemplateSize())) { + String msg = "Not enough space on the defined temporary location to download the template " + cmd.getTemplateId(); + s_logger.error(msg); + return new DirectDownloadAnswer(false, msg, true); + } + + KVMStoragePool destPool = storagePoolMgr.getStoragePool(pool.getPoolType(), pool.getUuid()); + downloader = getDirectTemplateDownloaderFromCommand(cmd, destPool, temporaryDownloadPath); + s_logger.debug("Trying to download template"); + Pair result = downloader.downloadTemplate(); + if (!result.first()) { s_logger.warn("Couldn't download template"); return new DirectDownloadAnswer(false, "Unable to download template", true); } + String tempFilePath = result.second(); if (!downloader.validateChecksum()) { s_logger.warn("Couldn't validate template checksum"); return new DirectDownloadAnswer(false, "Checksum validation failed", false); } - if (!downloader.extractAndInstallDownloadedTemplate()) { - s_logger.warn("Couldn't extract and install template"); - return new DirectDownloadAnswer(false, "Extraction and installation failed", false); - } + template = storagePoolMgr.createPhysicalDiskFromDirectDownloadTemplate(tempFilePath, destPool, cmd.isIso()); } catch (CloudRuntimeException e) { s_logger.warn("Error downloading template " + cmd.getTemplateId() + " due to: " + e.getMessage()); return new DirectDownloadAnswer(false, "Unable to download template: " + e.getMessage(), true); + } catch (IllegalArgumentException e) { + return new DirectDownloadAnswer(false, "Unable to create direct downloader: " + e.getMessage(), true); } - DirectTemplateInformation info = downloader.getTemplateInformation(); - return new DirectDownloadAnswer(true, info.getSize(), info.getInstallPath()); + return new DirectDownloadAnswer(true, template.getSize(), template.getName()); + } + + @Override + public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { + final DataTO srcData = cmd.getSrcTO(); + final DataTO destData = cmd.getDestTO(); + final VolumeObjectTO srcVol = (VolumeObjectTO)srcData; + final VolumeObjectTO destVol = (VolumeObjectTO)destData; + final ImageFormat srcFormat = srcVol.getFormat(); + final ImageFormat destFormat = destVol.getFormat(); + final DataStoreTO srcStore = srcData.getDataStore(); + final DataStoreTO destStore = destData.getDataStore(); + final PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)srcStore; + final PrimaryDataStoreTO primaryStoreDest = (PrimaryDataStoreTO)destStore; + final String srcVolumePath = srcData.getPath(); + final String destVolumePath = destData.getPath(); + KVMStoragePool destPool = null; + + try { + final String volumeName = UUID.randomUUID().toString(); + + final String destVolumeName = volumeName + "." + destFormat.getFileExtension(); + final KVMPhysicalDisk volume = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), srcVolumePath); + volume.setFormat(PhysicalDiskFormat.valueOf(srcFormat.toString())); + + destPool = storagePoolMgr.getStoragePool(primaryStoreDest.getPoolType(), primaryStoreDest.getUuid()); + storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds()); + final VolumeObjectTO newVol = new VolumeObjectTO(); + newVol.setPath(destVolumePath + File.separator + destVolumeName); + newVol.setFormat(destFormat); + return new CopyCmdAnswer(newVol); + } catch (final CloudRuntimeException e) { + s_logger.debug("Failed to copyVolumeFromPrimaryToPrimary: ", e); + return new CopyCmdAnswer(e.toString()); + } + } + + /** + * True if location exists + */ + private boolean isLocationAccessible(String temporaryDownloadPath) { + File dir = new File(temporaryDownloadPath); + return dir.exists(); + } + + /** + * Perform a free space check on the host for downloading the direct download templates + * @param templateSize template size obtained from remote server when registering the template (in bytes) + */ + protected boolean isEnoughSpaceForDownloadTemplateOnTemporaryLocation(Long templateSize) { + if (templateSize == null || templateSize == 0L) { + s_logger.info("The server did not provide the template size, assuming there is enough space to download it"); + return true; + } + String cmd = String.format("df --output=avail %s -B 1 | tail -1", resource.getDirectDownloadTemporaryDownloadPath()); + String resultInBytes = Script.runSimpleBashScript(cmd); + Long availableBytes; + try { + availableBytes = Long.parseLong(resultInBytes); + } catch (NumberFormatException e) { + String msg = "Could not parse the output " + resultInBytes + " as a number, therefore not able to check for free space"; + s_logger.error(msg, e); + return false; + } + return availableBytes >= templateSize; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index f858a4f1577..ce2199cc28d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -122,6 +122,64 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { return disk; } + /** + * Checks if downloaded template is extractable + * @return true if it should be extracted, false if not + */ + private boolean isTemplateExtractable(String templatePath) { + String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'"); + return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); + } + + /** + * Return extract command to execute given downloaded file + * @param downloadedTemplateFile + * @param templateUuid + */ + private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateUuid) { + if (downloadedTemplateFile.endsWith(".zip")) { + return "unzip -p " + downloadedTemplateFile + " | cat > " + templateUuid; + } else if (downloadedTemplateFile.endsWith(".bz2")) { + return "bunzip2 -c " + downloadedTemplateFile + " > " + templateUuid; + } else if (downloadedTemplateFile.endsWith(".gz")) { + return "gunzip -c " + downloadedTemplateFile + " > " + templateUuid; + } else { + throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile); + } + } + + /** + * Extract downloaded template into installPath, remove compressed file + */ + private void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, String destinationFile) { + String extractCommand = getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile); + Script.runSimpleBashScript(extractCommand); + Script.runSimpleBashScript("rm -f " + downloadedTemplateFile); + } + + @Override + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, KVMStoragePool destPool, boolean isIso) { + File sourceFile = new File(templateFilePath); + if (!sourceFile.exists()) { + throw new CloudRuntimeException("Direct download template file " + sourceFile + " does not exist on this host"); + } + String templateUuid = UUID.randomUUID().toString(); + if (isIso) { + templateUuid += ".iso"; + } + String destinationFile = destPool.getLocalPath() + File.separator + templateUuid; + + if (destPool.getType() == StoragePoolType.NetworkFilesystem || destPool.getType() == StoragePoolType.Filesystem + || destPool.getType() == StoragePoolType.SharedMountPoint) { + if (!isIso && isTemplateExtractable(templateFilePath)) { + extractDownloadedTemplate(templateFilePath, destPool, destinationFile); + } else { + Script.runSimpleBashScript("mv " + templateFilePath + " " + destinationFile); + } + } + return destPool.getPhysicalDisk(templateUuid); + } + public StorageVol getVolume(StoragePool pool, String volName) { StorageVol vol = null; @@ -1198,7 +1256,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { if (disk.getFormat() == PhysicalDiskFormat.TAR) { newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, Storage.ProvisioningType.THIN, disk.getVirtualSize()); } else { - newDisk = destPool.createPhysicalDisk(name, Storage.ProvisioningType.THIN, disk.getVirtualSize()); + newDisk = destPool.createPhysicalDisk(name, Storage.ProvisioningType.THIN, disk.getVirtualSize()); } } else { newDisk = new KVMPhysicalDisk(destPool.getSourceDir() + "/" + name, name, destPool); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java index 309308ae9c1..1ea4f626226 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java @@ -318,6 +318,11 @@ public class ManagedNfsStorageAdaptor implements StorageAdaptor { return null; } + @Override + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, KVMStoragePool destPool, boolean isIso) { + return null; + } + @Override public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, PhysicalDiskFormat format, ProvisioningType provisioningType, long size) { return null; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java index a3c1387aa6b..99f2876915c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java @@ -81,4 +81,12 @@ public interface StorageAdaptor { KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout); + + /** + * Create physical disk on Primary Storage from direct download template on the host (in temporary location) + * @param templateFilePath + * @param destPool + * @param isIso + */ + KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, KVMStoragePool destPool, boolean isIso); } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java index 63d46bc87e9..36d957038a2 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java @@ -21,13 +21,22 @@ package com.cloud.hypervisor.kvm.storage; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import javax.naming.ConfigurationException; +import com.cloud.utils.script.Script; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; import org.mockito.InjectMocks; +import org.mockito.Matchers; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; -import org.mockito.Spy; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +@PrepareForTest({ Script.class }) +@RunWith(PowerMockRunner.class) public class KVMStorageProcessorTest { @Mock @@ -35,26 +44,47 @@ public class KVMStorageProcessorTest { @Mock LibvirtComputingResource resource; - private static final Long TEMPLATE_ID = 202l; - private static final String EXPECTED_DIRECT_DOWNLOAD_DIR = "template/2/202"; - - @Spy @InjectMocks private KVMStorageProcessor storageProcessor; + private static final String directDownloadTemporaryPath = "/var/lib/libvirt/images/dd"; + private static final long templateSize = 80000L; + @Before public void setUp() throws ConfigurationException { MockitoAnnotations.initMocks(this); storageProcessor = new KVMStorageProcessor(storagePoolManager, resource); + PowerMockito.mockStatic(Script.class); + Mockito.when(resource.getDirectDownloadTemporaryDownloadPath()).thenReturn(directDownloadTemporaryPath); } @Test - public void testCloneVolumeFromBaseTemplate() throws Exception { - + public void testIsEnoughSpaceForDownloadTemplateOnTemporaryLocationAssumeEnoughSpaceWhenNotProvided() { + boolean result = storageProcessor.isEnoughSpaceForDownloadTemplateOnTemporaryLocation(null); + Assert.assertTrue(result); } @Test - public void testCopyVolumeFromImageCacheToPrimary() throws Exception { + public void testIsEnoughSpaceForDownloadTemplateOnTemporaryLocationNotEnoughSpace() { + String output = String.valueOf(templateSize - 30000L); + Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).thenReturn(output); + boolean result = storageProcessor.isEnoughSpaceForDownloadTemplateOnTemporaryLocation(templateSize); + Assert.assertFalse(result); + } + @Test + public void testIsEnoughSpaceForDownloadTemplateOnTemporaryLocationEnoughSpace() { + String output = String.valueOf(templateSize + 30000L); + Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).thenReturn(output); + boolean result = storageProcessor.isEnoughSpaceForDownloadTemplateOnTemporaryLocation(templateSize); + Assert.assertTrue(result); + } + + @Test + public void testIsEnoughSpaceForDownloadTemplateOnTemporaryLocationNotExistingLocation() { + String output = String.format("df: ‘%s’: No such file or directory", directDownloadTemporaryPath); + Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).thenReturn(output); + boolean result = storageProcessor.isEnoughSpaceForDownloadTemplateOnTemporaryLocation(templateSize); + Assert.assertFalse(result); } } diff --git a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java index 5f43c38f607..7915586fca3 100644 --- a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java +++ b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java @@ -826,6 +826,11 @@ public class Ovm3StorageProcessor implements StorageProcessor { return null; } + @Override + public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { + return null; + } + /** * Attach disks * @param cmd diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java index c2dfdbd4e56..e4ef4dfc1f6 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java @@ -264,4 +264,9 @@ public class SimulatorStorageProcessor implements StorageProcessor { // TODO Auto-generated method stub return null; } + + @Override + public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { + return null; + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index d98f3189147..796db94f0ed 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -3554,4 +3554,9 @@ public class VmwareStorageProcessor implements StorageProcessor { public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) { return null; } + + @Override + public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { + return null; + } } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java index 458b7d6377a..e4c07d4ba79 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java @@ -209,6 +209,11 @@ public class XenServerStorageProcessor implements StorageProcessor { return null; } + @Override + public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { + return null; + } + @Override public AttachAnswer attachIso(final AttachCommand cmd) { final DiskTO disk = cmd.getDisk(); diff --git a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java index 792a5a30ee8..a05c4b9e4aa 100644 --- a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java @@ -20,6 +20,32 @@ package org.apache.cloudstack.direct.download; import static com.cloud.storage.Storage.ImageFormat; +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; +import com.cloud.event.EventVO; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.exception.CloudRuntimeException; + import java.net.URI; import java.net.URISyntaxException; import java.security.cert.Certificate; @@ -27,7 +53,6 @@ import java.security.cert.CertificateException; import java.security.cert.CertificateExpiredException; import java.security.cert.CertificateNotYetValidException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -67,29 +92,6 @@ import org.apache.log4j.Logger; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import com.cloud.agent.AgentManager; -import com.cloud.agent.api.Answer; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.dao.DataCenterDao; -import com.cloud.event.ActionEventUtils; -import com.cloud.event.EventTypes; -import com.cloud.event.EventVO; -import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationTimedoutException; -import com.cloud.host.Host; -import com.cloud.host.HostVO; -import com.cloud.host.Status; -import com.cloud.host.dao.HostDao; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.VMTemplateStoragePoolVO; -import com.cloud.storage.VMTemplateStorageResourceAssoc; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VMTemplatePoolDao; -import com.cloud.utils.component.ManagerBase; -import com.cloud.utils.concurrency.NamedThreadFactory; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.security.CertificateHelper; import sun.security.x509.X509CertImpl; @@ -202,7 +204,7 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown */ protected Long[] createHostIdsList(List hostIds, long hostId) { if (CollectionUtils.isEmpty(hostIds)) { - return Arrays.asList(hostId).toArray(new Long[1]); + return Collections.singletonList(hostId).toArray(new Long[1]); } Long[] ids = new Long[hostIds.size() + 1]; ids[0] = hostId; @@ -215,11 +217,15 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown } /** - * Get hosts to retry download having hostId as the first element + * Get alternative hosts to retry downloading a template. The planner have previously selected a host and a storage pool + * @return array of host ids which can access the storage pool */ - protected Long[] getHostsToRetryOn(Long clusterId, long dataCenterId, HypervisorType hypervisorType, long hostId) { - List hostIds = getRunningHostIdsInTheSameCluster(clusterId, dataCenterId, hypervisorType, hostId); - return createHostIdsList(hostIds, hostId); + protected Long[] getHostsToRetryOn(Host host, StoragePoolVO storagePool) { + List clusterHostIds = new ArrayList<>(); + if (storagePool.getPoolType() != Storage.StoragePoolType.Filesystem || storagePool.getScope() != ScopeType.HOST) { + clusterHostIds = getRunningHostIdsInTheSameCluster(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType(), host.getId()); + } + return createHostIdsList(clusterHostIds, host.getId()); } @Override @@ -252,6 +258,8 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown DownloadProtocol protocol = getProtocolFromUrl(url); DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers); + cmd.setTemplateSize(template.getSize()); + cmd.setIso(template.getFormat() == ImageFormat.ISO); Answer answer = sendDirectDownloadCommand(cmd, template, poolId, host); @@ -284,7 +292,9 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown private Answer sendDirectDownloadCommand(DirectDownloadCommand cmd, VMTemplateVO template, long poolId, HostVO host) { boolean downloaded = false; int retry = 3; - Long[] hostsToRetry = getHostsToRetryOn(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType(), host.getId()); + + StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); + Long[] hostsToRetry = getHostsToRetryOn(host, storagePoolVO); int hostIndex = 0; Answer answer = null; Long hostToSendDownloadCmd = hostsToRetry[hostIndex]; diff --git a/test/integration/smoke/test_direct_download.py b/test/integration/smoke/test_direct_download.py index 132deb4f698..324fb5972cb 100644 --- a/test/integration/smoke/test_direct_download.py +++ b/test/integration/smoke/test_direct_download.py @@ -23,12 +23,14 @@ from marvin.lib.base import (ServiceOffering, NetworkOffering, Network, Template, - VirtualMachine) + VirtualMachine, + StoragePool) from marvin.lib.common import (get_pod, get_zone) from nose.plugins.attrib import attr from marvin.cloudstackAPI import (uploadTemplateDirectDownloadCertificate, revokeTemplateDirectDownloadCertificate) from marvin.lib.decoratorGenerators import skipTestIf +import uuid class TestUploadDirectDownloadCertificates(cloudstackTestCase): @@ -90,7 +92,7 @@ class TestUploadDirectDownloadCertificates(cloudstackTestCase): cmd = uploadTemplateDirectDownloadCertificate.uploadTemplateDirectDownloadCertificateCmd() cmd.hypervisor = self.hypervisor - cmd.name = "marvin-test-verify-certs" + cmd.name = "marvin-test-verify-certs" + str(uuid.uuid1()) cmd.certificate = self.certificates["invalid"] cmd.zoneid = self.zone.id @@ -125,7 +127,7 @@ class TestUploadDirectDownloadCertificates(cloudstackTestCase): cmd = uploadTemplateDirectDownloadCertificate.uploadTemplateDirectDownloadCertificateCmd() cmd.hypervisor = self.hypervisor - cmd.name = "marvin-test-verify-certs" + cmd.name = "marvin-test-verify-certs" + str(uuid.uuid1()) cmd.certificate = self.certificates["valid"] cmd.zoneid = self.zone.id @@ -160,11 +162,15 @@ class TestDirectDownloadTemplates(cloudstackTestCase): cls.services = cls.testClient.getParsedTestDataConfig() cls._cleanup = [] - cls.hypervisorNotSupported = False - if cls.hypervisor.lower() not in ['kvm', 'lxc']: - cls.hypervisorNotSupported = True + cls.hypervisorSupported = False + cls.nfsStorageFound = False + cls.localStorageFound = False + cls.sharedMountPointFound = False - if not cls.hypervisorNotSupported: + if cls.hypervisor.lower() in ['kvm', 'lxc']: + cls.hypervisorSupported = True + + if cls.hypervisorSupported: cls.services["test_templates"]["kvm"]["directdownload"] = "true" cls.template = Template.register(cls.apiclient, cls.services["test_templates"]["kvm"], zoneid=cls.zone.id, hypervisor=cls.hypervisor) @@ -192,6 +198,25 @@ class TestDirectDownloadTemplates(cloudstackTestCase): ) cls._cleanup.append(cls.l2_network) cls._cleanup.append(cls.network_offering) + + storage_pools = StoragePool.list( + cls.apiclient, + zoneid=cls.zone.id + ) + for pool in storage_pools: + if not cls.nfsStorageFound and pool.type == "NetworkFilesystem": + cls.nfsStorageFound = True + cls.nfsPoolId = pool.id + elif not cls.localStorageFound and pool.type == "Filesystem": + cls.localStorageFound = True + cls.localPoolId = pool.id + elif not cls.sharedMountPointFound and pool.type == "SharedMountPoint": + cls.sharedMountPointFound = True + cls.sharedPoolId = pool.id + + cls.nfsKvmNotAvailable = not cls.hypervisorSupported or not cls.nfsStorageFound + cls.localStorageKvmNotAvailable = not cls.hypervisorSupported or not cls.localStorageFound + cls.sharedMountPointKvmNotAvailable = not cls.hypervisorSupported or not cls.sharedMountPointFound return @classmethod @@ -215,26 +240,124 @@ class TestDirectDownloadTemplates(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return - @skipTestIf("hypervisorNotSupported") + def getCurrentStoragePoolTags(self, poolId): + local_pool = StoragePool.list( + self.apiclient, + id=poolId + ) + return local_pool[0].tags + + def updateStoragePoolTags(self, poolId, tags): + StoragePool.update( + self.apiclient, + id=poolId, + tags=tags + ) + + def createServiceOffering(self, name, type, tags): + services = { + "cpunumber": 1, + "cpuspeed": 512, + "memory": 256, + "displaytext": name, + "name": name, + "storagetype": type + } + return ServiceOffering.create( + self.apiclient, + services, + tags=tags + ) + + + @skipTestIf("nfsKvmNotAvailable") @attr(tags=["advanced", "basic", "eip", "advancedns", "sg"], required_hardware="false") - def test_01_deploy_vm_from_direct_download_template(self): - """Test Deploy VM from direct download template + def test_01_deploy_vm_from_direct_download_template_nfs_storage(self): + """Test Deploy VM from direct download template on NFS storage """ - # Validate the following - # 1. Register direct download template - # 2. Deploy VM from direct download template + # Create service offering for local storage using storage tags + tags = self.getCurrentStoragePoolTags(self.nfsPoolId) + test_tag = "marvin_test_nfs_storage_direct_download" + self.updateStoragePoolTags(self.nfsPoolId, test_tag) + nfs_storage_offering = self.createServiceOffering("TestNFSStorageDirectDownload", "shared", test_tag) vm = VirtualMachine.create( self.apiclient, self.services["virtual_machine"], - serviceofferingid=self.service_offering.id, + serviceofferingid=nfs_storage_offering.id, networkids=self.l2_network.id ) self.assertEqual( vm.state, "Running", - "Check VM deployed from direct download template is running" + "Check VM deployed from direct download template is running on NFS storage" ) + + # Revert storage tags for the storage pool used in this test + self.updateStoragePoolTags(self.nfsPoolId, tags) self.cleanup.append(vm) + self.cleanup.append(nfs_storage_offering) + return + + @skipTestIf("localStorageKvmNotAvailable") + @attr(tags=["advanced", "basic", "eip", "advancedns", "sg"], required_hardware="false") + def test_02_deploy_vm_from_direct_download_template_local_storage(self): + """Test Deploy VM from direct download template on local storage + """ + + # Create service offering for local storage using storage tags + tags = self.getCurrentStoragePoolTags(self.localPoolId) + test_tag = "marvin_test_local_storage_direct_download" + self.updateStoragePoolTags(self.localPoolId, test_tag) + local_storage_offering = self.createServiceOffering("TestLocalStorageDirectDownload", "local", test_tag) + + # Deploy VM + vm = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + serviceofferingid=local_storage_offering.id, + networkids=self.l2_network.id, + ) + self.assertEqual( + vm.state, + "Running", + "Check VM deployed from direct download template is running on local storage" + ) + + # Revert storage tags for the storage pool used in this test + self.updateStoragePoolTags(self.localPoolId, tags) + self.cleanup.append(vm) + self.cleanup.append(local_storage_offering) + return + + @skipTestIf("sharedMountPointKvmNotAvailable") + @attr(tags=["advanced", "basic", "eip", "advancedns", "sg"], required_hardware="false") + def test_03_deploy_vm_from_direct_download_template_shared_mount_point_storage(self): + """Test Deploy VM from direct download template on shared mount point + """ + + # Create service offering for local storage using storage tags + tags = self.getCurrentStoragePoolTags(self.sharedPoolId) + test_tag = "marvin_test_shared_mount_point_storage_direct_download" + self.updateStoragePoolTags(self.sharedPoolId, test_tag) + shared_offering = self.createServiceOffering("TestSharedMountPointStorageDirectDownload", "shared", test_tag) + + # Deploy VM + vm = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + serviceofferingid=shared_offering.id, + networkids=self.l2_network.id, + ) + self.assertEqual( + vm.state, + "Running", + "Check VM deployed from direct download template is running on shared mount point" + ) + + # Revert storage tags for the storage pool used in this test + self.updateStoragePoolTags(self.sharedPoolId, tags) + self.cleanup.append(vm) + self.cleanup.append(shared_offering) return