apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-10146: Bypass Secondary Storage for KVM templates (#2379)

Browse Source 
This feature allows using templates and ISOs avoiding secondary storage as intermediate cache on KVM. The virtual machine deployment process is enhanced to supported bypassed registered templates and ISOs, delegating the work of downloading them to primary storage to the KVM agent instead of the SSVM agent.

Template and ISO registration:
- When hypervisor is KVM, a checkbox is displayed with 'Direct Download' label.
- API methods registerTemplate and registerISO are both extended with this new parameter directdownload.
- On template or ISO registration, no download job is sent to SSVM agent, CloudStack would only persist an entry on template_store_ref indicating that template or ISO has been marked as 'Direct Download' (bypassing Secondary Storage). These entries are persisted as:
template_id = Template or ISO id on vm_template table
store_id NULL
download_state = BYPASSED
state = Ready
(Note: these entries allow users to deploy virtual machine from registered templates or ISOs)
- An URL validation command is sent to a random KVM host to check if template/ISO location can be reached. Metalink are also supported by this feature. In case of a metalink, it is fetched and URL check is performed on each of its URLs.
- Checksum should be provided as indicated on #2246: {ALGORITHM}CHKSUMHASH
- After template or ISO is registered, it would be displayed in the UI

Virtual machine deployment:
When a 'Direct Download' template is selected for deployment, CloudStack would delegate template downloading to destination storage pool via destination host by a new pluggable download manager.
Download manager would handle template downloading depending on URL protocol. In case of HTTP, request headers can be set by the user via vm_template_details. Those details should be persisted as:
Key: HTTP_HEADER
Value: HEADERNAME:HEADERVALUE

In case of HTTPS, a new API method is added uploadTemplateDirectDownloadCertificate to allow user importing a client certificate into all KVM hosts' keystore before deployment.
After template or ISO is downloaded to primary storage, usual entry would be persisted on template_spool_ref indicating the mapping between template/ISO and storage pool.
This commit is contained in:
Nicolas Vazquez 2018-01-09 03:52:18 -03:00 committed by Rohit Yadav
parent 0d0fa5e306
commit e86bb41e0e
103 changed files with 2726 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
agent/src/com/cloud/agent/Agent.java
View File

@ -37,6 +37,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate;
import org.apache.cloudstack.ca.SetupCertificateAnswer;
import org.apache.cloudstack.ca.SetupCertificateCommand;
import org.apache.cloudstack.ca.SetupKeyStoreCommand;
@ -551,6 +552,8 @@ public class Agent implements HandlerFactory, IAgentControl {
answer = setupAgentKeystore((SetupKeyStoreCommand) cmd);
} else if (cmd instanceof SetupCertificateCommand && ((SetupCertificateCommand) cmd).isHandleByAgent()) {
answer = setupAgentCertificate((SetupCertificateCommand) cmd);
} else if (cmd instanceof SetupDirectDownloadCertificate) {
answer = setupDirectDownloadCertificate((SetupDirectDownloadCertificate) cmd);
} else {
if (cmd instanceof ReadyCommand) {
processReadyCommand(cmd);
@ -600,6 +603,31 @@ public class Agent implements HandlerFactory, IAgentControl {
}
}
private Answer setupDirectDownloadCertificate(SetupDirectDownloadCertificate cmd) {
String certificate = cmd.getCertificate();
String certificateName = cmd.getCertificateName();
s_logger.info("Importing certificate " + certificateName + " into keystore");
final File agentFile = PropertiesUtil.findConfigFile("agent.properties");
if (agentFile == null) {
return new Answer(cmd, false, "Failed to find agent.properties file");
}
final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile;
String cerFile = agentFile.getParent() + "/" + certificateName + ".cer";
Script.runSimpleBashScript(String.format("echo '%s' > %s", certificate, cerFile));
String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null";
String privatePasswordCmd = String.format(privatePasswordFormat, agentFile.getAbsolutePath());
String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
String importCommandFormat = "keytool -importcert -file %s -keystore %s -alias '%s' -storepass '%s' -noprompt";
String importCmd = String.format(importCommandFormat, cerFile, keyStoreFile, certificateName, privatePassword);
Script.runSimpleBashScript(importCmd);
return new Answer(cmd, true, "Certificate " + certificateName + " imported");
}
public Answer setupAgentKeystore(final SetupKeyStoreCommand cmd) {
final String keyStorePassword = cmd.getKeystorePassword();
final long validityDays = cmd.getValidityDays();

71
agent/src/com/cloud/agent/direct/download/DirectTemplateDownloader.java Normal file
View File

@ -0,0 +1,71 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
public interface DirectTemplateDownloader {
class DirectTemplateInformation {
private String installPath;
private Long size;
private String checksum;
public DirectTemplateInformation(String installPath, Long size, String checksum) {
this.installPath = installPath;
this.size = size;
this.checksum = checksum;
}
public String getInstallPath() {
return installPath;
}
public Long getSize() {
return size;
}
public String getChecksum() {
return checksum;
}
}
/**
* Perform template download to pool specified on downloader creation
* @return true if successful, false if not
*/
boolean downloadTemplate();
/**
* Perform extraction (if necessary) and installation of previously downloaded template
* @return true if successful, false if not
*/
boolean extractAndInstallDownloadedTemplate();
/**
* Get template information after it is properly installed on pool
* @return template information
*/
DirectTemplateInformation getTemplateInformation();
/**
* Perform checksum validation of previously downloadeed template
* @return true if successful, false if not
*/
boolean validateChecksum();
}

185
agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java Normal file
View File

@ -0,0 +1,185 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import org.apache.cloudstack.utils.security.ChecksumValue;
import org.apache.commons.lang.StringUtils;
import java.io.File;
import java.util.UUID;
public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader {
private String url;
private String destPoolPath;
private Long templateId;
private String downloadedFilePath;
private String installPath;
private String checksum;
protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, final String checksum) {
this.url = url;
this.destPoolPath = destPoolPath;
this.templateId = templateId;
this.checksum = checksum;
}
private static String directDownloadDir = "template";
/**
* Return direct download temporary path to download template
*/
protected static String getDirectDownloadTempPath(Long templateId) {
String templateIdAsString = String.valueOf(templateId);
return directDownloadDir + File.separator + templateIdAsString.substring(0,1) +
File.separator + templateIdAsString;
}
/**
* Create folder on path if it does not exist
*/
protected void createFolder(String path) {
File dir = new File(path);
if (!dir.exists()) {
dir.mkdirs();
}
}
public String getUrl() {
return url;
}
public String getDestPoolPath() {
return destPoolPath;
}
public Long getTemplateId() {
return templateId;
}
public String getDownloadedFilePath() {
return downloadedFilePath;
}
public void setDownloadedFilePath(String filePath) {
this.downloadedFilePath = filePath;
}
/**
* Return filename from url
*/
public String getFileNameFromUrl() {
String[] urlParts = url.split("/");
return urlParts[urlParts.length - 1];
}
/**
* Checks if downloaded template is extractable
* @return true if it should be extracted, false if not
*/
private boolean isTemplateExtractable() {
String type = Script.runSimpleBashScript("file " + downloadedFilePath + " | awk -F' ' '{print $2}'");
return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
}
@Override
public boolean extractAndInstallDownloadedTemplate() {
installPath = UUID.randomUUID().toString();
if (isTemplateExtractable()) {
extractDownloadedTemplate();
} else {
Script.runSimpleBashScript("mv " + downloadedFilePath + " " + getInstallFullPath());
}
return true;
}
/**
* Return install full path
*/
private String getInstallFullPath() {
return destPoolPath + File.separator + installPath;
}
/**
* Return extract command to execute given downloaded file
*/
private String getExtractCommandForDownloadedFile() {
if (downloadedFilePath.endsWith(".zip")) {
return "unzip -p " + downloadedFilePath + " | cat > " + getInstallFullPath();
} else if (downloadedFilePath.endsWith(".bz2")) {
return "bunzip2 -c " + downloadedFilePath + " > " + getInstallFullPath();
} else if (downloadedFilePath.endsWith(".gz")) {
return "gunzip -c " + downloadedFilePath + " > " + getInstallFullPath();
} else {
throw new CloudRuntimeException("Unable to extract template " + templateId + " on " + downloadedFilePath);
}
}
/**
* Extract downloaded template into installPath, remove compressed file
*/
private void extractDownloadedTemplate() {
String extractCommand = getExtractCommandForDownloadedFile();
Script.runSimpleBashScript(extractCommand);
Script.runSimpleBashScript("rm -f " + downloadedFilePath);
}
@Override
public DirectTemplateInformation getTemplateInformation() {
String sizeResult = Script.runSimpleBashScript("ls -als " + getInstallFullPath() + " | awk '{print $1}'");
long size = Long.parseLong(sizeResult);
return new DirectTemplateInformation(installPath, size, checksum);
}
/**
* Return checksum command from algorithm
*/
private String getChecksumCommandFromAlgorithm(String algorithm) {
if (algorithm.equalsIgnoreCase("MD5")) {
return "md5sum";
} else if (algorithm.equalsIgnoreCase("SHA-1")) {
return "sha1sum";
} else if (algorithm.equalsIgnoreCase("SHA-224")) {
return "sha224sum";
} else if (algorithm.equalsIgnoreCase("SHA-256")) {
return "sha256sum";
} else if (algorithm.equalsIgnoreCase("SHA-384")) {
return "sha384sum";
} else if (algorithm.equalsIgnoreCase("SHA-512")) {
return "sha512sum";
} else {
throw new CloudRuntimeException("Unknown checksum algorithm: " + algorithm);
}
}
@Override
public boolean validateChecksum() {
if (StringUtils.isNotBlank(checksum)) {
ChecksumValue providedChecksum = new ChecksumValue(checksum);
String algorithm = providedChecksum.getAlgorithm();
String checksumCommand = "echo '%s %s' | %s -c --quiet";
String cmd = String.format(checksumCommand, providedChecksum.getChecksum(), downloadedFilePath, getChecksumCommandFromAlgorithm(algorithm));
int result = Script.runSimpleBashScriptForExitValue(cmd);
return result == 0;
}
return true;
}
}

122
agent/src/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java Normal file
View File

@ -0,0 +1,122 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpMethodRetryHandler;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.NoHttpResponseException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.io.IOUtils;
import org.apache.log4j.Logger;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;
import java.util.Map;
public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
private HttpClient client;
private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
private static final int CHUNK_SIZE = 1024 * 1024; //1M
protected HttpMethodRetryHandler myretryhandler;
public static final Logger s_logger = Logger.getLogger(HttpDirectTemplateDownloader.class.getName());
protected GetMethod request;
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers) {
super(url, destPoolPath, templateId, checksum);
client = new HttpClient(s_httpClientManager);
myretryhandler = createRetryTwiceHandler();
request = createRequest(url, headers);
String downloadDir = getDirectDownloadTempPath(templateId);
createTemporaryDirectoryAndFile(downloadDir);
}
protected void createTemporaryDirectoryAndFile(String downloadDir) {
createFolder(getDestPoolPath() + File.separator + downloadDir);
File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl());
setDownloadedFilePath(f.getAbsolutePath());
}
protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
GetMethod request = new GetMethod(downloadUrl);
request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
request.setFollowRedirects(true);
if (MapUtils.isNotEmpty(headers)) {
for (String key : headers.keySet()) {
request.setRequestHeader(key, headers.get(key));
}
}
return request;
}
protected HttpMethodRetryHandler createRetryTwiceHandler() {
return new HttpMethodRetryHandler() {
@Override
public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
if (executionCount >= 2) {
// Do not retry if over max retry count
return false;
}
if (exception instanceof NoHttpResponseException) {
// Retry if the server dropped connection on us
return true;
}
if (!method.isRequestSent()) {
// Retry if the request has not been sent fully or
// if it's OK to retry methods that have been sent
return true;
}
// otherwise do not retry
return false;
}
};
}
@Override
public boolean downloadTemplate() {
try {
client.executeMethod(request);
} catch (IOException e) {
throw new CloudRuntimeException("Error on HTTP request: " + e.getMessage());
}
return performDownload();
}
protected boolean performDownload() {
try (
InputStream in = request.getResponseBodyAsStream();
OutputStream out = new FileOutputStream(getDownloadedFilePath());
) {
IOUtils.copy(in, out);
} catch (IOException e) {
s_logger.error("Error downloading template " + getTemplateId() + " due to: " + e.getMessage());
return false;
}
return true;
}
}

89
agent/src/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java Normal file
View File

@ -0,0 +1,89 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.ssl.SSLContexts;
import javax.net.ssl.SSLContext;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader {
private CloseableHttpClient httpsClient;
private HttpUriRequest req;
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum) {
super(url, templateId, destPoolPath, checksum, null);
SSLContext sslcontext = null;
try {
sslcontext = getSSLContext();
} catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) {
throw new CloudRuntimeException("Failure getting SSL context for HTTPS downloader: " + e.getMessage());
}
SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
httpsClient = HttpClients.custom().setSSLSocketFactory(factory).build();
req = createUriRequest(url);
}
protected HttpUriRequest createUriRequest(String downloadUrl) {
return new HttpGet(downloadUrl);
}
private SSLContext getSSLContext() throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException {
KeyStore trustStore = KeyStore.getInstance("jks");
FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks"));
try {
String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null";
String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties");
String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
trustStore.load(instream, privatePassword.toCharArray());
} finally {
instream.close();
}
return SSLContexts.custom()
.loadTrustMaterial(trustStore, new TrustSelfSignedStrategy())
.build();
}
@Override
public boolean downloadTemplate() {
try {
httpsClient.execute(req);
} catch (IOException e) {
throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage());
}
return performDownload();
}
}

49
agent/src/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java Normal file
View File

@ -0,0 +1,49 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
import com.cloud.utils.script.Script;
import java.io.File;
public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
private String downloadDir;
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum) {
super(url, destPoolPath, templateId, checksum);
String relativeDir = getDirectDownloadTempPath(templateId);
downloadDir = getDestPoolPath() + File.separator + relativeDir;
createFolder(downloadDir);
}
@Override
public boolean downloadTemplate() {
String downloadCommand = "aria2c " + getUrl() + " -d " + downloadDir + " --check-integrity=true";
Script.runSimpleBashScript(downloadCommand);
//Remove .metalink file
Script.runSimpleBashScript("rm -f " + downloadDir + File.separator + getFileNameFromUrl());
String fileName = Script.runSimpleBashScript("ls " + downloadDir);
if (fileName == null) {
return false;
}
setDownloadedFilePath(downloadDir + File.separator + fileName);
return true;
}
}

70
agent/src/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java Normal file
View File

@ -0,0 +1,70 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import java.io.File;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.UUID;
public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
private String srcHost;
private String srcPath;
private static final String mountCommand = "mount -t nfs %s %s";
/**
* Parse url and set srcHost and srcPath
*/
private void parseUrl() {
URI uri = null;
String url = getUrl();
try {
uri = new URI(UriUtils.encodeURIComponent(url));
if (uri.getScheme() != null && uri.getScheme().equalsIgnoreCase("nfs")) {
srcHost = uri.getHost();
srcPath = uri.getPath();
}
} catch (URISyntaxException e) {
throw new CloudRuntimeException("Invalid NFS url " + url + " caused error: " + e.getMessage());
}
}
public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum) {
super(url, destPool, templateId, checksum);
parseUrl();
}
@Override
public boolean downloadTemplate() {
String mountSrcUuid = UUID.randomUUID().toString();
String mount = String.format(mountCommand, srcHost + ":" + srcPath, "/mnt/" + mountSrcUuid);
Script.runSimpleBashScript(mount);
String downloadDir = getDestPoolPath() + File.separator + getDirectDownloadTempPath(getTemplateId());
setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl());
Script.runSimpleBashScript("cp /mnt/" + mountSrcUuid + srcPath + " " + getDownloadedFilePath());
Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid);
return true;
}
}

36
agent/test/com/cloud/agent/direct/download/DirectTemplateDownloaderImplTest.java Normal file
View File

@ -0,0 +1,36 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.direct.download;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.runners.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public class DirectTemplateDownloaderImplTest {
private static final Long templateId = 202l;
@Test
public void testGetDirectDownloadTempPath() {
String path = DirectTemplateDownloaderImpl.getDirectDownloadTempPath(templateId);
Assert.assertEquals("template/2/202", path);
}
}

5
api/pom.xml
View File

@ -61,6 +61,11 @@
<artifactId>commons-lang3</artifactId>
<version>${cs.commons-lang3.version}</version>
</dependency>
<dependency>
<groupId>org.apache.cloudstack</groupId>
<artifactId>cloud-framework-direct-download</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<build>
<plugins>

2
api/src/com/cloud/storage/VMTemplateStorageResourceAssoc.java
View File

@ -22,7 +22,7 @@ import org.apache.cloudstack.api.InternalIdentity;
public interface VMTemplateStorageResourceAssoc extends InternalIdentity {
public static enum Status {
UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED
UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED
}
String getInstallPath();

1
api/src/org/apache/cloudstack/api/ApiConstants.java
View File

@ -83,6 +83,7 @@ public class ApiConstants {
public static final String DESTINATION_ZONE_ID = "destzoneid";
public static final String DETAILS = "details";
public static final String DEVICE_ID = "deviceid";
public static final String DIRECT_DOWNLOAD = "directdownload";
public static final String DISK_OFFERING_ID = "diskofferingid";
public static final String DISK_SIZE = "disksize";
public static final String UTILIZATION = "utilization";

89
api/src/org/apache/cloudstack/api/command/admin/direct/download/UploadTemplateDirectDownloadCertificate.java Executable file
View File

@ -0,0 +1,89 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.direct.download;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.exception.NetworkRuleConflictException;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.response.SuccessResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.direct.download.DirectDownloadManager;
import org.apache.log4j.Logger;
import javax.inject.Inject;
@APICommand(name = UploadTemplateDirectDownloadCertificate.APINAME,
description = "Upload a certificate for HTTPS direct template download on KVM hosts",
responseObject = SuccessResponse.class,
requestHasSensitiveInfo = true,
responseHasSensitiveInfo = true,
since = "4.11.0",
authorized = {RoleType.Admin})
public class UploadTemplateDirectDownloadCertificate extends BaseCmd {
@Inject
DirectDownloadManager directDownloadManager;
private static final Logger LOG = Logger.getLogger(UploadTemplateDirectDownloadCertificate.class);
public static final String APINAME = "uploadTemplateDirectDownloadCertificate";
@Parameter(name = ApiConstants.CERTIFICATE, type = BaseCmd.CommandType.STRING, required = true, length = 65535,
description = "SSL certificate")
private String certificate;
@Parameter(name = ApiConstants.NAME , type = BaseCmd.CommandType.STRING, required = true,
description = "Name for the uploaded certificate")
private String name;
@Parameter(name = ApiConstants.HYPERVISOR, type = BaseCmd.CommandType.STRING, required = true, description = "Hypervisor type")
private String hypervisor;
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
if (!hypervisor.equalsIgnoreCase("kvm")) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Currently supporting KVM hosts only");
}
try {
directDownloadManager.uploadCertificateToHosts(certificate, name);;
setResponseObject(new SuccessResponse(getCommandName()));
} catch (Exception e) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage());
}
}
@Override
public String getCommandName() {
return UploadTemplateDirectDownloadCertificate.APINAME;
}
@Override
public long getEntityOwnerId() {
return CallContext.current().getCallingAccount().getId();
}
}

12
api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
View File

@ -40,17 +40,7 @@ public class RegisterTemplateCmdByAdmin extends RegisterTemplateCmd {
@Override
public void execute() throws ResourceAllocationException{
try {
if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty()))
throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
"Both zoneid and zoneids cannot be specified at the same time");
if (zoneId == null && (zoneIds == null || zoneIds.isEmpty()))
throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
"Either zoneid or zoneids is required. Both cannot be null.");
if (zoneIds != null && zoneIds.size() > 1 && zoneIds.contains(-1L))
throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
"Parameter zoneids cannot combine all zones (-1) option with other zones");
validateParameters();
VirtualMachineTemplate template = _templateService.registerTemplate(this);
if (template != null){

9
api/src/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java
View File

@ -108,6 +108,11 @@ public class RegisterIsoCmd extends BaseCmd {
description = "true if ISO contains XS/VMWare tools inorder to support dynamic scaling of VM CPU/memory")
protected Boolean isDynamicallyScalable;
@Parameter(name=ApiConstants.DIRECT_DOWNLOAD,
type = CommandType.BOOLEAN,
description = "true if ISO should bypass Secondary Storage and be downloaded to Primary Storage on deployment")
private Boolean directDownload;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@ -168,6 +173,10 @@ public class RegisterIsoCmd extends BaseCmd {
return isDynamicallyScalable == null ? Boolean.FALSE : isDynamicallyScalable;
}
public boolean isDirectDownload() {
return directDownload == null ? false : directDownload;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////

41
api/src/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java
View File

@ -16,6 +16,7 @@
// under the License.
package org.apache.cloudstack.api.command.user.template;
import com.cloud.hypervisor.Hypervisor;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
@ -155,6 +156,11 @@ public class RegisterTemplateCmd extends BaseCmd {
"zone template and will be copied to all zones. ")
protected List<Long> zoneIds;
@Parameter(name=ApiConstants.DIRECT_DOWNLOAD,
type = CommandType.BOOLEAN,
description = "true if template should bypass Secondary Storage and be downloaded to Primary Storage on deployment")
private Boolean directDownload;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@ -263,6 +269,10 @@ public class RegisterTemplateCmd extends BaseCmd {
return isRoutingType;
}
public boolean isDirectDownload() {
return directDownload == null ? false : directDownload;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@ -289,17 +299,7 @@ public class RegisterTemplateCmd extends BaseCmd {
@Override
public void execute() throws ResourceAllocationException {
try {
if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty()))
throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
"Both zoneid and zoneids cannot be specified at the same time");
if (zoneId == null && (zoneIds == null || zoneIds.isEmpty()))
throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
"Either zoneid or zoneids is required. Both cannot be null.");
if (zoneIds != null && zoneIds.size() > 1 && zoneIds.contains(-1L))
throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
"Parameter zoneids cannot combine all zones (-1) option with other zones");
validateParameters();
VirtualMachineTemplate template = _templateService.registerTemplate(this);
if (template != null) {
@ -317,4 +317,23 @@ public class RegisterTemplateCmd extends BaseCmd {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, ex1.getMessage());
}
}
protected void validateParameters() {
if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty()))
throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
"Both zoneid and zoneids cannot be specified at the same time");
if (zoneId == null && (zoneIds == null || zoneIds.isEmpty()))
throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
"Either zoneid or zoneids is required. Both cannot be null.");
if (zoneIds != null && zoneIds.size() > 1 && zoneIds.contains(-1L))
throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
"Parameter zoneids cannot combine all zones (-1) option with other zones");
if (isDirectDownload() && !getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.KVM.toString())) {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
"Parameter directdownload is only allowed for KVM templates");
}
}
}

12
api/src/org/apache/cloudstack/api/response/TemplateResponse.java
View File

@ -185,6 +185,10 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements
@Param(description = "true if template contains XS/VMWare tools inorder to support dynamic scaling of VM cpu/memory")
private Boolean isDynamicallyScalable;
@SerializedName(ApiConstants.DIRECT_DOWNLOAD)
@Param(description = "KVM Only: true if template is directly downloaded to Primary Storage bypassing Secondary Storage")
private Boolean directDownload;
public TemplateResponse() {
tags = new LinkedHashSet<ResourceTagResponse>();
}
@ -362,4 +366,12 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements
public void setBits(int bits) {
this.bits = bits;
}
public void setDirectDownload(Boolean directDownload) {
this.directDownload = directDownload;
}
public Boolean getDirectDownload() {
return directDownload;
}
}

25
api/src/org/apache/cloudstack/direct/download/DirectDownloadManager.java Normal file
View File

@ -0,0 +1,25 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.direct.download;
import com.cloud.utils.component.PluggableService;
import org.apache.cloudstack.framework.agent.direct.download.DirectDownloadService;
public interface DirectDownloadManager extends DirectDownloadService, PluggableService {
}

5
client/pom.xml
View File

@ -318,6 +318,11 @@
<artifactId>cloud-framework-ca</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.cloudstack</groupId>
<artifactId>cloud-framework-direct-download</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.cloudstack</groupId>
<artifactId>cloud-framework-ipc</artifactId>

4
core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml
View File

@ -319,4 +319,8 @@
class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
</bean>
<bean id="directDownloadRegistry"
class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
</bean>
</beans>

21
core/resources/META-INF/cloudstack/direct-download/module.properties Normal file
View File

@ -0,0 +1,21 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name=direct-download
parent=backend

32
core/resources/META-INF/cloudstack/direct-download/spring-lifecycle-direct-download-context-inheritable.xml Normal file
View File

@ -0,0 +1,32 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd"
>
<bean class="org.apache.cloudstack.spring.lifecycle.registry.RegistryLifecycle">
<property name="registry" ref="directDownloadRegistry" />
<property name="typeClass" value="org.apache.cloudstack.framework.agent.direct.download.DirectDownloadService" />
</bean>
</beans>

3
core/src/com/cloud/storage/resource/StorageProcessor.java
View File

@ -19,6 +19,7 @@
package com.cloud.storage.resource;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.storage.command.AttachCommand;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.CreateObjectCommand;
@ -71,4 +72,6 @@ public interface StorageProcessor {
public Answer snapshotAndCopy(SnapshotAndCopyCommand cmd);
public Answer resignature(ResignatureCommand cmd);
public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd);
}

5
core/src/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java
View File

@ -19,6 +19,7 @@
package com.cloud.storage.resource;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.log4j.Logger;
import org.apache.cloudstack.storage.command.AttachCommand;
@ -66,7 +67,9 @@ public class StorageSubsystemCommandHandlerBase implements StorageSubsystemComma
} else if (command instanceof SnapshotAndCopyCommand) {
return processor.snapshotAndCopy((SnapshotAndCopyCommand)command);
} else if (command instanceof ResignatureCommand) {
return processor.resignature((ResignatureCommand)command);
return processor.resignature((ResignatureCommand) command);
} else if (command instanceof DirectDownloadCommand) {
return processor.handleDownloadTemplateToPrimaryStorage((DirectDownloadCommand) command);
}
return new Answer((Command)command, false, "not implemented yet");

66
core/src/com/cloud/storage/template/MetalinkTemplateDownloader.java Normal file
View File

@ -0,0 +1,66 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.storage.template;
import com.cloud.storage.StorageLayer;
import com.cloud.utils.script.Script;
import org.apache.log4j.Logger;
import java.io.File;
public class MetalinkTemplateDownloader extends TemplateDownloaderBase implements TemplateDownloader {
private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
private static final Logger LOGGER = Logger.getLogger(MetalinkTemplateDownloader.class.getName());
public MetalinkTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSize) {
super(storageLayer, downloadUrl, toDir, maxTemplateSize, callback);
String[] parts = _downloadUrl.split("/");
String filename = parts[parts.length - 1];
_toFile = toDir + File.separator + filename;
}
@Override
public long download(boolean resume, DownloadCompleteCallback callback) {
if (!status.equals(Status.NOT_STARTED)) {
// Only start downloading if we haven't started yet.
LOGGER.debug("Template download is already started, not starting again. Template: " + _downloadUrl);
return 0;
}
status = Status.IN_PROGRESS;
Script.runSimpleBashScript("aria2c " + _downloadUrl + " -d " + _toDir);
status = Status.DOWNLOAD_FINISHED;
String sizeResult = Script.runSimpleBashScript("ls -als " + _toFile + " | awk '{print $1}'");
long size = Long.parseLong(sizeResult);
return size;
}
@Override
public int getDownloadPercent() {
if (status == Status.DOWNLOAD_FINISHED) {
return 100;
} else if (status == Status.IN_PROGRESS) {
return 50;
} else {
return 0;
}
}
}

37
core/src/org/apache/cloudstack/agent/directdownload/CheckUrlAnswer.java Normal file
View File

@ -0,0 +1,37 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.agent.directdownload;
import com.cloud.agent.api.Answer;
public class CheckUrlAnswer extends Answer {
private Long templateSize;
public CheckUrlAnswer(final boolean result, final Long templateSize) {
super(null);
this.result = result;
this.templateSize = templateSize;
}
public Long getTemplateSize() {
return templateSize;
}
}

42
core/src/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java Normal file
View File

@ -0,0 +1,42 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.agent.directdownload;
import com.cloud.agent.api.Command;
public class CheckUrlCommand extends Command {
private String url;
public String getUrl() {
return url;
}
public CheckUrlCommand(final String url) {
super();
this.url = url;
}
@Override
public boolean executeInSequence() {
return false;
}
}

48
core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadAnswer.java Normal file
View File

@ -0,0 +1,48 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.agent.directdownload;
import com.cloud.agent.api.Answer;
public class DirectDownloadAnswer extends Answer {
private Long templateSize;
private String installPath;
public DirectDownloadAnswer(final boolean result, final String msg) {
super(null);
this.result = result;
this.details = msg;
}
public DirectDownloadAnswer(final boolean result, final Long size, final String installPath) {
super(null);
this.result = result;
this.templateSize = size;
this.installPath = installPath;
}
public long getTemplateSize() {
return templateSize;
}
public String getInstallPath() {
return installPath;
}
}

67
core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java Normal file
View File

@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cloudstack.agent.directdownload;
import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
public enum DownloadProtocol {
HTTP, HTTPS, NFS, METALINK
}
private String url;
private Long templateId;
private PrimaryDataStoreTO destPool;
private String checksum;
protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum) {
this.url = url;
this.templateId = templateId;
this.destPool = destPool;
this.checksum = checksum;
}
public String getUrl() {
return url;
}
public Long getTemplateId() {
return templateId;
}
public PrimaryDataStoreTO getDestPool() {
return destPool;
}
public String getChecksum() {
return checksum;
}
@Override
public void setExecuteInSequence(boolean inSeq) {
}
@Override
public boolean executeInSequence() {
return false;
}
}

38
core/src/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cloudstack.agent.directdownload;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import java.util.Map;
public class HttpDirectDownloadCommand extends DirectDownloadCommand {
private Map<String, String> headers;
public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
super(url, templateId, destPool, checksum);
this.headers = headers;
}
public Map<String, String> getHeaders() {
return headers;
}
}

31
core/src/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java Normal file
View File

@ -0,0 +1,31 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.agent.directdownload;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import java.util.Map;
public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
super(url, templateId, destPool, checksum);
}
}

29
core/src/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java Normal file
View File

@ -0,0 +1,29 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.agent.directdownload;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum) {
super(url, templateId, destPool, checksum);
}
}

29
core/src/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java Normal file
View File

@ -0,0 +1,29 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.agent.directdownload;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
public class NfsDirectDownloadCommand extends DirectDownloadCommand {
public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum) {
super(url, templateId, destPool, checksum);
}
}

45
core/src/org/apache/cloudstack/agent/directdownload/SetupDirectDownloadCertificate.java Normal file
View File

@ -0,0 +1,45 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.agent.directdownload;
import com.cloud.agent.api.Command;
public class SetupDirectDownloadCertificate extends Command {
private String certificate;
private String certificateName;
public SetupDirectDownloadCertificate(String certificate, String name) {
this.certificate = certificate;
this.certificateName = name;
}
public String getCertificate() {
return certificate;
}
public String getCertificateName() {
return certificateName;
}
@Override
public boolean executeInSequence() {
return false;
}
}

2
debian/control vendored
View File

@ -22,7 +22,7 @@ Description: CloudStack server library
Package: cloudstack-agent
Architecture: all
Depends: ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 1.2.2), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, lsb-release, init-system-helpers (>= 1.14~)
Depends: ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 1.2.2), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, lsb-release, init-system-helpers (>= 1.14~), aria2
Recommends: init-system-helpers
Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
Description: CloudStack agent

4
engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java
View File

@ -36,4 +36,8 @@ public interface TemplateDataFactory {
TemplateInfo getReadyTemplateOnCache(long templateId);
List<TemplateInfo> listTemplateOnCache(long templateId);
TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long poolId, Long hostId);
boolean isTemplateMarkedForDirectDownload(long templateId);
}

2
engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java
View File

@ -25,4 +25,6 @@ public interface TemplateInfo extends DataObject, VirtualMachineTemplate {
String getUniqueName();
String getInstallPath();
boolean isDirectDownload();
}

1
engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java
View File

@ -108,4 +108,5 @@ public interface VolumeService {
SnapshotInfo takeSnapshot(VolumeInfo volume);
VolumeInfo updateHypervisorSnapshotReserveForVolume(DiskOffering diskOffering, long volumeId, HypervisorType hyperType);
}

2
engine/components-api/src/com/cloud/resource/ResourceManager.java
View File

@ -195,4 +195,6 @@ public interface ResourceManager extends ResourceService {
* @return Details of groupNames and enabled VGPU type with remaining capacity.
*/
HashMap<String, HashMap<String, VgpuTypesInfo>> getGPUStatistics(HostVO host);
HostVO findOneRandomRunningHostByHypervisor(HypervisorType type);
}

5
engine/components-api/src/com/cloud/template/TemplateManager.java
View File

@ -18,6 +18,7 @@ package com.cloud.template;
import java.util.List;
import com.cloud.deploy.DeployDestination;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.framework.config.ConfigKey;
@ -119,7 +120,7 @@ public interface TemplateManager {
List<DataStore> getImageStoreByTemplate(long templateId, Long zoneId);
TemplateInfo prepareIso(long isoId, long dcId);
TemplateInfo prepareIso(long isoId, long dcId, Long hostId, Long poolId);
/**
@ -127,7 +128,7 @@ public interface TemplateManager {
*
* @param VirtualMachineProfile
*/
void prepareIsoForVmProfile(VirtualMachineProfile profile);
void prepareIsoForVmProfile(VirtualMachineProfile profile, DeployDestination dest);
public static final String MESSAGE_REGISTER_PUBLIC_TEMPLATE_EVENT = "Message.RegisterPublicTemplate.Event";
public static final String MESSAGE_RESET_TEMPLATE_PERMISSION_EVENT = "Message.ResetTemplatePermission.Event";

7
engine/orchestration/pom.xml
View File

@ -58,13 +58,6 @@
<artifactId>cloud-utils</artifactId>
<version>${project.version}</version>
</dependency>
<!--
<dependency>
<groupId>org.apache.cloudstack</groupId>
<artifactId>cloud-server</artifactId>
<version>${project.version}</version>
</dependency>
-->
</dependencies>
<build>
<plugins>

12
engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java
View File

@ -1108,7 +1108,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
//if (vm.getType() == VirtualMachine.Type.User && vm.getTemplate().getFormat() == ImageFormat.ISO) {
if (vm.getType() == VirtualMachine.Type.User) {
_tmpltMgr.prepareIsoForVmProfile(vm);
_tmpltMgr.prepareIsoForVmProfile(vm, dest);
//DataTO dataTO = tmplFactory.getTemplate(vm.getTemplate().getId(), DataStoreRole.Image, vm.getVirtualMachine().getDataCenterId()).getTO();
//DiskTO iso = new DiskTO(dataTO, 3L, null, Volume.Type.ISO);
//vm.addDisk(iso);
@ -1287,9 +1287,13 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
TemplateInfo templ = tmplFactory.getReadyTemplateOnImageStore(templateId, dest.getDataCenter().getId());
if (templ == null) {
s_logger.debug("can't find ready template: " + templateId + " for data center " + dest.getDataCenter().getId());
throw new CloudRuntimeException("can't find ready template: " + templateId + " for data center " + dest.getDataCenter().getId());
if (tmplFactory.isTemplateMarkedForDirectDownload(templateId)) {
// Template is marked for direct download bypassing Secondary Storage
templ = tmplFactory.getReadyBypassedTemplateOnPrimaryStore(templateId, destPool.getId(), dest.getHost().getId());
} else {
s_logger.debug("can't find ready template: " + templateId + " for data center " + dest.getDataCenter().getId());
throw new CloudRuntimeException("can't find ready template: " + templateId + " for data center " + dest.getDataCenter().getId());
}
}
PrimaryDataStore primaryDataStore = (PrimaryDataStore)destPool;

105
engine/schema/resources/META-INF/db/schema-41000to41100.sql
View File

@ -524,6 +524,111 @@ ADD COLUMN `forsystemvms` TINYINT(1) NOT NULL DEFAULT '0' COMMENT 'Indicates if
ALTER TABLE `cloud`.`op_dc_ip_address_alloc`
ADD COLUMN `vlan` INT(10) UNSIGNED NULL COMMENT 'Vlan the management network range is on';
-- CLOUDSTACK-10146: Bypass Secondary Storage for KVM templates
ALTER TABLE `cloud`.`vm_template`
ADD COLUMN `direct_download` TINYINT(1) DEFAULT '0' COMMENT 'Indicates if Secondary Storage is bypassed and template is downloaded to Primary Storage';
CREATE OR REPLACE VIEW `template_view` AS
SELECT
`vm_template`.`id` AS `id`,
`vm_template`.`uuid` AS `uuid`,
`vm_template`.`unique_name` AS `unique_name`,
`vm_template`.`name` AS `name`,
`vm_template`.`public` AS `public`,
`vm_template`.`featured` AS `featured`,
`vm_template`.`type` AS `type`,
`vm_template`.`hvm` AS `hvm`,
`vm_template`.`bits` AS `bits`,
`vm_template`.`url` AS `url`,
`vm_template`.`format` AS `format`,
`vm_template`.`created` AS `created`,
`vm_template`.`checksum` AS `checksum`,
`vm_template`.`display_text` AS `display_text`,
`vm_template`.`enable_password` AS `enable_password`,
`vm_template`.`dynamically_scalable` AS `dynamically_scalable`,
`vm_template`.`state` AS `template_state`,
`vm_template`.`guest_os_id` AS `guest_os_id`,
`guest_os`.`uuid` AS `guest_os_uuid`,
`guest_os`.`display_name` AS `guest_os_name`,
`vm_template`.`bootable` AS `bootable`,
`vm_template`.`prepopulate` AS `prepopulate`,
`vm_template`.`cross_zones` AS `cross_zones`,
`vm_template`.`hypervisor_type` AS `hypervisor_type`,
`vm_template`.`extractable` AS `extractable`,
`vm_template`.`template_tag` AS `template_tag`,
`vm_template`.`sort_key` AS `sort_key`,
`vm_template`.`removed` AS `removed`,
`vm_template`.`enable_sshkey` AS `enable_sshkey`,
`source_template`.`id` AS `source_template_id`,
`source_template`.`uuid` AS `source_template_uuid`,
`account`.`id` AS `account_id`,
`account`.`uuid` AS `account_uuid`,
`account`.`account_name` AS `account_name`,
`account`.`type` AS `account_type`,
`domain`.`id` AS `domain_id`,
`domain`.`uuid` AS `domain_uuid`,
`domain`.`name` AS `domain_name`,
`domain`.`path` AS `domain_path`,
`projects`.`id` AS `project_id`,
`projects`.`uuid` AS `project_uuid`,
`projects`.`name` AS `project_name`,
`data_center`.`id` AS `data_center_id`,
`data_center`.`uuid` AS `data_center_uuid`,
`data_center`.`name` AS `data_center_name`,
`launch_permission`.`account_id` AS `lp_account_id`,
`template_store_ref`.`store_id` AS `store_id`,
`image_store`.`scope` AS `store_scope`,
`template_store_ref`.`state` AS `state`,
`template_store_ref`.`download_state` AS `download_state`,
`template_store_ref`.`download_pct` AS `download_pct`,
`template_store_ref`.`error_str` AS `error_str`,
`template_store_ref`.`size` AS `size`,
`template_store_ref`.physical_size AS `physical_size`,
`template_store_ref`.`destroyed` AS `destroyed`,
`template_store_ref`.`created` AS `created_on_store`,
`vm_template_details`.`name` AS `detail_name`,
`vm_template_details`.`value` AS `detail_value`,
`resource_tags`.`id` AS `tag_id`,
`resource_tags`.`uuid` AS `tag_uuid`,
`resource_tags`.`key` AS `tag_key`,
`resource_tags`.`value` AS `tag_value`,
`resource_tags`.`domain_id` AS `tag_domain_id`,
`domain`.`uuid` AS `tag_domain_uuid`,
`domain`.`name` AS `tag_domain_name`,
`resource_tags`.`account_id` AS `tag_account_id`,
`account`.`account_name` AS `tag_account_name`,
`resource_tags`.`resource_id` AS `tag_resource_id`,
`resource_tags`.`resource_uuid` AS `tag_resource_uuid`,
`resource_tags`.`resource_type` AS `tag_resource_type`,
`resource_tags`.`customer` AS `tag_customer`,
CONCAT(`vm_template`.`id`,
'_',
IFNULL(`data_center`.`id`, 0)) AS `temp_zone_pair`,
`vm_template`.`direct_download` AS `direct_download`
FROM
((((((((((((`vm_template`
JOIN `guest_os` ON ((`guest_os`.`id` = `vm_template`.`guest_os_id`)))
JOIN `account` ON ((`account`.`id` = `vm_template`.`account_id`)))
JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`)))
LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`)))
LEFT JOIN `vm_template_details` ON ((`vm_template_details`.`template_id` = `vm_template`.`id`)))
LEFT JOIN `vm_template` `source_template` ON ((`source_template`.`id` = `vm_template`.`source_template_id`)))
LEFT JOIN `template_store_ref` ON (((`template_store_ref`.`template_id` = `vm_template`.`id`)
AND (`template_store_ref`.`store_role` = 'Image')
AND (`template_store_ref`.`destroyed` = 0))))
LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`)
AND (`template_store_ref`.`store_id` IS NOT NULL)
AND (`image_store`.`id` = `template_store_ref`.`store_id`))))
LEFT JOIN `template_zone_ref` ON (((`template_zone_ref`.`template_id` = `vm_template`.`id`)
AND ISNULL(`template_store_ref`.`store_id`)
AND ISNULL(`template_zone_ref`.`removed`))))
LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`)
OR (`template_zone_ref`.`zone_id` = `data_center`.`id`))))
LEFT JOIN `launch_permission` ON ((`launch_permission`.`template_id` = `vm_template`.`id`)))
LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_template`.`id`)
AND ((`resource_tags`.`resource_type` = 'Template')
OR (`resource_tags`.`resource_type` = 'ISO')))));
-- CLOUDSTACK-10109: Enable dedication of public IPs to SSVM and CPVM
ALTER TABLE `cloud`.`user_ip_address`
ADD COLUMN `forsystemvms` TINYINT(1) NOT NULL DEFAULT '0' COMMENT 'true if IP is set to system vms, false if not';

10
engine/schema/src/com/cloud/storage/VMTemplateVO.java
View File

@ -146,6 +146,9 @@ public class VMTemplateVO implements VirtualMachineTemplate {
@Column(name = "dynamically_scalable")
protected boolean dynamicallyScalable;
@Column(name = "direct_download")
private boolean directDownload;
@Override
public String getUniqueName() {
return uniqueName;
@ -188,7 +191,7 @@ public class VMTemplateVO implements VirtualMachineTemplate {
public VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic, boolean featured, boolean isExtractable, TemplateType type, String url,
boolean requiresHvm, int bits, long accountId, String cksum, String displayText, boolean enablePassword, long guestOSId, boolean bootable,
HypervisorType hyperType, String templateTag, Map<String, String> details, boolean sshKeyEnabled, boolean isDynamicallyScalable) {
HypervisorType hyperType, String templateTag, Map<String, String> details, boolean sshKeyEnabled, boolean isDynamicallyScalable, boolean directDownload) {
this(id,
name,
format,
@ -212,6 +215,7 @@ public class VMTemplateVO implements VirtualMachineTemplate {
enableSshKey = sshKeyEnabled;
dynamicallyScalable = isDynamicallyScalable;
state = State.Active;
this.directDownload = directDownload;
}
public static VMTemplateVO createPreHostIso(Long id, String uniqueName, String name, ImageFormat format, boolean isPublic, boolean featured, TemplateType type,
@ -605,6 +609,10 @@ public class VMTemplateVO implements VirtualMachineTemplate {
this.updated = updated;
}
public boolean isDirectDownload() {
return directDownload;
}
@Override
public Class<?> getEntityType() {
return VirtualMachineTemplate.class;

5
engine/schema/src/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java
View File

@ -23,6 +23,8 @@ import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@ -44,6 +46,9 @@ import com.cloud.utils.db.UpdateBuilder;
public class VMTemplatePoolDaoImpl extends GenericDaoBase<VMTemplateStoragePoolVO, Long> implements VMTemplatePoolDao {
public static final Logger s_logger = Logger.getLogger(VMTemplatePoolDaoImpl.class.getName());
@Inject
DataStoreManager dataStoreManager;
protected final SearchBuilder<VMTemplateStoragePoolVO> PoolSearch;
protected final SearchBuilder<VMTemplateStoragePoolVO> TemplateSearch;
protected final SearchBuilder<VMTemplateStoragePoolVO> PoolTemplateSearch;

6
engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
View File

@ -83,4 +83,10 @@ public interface TemplateDataStoreDao extends GenericDao<TemplateDataStoreVO, Lo
void expireDnldUrlsForZone(Long dcId);
List<TemplateDataStoreVO> listByTemplateState(VirtualMachineTemplate.State... states);
TemplateDataStoreVO createTemplateDirectDownloadEntry(long templateId, Long size);
TemplateDataStoreVO getReadyBypassedTemplate(long templateId);
boolean isTemplateMarkedForDirectDownload(long templateId);
}

82
engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java
View File

@ -18,16 +18,23 @@
*/
package org.apache.cloudstack.storage.image;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.direct.download.DirectDownloadManager;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.image.store.TemplateObject;
@ -51,6 +58,12 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
VMTemplatePoolDao templatePoolDao;
@Inject
TemplateDataStoreDao templateStoreDao;
@Inject
DirectDownloadManager directDownloadManager;
@Inject
HostDao hostDao;
@Inject
PrimaryDataStoreDao primaryDataStoreDao;
@Override
public TemplateInfo getTemplate(long templateId, DataStore store) {
@ -137,7 +150,6 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
} else {
return null;
}
}
@Override
@ -153,4 +165,70 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
return tmplObjs;
}
/**
* Given existing spool refs, return one pool id existing on pools and refs
*/
private Long getOneMatchingPoolIdFromRefs(List<VMTemplateStoragePoolVO> existingRefs, List<StoragePoolVO> pools) {
if (pools.isEmpty()) {
throw new CloudRuntimeException("No storage pools found");
}
if (existingRefs.isEmpty()) {
return pools.get(0).getId();
} else {
for (VMTemplateStoragePoolVO ref : existingRefs) {
for (StoragePoolVO p : pools) {
if (ref.getPoolId() == p.getId()) {
return p.getId();
}
}
}
}
return null;
}
/**
* Retrieve storage pools with scope = cluster or zone matching clusterId or dataCenterId depending on their scope
*/
private List<StoragePoolVO> getStoragePoolsFromClusterOrZone(Long clusterId, long dataCenterId, Hypervisor.HypervisorType hypervisorType) {
List<StoragePoolVO> pools = new ArrayList<>();
if (clusterId != null) {
List<StoragePoolVO> clusterPools = primaryDataStoreDao.listPoolsByCluster(clusterId);
pools.addAll(clusterPools);
}
List<StoragePoolVO> zonePools = primaryDataStoreDao.findZoneWideStoragePoolsByHypervisor(dataCenterId, hypervisorType);
pools.addAll(zonePools);
return pools;
}
@Override
public TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long poolId, Long hostId) {
VMTemplateVO templateVO = imageDataDao.findById(templateId);
if (templateVO == null || !templateVO.isDirectDownload()) {
return null;
}
Long pool = poolId;
if (poolId == null) {
//Get ISO from existing pool ref
HostVO host = hostDao.findById(hostId);
List<StoragePoolVO> pools = getStoragePoolsFromClusterOrZone(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType());
List<VMTemplateStoragePoolVO> existingRefs = templatePoolDao.listByTemplateId(templateId);
pool = getOneMatchingPoolIdFromRefs(existingRefs, pools);
}
if (pool == null) {
throw new CloudRuntimeException("No storage pool found where to download template: " + templateId);
}
VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(pool, templateId);
if (spoolRef == null) {
directDownloadManager.downloadTemplate(templateId, pool, hostId);
}
DataStore store = storeMgr.getDataStore(pool, DataStoreRole.Primary);
return this.getTemplate(templateId, store);
}
@Override
public boolean isTemplateMarkedForDirectDownload(long templateId) {
VMTemplateVO templateVO = imageDataDao.findById(templateId);
return templateVO.isDirectDownload();
}
}

3
engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
View File

@ -445,6 +445,9 @@ public class TemplateServiceImpl implements TemplateService {
} catch (NoTransitionException e) {
s_logger.error("Unexpected state transition exception for template " + tmplt.getName() + ". Details: " + e.getMessage());
}
} else if (tmplt.isDirectDownload()) {
s_logger.info("Template " + tmplt.getName() + ":" + tmplt.getId() + " is marked for direct download, discarding it for download on image stores");
toBeDownloaded.remove(tmplt);
} else {
s_logger.info("Template Sync did not find " + uniqueName + " on image store " + storeId + ", may request download based on available hypervisor types");
if (tmpltStore != null) {

8
engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java
View File

@ -344,6 +344,14 @@ public class TemplateObject implements TemplateInfo {
return obj != null ? obj.getInstallPath() : null;
}
@Override
public boolean isDirectDownload() {
if (this.imageVO == null) {
return false;
}
return this.imageVO.isDirectDownload();
}
public void setInstallPath(String installPath) {
this.installPath = installPath;
}

8
engine/storage/src/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java
View File

@ -22,12 +22,12 @@ import java.util.List;
import javax.inject.Inject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager;
import com.cloud.storage.DataStoreRole;

39
engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
View File

@ -26,6 +26,7 @@ import java.util.Map;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
@ -67,6 +68,7 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
private SearchBuilder<TemplateDataStoreVO> storeTemplateDownloadStatusSearch;
private SearchBuilder<TemplateDataStoreVO> downloadTemplateSearch;
private SearchBuilder<TemplateDataStoreVO> uploadTemplateStateSearch;
private SearchBuilder<TemplateDataStoreVO> directDownloadTemplateSeach;
private SearchBuilder<VMTemplateVO> templateOnlySearch;
private static final String EXPIRE_DOWNLOAD_URLS_FOR_ZONE = "update template_store_ref set download_url_created=? where download_url_created is not null and store_id in (select id from image_store where data_center_id=?)";
@ -143,6 +145,13 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
downloadTemplateSearch.and("destroyed", downloadTemplateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
downloadTemplateSearch.done();
directDownloadTemplateSeach = createSearchBuilder();
directDownloadTemplateSeach.and("template_id", directDownloadTemplateSeach.entity().getTemplateId(), Op.EQ);
directDownloadTemplateSeach.and("download_state", directDownloadTemplateSeach.entity().getDownloadState(), Op.EQ);
directDownloadTemplateSeach.and("store_id", directDownloadTemplateSeach.entity().getDataStoreId(), Op.NULL);
directDownloadTemplateSeach.and("state", directDownloadTemplateSeach.entity().getState(), Op.EQ);
directDownloadTemplateSeach.done();
templateOnlySearch = _tmpltDao.createSearchBuilder();
templateOnlySearch.and("states", templateOnlySearch.entity().getState(), SearchCriteria.Op.IN);
uploadTemplateStateSearch = createSearchBuilder();
@ -549,4 +558,34 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
sc.setParameters("destroyed", false);
return listIncludingRemovedBy(sc);
}
@Override
public TemplateDataStoreVO createTemplateDirectDownloadEntry(long templateId, Long size) {
TemplateDataStoreVO templateDataStoreVO = new TemplateDataStoreVO();
templateDataStoreVO.setTemplateId(templateId);
templateDataStoreVO.setDataStoreRole(DataStoreRole.Image);
templateDataStoreVO.setState(State.Ready);
templateDataStoreVO.setDownloadState(Status.BYPASSED);
templateDataStoreVO.setSize(size == null ? 0l : size);
return templateDataStoreVO;
}
@Override
public TemplateDataStoreVO getReadyBypassedTemplate(long templateId) {
SearchCriteria<TemplateDataStoreVO> sc = directDownloadTemplateSeach.create();
sc.setParameters("template_id", templateId);
sc.setParameters("download_state", Status.BYPASSED);
sc.setParameters("state", State.Ready);
List<TemplateDataStoreVO> list = search(sc, null);
if (CollectionUtils.isEmpty(list) || list.size() > 1) {
return null;
}
return list.get(0);
}
@Override
public boolean isTemplateMarkedForDirectDownload(long templateId) {
TemplateDataStoreVO templateRef = getReadyBypassedTemplate(templateId);
return templateRef != null;
}
}

31
framework/direct-download/pom.xml Normal file
View File

@ -0,0 +1,31 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>cloud-framework-direct-download</artifactId>
<name>Apache CloudStack Framework - Direct Download to Primary Storage</name>
<parent>
<artifactId>cloudstack-framework</artifactId>
<groupId>org.apache.cloudstack</groupId>
<version>4.11.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
</project>

31
framework/direct-download/src/org/apache/cloudstack/framework/agent/direct/download/DirectDownloadService.java Normal file
View File

@ -0,0 +1,31 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.framework.agent.direct.download;
public interface DirectDownloadService {
/**
* Download template/ISO into poolId bypassing secondary storage. Download performed by hostId
*/
void downloadTemplate(long templateId, long poolId, long hostId);
/**
* Upload client certificate to each running host
*/
boolean uploadCertificateToHosts(String certificateCer, String certificateName);
}

1
framework/pom.xml
View File

@ -56,5 +56,6 @@
<module>spring/lifecycle</module>
<module>spring/module</module>
<module>security</module>
<module>direct-download</module>
</modules>
</project>

2
packaging/centos63/cloud.spec
View File

@ -130,6 +130,8 @@ Requires: perl
Requires: libvirt-python
Requires: qemu-img
Requires: qemu-kvm
Requires: epel-release
Requires: aria2
Provides: cloud-agent
Obsoletes: cloud-agent < 4.1.0
Obsoletes: cloud-agent-libs < 4.1.0

2
packaging/centos7/cloud.spec
View File

@ -111,6 +111,8 @@ Requires: perl
Requires: libvirt-python
Requires: qemu-img
Requires: qemu-kvm
Requires: epel-release
Requires: aria2
Provides: cloud-agent
Group: System Environment/Libraries
%description agent

14
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View File

@ -2200,8 +2200,18 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
KVMStoragePool pool = null;
final DataTO data = volume.getData();
if (volume.getType() == Volume.Type.ISO && data.getPath() != null) {
final NfsTO nfsStore = (NfsTO)data.getDataStore();
final String volPath = nfsStore.getUrl() + File.separator + data.getPath();
DataStoreTO dataStore = data.getDataStore();
String dataStoreUrl = null;
if (dataStore instanceof NfsTO) {
NfsTO nfsStore = (NfsTO)data.getDataStore();
dataStoreUrl = nfsStore.getUrl();
} else if (dataStore instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO) dataStore).getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
//In order to support directly downloaded ISOs
String psHost = ((PrimaryDataStoreTO) dataStore).getHost();
String psPath = ((PrimaryDataStoreTO) dataStore).getPath();
dataStoreUrl = "nfs://" + psHost + File.separator + psPath;
}
final String volPath = dataStoreUrl + File.separator + data.getPath();
final int index = volPath.lastIndexOf("/");
final String volDir = volPath.substring(0, index);
final String volName = volPath.substring(index + 1);

50
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java Normal file
View File

@ -0,0 +1,50 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.hypervisor.kvm.resource.wrapper;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.resource.CommandWrapper;
import com.cloud.resource.ResourceWrapper;
import com.cloud.utils.UriUtils;
import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer;
import org.apache.cloudstack.agent.directdownload.CheckUrlCommand;
import org.apache.log4j.Logger;
@ResourceWrapper(handles = CheckUrlCommand.class)
public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, CheckUrlAnswer, LibvirtComputingResource> {
private static final Logger s_logger = Logger.getLogger(LibvirtCheckUrlCommand.class);
@Override
public CheckUrlAnswer execute(CheckUrlCommand cmd, LibvirtComputingResource serverResource) {
final String url = cmd.getUrl();
s_logger.info("Checking URL: " + url);
boolean checkResult = true;
Long remoteSize = null;
try {
UriUtils.checkUrlExistence(url);
remoteSize = UriUtils.getRemoteSize(url);
}
catch (IllegalArgumentException e) {
s_logger.warn(e.getMessage());
checkResult = false;
}
return new CheckUrlAnswer(checkResult, remoteSize);
}
}

59
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View File

@ -36,6 +36,18 @@ import java.util.UUID;
import javax.naming.ConfigurationException;
import com.cloud.agent.direct.download.DirectTemplateDownloader;
import com.cloud.agent.direct.download.DirectTemplateDownloader.DirectTemplateInformation;
import com.cloud.agent.direct.download.HttpDirectTemplateDownloader;
import com.cloud.agent.direct.download.MetalinkDirectTemplateDownloader;
import com.cloud.agent.direct.download.NfsDirectTemplateDownloader;
import com.cloud.agent.direct.download.HttpsDirectTemplateDownloader;
import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
@ -894,13 +906,21 @@ public class KVMStorageProcessor implements StorageProcessor {
final DiskTO disk = cmd.getDisk();
final TemplateObjectTO isoTO = (TemplateObjectTO)disk.getData();
final DataStoreTO store = isoTO.getDataStore();
if (!(store instanceof NfsTO)) {
String dataStoreUrl = null;
if (store instanceof NfsTO) {
NfsTO nfsStore = (NfsTO)store;
dataStoreUrl = nfsStore.getUrl();
} else if (store instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO) store).getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
//In order to support directly downloaded ISOs
String psHost = ((PrimaryDataStoreTO) store).getHost();
String psPath = ((PrimaryDataStoreTO) store).getPath();
dataStoreUrl = "nfs://" + psHost + File.separator + psPath;
} else {
return new AttachAnswer("unsupported protocol");
}
final NfsTO nfsStore = (NfsTO)store;
try {
final Connect conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName());
attachOrDetachISO(conn, cmd.getVmName(), nfsStore.getUrl() + File.separator + isoTO.getPath(), true);
attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), true);
} catch (final LibvirtException e) {
return new Answer(cmd, false, e.toString());
} catch (final URISyntaxException e) {
@ -1387,4 +1407,37 @@ public class KVMStorageProcessor implements StorageProcessor {
return new Answer(cmd, false, "not implememented yet");
}
@Override
public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
final PrimaryDataStoreTO pool = cmd.getDestPool();
if (!pool.getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
return new DirectDownloadAnswer(false, "Unsupported pool type " + pool.getPoolType().toString());
}
KVMStoragePool destPool = storagePoolMgr.getStoragePool(pool.getPoolType(), pool.getUuid());
DirectTemplateDownloader downloader;
if (cmd instanceof HttpDirectDownloadCommand) {
downloader = new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), ((HttpDirectDownloadCommand) cmd).getHeaders());
} else if (cmd instanceof HttpsDirectDownloadCommand) {
downloader = new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum());
} else if (cmd instanceof NfsDirectDownloadCommand) {
downloader = new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum());
} else if (cmd instanceof MetalinkDirectDownloadCommand) {
downloader = new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum());
} else {
return new DirectDownloadAnswer(false, "Unsupported protocol, please provide HTTP(S), NFS or a metalink");
}
if (!downloader.downloadTemplate()) {
return new DirectDownloadAnswer(false, "Could not download template " + cmd.getTemplateId() + " on " + destPool.getLocalPath());
}
if (!downloader.validateChecksum()) {
return new DirectDownloadAnswer(false, "Checksum validation failed for template " + cmd.getTemplateId());
}
if (!downloader.extractAndInstallDownloadedTemplate()) {
return new DirectDownloadAnswer(false, "Template downloaded but there was an error on installation");
}
DirectTemplateInformation info = downloader.getTemplateInformation();
return new DirectDownloadAnswer(true, info.getSize(), info.getInstallPath());
}
}

1
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java
View File

@ -1341,4 +1341,5 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
private void deleteDirVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtException {
Script.runSimpleBashScript("rm -r --interactive=never " + vol.getPath());
}
}

1
plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java
View File

@ -71,5 +71,4 @@ public interface StorageAdaptor {
public boolean deleteStoragePool(KVMStoragePool pool);
public boolean createFolder(String uuid, String path);
}

20
plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java
View File

@ -18,14 +18,34 @@
*/
package com.cloud.hypervisor.kvm.storage;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import javax.naming.ConfigurationException;
import org.junit.Before;
import org.junit.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.mockito.Spy;
public class KVMStorageProcessorTest {
@Mock
KVMStoragePoolManager storagePoolManager;
@Mock
LibvirtComputingResource resource;
private static final Long TEMPLATE_ID = 202l;
private static final String EXPECTED_DIRECT_DOWNLOAD_DIR = "template/2/202";
@Spy
@InjectMocks
private KVMStorageProcessor storageProcessor;
@Before
public void setUp() throws ConfigurationException {
MockitoAnnotations.initMocks(this);
storageProcessor = new KVMStorageProcessor(storagePoolManager, resource);
}
@Test

6
plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java
View File

@ -21,6 +21,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.UUID;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
@ -820,6 +821,11 @@ public class Ovm3StorageProcessor implements StorageProcessor {
return new ResignatureAnswer("Not implemented");
}
@Override
public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
return null;
}
/**
* Attach disks
* @param cmd

6
plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorStorageProcessor.java Normal file → Executable file
View File

@ -22,6 +22,7 @@ package com.cloud.resource;
import java.io.File;
import java.util.UUID;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.log4j.Logger;
import org.apache.cloudstack.storage.command.AttachAnswer;
@ -75,6 +76,11 @@ public class SimulatorStorageProcessor implements StorageProcessor {
return new ResignatureAnswer();
}
@Override
public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
return null;
}
@Override
public Answer copyTemplateToPrimaryStorage(CopyCommand cmd) {
TemplateObjectTO template = new TemplateObjectTO();

6
plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java
View File

@ -34,6 +34,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Strings;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
@ -2425,4 +2426,9 @@ public class VmwareStorageProcessor implements StorageProcessor {
this._fullCloneFlag = value;
s_logger.debug("VmwareProcessor instance - create full clone = " + (value ? "TRUE" : "FALSE"));
}
@Override
public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
return null;
}
}

7
plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
View File

@ -44,6 +44,7 @@ import com.xensource.xenapi.Types.XenAPIException;
import com.xensource.xenapi.VBD;
import com.xensource.xenapi.VDI;
import com.xensource.xenapi.VM;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
@ -195,6 +196,12 @@ public class XenServerStorageProcessor implements StorageProcessor {
}
}
@Override
public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
//Not implemented for Xen
return null;
}
@Override
public AttachAnswer attachIso(final AttachCommand cmd) {
final DiskTO disk = cmd.getDisk();

12
scripts/vm/hypervisor/kvm/setup_agent.sh
View File

@ -224,5 +224,17 @@ then
setenforce 0
fi
which aria2c
if [ $? -gt 0 ]
then
yum install epel-release -y
yum install aria2 -y
if [ $? -gt 0 ]
then
printf "failed to install aria2"
exit 1
fi
fi
cloudstack-setup-agent --host=$host --zone=$zone --pod=$pod --cluster=$cluster --guid=$guid $paramters -a > /dev/null
#cloud_consoleP_setup $host $zone $pod

1
server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
View File

@ -295,4 +295,5 @@
<bean id="annotationService" class="org.apache.cloudstack.annotation.AnnotationManagerImpl" />
<bean id="directDownloadManager" class="org.apache.cloudstack.direct.download.DirectDownloadManagerImpl" />
</beans>

8
server/src/com/cloud/api/query/dao/TemplateJoinDaoImpl.java
View File

@ -105,6 +105,8 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation<Templa
} else {
templateStatus = template.getDownloadPercent() + "% Downloaded";
}
} else if (template.getDownloadState() == Status.BYPASSED) {
templateStatus = "Bypassed Secondary Storage";
}else if (template.getErrorString()==null){
templateStatus = template.getTemplateState().toString();
}else {
@ -197,6 +199,8 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation<Templa
addTagInformation(template, templateResponse);
}
templateResponse.setDirectDownload(template.isDirectDownload());
templateResponse.setObjectName("template");
return templateResponse;
}
@ -320,6 +324,8 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation<Templa
} else {
isoStatus = iso.getDownloadPercent() + "% Downloaded";
}
} else if (iso.getDownloadState() == Status.BYPASSED) {
isoStatus = "Bypassed Secondary Storage";
} else {
isoStatus = iso.getErrorString();
}
@ -348,6 +354,8 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation<Templa
}
}
isoResponse.setDirectDownload(iso.isDirectDownload());
isoResponse.setObjectName("iso");
return isoResponse;

6
server/src/com/cloud/api/query/vo/TemplateJoinVO.java
View File

@ -222,6 +222,9 @@ public class TemplateJoinVO extends BaseViewWithTagInformationVO implements Cont
@Column(name = "temp_zone_pair")
private String tempZonePair; // represent a distinct (templateId, data_center_id) pair
@Column(name = "direct_download")
private boolean directDownload;
public TemplateJoinVO() {
}
@ -477,4 +480,7 @@ public class TemplateJoinVO extends BaseViewWithTagInformationVO implements Cont
this.accountId = accountId;
}
public boolean isDirectDownload() {
return directDownload;
}
}

18
server/src/com/cloud/resource/ResourceManagerImpl.java
View File

@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
@ -2797,6 +2798,23 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
return null;
}
@Override
public HostVO findOneRandomRunningHostByHypervisor(HypervisorType type) {
final QueryBuilder<HostVO> sc = QueryBuilder.create(HostVO.class);
sc.and(sc.entity().getHypervisorType(), Op.EQ, type);
sc.and(sc.entity().getType(),Op.EQ, Type.Routing);
sc.and(sc.entity().getStatus(), Op.EQ, Status.Up);
sc.and(sc.entity().getResourceState(), Op.EQ, ResourceState.Enabled);
sc.and(sc.entity().getRemoved(), Op.NULL);
List<HostVO> hosts = sc.list();
if (CollectionUtils.isEmpty(hosts)) {
return null;
} else {
Collections.shuffle(hosts, new Random(System.currentTimeMillis()));
return hosts.get(0);
}
}
@Override
@DB
@ActionEvent(eventType = EventTypes.EVENT_HOST_RESERVATION_RELEASE, eventDescription = "releasing host reservation", async = true)

3
server/src/com/cloud/server/ManagementServerImpl.java
View File

@ -65,6 +65,7 @@ import org.apache.cloudstack.api.command.admin.config.ListDeploymentPlannersCmd;
import org.apache.cloudstack.api.command.admin.config.ListHypervisorCapabilitiesCmd;
import org.apache.cloudstack.api.command.admin.config.UpdateCfgCmd;
import org.apache.cloudstack.api.command.admin.config.UpdateHypervisorCapabilitiesCmd;
import org.apache.cloudstack.api.command.admin.direct.download.UploadTemplateDirectDownloadCertificate;
import org.apache.cloudstack.api.command.admin.domain.CreateDomainCmd;
import org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd;
import org.apache.cloudstack.api.command.admin.domain.ListDomainChildrenCmd;
@ -3030,6 +3031,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
cmdList.add(ReleasePodIpCmdByAdmin.class);
cmdList.add(CreateManagementNetworkIpRangeCmd.class);
cmdList.add(DeleteManagementNetworkIpRangeCmd.class);
cmdList.add(UploadTemplateDirectDownloadCertificate.class);
// Out-of-band management APIs for admins
cmdList.add(EnableOutOfBandManagementForHostCmd.class);
@ -3042,7 +3044,6 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
cmdList.add(IssueOutOfBandManagementPowerActionCmd.class);
cmdList.add(ChangeOutOfBandManagementPasswordCmd.class);
cmdList.add(GetUserKeysCmd.class);
return cmdList;
}

17
server/src/com/cloud/storage/TemplateProfile.java
View File

@ -51,6 +51,8 @@ public class TemplateProfile {
Map details;
Boolean isDynamicallyScalable;
TemplateType templateType;
Boolean directDownload;
Long size;
public TemplateProfile(Long templateId, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHvm, String url,
Boolean isPublic, Boolean featured, Boolean isExtractable, ImageFormat format, Long guestOsId, List<Long> zoneIdList, HypervisorType hypervisorType,
@ -93,7 +95,7 @@ public class TemplateProfile {
Boolean isPublic, Boolean featured, Boolean isExtractable, ImageFormat format, Long guestOsId, List<Long> zoneId,
HypervisorType hypervisorType, String accountName, Long domainId, Long accountId, String chksum, Boolean bootable, String templateTag, Map details,
Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType) {
Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType, Boolean directDownload) {
this(templateId,
userId,
name,
@ -119,6 +121,7 @@ public class TemplateProfile {
this.templateTag = templateTag;
this.isDynamicallyScalable = isDynamicallyScalable;
this.templateType = templateType;
this.directDownload = directDownload;
}
public Long getTemplateId() {
@ -316,4 +319,16 @@ public class TemplateProfile {
public void setTemplateType(TemplateType templateType) {
this.templateType = templateType;
}
public boolean isDirectDownload() {
return directDownload == null ? false : directDownload;
}
public Long getSize() {
return size;
}
public void setSize(Long size) {
this.size = size;
}
}

71
server/src/com/cloud/template/HypervisorTemplateAdapter.java
View File

@ -16,6 +16,10 @@
// under the License.
package com.cloud.template;
import com.cloud.agent.api.Answer;
import com.cloud.host.HostVO;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.resource.ResourceManager;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
@ -29,6 +33,8 @@ import com.cloud.configuration.Config;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionStatus;
import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer;
import org.apache.cloudstack.agent.directdownload.CheckUrlCommand;
import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@ -68,6 +74,9 @@ import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.org.Grouping;
import com.cloud.server.StatsCollector;
import com.cloud.template.VirtualMachineTemplate.State;
import com.cloud.user.Account;
import com.cloud.utils.Pair;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
@ -77,9 +86,6 @@ import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VMTemplateZoneVO;
import com.cloud.storage.dao.VMTemplateZoneDao;
import com.cloud.storage.download.DownloadMonitor;
import com.cloud.template.VirtualMachineTemplate.State;
import com.cloud.user.Account;
import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.EntityManager;
@ -113,17 +119,42 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
DataCenterDao _dcDao;
@Inject
MessageBus _messageBus;
@Inject
ResourceManager resourceManager;
@Override
public String getName() {
return TemplateAdapterType.Hypervisor.getName();
}
/**
* Validate on random running KVM host that URL is reachable
* @param url url
*/
private Long performDirectDownloadUrlValidation(final String url) {
HostVO host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM);
if (host == null) {
throw new CloudRuntimeException("Couldn't find a host to validate URL " + url);
}
CheckUrlCommand cmd = new CheckUrlCommand(url);
s_logger.debug("Performing URL " + url + " validation on host " + host.getId());
Answer answer = _agentMgr.easySend(host.getId(), cmd);
if (answer == null || !answer.getResult()) {
throw new CloudRuntimeException("URL: " + url + " validation failed on host id " + host.getId());
}
CheckUrlAnswer ans = (CheckUrlAnswer) answer;
return ans.getTemplateSize();
}
@Override
public TemplateProfile prepare(RegisterIsoCmd cmd) throws ResourceAllocationException {
TemplateProfile profile = super.prepare(cmd);
String url = profile.getUrl();
UriUtils.validateUrl(ImageFormat.ISO.getFileExtension(), url);
if (cmd.isDirectDownload()) {
Long templateSize = performDirectDownloadUrlValidation(url);
profile.setSize(templateSize);
}
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
@ -135,6 +166,10 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
TemplateProfile profile = super.prepare(cmd);
String url = profile.getUrl();
UriUtils.validateUrl(cmd.getFormat(), url);
if (cmd.isDirectDownload()) {
Long templateSize = performDirectDownloadUrlValidation(url);
profile.setSize(templateSize);
}
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
@ -150,6 +185,14 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
return profile;
}
/**
* Persist template marking it for direct download to Primary Storage, skipping Secondary Storage
*/
private void persistDirectDownloadTemplate(long templateId, Long size) {
TemplateDataStoreVO directDownloadEntry = templateDataStoreDao.createTemplateDirectDownloadEntry(templateId, size);
templateDataStoreDao.persist(directDownloadEntry);
}
@Override
public VMTemplateVO create(TemplateProfile profile) {
// persist entry in vm_template, vm_template_details and template_zone_ref tables, not that entry at template_store_ref is not created here, and created in createTemplateAsync.
@ -159,17 +202,23 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
throw new CloudRuntimeException("Unable to persist the template " + profile.getTemplate());
}
List<Long> zones = profile.getZoneIdList();
if (!profile.isDirectDownload()) {
List<Long> zones = profile.getZoneIdList();
//zones is null when this template is to be registered to all zones
if (zones == null){
createTemplateWithinZone(null, profile, template);
}
else {
for (Long zId : zones) {
createTemplateWithinZone(zId, profile, template);
//zones is null when this template is to be registered to all zones
if (zones == null){
createTemplateWithinZone(null, profile, template);
}
else {
for (Long zId : zones) {
createTemplateWithinZone(zId, profile, template);
}
}
} else {
//KVM direct download templates bypassing Secondary Storage
persistDirectDownloadTemplate(template.getId(), profile.getSize());
}
_resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template);
return template;
}

4
server/src/com/cloud/template/TemplateAdapter.java
View File

@ -71,11 +71,11 @@ public interface TemplateAdapter extends Adapter {
public TemplateProfile prepare(boolean isIso, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url,
Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List<Long> zoneId, HypervisorType hypervisorType, String accountName,
Long domainId, String chksum, Boolean bootable, Map details) throws ResourceAllocationException;
Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload) throws ResourceAllocationException;
public TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url,
Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List<Long> zoneId, HypervisorType hypervisorType, String chksum,
Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshKeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable,
TemplateType templateType) throws ResourceAllocationException;
TemplateType templateType, boolean directDownload) throws ResourceAllocationException;
}

22
server/src/com/cloud/template/TemplateAdapterBase.java
View File

@ -112,6 +112,8 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
ConfigurationServer _configServer;
@Inject
ProjectManager _projectMgr;
@Inject
private TemplateDataStoreDao templateDataStoreDao;
@Override
public boolean stop() {
@ -121,16 +123,16 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
@Override
public TemplateProfile prepare(boolean isIso, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url,
Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List<Long> zoneId, HypervisorType hypervisorType, String accountName,
Long domainId, String chksum, Boolean bootable, Map details) throws ResourceAllocationException {
Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload) throws ResourceAllocationException {
return prepare(isIso, userId, name, displayText, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, format, guestOSId, zoneId,
hypervisorType, chksum, bootable, null, null, details, false, null, false, TemplateType.USER);
hypervisorType, chksum, bootable, null, null, details, false, null, false, TemplateType.USER, directDownload);
}
@Override
public TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url,
Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List<Long> zoneIdList, HypervisorType hypervisorType, String chksum,
Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshkeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable,
TemplateType templateType) throws ResourceAllocationException {
TemplateType templateType, boolean directDownload) throws ResourceAllocationException {
//Long accountId = null;
// parameters verification
@ -249,7 +251,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
CallContext.current().setEventDetails("Id: " + id + " name: " + name);
return new TemplateProfile(id, userId, name, displayText, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, imgfmt, guestOSId, zoneIdList,
hypervisorType, templateOwner.getAccountName(), templateOwner.getDomainId(), templateOwner.getAccountId(), chksum, bootable, templateTag, details,
sshkeyEnabled, null, isDynamicallyScalable, templateType);
sshkeyEnabled, null, isDynamicallyScalable, templateType, directDownload);
}
@ -277,7 +279,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
return prepare(false, CallContext.current().getCallingUserId(), cmd.getTemplateName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(), cmd.getRequiresHvm(),
cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneId, hypervisorType, cmd.getChecksum(), true,
cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER);
cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, cmd.isDirectDownload());
}
@ -308,7 +310,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
return prepare(false, CallContext.current().getCallingUserId(), cmd.getName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(),
cmd.getRequiresHvm(), null, cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneList,
hypervisorType, cmd.getChecksum(), true, cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null,
cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER);
cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, false);
}
@ -330,7 +332,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
return prepare(true, CallContext.current().getCallingUserId(), cmd.getIsoName(), cmd.getDisplayText(), 64, false, true, cmd.getUrl(), cmd.isPublic(),
cmd.isFeatured(), cmd.isExtractable(), ImageFormat.ISO.toString(), cmd.getOsTypeId(), zoneList, HypervisorType.None, cmd.getChecksum(), cmd.isBootable(), null,
owner, null, false, cmd.getImageStoreUuid(), cmd.isDynamicallyScalable(), TemplateType.USER);
owner, null, false, cmd.getImageStoreUuid(), cmd.isDynamicallyScalable(), TemplateType.USER, cmd.isDirectDownload());
}
protected VMTemplateVO persistTemplate(TemplateProfile profile, VirtualMachineTemplate.State initialState) {
@ -339,9 +341,13 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
new VMTemplateVO(profile.getTemplateId(), profile.getName(), profile.getFormat(), profile.getIsPublic(), profile.getFeatured(), profile.getIsExtractable(),
profile.getTemplateType(), profile.getUrl(), profile.getRequiresHVM(), profile.getBits(), profile.getAccountId(), profile.getCheckSum(),
profile.getDisplayText(), profile.getPasswordEnabled(), profile.getGuestOsId(), profile.getBootable(), profile.getHypervisorType(),
profile.getTemplateTag(), profile.getDetails(), profile.getSshKeyEnabled(), profile.IsDynamicallyScalable());
profile.getTemplateTag(), profile.getDetails(), profile.getSshKeyEnabled(), profile.IsDynamicallyScalable(), profile.isDirectDownload());
template.setState(initialState);
if (profile.isDirectDownload()) {
template.setSize(profile.getSize());
}
if (zoneIdList == null) {
List<DataCenterVO> dcs = _dcDao.listAll();

35
server/src/com/cloud/template/TemplateManagerImpl.java
View File

@ -32,6 +32,7 @@ import java.util.concurrent.Executors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.deploy.DeployDestination;
import com.cloud.storage.ImageStoreUploadMonitorImpl;
import com.cloud.utils.StringUtils;
import com.cloud.utils.EncryptionUtil;
@ -551,10 +552,18 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
}
@Override
public void prepareIsoForVmProfile(VirtualMachineProfile profile) {
public void prepareIsoForVmProfile(VirtualMachineProfile profile, DeployDestination dest) {
UserVmVO vm = _userVmDao.findById(profile.getId());
if (vm.getIsoId() != null) {
TemplateInfo template = prepareIso(vm.getIsoId(), vm.getDataCenterId());
Map<Volume, StoragePool> storageForDisks = dest.getStorageForDisks();
Long poolId = null;
for (StoragePool storagePool : storageForDisks.values()) {
if (poolId != null && storagePool.getId() != poolId) {
throw new CloudRuntimeException("Cannot determine where to download iso");
}
poolId = storagePool.getId();
}
TemplateInfo template = prepareIso(vm.getIsoId(), vm.getDataCenterId(), dest.getHost().getId(), poolId);
if (template == null){
s_logger.error("Failed to prepare ISO on secondary or cache storage");
throw new CloudRuntimeException("Failed to prepare ISO on secondary or cache storage");
@ -1156,14 +1165,22 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
// for ISO, we need to consider whether to copy to cache storage or not if it is not on NFS, since our hypervisor resource always assumes that they are in NFS
@Override
public TemplateInfo prepareIso(long isoId, long dcId) {
TemplateInfo tmplt = _tmplFactory.getTemplate(isoId, DataStoreRole.Image, dcId);
public TemplateInfo prepareIso(long isoId, long dcId, Long hostId, Long poolId) {
TemplateInfo tmplt;
boolean bypassed = false;
if (_tmplFactory.isTemplateMarkedForDirectDownload(isoId)) {
tmplt = _tmplFactory.getReadyBypassedTemplateOnPrimaryStore(isoId, poolId, hostId);
bypassed = true;
} else {
tmplt = _tmplFactory.getTemplate(isoId, DataStoreRole.Image, dcId);
}
if (tmplt == null || tmplt.getFormat() != ImageFormat.ISO) {
s_logger.warn("ISO: " + isoId + " does not exist in vm_template table");
return null;
}
if (tmplt.getDataStore() != null && !(tmplt.getDataStore().getTO() instanceof NfsTO)) {
if (!bypassed && tmplt.getDataStore() != null && !(tmplt.getDataStore().getTO() instanceof NfsTO)) {
// if it is s3, need to download into cache storage first
Scope destScope = new ZoneScope(dcId);
TemplateInfo cacheData = (TemplateInfo)cacheMgr.createCacheObject(tmplt, destScope);
@ -1187,7 +1204,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
}
// prepare ISO ready to mount on hypervisor resource level
TemplateInfo tmplt = prepareIso(isoId, vm.getDataCenterId());
TemplateInfo tmplt = prepareIso(isoId, vm.getDataCenterId(), vm.getHostId(), null);
String vmName = vm.getInstanceName();
@ -1804,7 +1821,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
}
privateTemplate = new VMTemplateVO(nextTemplateId, name, ImageFormat.RAW, isPublic, featured, isExtractable,
TemplateType.USER, null, requiresHvmValue, bitsValue, templateOwner.getId(), null, description,
passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), false, isDynamicScalingEnabled);
passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), false, isDynamicScalingEnabled, false);
if (sourceTemplateId != null) {
if (s_logger.isDebugEnabled()) {
@ -1912,6 +1929,10 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
@Override
public Long getTemplateSize(long templateId, long zoneId) {
if (_tmplStoreDao.isTemplateMarkedForDirectDownload(templateId)) {
// check if template is marked for direct download
return _tmplStoreDao.getReadyBypassedTemplate(templateId).getSize();
}
TemplateDataStoreVO templateStoreRef = _tmplStoreDao.findByTemplateZoneDownloadStatus(templateId, zoneId, VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
if (templateStoreRef == null) {
// check if it is ready on image cache stores

2
server/src/com/cloud/vm/UserVmManagerImpl.java
View File

@ -4055,7 +4055,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
_templateMgr.prepareIsoForVmProfile(profile);
_templateMgr.prepareIsoForVmProfile(profile, dest);
return true;
}

245
server/src/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java Executable file
View File

@ -0,0 +1,245 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.Status;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.exception.CloudRuntimeException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.inject.Inject;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand.DownloadProtocol;
import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.commons.collections.MapUtils;
import org.apache.log4j.Logger;
public class DirectDownloadManagerImpl extends ManagerBase implements DirectDownloadManager {
private static final Logger s_logger = Logger.getLogger(DirectDownloadManagerImpl.class);
protected static final String httpHeaderDetailKey = "HTTP_HEADER";
protected static final String BEGIN_CERT = "-----BEGIN CERTIFICATE-----";
protected static final String END_CERT = "-----END CERTIFICATE-----";
protected final static String LINE_SEPARATOR = "\n";
@Inject
VMTemplateDao vmTemplateDao;
@Inject
PrimaryDataStoreDao primaryDataStoreDao;
@Inject
HostDao hostDao;
@Inject
AgentManager agentManager;
@Inject
VMTemplatePoolDao vmTemplatePoolDao;
@Inject
DataStoreManager dataStoreManager;
@Override
public List<Class<?>> getCommands() {
final List<Class<?>> cmdList = new ArrayList<Class<?>>();
return cmdList;
}
/**
* Return protocol to use from provided URL
* @param url
* @return
*/
public static DownloadProtocol getProtocolFromUrl(String url) {
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
throw new CloudRuntimeException("URI is incorrect: " + url);
}
if ((uri != null) && (uri.getScheme() != null)) {
if (uri.getPath().endsWith(".metalink")) {
return DownloadProtocol.METALINK;
} else if (uri.getScheme().equalsIgnoreCase("http")) {
return DownloadProtocol.HTTP;
} else if (uri.getScheme().equalsIgnoreCase("https")) {
return DownloadProtocol.HTTPS;
} else if (uri.getScheme().equalsIgnoreCase("nfs")) {
return DownloadProtocol.NFS;
} else {
throw new CloudRuntimeException("Scheme is not supported " + url);
}
} else {
throw new CloudRuntimeException("URI is incorrect: " + url);
}
}
/**
* Return HTTP headers from template details
* @param templateDetails
* @return
*/
protected Map<String, String> getHeadersFromDetails(Map<String, String> templateDetails) {
if (MapUtils.isEmpty(templateDetails)) {
return new HashMap<>();
}
Map<String, String> headers = new HashMap<>();
for (String key : templateDetails.keySet()) {
if (key.startsWith(httpHeaderDetailKey)) {
String header = key.split(":")[1];
String value = templateDetails.get(key);
headers.put(header, value);
}
}
return headers;
}
@Override
public void downloadTemplate(long templateId, long poolId, long hostId) {
VMTemplateVO template = vmTemplateDao.findById(templateId);
StoragePoolVO pool = primaryDataStoreDao.findById(poolId);
HostVO host = hostDao.findById(hostId);
if (pool == null) {
throw new CloudRuntimeException("Storage pool " + poolId + " could not be found");
}
if (template == null) {
throw new CloudRuntimeException("Template " + templateId + " could not be found");
}
if (host == null) {
throw new CloudRuntimeException("Host " + hostId + " could not be found");
}
if (!template.isDirectDownload()) {
throw new CloudRuntimeException("Template " + templateId + " is not marked for direct download");
}
Map<String, String> details = template.getDetails();
String url = template.getUrl();
String checksum = template.getChecksum();
Map<String, String> headers = getHeadersFromDetails(details);
DataStore store = dataStoreManager.getDataStore(poolId, DataStoreRole.Primary);
if (store == null) {
throw new CloudRuntimeException("Data store " + poolId + " could not be found");
}
PrimaryDataStore primaryDataStore = (PrimaryDataStore) store;
PrimaryDataStoreTO to = (PrimaryDataStoreTO) primaryDataStore.getTO();
DownloadProtocol protocol = getProtocolFromUrl(url);
DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers);
Answer answer = agentManager.easySend(hostId, cmd);
if (answer == null || !answer.getResult()) {
throw new CloudRuntimeException("Host " + hostId + " could not download template " +
templateId + " on pool " + poolId);
}
VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, templateId);
if (sPoolRef == null) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Not found (templateId:" + templateId + " poolId: " + poolId + ") in template_spool_ref, persisting it");
}
DirectDownloadAnswer ans = (DirectDownloadAnswer) answer;
sPoolRef = new VMTemplateStoragePoolVO(poolId, templateId);
sPoolRef.setDownloadPercent(100);
sPoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
sPoolRef.setState(ObjectInDataStoreStateMachine.State.Ready);
sPoolRef.setTemplateSize(ans.getTemplateSize());
sPoolRef.setLocalDownloadPath(ans.getInstallPath());
sPoolRef.setInstallPath(ans.getInstallPath());
vmTemplatePoolDao.persist(sPoolRef);
}
}
/**
* Return DirectDownloadCommand according to the protocol
* @param protocol
* @param url
* @param templateId
* @param destPool
* @return
*/
private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool,
String checksum, Map<String, String> httpHeaders) {
if (protocol.equals(DownloadProtocol.HTTP)) {
return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
} else if (protocol.equals(DownloadProtocol.HTTPS)) {
return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
} else if (protocol.equals(DownloadProtocol.NFS)) {
return new NfsDirectDownloadCommand(url, templateId, destPool, checksum);
} else if (protocol.equals(DownloadProtocol.METALINK)) {
return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum);
} else {
return null;
}
}
@Override
public boolean uploadCertificateToHosts(String certificateCer, String certificateName) {
List<HostVO> hosts = hostDao.listAllHostsByType(Host.Type.Routing)
.stream()
.filter(x -> x.getStatus().equals(Status.Up) &&
x.getHypervisorType().equals(Hypervisor.HypervisorType.KVM))
.collect(Collectors.toList());
for (HostVO host : hosts) {
if (!uploadCertificate(certificateCer, certificateName, host.getId())) {
throw new CloudRuntimeException("Uploading certificate " + certificateName + " failed on host: " + host.getId());
}
}
return true;
}
/**
* Upload and import certificate to hostId on keystore
*/
protected boolean uploadCertificate(String certificate, String certificateName, long hostId) {
String cert = certificate.replaceAll("(.{64})", "$1\n");
final String prettified_cert = BEGIN_CERT + LINE_SEPARATOR + cert + LINE_SEPARATOR + END_CERT;
SetupDirectDownloadCertificate cmd = new SetupDirectDownloadCertificate(prettified_cert, certificateName);
Answer answer = agentManager.easySend(hostId, cmd);
if (answer == null || !answer.getResult()) {
return false;
}
s_logger.info("Certificate " + certificateName + " successfully uploaded to host: " + hostId);
return true;
}
}

6
server/test/com/cloud/resource/MockResourceManagerImpl.java
View File

@ -614,6 +614,12 @@ public class MockResourceManagerImpl extends ManagerBase implements ResourceMana
return null;
}
@Override
public HostVO findOneRandomRunningHostByHypervisor(HypervisorType type) {
// TODO Auto-generated method stub
return null;
}
@Override
public boolean isHostGpuEnabled(final long hostId) {
// TODO Auto-generated method stub

106
server/test/org/apache/cloudstack/direct/download/DirectDownloadManagerImplTest.java Normal file
View File

@ -0,0 +1,106 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.direct.download;
import com.cloud.agent.AgentManager;
import com.cloud.host.dao.HostDao;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand.DownloadProtocol;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.runners.MockitoJUnitRunner;
import java.util.HashMap;
import java.util.Map;
@RunWith(MockitoJUnitRunner.class)
public class DirectDownloadManagerImplTest {
@Mock
HostDao hostDao;
@Mock
AgentManager agentManager;
@Spy
@InjectMocks
private DirectDownloadManagerImpl manager = new DirectDownloadManagerImpl();
private static final String HTTP_HEADER_1 = "Content-Type";
private static final String HTTP_VALUE_1 = "application/x-www-form-urlencoded";
private static final String HTTP_HEADER_2 = "Accept-Encoding";
private static final String HTTP_VALUE_2 = "gzip";
@Before
public void setUp() {
}
@Test
public void testGetProtocolMetalink() {
String url = "http://192.168.1.2/tmpl.metalink";
DownloadProtocol protocol = DirectDownloadManagerImpl.getProtocolFromUrl(url);
Assert.assertEquals(DownloadProtocol.METALINK, protocol);
}
@Test
public void testGetProtocolHttp() {
String url = "http://192.168.1.2/tmpl.qcow2";
DownloadProtocol protocol = DirectDownloadManagerImpl.getProtocolFromUrl(url);
Assert.assertEquals(DownloadProtocol.HTTP, protocol);
}
@Test
public void testGetProtocolHttps() {
String url = "https://192.168.1.2/tmpl.qcow2";
DownloadProtocol protocol = DirectDownloadManagerImpl.getProtocolFromUrl(url);
Assert.assertEquals(DownloadProtocol.HTTPS, protocol);
}
@Test
public void testGetProtocolNfs() {
String url = "nfs://192.168.1.2/tmpl.qcow2";
DownloadProtocol protocol = DirectDownloadManagerImpl.getProtocolFromUrl(url);
Assert.assertEquals(DownloadProtocol.NFS, protocol);
}
@Test
public void testGetHeadersFromDetailsHttpHeaders() {
Map<String, String> details = new HashMap<>();
details.put("Message.ReservedCapacityFreed.Flag", "false");
details.put(DirectDownloadManagerImpl.httpHeaderDetailKey + ":" + HTTP_HEADER_1, HTTP_VALUE_1);
details.put(DirectDownloadManagerImpl.httpHeaderDetailKey + ":" + HTTP_HEADER_2, HTTP_VALUE_2);
Map<String, String> headers = manager.getHeadersFromDetails(details);
Assert.assertEquals(2, headers.keySet().size());
Assert.assertTrue(headers.containsKey(HTTP_HEADER_1));
Assert.assertTrue(headers.containsKey(HTTP_HEADER_2));
Assert.assertEquals(HTTP_VALUE_1, headers.get(HTTP_HEADER_1));
Assert.assertEquals(HTTP_VALUE_2, headers.get(HTTP_HEADER_2));
}
@Test
public void testGetHeadersFromDetailsNonHttpHeaders() {
Map<String, String> details = new HashMap<>();
details.put("Message.ReservedCapacityFreed.Flag", "false");
Map<String, String> headers = manager.getHeadersFromDetails(details);
Assert.assertTrue(headers.isEmpty());
}
}

37
services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
View File

@ -38,6 +38,23 @@ import java.util.concurrent.Executors;
import javax.naming.ConfigurationException;
import com.cloud.storage.template.Processor;
import com.cloud.storage.template.S3TemplateDownloader;
import com.cloud.storage.template.TemplateDownloader;
import com.cloud.storage.template.TemplateLocation;
import com.cloud.storage.template.MetalinkTemplateDownloader;
import com.cloud.storage.template.HttpTemplateDownloader;
import com.cloud.storage.template.LocalTemplateDownloader;
import com.cloud.storage.template.ScpTemplateDownloader;
import com.cloud.storage.template.TemplateProp;
import com.cloud.storage.template.OVAProcessor;
import com.cloud.storage.template.IsoProcessor;
import com.cloud.storage.template.QCOW2Processor;
import com.cloud.storage.template.VmdkProcessor;
import com.cloud.storage.template.RawImageProcessor;
import com.cloud.storage.template.TARProcessor;
import com.cloud.storage.template.VhdProcessor;
import com.cloud.storage.template.TemplateConstants;
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
@ -56,25 +73,9 @@ import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.StorageLayer;
import com.cloud.storage.VMTemplateHostVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.template.HttpTemplateDownloader;
import com.cloud.storage.template.IsoProcessor;
import com.cloud.storage.template.LocalTemplateDownloader;
import com.cloud.storage.template.OVAProcessor;
import com.cloud.storage.template.Processor;
import com.cloud.storage.template.Processor.FormatInfo;
import com.cloud.storage.template.QCOW2Processor;
import com.cloud.storage.template.RawImageProcessor;
import com.cloud.storage.template.S3TemplateDownloader;
import com.cloud.storage.template.ScpTemplateDownloader;
import com.cloud.storage.template.TARProcessor;
import com.cloud.storage.template.TemplateConstants;
import com.cloud.storage.template.TemplateDownloader;
import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback;
import com.cloud.storage.template.TemplateDownloader.Status;
import com.cloud.storage.template.TemplateLocation;
import com.cloud.storage.template.TemplateProp;
import com.cloud.storage.template.VhdProcessor;
import com.cloud.storage.template.VmdkProcessor;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.StringUtils;
import com.cloud.utils.component.ManagerBase;
@ -560,7 +561,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
TemplateDownloader td;
if ((uri != null) && (uri.getScheme() != null)) {
if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
if (uri.getPath().endsWith(".metalink")) {
td = new MetalinkTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes);
} else if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType);
} else if (uri.getScheme().equalsIgnoreCase("file")) {
td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId));

160
test/integration/smoke/test_templates.py
View File

@ -1197,3 +1197,163 @@ class TestCopyDeleteTemplate(cloudstackTestCase):
"Removed state is not correct."
)
return
class TestCreateTemplateWithDirectDownload(cloudstackTestCase):
@classmethod
def setUpClass(self):
self.testClient = super(TestCreateTemplateWithDirectDownload, self).getClsTestClient()
self.apiclient = self.testClient.getApiClient()
self.dbclient = self.testClient.getDbConnection()
self._cleanup = []
self.templates = []
self.services = self.testClient.getParsedTestDataConfig()
self.unsupportedHypervisor = False
self.hypervisor = self.testClient.getHypervisorInfo()
if self.hypervisor.lower() not in ['kvm']:
# Direct Download is only available for KVM hypervisor
self.unsupportedHypervisor = True
self.skipTest("Skipping test because unsupported hypervisor\
%s" % self.hypervisor)
return
# Get Zone, Domain and templates
self.domain = get_domain(self.apiclient)
self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests())
self.services["mode"] = self.zone.networktype
self.services["virtual_machine"]["zoneid"] = self.zone.id
self.account = Account.create(
self.apiclient,
self.services["account"],
admin=True,
domainid=self.domain.id
)
self._cleanup.append(self.account)
self.user = Account.create(
self.apiclient,
self.services["account"],
domainid=self.domain.id
)
self._cleanup.append(self.user)
self.service_offering = ServiceOffering.create(
self.apiclient,
self.services["service_offerings"]["tiny"]
)
self._cleanup.append(self.service_offering)
self.template = {
"name": "tiny-kvm",
"displaytext": "tiny kvm",
"format": "QCOW2",
"url": "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-kvm.qcow2.bz2",
"requireshvm": "True",
"ispublic": "True",
"isextractable": "True",
"checksum": "{SHA-1}" + "6952e58f39b470bd166ace11ffd20bf479bed936",
"hypervisor": self.hypervisor,
"zoneid": self.zone.id,
"ostype": "Other Linux (64-bit)",
"directdownload": True
}
return
@classmethod
def tearDownClass(cls):
try:
cleanup_resources(cls.apiclient, cls._cleanup)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
def setUp(self):
self.apiclient = self.testClient.getApiClient()
self.dbclient = self.testClient.getDbConnection()
self.cleanup = []
if self.unsupportedHypervisor:
self.skipTest("Skipping test because unsupported hypervisor\
%s" % self.hypervisor)
return
def tearDown(self):
try:
#Clean up, terminate the created templates
cleanup_resources(self.apiclient, self.cleanup)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_01_register_template_direct_download_flag(self):
"""
Register a template using Direct Download flag
"""
self.bypassed_template = Template.register(self.apiclient, self.template, zoneid=self.zone.id, hypervisor=self.hypervisor, randomize_name=False)
self._cleanup.append(self.bypassed_template)
self.templates.append(self.bypassed_template)
tmplt = self.dbclient.execute("select id, direct_download from vm_template where uuid='%s';" % self.bypassed_template.id)
det = tmplt[0]
self.assertEqual(det[1],
1,
"Template should be marked as Direct Download"
)
qresultset = self.dbclient.execute("select download_state, state from template_store_ref where template_id='%s' and store_id is NULL;"
% det[0])
ref = qresultset[0]
self.assertEqual(ref[0],
"BYPASSED",
"Template store ref download state should be marked as BYPASSED"
)
self.assertEqual(ref[1],
"Ready",
"Template store ref state should be marked as Ready"
)
return
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_02_deploy_vm_from_direct_download_template(self):
"""
Deploy a VM from a Direct Download registered template
"""
bp = self.templates[0]
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
templateid=bp.id,
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
self.cleanup.append(virtual_machine)
return
@attr(tags=["advanced", "smoke"], required_hardware="true")
def test_03_deploy_vm_wrong_checksum(self):
"""
Deploy a VM from a Direct Download registered template with wrong checksum
"""
self.template["checksum"]="{MD5}XXXXXXX"
tmpl = Template.register(self.apiclient, self.template, zoneid=self.zone.id, hypervisor=self.hypervisor, randomize_name=False)
try:
virtual_machine = VirtualMachine.create(
self.apiclient,
self.services["virtual_machine"],
templateid=tmpl.id,
accountid=self.account.name,
domainid=self.account.domainid,
serviceofferingid=self.service_offering.id
)
self.cleanup.append(tmpl)
self.fail("Expected to fail deployment")
except Exception as e:
self.debug("Expected exception")
self.cleanup.append(virtual_machine)
self.cleanup.append(tmpl)
return

2
tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh
View File

@ -68,7 +68,7 @@ function install_packages() {
python-flask \
haproxy \
radvd \
sharutils genisoimage \
sharutils genisoimage aria2 \
strongswan libcharon-extra-plugins libstrongswan-extra-plugins \
virt-what open-vm-tools qemu-guest-agent hyperv-daemons

4
tools/marvin/marvin/lib/base.py
View File

@ -1302,6 +1302,10 @@ class Template:
if details:
cmd.details = details
if "directdownload" in services:
cmd.directdownload = services["directdownload"]
# Register Template
template = apiclient.registerTemplate(cmd)

1
ui/l10n/ar.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "الأجهزة",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direct Attached Public IP",
"label.direct.download":"Direct Download",
"label.direct.ips": "الشبكة المشتركة IPs",
"label.disable.autoscale": "Disable Autoscale",
"label.disable.host": "Disable Host",

1
ui/l10n/ca.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Devices",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direct Attached Public IP",
"label.direct.download":"Direct Download",
"label.direct.ips": "Shared Network IPs",
"label.disable.autoscale": "Disable Autoscale",
"label.disable.host": "Disable Host",

1
ui/l10n/de_DE.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Geräte",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direkt angeschlossene öffentliche IP",
"label.direct.download":"Direct Download",
"label.direct.ips": "Gemeinsame Netzwerk-IPs",
"label.disable.autoscale": "Automatische Skalierung deaktivieren",
"label.disable.host": "Host deaktivieren",

1
ui/l10n/en.js
View File

@ -647,6 +647,7 @@ var dictionary = {"ICMP.code":"ICMP Code",
"label.devices":"Devices",
"label.dhcp":"DHCP",
"label.direct.attached.public.ip":"Direct Attached Public IP",
"label.direct.download":"Direct Download",
"label.direct.ips":"Shared Network IPs",
"label.disable.autoscale":"Disable Autoscale",
"label.disable.host":"Disable Host",

1
ui/l10n/es.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Dispositivos",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "IP Pública Conectada en forma Directa",
"label.direct.download":"Direct Download",
"label.direct.ips": "IPs de la Red Compartida",
"label.disable.autoscale": "Deshabilitar Escalado Automático",
"label.disable.host": "Deshabitar Anfitrión",

1
ui/l10n/fr_FR.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Machines",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "IP publique attachée directement",
"label.direct.download":"Direct Download",
"label.direct.ips": "Adresses IP du réseau partagé",
"label.disable.autoscale": "Désactiver Autoscale",
"label.disable.host": "Désactiver Hôte",

1
ui/l10n/hu.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Eszközök",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direct Attached Public IP",
"label.direct.download":"Direct Download",
"label.direct.ips": "Osztott hálózati IP címek",
"label.disable.autoscale": "Automatikus skálázás kikapcsolása",
"label.disable.host": "Kiszolgáló kikapcsolása",

1
ui/l10n/it_IT.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Device",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direct Attached Public IP",
"label.direct.download":"Direct Download",
"label.direct.ips": "Indirizzi IP di Rete condivisi",
"label.disable.autoscale": "Disable Autoscale",
"label.disable.host": "Disable Host",

1
ui/l10n/ja_JP.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "デバイス",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "直接アタッチされているパブリック IP アドレス",
"label.direct.download":"Direct Download",
"label.direct.ips": "共有ネットワークの IP アドレス",
"label.disable.autoscale": "自動サイズ設定の無効化",
"label.disable.host": "ホストの無効化",

1
ui/l10n/ko_KR.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "기기",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direct Attached Public IP",
"label.direct.download":"Direct Download",
"label.direct.ips": "직접 IP 주소",
"label.disable.autoscale": "Disable Autoscale",
"label.disable.host": "Disable Host",

1
ui/l10n/nb_NO.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Enheter",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direkte Tilknyttet Offentlig IP-adresse",
"label.direct.download":"Direct Download",
"label.direct.ips": "Deltnettverk-IPadresser",
"label.disable.autoscale": "Deaktiver autoskalering",
"label.disable.host": "Deaktiver host",

1
ui/l10n/nl_NL.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Apparaten",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "direct verbonden publieke IP",
"label.direct.download":"Direct Download",
"label.direct.ips": "Shared Netwerk IPs",
"label.disable.autoscale": "Autoscale uitschakelen",
"label.disable.host": "schakel host uit",

1
ui/l10n/pl.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Devices",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direct Attached Public IP",
"label.direct.download":"Direct Download",
"label.direct.ips": "Shared Network IPs",
"label.disable.autoscale": "Disable Autoscale",
"label.disable.host": "Disable Host",

1
ui/l10n/pt_BR.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Dispositivos",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "IP Público COnectado Diretamente",
"label.direct.download":"Direct Download",
"label.direct.ips": "IPs Diretos",
"label.disable.autoscale": "Desabilita Auto-escala",
"label.disable.host": "Desabilita Host",

1
ui/l10n/ru_RU.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Устройство",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Выданные публичные IP",
"label.direct.download":"Direct Download",
"label.direct.ips": "Прямые IP-адреса",
"label.disable.autoscale": "Выключить автоматическое маштабирование",
"label.disable.host": "Отключить хост",

1
ui/l10n/zh_CN.js
View File

@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "设备",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "直连公用 IP",
"label.direct.download":"Direct Download",
"label.direct.ips": "共享网络 IP",
"label.disable.autoscale": "禁用自动缩放",
"label.disable.host": "禁用主机",

Some files were not shown because too many files have changed in this diff Show More