CLOUDSTACK-10146: Bypass Secondary Storage for KVM templates (#2379)

This feature allows using templates and ISOs avoiding secondary storage as intermediate cache on KVM. The virtual machine deployment process is enhanced to supported bypassed registered templates and ISOs, delegating the work of downloading them to primary storage to the KVM agent instead of the SSVM agent.

Template and ISO registration:
- When hypervisor is KVM, a checkbox is displayed with 'Direct Download' label.
- API methods registerTemplate and registerISO are both extended with this new parameter directdownload.
- On template or ISO registration, no download job is sent to SSVM agent, CloudStack would only persist an entry on template_store_ref indicating that template or ISO has been marked as 'Direct Download' (bypassing Secondary Storage). These entries are persisted as:
template_id = Template or ISO id on vm_template table
store_id NULL
download_state = BYPASSED
state = Ready
(Note: these entries allow users to deploy virtual machine from registered templates or ISOs)
- An URL validation command is sent to a random KVM host to check if template/ISO location can be reached. Metalink are also supported by this feature. In case of a metalink, it is fetched and URL check is performed on each of its URLs.
- Checksum should be provided as indicated on #2246: {ALGORITHM}CHKSUMHASH
- After template or ISO is registered, it would be displayed in the UI

Virtual machine deployment:
When a 'Direct Download' template is selected for deployment, CloudStack would delegate template downloading to destination storage pool via destination host by a new pluggable download manager.
Download manager would handle template downloading depending on URL protocol. In case of HTTP, request headers can be set by the user via vm_template_details. Those details should be persisted as:
Key: HTTP_HEADER
Value: HEADERNAME:HEADERVALUE

In case of HTTPS, a new API method is added uploadTemplateDirectDownloadCertificate to allow user importing a client certificate into all KVM hosts' keystore before deployment.
After template or ISO is downloaded to primary storage, usual entry would be persisted on template_spool_ref indicating the mapping between template/ISO and storage pool.

agent/src/com/cloud/agent/Agent.java

@@ -37,6 +37,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 import javax.naming.ConfigurationException;
 
+import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate;
 import org.apache.cloudstack.ca.SetupCertificateAnswer;
 import org.apache.cloudstack.ca.SetupCertificateCommand;
 import org.apache.cloudstack.ca.SetupKeyStoreCommand;
@@ -551,6 +552,8 @@ public class Agent implements HandlerFactory, IAgentControl {
                         answer = setupAgentKeystore((SetupKeyStoreCommand) cmd);
                     } else if (cmd instanceof SetupCertificateCommand && ((SetupCertificateCommand) cmd).isHandleByAgent()) {
                         answer = setupAgentCertificate((SetupCertificateCommand) cmd);
+                    } else if (cmd instanceof SetupDirectDownloadCertificate) {
+                        answer = setupDirectDownloadCertificate((SetupDirectDownloadCertificate) cmd);
                     } else {
                         if (cmd instanceof ReadyCommand) {
                             processReadyCommand(cmd);
@@ -600,6 +603,31 @@ public class Agent implements HandlerFactory, IAgentControl {
         }
     }
 
+    private Answer setupDirectDownloadCertificate(SetupDirectDownloadCertificate cmd) {
+        String certificate = cmd.getCertificate();
+        String certificateName = cmd.getCertificateName();
+        s_logger.info("Importing certificate " + certificateName + " into keystore");
+
+        final File agentFile = PropertiesUtil.findConfigFile("agent.properties");
+        if (agentFile == null) {
+            return new Answer(cmd, false, "Failed to find agent.properties file");
+        }
+
+        final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile;
+
+        String cerFile = agentFile.getParent() + "/" + certificateName + ".cer";
+        Script.runSimpleBashScript(String.format("echo '%s' > %s", certificate, cerFile));
+
+        String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null  | sed 's/keystore.passphrase=//g' 2>/dev/null";
+        String privatePasswordCmd = String.format(privatePasswordFormat, agentFile.getAbsolutePath());
+        String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
+
+        String importCommandFormat = "keytool -importcert -file %s -keystore %s -alias '%s' -storepass '%s' -noprompt";
+        String importCmd = String.format(importCommandFormat, cerFile, keyStoreFile, certificateName, privatePassword);
+        Script.runSimpleBashScript(importCmd);
+        return new Answer(cmd, true, "Certificate " + certificateName + " imported");
+    }
+
     public Answer setupAgentKeystore(final SetupKeyStoreCommand cmd) {
         final String keyStorePassword = cmd.getKeystorePassword();
         final long validityDays = cmd.getValidityDays();

agent/src/com/cloud/agent/direct/download/DirectTemplateDownloader.java

@@ -0,0 +1,71 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.direct.download;
+
+public interface DirectTemplateDownloader {
+
+    class DirectTemplateInformation {
+        private String installPath;
+        private Long size;
+        private String checksum;
+
+        public DirectTemplateInformation(String installPath, Long size, String checksum) {
+            this.installPath = installPath;
+            this.size = size;
+            this.checksum = checksum;
+        }
+
+        public String getInstallPath() {
+            return installPath;
+        }
+
+        public Long getSize() {
+            return size;
+        }
+
+        public String getChecksum() {
+            return checksum;
+        }
+    }
+
+    /**
+     * Perform template download to pool specified on downloader creation
+     * @return true if successful, false if not
+     */
+    boolean downloadTemplate();
+
+    /**
+     * Perform extraction (if necessary) and installation of previously downloaded template
+     * @return true if successful, false if not
+     */
+    boolean extractAndInstallDownloadedTemplate();
+
+    /**
+     * Get template information after it is properly installed on pool
+     * @return template information
+     */
+    DirectTemplateInformation getTemplateInformation();
+
+    /**
+     * Perform checksum validation of previously downloadeed template
+     * @return true if successful, false if not
+     */
+    boolean validateChecksum();
+}

agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java

@@ -0,0 +1,185 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.agent.direct.download;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+import org.apache.cloudstack.utils.security.ChecksumValue;
+import org.apache.commons.lang.StringUtils;
+
+import java.io.File;
+import java.util.UUID;
+
+public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader {
+
+    private String url;
+    private String destPoolPath;
+    private Long templateId;
+    private String downloadedFilePath;
+    private String installPath;
+    private String checksum;
+
+    protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, final String checksum) {
+        this.url = url;
+        this.destPoolPath = destPoolPath;
+        this.templateId = templateId;
+        this.checksum = checksum;
+    }
+
+    private static String directDownloadDir = "template";
+
+    /**
+     * Return direct download temporary path to download template
+     */
+    protected static String getDirectDownloadTempPath(Long templateId) {
+        String templateIdAsString = String.valueOf(templateId);
+        return directDownloadDir + File.separator + templateIdAsString.substring(0,1) +
+                File.separator + templateIdAsString;
+    }
+
+    /**
+     * Create folder on path if it does not exist
+     */
+    protected void createFolder(String path) {
+        File dir = new File(path);
+        if (!dir.exists()) {
+            dir.mkdirs();
+        }
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public String getDestPoolPath() {
+        return destPoolPath;
+    }
+
+    public Long getTemplateId() {
+        return templateId;
+    }
+
+    public String getDownloadedFilePath() {
+        return downloadedFilePath;
+    }
+
+    public void setDownloadedFilePath(String filePath) {
+        this.downloadedFilePath = filePath;
+    }
+
+    /**
+     * Return filename from url
+     */
+    public String getFileNameFromUrl() {
+        String[] urlParts = url.split("/");
+        return urlParts[urlParts.length - 1];
+    }
+
+    /**
+     * Checks if downloaded template is extractable
+     * @return true if it should be extracted, false if not
+     */
+    private boolean isTemplateExtractable() {
+        String type = Script.runSimpleBashScript("file " + downloadedFilePath + " | awk -F' ' '{print $2}'");
+        return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
+    }
+
+    @Override
+    public boolean extractAndInstallDownloadedTemplate() {
+        installPath = UUID.randomUUID().toString();
+        if (isTemplateExtractable()) {
+            extractDownloadedTemplate();
+        } else {
+            Script.runSimpleBashScript("mv " + downloadedFilePath + " " + getInstallFullPath());
+        }
+        return true;
+    }
+
+    /**
+     * Return install full path
+     */
+    private String getInstallFullPath() {
+        return destPoolPath + File.separator + installPath;
+    }
+
+    /**
+     * Return extract command to execute given downloaded file
+     */
+    private String getExtractCommandForDownloadedFile() {
+        if (downloadedFilePath.endsWith(".zip")) {
+            return "unzip -p " + downloadedFilePath + " | cat > " + getInstallFullPath();
+        } else if (downloadedFilePath.endsWith(".bz2")) {
+            return "bunzip2 -c " + downloadedFilePath + " > " + getInstallFullPath();
+        } else if (downloadedFilePath.endsWith(".gz")) {
+            return "gunzip -c " + downloadedFilePath + " > " + getInstallFullPath();
+        } else {
+            throw new CloudRuntimeException("Unable to extract template " + templateId + " on " + downloadedFilePath);
+        }
+    }
+
+    /**
+     * Extract downloaded template into installPath, remove compressed file
+     */
+    private void extractDownloadedTemplate() {
+        String extractCommand = getExtractCommandForDownloadedFile();
+        Script.runSimpleBashScript(extractCommand);
+        Script.runSimpleBashScript("rm -f " + downloadedFilePath);
+    }
+
+    @Override
+    public DirectTemplateInformation getTemplateInformation() {
+        String sizeResult = Script.runSimpleBashScript("ls -als " + getInstallFullPath() + " | awk '{print $1}'");
+        long size = Long.parseLong(sizeResult);
+        return new DirectTemplateInformation(installPath, size, checksum);
+    }
+
+    /**
+     * Return checksum command from algorithm
+     */
+    private String getChecksumCommandFromAlgorithm(String algorithm) {
+        if (algorithm.equalsIgnoreCase("MD5")) {
+            return "md5sum";
+        } else if (algorithm.equalsIgnoreCase("SHA-1")) {
+            return "sha1sum";
+        } else if (algorithm.equalsIgnoreCase("SHA-224")) {
+            return "sha224sum";
+        } else if (algorithm.equalsIgnoreCase("SHA-256")) {
+            return "sha256sum";
+        } else if (algorithm.equalsIgnoreCase("SHA-384")) {
+            return "sha384sum";
+        } else if (algorithm.equalsIgnoreCase("SHA-512")) {
+            return "sha512sum";
+        } else {
+            throw new CloudRuntimeException("Unknown checksum algorithm: " + algorithm);
+        }
+    }
+
+    @Override
+    public boolean validateChecksum() {
+        if (StringUtils.isNotBlank(checksum)) {
+            ChecksumValue providedChecksum = new ChecksumValue(checksum);
+            String algorithm = providedChecksum.getAlgorithm();
+            String checksumCommand = "echo '%s %s' | %s -c --quiet";
+            String cmd = String.format(checksumCommand, providedChecksum.getChecksum(), downloadedFilePath, getChecksumCommandFromAlgorithm(algorithm));
+            int result = Script.runSimpleBashScriptForExitValue(cmd);
+            return result == 0;
+        }
+        return true;
+    }
+}

agent/src/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java

@@ -0,0 +1,122 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.direct.download;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpMethod;
+import org.apache.commons.httpclient.HttpMethodRetryHandler;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.NoHttpResponseException;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.params.HttpMethodParams;
+import org.apache.commons.io.IOUtils;
+import org.apache.log4j.Logger;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.util.Map;
+
+public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
+
+    private HttpClient client;
+    private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
+    private static final int CHUNK_SIZE = 1024 * 1024; //1M
+    protected HttpMethodRetryHandler myretryhandler;
+    public static final Logger s_logger = Logger.getLogger(HttpDirectTemplateDownloader.class.getName());
+    protected GetMethod request;
+
+    public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers) {
+        super(url, destPoolPath, templateId, checksum);
+        client = new HttpClient(s_httpClientManager);
+        myretryhandler = createRetryTwiceHandler();
+        request = createRequest(url, headers);
+        String downloadDir = getDirectDownloadTempPath(templateId);
+        createTemporaryDirectoryAndFile(downloadDir);
+    }
+
+    protected void createTemporaryDirectoryAndFile(String downloadDir) {
+        createFolder(getDestPoolPath() + File.separator + downloadDir);
+        File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl());
+        setDownloadedFilePath(f.getAbsolutePath());
+    }
+
+    protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
+        GetMethod request = new GetMethod(downloadUrl);
+        request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
+        request.setFollowRedirects(true);
+        if (MapUtils.isNotEmpty(headers)) {
+            for (String key : headers.keySet()) {
+                request.setRequestHeader(key, headers.get(key));
+            }
+        }
+        return request;
+    }
+
+    protected HttpMethodRetryHandler createRetryTwiceHandler() {
+        return new HttpMethodRetryHandler() {
+            @Override
+            public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
+                if (executionCount >= 2) {
+                    // Do not retry if over max retry count
+                    return false;
+                }
+                if (exception instanceof NoHttpResponseException) {
+                    // Retry if the server dropped connection on us
+                    return true;
+                }
+                if (!method.isRequestSent()) {
+                    // Retry if the request has not been sent fully or
+                    // if it's OK to retry methods that have been sent
+                    return true;
+                }
+                // otherwise do not retry
+                return false;
+            }
+        };
+    }
+
+    @Override
+    public boolean downloadTemplate() {
+        try {
+            client.executeMethod(request);
+        } catch (IOException e) {
+            throw new CloudRuntimeException("Error on HTTP request: " + e.getMessage());
+        }
+        return performDownload();
+    }
+
+    protected boolean performDownload() {
+        try (
+                InputStream in = request.getResponseBodyAsStream();
+                OutputStream out = new FileOutputStream(getDownloadedFilePath());
+        ) {
+            IOUtils.copy(in, out);
+        } catch (IOException e) {
+            s_logger.error("Error downloading template " + getTemplateId() + " due to: " + e.getMessage());
+            return false;
+        }
+        return true;
+    }
+}

agent/src/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java

@@ -0,0 +1,89 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.direct.download;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.ssl.SSLContexts;
+
+import javax.net.ssl.SSLContext;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+
+public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader {
+
+    private CloseableHttpClient httpsClient;
+    private HttpUriRequest req;
+
+    public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum) {
+        super(url, templateId, destPoolPath, checksum, null);
+        SSLContext sslcontext = null;
+        try {
+            sslcontext = getSSLContext();
+        } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) {
+            throw new CloudRuntimeException("Failure getting SSL context for HTTPS downloader: " + e.getMessage());
+        }
+        SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+        httpsClient = HttpClients.custom().setSSLSocketFactory(factory).build();
+        req = createUriRequest(url);
+    }
+
+    protected HttpUriRequest createUriRequest(String downloadUrl) {
+        return new HttpGet(downloadUrl);
+    }
+
+    private SSLContext getSSLContext() throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException {
+        KeyStore trustStore  = KeyStore.getInstance("jks");
+        FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks"));
+        try {
+            String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null  | sed 's/keystore.passphrase=//g' 2>/dev/null";
+            String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties");
+            String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
+            trustStore.load(instream, privatePassword.toCharArray());
+        } finally {
+            instream.close();
+        }
+        return SSLContexts.custom()
+                .loadTrustMaterial(trustStore, new TrustSelfSignedStrategy())
+                .build();
+    }
+
+    @Override
+    public boolean downloadTemplate() {
+        try {
+            httpsClient.execute(req);
+        } catch (IOException e) {
+            throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage());
+        }
+        return performDownload();
+    }
+}

agent/src/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java

@@ -0,0 +1,49 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.agent.direct.download;
+
+import com.cloud.utils.script.Script;
+
+import java.io.File;
+
+public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
+
+    private String downloadDir;
+
+    public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum) {
+        super(url, destPoolPath, templateId, checksum);
+        String relativeDir = getDirectDownloadTempPath(templateId);
+        downloadDir = getDestPoolPath() + File.separator + relativeDir;
+        createFolder(downloadDir);
+    }
+
+    @Override
+    public boolean downloadTemplate() {
+        String downloadCommand = "aria2c " + getUrl() + " -d " + downloadDir + " --check-integrity=true";
+        Script.runSimpleBashScript(downloadCommand);
+        //Remove .metalink file
+        Script.runSimpleBashScript("rm -f " + downloadDir + File.separator + getFileNameFromUrl());
+        String fileName = Script.runSimpleBashScript("ls " + downloadDir);
+        if (fileName == null) {
+            return false;
+        }
+        setDownloadedFilePath(downloadDir + File.separator + fileName);
+        return true;
+    }
+}

agent/src/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java

@@ -0,0 +1,70 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.agent.direct.download;
+
+import com.cloud.utils.UriUtils;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+
+import java.io.File;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.UUID;
+
+public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
+
+    private String srcHost;
+    private String srcPath;
+
+    private static final String mountCommand = "mount -t nfs %s %s";
+
+    /**
+     * Parse url and set srcHost and srcPath
+     */
+    private void parseUrl() {
+        URI uri = null;
+        String url = getUrl();
+        try {
+            uri = new URI(UriUtils.encodeURIComponent(url));
+            if (uri.getScheme() != null && uri.getScheme().equalsIgnoreCase("nfs")) {
+                srcHost = uri.getHost();
+                srcPath = uri.getPath();
+            }
+        } catch (URISyntaxException e) {
+            throw new CloudRuntimeException("Invalid NFS url " + url + " caused error: " + e.getMessage());
+        }
+    }
+
+    public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum) {
+        super(url, destPool, templateId, checksum);
+        parseUrl();
+    }
+
+    @Override
+    public boolean downloadTemplate() {
+        String mountSrcUuid = UUID.randomUUID().toString();
+        String mount = String.format(mountCommand, srcHost + ":" + srcPath, "/mnt/" + mountSrcUuid);
+        Script.runSimpleBashScript(mount);
+        String downloadDir = getDestPoolPath() + File.separator + getDirectDownloadTempPath(getTemplateId());
+        setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl());
+        Script.runSimpleBashScript("cp /mnt/" + mountSrcUuid + srcPath + " " + getDownloadedFilePath());
+        Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid);
+        return true;
+    }
+}

agent/test/com/cloud/agent/direct/download/DirectTemplateDownloaderImplTest.java

@@ -0,0 +1,36 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.agent.direct.download;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.runners.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class DirectTemplateDownloaderImplTest {
+
+    private static final Long templateId = 202l;
+
+    @Test
+    public void testGetDirectDownloadTempPath() {
+        String path = DirectTemplateDownloaderImpl.getDirectDownloadTempPath(templateId);
+        Assert.assertEquals("template/2/202", path);
+    }
+}

api/pom.xml

@@ -61,6 +61,11 @@
       <artifactId>commons-lang3</artifactId>
       <version>${cs.commons-lang3.version}</version>
     </dependency>
+    <dependency>
+        <groupId>org.apache.cloudstack</groupId>
+        <artifactId>cloud-framework-direct-download</artifactId>
+        <version>${project.version}</version>
+    </dependency>
   </dependencies>
   <build>
     <plugins>

api/src/com/cloud/storage/VMTemplateStorageResourceAssoc.java

@@ -22,7 +22,7 @@ import org.apache.cloudstack.api.InternalIdentity;
 
 public interface VMTemplateStorageResourceAssoc extends InternalIdentity {
     public static enum Status {
-        UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED
+        UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED
     }
 
     String getInstallPath();

api/src/org/apache/cloudstack/api/ApiConstants.java

@@ -83,6 +83,7 @@ public class ApiConstants {
     public static final String DESTINATION_ZONE_ID = "destzoneid";
     public static final String DETAILS = "details";
     public static final String DEVICE_ID = "deviceid";
+    public static final String DIRECT_DOWNLOAD = "directdownload";
     public static final String DISK_OFFERING_ID = "diskofferingid";
     public static final String DISK_SIZE = "disksize";
     public static final String UTILIZATION = "utilization";

api/src/org/apache/cloudstack/api/command/admin/direct/download/UploadTemplateDirectDownloadCertificate.java

@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.direct.download;
+
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.exception.NetworkRuleConflictException;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.direct.download.DirectDownloadManager;
+import org.apache.log4j.Logger;
+
+import javax.inject.Inject;
+
+@APICommand(name = UploadTemplateDirectDownloadCertificate.APINAME,
+        description = "Upload a certificate for HTTPS direct template download on KVM hosts",
+        responseObject = SuccessResponse.class,
+        requestHasSensitiveInfo = true,
+        responseHasSensitiveInfo = true,
+        since = "4.11.0",
+        authorized = {RoleType.Admin})
+public class UploadTemplateDirectDownloadCertificate extends BaseCmd {
+
+    @Inject
+    DirectDownloadManager directDownloadManager;
+
+    private static final Logger LOG = Logger.getLogger(UploadTemplateDirectDownloadCertificate.class);
+    public static final String APINAME = "uploadTemplateDirectDownloadCertificate";
+
+    @Parameter(name = ApiConstants.CERTIFICATE, type = BaseCmd.CommandType.STRING, required = true, length = 65535,
+            description = "SSL certificate")
+    private String certificate;
+
+    @Parameter(name = ApiConstants.NAME , type = BaseCmd.CommandType.STRING, required = true,
+            description = "Name for the uploaded certificate")
+    private String name;
+
+    @Parameter(name = ApiConstants.HYPERVISOR, type = BaseCmd.CommandType.STRING, required = true, description = "Hypervisor type")
+    private String hypervisor;
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        if (!hypervisor.equalsIgnoreCase("kvm")) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Currently supporting KVM hosts only");
+        }
+
+        try {
+            directDownloadManager.uploadCertificateToHosts(certificate, name);;
+            setResponseObject(new SuccessResponse(getCommandName()));
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage());
+        }
+    }
+
+    @Override
+    public String getCommandName() {
+        return UploadTemplateDirectDownloadCertificate.APINAME;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccount().getId();
+    }
+}
+
+

api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java

@@ -40,17 +40,7 @@ public class RegisterTemplateCmdByAdmin extends RegisterTemplateCmd {
     @Override
     public void execute() throws ResourceAllocationException{
         try {
-            if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty()))
-                throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
-                        "Both zoneid and zoneids cannot be specified at the same time");
-
-            if (zoneId == null && (zoneIds == null || zoneIds.isEmpty()))
-                throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
-                        "Either zoneid or zoneids is required. Both cannot be null.");
-
-            if (zoneIds != null && zoneIds.size() > 1 && zoneIds.contains(-1L))
-                throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
-                        "Parameter zoneids cannot combine all zones (-1) option with other zones");
+            validateParameters();
 
             VirtualMachineTemplate template = _templateService.registerTemplate(this);
             if (template != null){

api/src/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java

@@ -108,6 +108,11 @@ public class RegisterIsoCmd extends BaseCmd {
                description = "true if ISO contains XS/VMWare tools inorder to support dynamic scaling of VM CPU/memory")
     protected Boolean isDynamicallyScalable;
 
+    @Parameter(name=ApiConstants.DIRECT_DOWNLOAD,
+            type = CommandType.BOOLEAN,
+            description = "true if ISO should bypass Secondary Storage and be downloaded to Primary Storage on deployment")
+    private Boolean directDownload;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -168,6 +173,10 @@ public class RegisterIsoCmd extends BaseCmd {
         return isDynamicallyScalable ==  null ? Boolean.FALSE : isDynamicallyScalable;
     }
 
+    public boolean isDirectDownload() {
+        return directDownload == null ? false : directDownload;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java

@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.template;
 
+import com.cloud.hypervisor.Hypervisor;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -155,6 +156,11 @@ public class RegisterTemplateCmd extends BaseCmd {
                     "zone template and will be copied to all zones. ")
     protected List<Long> zoneIds;
 
+    @Parameter(name=ApiConstants.DIRECT_DOWNLOAD,
+                type = CommandType.BOOLEAN,
+                description = "true if template should bypass Secondary Storage and be downloaded to Primary Storage on deployment")
+    private Boolean directDownload;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -263,6 +269,10 @@ public class RegisterTemplateCmd extends BaseCmd {
         return isRoutingType;
     }
 
+    public boolean isDirectDownload() {
+        return directDownload == null ? false : directDownload;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -289,17 +299,7 @@ public class RegisterTemplateCmd extends BaseCmd {
     @Override
     public void execute() throws ResourceAllocationException {
         try {
-            if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty()))
-                throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
-                        "Both zoneid and zoneids cannot be specified at the same time");
-
-            if (zoneId == null && (zoneIds == null || zoneIds.isEmpty()))
-                throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
-                        "Either zoneid or zoneids is required. Both cannot be null.");
-
-            if (zoneIds != null && zoneIds.size() > 1 && zoneIds.contains(-1L))
-                throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
-                        "Parameter zoneids cannot combine all zones (-1) option with other zones");
+            validateParameters();
 
             VirtualMachineTemplate template = _templateService.registerTemplate(this);
             if (template != null) {
@@ -317,4 +317,23 @@ public class RegisterTemplateCmd extends BaseCmd {
             throw new ServerApiException(ApiErrorCode.PARAM_ERROR, ex1.getMessage());
         }
     }
+
+    protected void validateParameters() {
+        if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty()))
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
+                    "Both zoneid and zoneids cannot be specified at the same time");
+
+        if (zoneId == null && (zoneIds == null || zoneIds.isEmpty()))
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
+                    "Either zoneid or zoneids is required. Both cannot be null.");
+
+        if (zoneIds != null && zoneIds.size() > 1 && zoneIds.contains(-1L))
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
+                    "Parameter zoneids cannot combine all zones (-1) option with other zones");
+
+        if (isDirectDownload() && !getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.KVM.toString())) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
+                    "Parameter directdownload is only allowed for KVM templates");
+        }
+    }
 }

api/src/org/apache/cloudstack/api/response/TemplateResponse.java

@@ -185,6 +185,10 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements
     @Param(description = "true if template contains XS/VMWare tools inorder to support dynamic scaling of VM cpu/memory")
     private Boolean isDynamicallyScalable;
 
+    @SerializedName(ApiConstants.DIRECT_DOWNLOAD)
+    @Param(description = "KVM Only: true if template is directly downloaded to Primary Storage bypassing Secondary Storage")
+    private Boolean directDownload;
+
     public TemplateResponse() {
         tags = new LinkedHashSet<ResourceTagResponse>();
     }
@@ -362,4 +366,12 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements
     public void setBits(int bits) {
         this.bits = bits;
     }
+
+    public void setDirectDownload(Boolean directDownload) {
+        this.directDownload = directDownload;
+    }
+
+    public Boolean getDirectDownload() {
+        return directDownload;
+    }
 }

api/src/org/apache/cloudstack/direct/download/DirectDownloadManager.java

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.direct.download;
+
+import com.cloud.utils.component.PluggableService;
+import org.apache.cloudstack.framework.agent.direct.download.DirectDownloadService;
+
+public interface DirectDownloadManager extends DirectDownloadService, PluggableService {
+
+}

client/pom.xml

@@ -318,6 +318,11 @@
       <artifactId>cloud-framework-ca</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-framework-direct-download</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.cloudstack</groupId>
       <artifactId>cloud-framework-ipc</artifactId>

core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml

@@ -319,4 +319,8 @@
           class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
     </bean>
 
+    <bean id="directDownloadRegistry"
+          class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
+    </bean>
+
 </beans>

core/resources/META-INF/cloudstack/direct-download/module.properties

@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name=direct-download
+parent=backend

core/resources/META-INF/cloudstack/direct-download/spring-lifecycle-direct-download-context-inheritable.xml

@@ -0,0 +1,32 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+                      http://www.springframework.org/schema/beans/spring-beans-3.0.xsd"
+>
+
+    <bean class="org.apache.cloudstack.spring.lifecycle.registry.RegistryLifecycle">
+        <property name="registry" ref="directDownloadRegistry" />
+        <property name="typeClass" value="org.apache.cloudstack.framework.agent.direct.download.DirectDownloadService" />
+    </bean>
+
+</beans>

core/src/com/cloud/storage/resource/StorageProcessor.java

@@ -19,6 +19,7 @@
 
 package com.cloud.storage.resource;
 
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
 import org.apache.cloudstack.storage.command.AttachCommand;
 import org.apache.cloudstack.storage.command.CopyCommand;
 import org.apache.cloudstack.storage.command.CreateObjectCommand;
@@ -71,4 +72,6 @@ public interface StorageProcessor {
     public Answer snapshotAndCopy(SnapshotAndCopyCommand cmd);
 
     public Answer resignature(ResignatureCommand cmd);
+
+    public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd);
 }

core/src/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java

@@ -19,6 +19,7 @@
 
 package com.cloud.storage.resource;
 
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.storage.command.AttachCommand;
@@ -66,7 +67,9 @@ public class StorageSubsystemCommandHandlerBase implements StorageSubsystemComma
         } else if (command instanceof SnapshotAndCopyCommand) {
             return processor.snapshotAndCopy((SnapshotAndCopyCommand)command);
         } else if (command instanceof ResignatureCommand) {
-            return processor.resignature((ResignatureCommand)command);
+            return processor.resignature((ResignatureCommand) command);
+        } else if (command instanceof DirectDownloadCommand) {
+            return processor.handleDownloadTemplateToPrimaryStorage((DirectDownloadCommand) command);
         }
 
         return new Answer((Command)command, false, "not implemented yet");

core/src/com/cloud/storage/template/MetalinkTemplateDownloader.java

@@ -0,0 +1,66 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.storage.template;
+
+import com.cloud.storage.StorageLayer;
+import com.cloud.utils.script.Script;
+import org.apache.log4j.Logger;
+
+import java.io.File;
+
+public class MetalinkTemplateDownloader extends TemplateDownloaderBase implements TemplateDownloader {
+
+    private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
+
+    private static final Logger LOGGER = Logger.getLogger(MetalinkTemplateDownloader.class.getName());
+
+    public MetalinkTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSize) {
+        super(storageLayer, downloadUrl, toDir, maxTemplateSize, callback);
+        String[] parts = _downloadUrl.split("/");
+        String filename = parts[parts.length - 1];
+        _toFile = toDir + File.separator + filename;
+    }
+
+    @Override
+    public long download(boolean resume, DownloadCompleteCallback callback) {
+        if (!status.equals(Status.NOT_STARTED)) {
+            // Only start downloading if we haven't started yet.
+            LOGGER.debug("Template download is already started, not starting again. Template: " + _downloadUrl);
+            return 0;
+        }
+        status = Status.IN_PROGRESS;
+        Script.runSimpleBashScript("aria2c " + _downloadUrl + " -d " + _toDir);
+        status = Status.DOWNLOAD_FINISHED;
+        String sizeResult = Script.runSimpleBashScript("ls -als " + _toFile + " | awk '{print $1}'");
+        long size = Long.parseLong(sizeResult);
+        return size;
+    }
+
+    @Override
+    public int getDownloadPercent() {
+        if (status == Status.DOWNLOAD_FINISHED) {
+            return 100;
+        } else if (status == Status.IN_PROGRESS) {
+            return 50;
+        } else {
+            return 0;
+        }
+    }
+
+}

core/src/org/apache/cloudstack/agent/directdownload/CheckUrlAnswer.java

@@ -0,0 +1,37 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.agent.directdownload;
+
+import com.cloud.agent.api.Answer;
+
+public class CheckUrlAnswer extends Answer {
+
+    private Long templateSize;
+
+    public CheckUrlAnswer(final boolean result, final Long templateSize) {
+        super(null);
+        this.result = result;
+        this.templateSize = templateSize;
+    }
+
+    public Long getTemplateSize() {
+        return templateSize;
+    }
+}

core/src/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java

@@ -0,0 +1,42 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.agent.directdownload;
+
+import com.cloud.agent.api.Command;
+
+public class CheckUrlCommand extends Command {
+
+    private String url;
+
+    public String getUrl() {
+        return url;
+    }
+
+    public CheckUrlCommand(final String url) {
+        super();
+        this.url = url;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+
+}

core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadAnswer.java

@@ -0,0 +1,48 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.agent.directdownload;
+
+import com.cloud.agent.api.Answer;
+
+public class DirectDownloadAnswer extends Answer {
+
+    private Long templateSize;
+    private String installPath;
+
+    public DirectDownloadAnswer(final boolean result, final String msg) {
+        super(null);
+        this.result = result;
+        this.details = msg;
+    }
+
+    public DirectDownloadAnswer(final boolean result, final Long size, final String installPath) {
+        super(null);
+        this.result = result;
+        this.templateSize = size;
+        this.installPath = installPath;
+    }
+
+    public long getTemplateSize() {
+        return templateSize;
+    }
+
+    public String getInstallPath() {
+        return installPath;
+    }
+}

core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.agent.directdownload;
+
+import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+
+public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
+
+    public enum DownloadProtocol {
+        HTTP, HTTPS, NFS, METALINK
+    }
+
+    private String url;
+    private Long templateId;
+    private PrimaryDataStoreTO destPool;
+    private String checksum;
+
+    protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum) {
+        this.url = url;
+        this.templateId = templateId;
+        this.destPool = destPool;
+        this.checksum = checksum;
+    }
+
+    public String getUrl() {
+        return url;
+    }
+
+    public Long getTemplateId() {
+        return templateId;
+    }
+
+    public PrimaryDataStoreTO getDestPool() {
+        return destPool;
+    }
+
+    public String getChecksum() {
+        return checksum;
+    }
+
+    @Override
+    public void setExecuteInSequence(boolean inSeq) {
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+}

core/src/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.agent.directdownload;
+
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+
+import java.util.Map;
+
+public class HttpDirectDownloadCommand extends DirectDownloadCommand {
+
+    private Map<String, String> headers;
+
+    public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
+        super(url, templateId, destPool, checksum);
+        this.headers = headers;
+    }
+
+    public Map<String, String> getHeaders() {
+        return headers;
+    }
+
+}

core/src/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java

@@ -0,0 +1,31 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.agent.directdownload;
+
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+
+import java.util.Map;
+
+public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
+
+    public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
+        super(url, templateId, destPool, checksum);
+    }
+}

core/src/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java

@@ -0,0 +1,29 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.agent.directdownload;
+
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+
+public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
+
+    public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum) {
+        super(url, templateId, destPool, checksum);
+    }
+
+}

core/src/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java

@@ -0,0 +1,29 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.agent.directdownload;
+
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+
+public class NfsDirectDownloadCommand extends DirectDownloadCommand {
+
+    public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum) {
+        super(url, templateId, destPool, checksum);
+    }
+
+}

core/src/org/apache/cloudstack/agent/directdownload/SetupDirectDownloadCertificate.java

@@ -0,0 +1,45 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.agent.directdownload;
+
+import com.cloud.agent.api.Command;
+
+public class SetupDirectDownloadCertificate extends Command {
+
+    private String certificate;
+    private String certificateName;
+
+    public SetupDirectDownloadCertificate(String certificate, String name) {
+        this.certificate = certificate;
+        this.certificateName = name;
+    }
+
+    public String getCertificate() {
+        return certificate;
+    }
+
+    public String getCertificateName() {
+        return certificateName;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+}

debian/control

@@ -22,7 +22,7 @@ Description: CloudStack server library
 
 Package: cloudstack-agent
 Architecture: all
-Depends: ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 1.2.2), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, lsb-release, init-system-helpers (>= 1.14~)
+Depends: ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 1.2.2), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, lsb-release, init-system-helpers (>= 1.14~), aria2
 Recommends: init-system-helpers
 Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
 Description: CloudStack agent

engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java

@@ -36,4 +36,8 @@ public interface TemplateDataFactory {
     TemplateInfo getReadyTemplateOnCache(long templateId);
 
     List<TemplateInfo> listTemplateOnCache(long templateId);
+
+    TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long poolId, Long hostId);
+
+    boolean isTemplateMarkedForDirectDownload(long templateId);
 }

engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java

@@ -25,4 +25,6 @@ public interface TemplateInfo extends DataObject, VirtualMachineTemplate {
     String getUniqueName();
 
     String getInstallPath();
+
+    boolean isDirectDownload();
 }

engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java

@@ -108,4 +108,5 @@ public interface VolumeService {
     SnapshotInfo takeSnapshot(VolumeInfo volume);
 
     VolumeInfo updateHypervisorSnapshotReserveForVolume(DiskOffering diskOffering, long volumeId, HypervisorType hyperType);
+
 }

engine/components-api/src/com/cloud/resource/ResourceManager.java

@@ -195,4 +195,6 @@ public interface ResourceManager extends ResourceService {
      * @return Details of groupNames and enabled VGPU type with remaining capacity.
      */
     HashMap<String, HashMap<String, VgpuTypesInfo>> getGPUStatistics(HostVO host);
+
+    HostVO findOneRandomRunningHostByHypervisor(HypervisorType type);
 }

engine/components-api/src/com/cloud/template/TemplateManager.java

@@ -18,6 +18,7 @@ package com.cloud.template;
 
 import java.util.List;
 
+import com.cloud.deploy.DeployDestination;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.framework.config.ConfigKey;
@@ -119,7 +120,7 @@ public interface TemplateManager {
 
     List<DataStore> getImageStoreByTemplate(long templateId, Long zoneId);
 
-    TemplateInfo prepareIso(long isoId, long dcId);
+    TemplateInfo prepareIso(long isoId, long dcId, Long hostId, Long poolId);
 
 
     /**
@@ -127,7 +128,7 @@ public interface TemplateManager {
      *
      * @param VirtualMachineProfile
      */
-    void prepareIsoForVmProfile(VirtualMachineProfile profile);
+    void prepareIsoForVmProfile(VirtualMachineProfile profile, DeployDestination dest);
 
     public static final String MESSAGE_REGISTER_PUBLIC_TEMPLATE_EVENT = "Message.RegisterPublicTemplate.Event";
     public static final String MESSAGE_RESET_TEMPLATE_PERMISSION_EVENT = "Message.ResetTemplatePermission.Event";

engine/orchestration/pom.xml

@@ -58,13 +58,6 @@
       <artifactId>cloud-utils</artifactId>
       <version>${project.version}</version>
     </dependency>
-    <!-- 
-    <dependency>
-      <groupId>org.apache.cloudstack</groupId>
-      <artifactId>cloud-server</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    -->
   </dependencies>
   <build>
     <plugins>

engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java

@@ -1108,7 +1108,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
 
         //if (vm.getType() == VirtualMachine.Type.User && vm.getTemplate().getFormat() == ImageFormat.ISO) {
         if (vm.getType() == VirtualMachine.Type.User) {
-            _tmpltMgr.prepareIsoForVmProfile(vm);
+            _tmpltMgr.prepareIsoForVmProfile(vm, dest);
             //DataTO dataTO = tmplFactory.getTemplate(vm.getTemplate().getId(), DataStoreRole.Image, vm.getVirtualMachine().getDataCenterId()).getTO();
             //DiskTO iso = new DiskTO(dataTO, 3L, null, Volume.Type.ISO);
             //vm.addDisk(iso);
@@ -1287,10 +1287,14 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
                 TemplateInfo templ = tmplFactory.getReadyTemplateOnImageStore(templateId, dest.getDataCenter().getId());
 
                 if (templ == null) {
+                    if (tmplFactory.isTemplateMarkedForDirectDownload(templateId)) {
+                        // Template is marked for direct download bypassing Secondary Storage
+                        templ = tmplFactory.getReadyBypassedTemplateOnPrimaryStore(templateId, destPool.getId(), dest.getHost().getId());
+                    } else {
                         s_logger.debug("can't find ready template: " + templateId + " for data center " + dest.getDataCenter().getId());
-
                         throw new CloudRuntimeException("can't find ready template: " + templateId + " for data center " + dest.getDataCenter().getId());
                     }
+                }
 
                 PrimaryDataStore primaryDataStore = (PrimaryDataStore)destPool;
 

engine/schema/resources/META-INF/db/schema-41000to41100.sql

@@ -524,6 +524,111 @@ ADD COLUMN `forsystemvms` TINYINT(1) NOT NULL DEFAULT '0' COMMENT 'Indicates if
 ALTER TABLE `cloud`.`op_dc_ip_address_alloc`
 ADD COLUMN `vlan` INT(10) UNSIGNED NULL COMMENT 'Vlan the management network range is on';
 
+-- CLOUDSTACK-10146: Bypass Secondary Storage for KVM templates
+ALTER TABLE `cloud`.`vm_template`
+ADD COLUMN `direct_download` TINYINT(1) DEFAULT '0' COMMENT 'Indicates if Secondary Storage is bypassed and template is downloaded to Primary Storage';
+
+CREATE OR REPLACE VIEW `template_view` AS
+     SELECT
+         `vm_template`.`id` AS `id`,
+         `vm_template`.`uuid` AS `uuid`,
+         `vm_template`.`unique_name` AS `unique_name`,
+         `vm_template`.`name` AS `name`,
+         `vm_template`.`public` AS `public`,
+         `vm_template`.`featured` AS `featured`,
+         `vm_template`.`type` AS `type`,
+         `vm_template`.`hvm` AS `hvm`,
+         `vm_template`.`bits` AS `bits`,
+         `vm_template`.`url` AS `url`,
+         `vm_template`.`format` AS `format`,
+         `vm_template`.`created` AS `created`,
+         `vm_template`.`checksum` AS `checksum`,
+         `vm_template`.`display_text` AS `display_text`,
+         `vm_template`.`enable_password` AS `enable_password`,
+         `vm_template`.`dynamically_scalable` AS `dynamically_scalable`,
+         `vm_template`.`state` AS `template_state`,
+         `vm_template`.`guest_os_id` AS `guest_os_id`,
+         `guest_os`.`uuid` AS `guest_os_uuid`,
+         `guest_os`.`display_name` AS `guest_os_name`,
+         `vm_template`.`bootable` AS `bootable`,
+         `vm_template`.`prepopulate` AS `prepopulate`,
+         `vm_template`.`cross_zones` AS `cross_zones`,
+         `vm_template`.`hypervisor_type` AS `hypervisor_type`,
+         `vm_template`.`extractable` AS `extractable`,
+         `vm_template`.`template_tag` AS `template_tag`,
+         `vm_template`.`sort_key` AS `sort_key`,
+         `vm_template`.`removed` AS `removed`,
+         `vm_template`.`enable_sshkey` AS `enable_sshkey`,
+         `source_template`.`id` AS `source_template_id`,
+         `source_template`.`uuid` AS `source_template_uuid`,
+         `account`.`id` AS `account_id`,
+         `account`.`uuid` AS `account_uuid`,
+         `account`.`account_name` AS `account_name`,
+         `account`.`type` AS `account_type`,
+         `domain`.`id` AS `domain_id`,
+         `domain`.`uuid` AS `domain_uuid`,
+         `domain`.`name` AS `domain_name`,
+         `domain`.`path` AS `domain_path`,
+         `projects`.`id` AS `project_id`,
+         `projects`.`uuid` AS `project_uuid`,
+         `projects`.`name` AS `project_name`,
+         `data_center`.`id` AS `data_center_id`,
+         `data_center`.`uuid` AS `data_center_uuid`,
+         `data_center`.`name` AS `data_center_name`,
+         `launch_permission`.`account_id` AS `lp_account_id`,
+         `template_store_ref`.`store_id` AS `store_id`,
+         `image_store`.`scope` AS `store_scope`,
+         `template_store_ref`.`state` AS `state`,
+         `template_store_ref`.`download_state` AS `download_state`,
+         `template_store_ref`.`download_pct` AS `download_pct`,
+         `template_store_ref`.`error_str` AS `error_str`,
+         `template_store_ref`.`size` AS `size`,
+         `template_store_ref`.physical_size AS `physical_size`,
+         `template_store_ref`.`destroyed` AS `destroyed`,
+         `template_store_ref`.`created` AS `created_on_store`,
+         `vm_template_details`.`name` AS `detail_name`,
+         `vm_template_details`.`value` AS `detail_value`,
+         `resource_tags`.`id` AS `tag_id`,
+         `resource_tags`.`uuid` AS `tag_uuid`,
+         `resource_tags`.`key` AS `tag_key`,
+         `resource_tags`.`value` AS `tag_value`,
+         `resource_tags`.`domain_id` AS `tag_domain_id`,
+         `domain`.`uuid` AS `tag_domain_uuid`,
+         `domain`.`name` AS `tag_domain_name`,
+         `resource_tags`.`account_id` AS `tag_account_id`,
+         `account`.`account_name` AS `tag_account_name`,
+         `resource_tags`.`resource_id` AS `tag_resource_id`,
+         `resource_tags`.`resource_uuid` AS `tag_resource_uuid`,
+         `resource_tags`.`resource_type` AS `tag_resource_type`,
+         `resource_tags`.`customer` AS `tag_customer`,
+          CONCAT(`vm_template`.`id`,
+                 '_',
+                 IFNULL(`data_center`.`id`, 0)) AS `temp_zone_pair`,
+          `vm_template`.`direct_download` AS `direct_download`
+     FROM
+         ((((((((((((`vm_template`
+         JOIN `guest_os` ON ((`guest_os`.`id` = `vm_template`.`guest_os_id`)))
+         JOIN `account` ON ((`account`.`id` = `vm_template`.`account_id`)))
+         JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`)))
+         LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`)))
+         LEFT JOIN `vm_template_details` ON ((`vm_template_details`.`template_id` = `vm_template`.`id`)))
+         LEFT JOIN `vm_template` `source_template` ON ((`source_template`.`id` = `vm_template`.`source_template_id`)))
+         LEFT JOIN `template_store_ref` ON (((`template_store_ref`.`template_id` = `vm_template`.`id`)
+             AND (`template_store_ref`.`store_role` = 'Image')
+             AND (`template_store_ref`.`destroyed` = 0))))
+         LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`)
+             AND (`template_store_ref`.`store_id` IS NOT NULL)
+             AND (`image_store`.`id` = `template_store_ref`.`store_id`))))
+         LEFT JOIN `template_zone_ref` ON (((`template_zone_ref`.`template_id` = `vm_template`.`id`)
+             AND ISNULL(`template_store_ref`.`store_id`)
+             AND ISNULL(`template_zone_ref`.`removed`))))
+         LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`)
+             OR (`template_zone_ref`.`zone_id` = `data_center`.`id`))))
+         LEFT JOIN `launch_permission` ON ((`launch_permission`.`template_id` = `vm_template`.`id`)))
+         LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_template`.`id`)
+             AND ((`resource_tags`.`resource_type` = 'Template')
+             OR (`resource_tags`.`resource_type` = 'ISO')))));
+
 -- CLOUDSTACK-10109: Enable dedication of public IPs to SSVM and CPVM
 ALTER TABLE `cloud`.`user_ip_address`
 ADD COLUMN `forsystemvms` TINYINT(1) NOT NULL DEFAULT '0' COMMENT 'true if IP is set to system vms, false if not';

engine/schema/src/com/cloud/storage/VMTemplateVO.java

@@ -146,6 +146,9 @@ public class VMTemplateVO implements VirtualMachineTemplate {
     @Column(name = "dynamically_scalable")
     protected boolean dynamicallyScalable;
 
+    @Column(name = "direct_download")
+    private boolean directDownload;
+
     @Override
     public String getUniqueName() {
         return uniqueName;
@@ -188,7 +191,7 @@ public class VMTemplateVO implements VirtualMachineTemplate {
 
     public VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic, boolean featured, boolean isExtractable, TemplateType type, String url,
             boolean requiresHvm, int bits, long accountId, String cksum, String displayText, boolean enablePassword, long guestOSId, boolean bootable,
-            HypervisorType hyperType, String templateTag, Map<String, String> details, boolean sshKeyEnabled, boolean isDynamicallyScalable) {
+            HypervisorType hyperType, String templateTag, Map<String, String> details, boolean sshKeyEnabled, boolean isDynamicallyScalable, boolean directDownload) {
         this(id,
             name,
             format,
@@ -212,6 +215,7 @@ public class VMTemplateVO implements VirtualMachineTemplate {
         enableSshKey = sshKeyEnabled;
         dynamicallyScalable = isDynamicallyScalable;
         state = State.Active;
+        this.directDownload = directDownload;
     }
 
     public static VMTemplateVO createPreHostIso(Long id, String uniqueName, String name, ImageFormat format, boolean isPublic, boolean featured, TemplateType type,
@@ -605,6 +609,10 @@ public class VMTemplateVO implements VirtualMachineTemplate {
         this.updated = updated;
     }
 
+    public boolean isDirectDownload() {
+        return directDownload;
+    }
+
     @Override
     public Class<?> getEntityType() {
         return VirtualMachineTemplate.class;

engine/schema/src/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java

@@ -23,6 +23,8 @@ import java.util.Date;
 import java.util.List;
 
 
+import javax.inject.Inject;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
@@ -44,6 +46,9 @@ import com.cloud.utils.db.UpdateBuilder;
 public class VMTemplatePoolDaoImpl extends GenericDaoBase<VMTemplateStoragePoolVO, Long> implements VMTemplatePoolDao {
     public static final Logger s_logger = Logger.getLogger(VMTemplatePoolDaoImpl.class.getName());
 
+    @Inject
+    DataStoreManager dataStoreManager;
+
     protected final SearchBuilder<VMTemplateStoragePoolVO> PoolSearch;
     protected final SearchBuilder<VMTemplateStoragePoolVO> TemplateSearch;
     protected final SearchBuilder<VMTemplateStoragePoolVO> PoolTemplateSearch;

engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java

@@ -83,4 +83,10 @@ public interface TemplateDataStoreDao extends GenericDao<TemplateDataStoreVO, Lo
     void expireDnldUrlsForZone(Long dcId);
 
     List<TemplateDataStoreVO> listByTemplateState(VirtualMachineTemplate.State... states);
+
+    TemplateDataStoreVO createTemplateDirectDownloadEntry(long templateId, Long size);
+
+    TemplateDataStoreVO getReadyBypassedTemplate(long templateId);
+
+    boolean isTemplateMarkedForDirectDownload(long templateId);
 }

engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java

@@ -18,16 +18,23 @@
  */
 package org.apache.cloudstack.storage.image;
 
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
 import java.util.ArrayList;
 import java.util.List;
 
 import javax.inject.Inject;
 
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.direct.download.DirectDownloadManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.cloudstack.storage.image.store.TemplateObject;
@@ -51,6 +58,12 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
     VMTemplatePoolDao templatePoolDao;
     @Inject
     TemplateDataStoreDao templateStoreDao;
+    @Inject
+    DirectDownloadManager directDownloadManager;
+    @Inject
+    HostDao hostDao;
+    @Inject
+    PrimaryDataStoreDao primaryDataStoreDao;
 
     @Override
     public TemplateInfo getTemplate(long templateId, DataStore store) {
@@ -137,7 +150,6 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
         } else {
             return null;
         }
-
     }
 
     @Override
@@ -153,4 +165,70 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
         return tmplObjs;
     }
 
+    /**
+     * Given existing spool refs, return one pool id existing on pools and refs
+     */
+    private Long getOneMatchingPoolIdFromRefs(List<VMTemplateStoragePoolVO> existingRefs, List<StoragePoolVO> pools) {
+        if (pools.isEmpty()) {
+            throw new CloudRuntimeException("No storage pools found");
+        }
+        if (existingRefs.isEmpty()) {
+            return pools.get(0).getId();
+        } else {
+            for (VMTemplateStoragePoolVO ref : existingRefs) {
+                for (StoragePoolVO p : pools) {
+                    if (ref.getPoolId() == p.getId()) {
+                        return p.getId();
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Retrieve storage pools with scope = cluster or zone matching clusterId or dataCenterId depending on their scope
+     */
+    private List<StoragePoolVO> getStoragePoolsFromClusterOrZone(Long clusterId, long dataCenterId, Hypervisor.HypervisorType hypervisorType) {
+        List<StoragePoolVO> pools = new ArrayList<>();
+        if (clusterId != null) {
+            List<StoragePoolVO> clusterPools = primaryDataStoreDao.listPoolsByCluster(clusterId);
+            pools.addAll(clusterPools);
+        }
+        List<StoragePoolVO> zonePools = primaryDataStoreDao.findZoneWideStoragePoolsByHypervisor(dataCenterId, hypervisorType);
+        pools.addAll(zonePools);
+        return pools;
+    }
+
+    @Override
+    public TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long poolId, Long hostId) {
+        VMTemplateVO templateVO = imageDataDao.findById(templateId);
+        if (templateVO == null || !templateVO.isDirectDownload()) {
+            return null;
+        }
+        Long pool = poolId;
+        if (poolId == null) {
+            //Get ISO from existing pool ref
+            HostVO host = hostDao.findById(hostId);
+            List<StoragePoolVO> pools = getStoragePoolsFromClusterOrZone(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType());
+            List<VMTemplateStoragePoolVO> existingRefs = templatePoolDao.listByTemplateId(templateId);
+            pool = getOneMatchingPoolIdFromRefs(existingRefs, pools);
+        }
+        if (pool == null) {
+            throw new CloudRuntimeException("No storage pool found where to download template: " + templateId);
+        }
+        VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(pool, templateId);
+        if (spoolRef == null) {
+            directDownloadManager.downloadTemplate(templateId, pool, hostId);
+        }
+        DataStore store = storeMgr.getDataStore(pool, DataStoreRole.Primary);
+        return this.getTemplate(templateId, store);
+    }
+
+    @Override
+    public boolean isTemplateMarkedForDirectDownload(long templateId) {
+        VMTemplateVO templateVO = imageDataDao.findById(templateId);
+        return templateVO.isDirectDownload();
+    }
+
 }

engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java

@@ -445,6 +445,9 @@ public class TemplateServiceImpl implements TemplateService {
                             } catch (NoTransitionException e) {
                                 s_logger.error("Unexpected state transition exception for template " + tmplt.getName() + ". Details: " + e.getMessage());
                             }
+                        } else if (tmplt.isDirectDownload()) {
+                            s_logger.info("Template " + tmplt.getName() + ":" + tmplt.getId() + " is marked for direct download, discarding it for download on image stores");
+                            toBeDownloaded.remove(tmplt);
                         } else {
                             s_logger.info("Template Sync did not find " + uniqueName + " on image store " + storeId + ", may request download based on available hypervisor types");
                             if (tmpltStore != null) {

engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java

@@ -344,6 +344,14 @@ public class TemplateObject implements TemplateInfo {
         return obj != null ? obj.getInstallPath() : null;
     }
 
+    @Override
+    public boolean isDirectDownload() {
+        if (this.imageVO == null) {
+            return false;
+        }
+        return this.imageVO.isDirectDownload();
+    }
+
     public void setInstallPath(String installPath) {
         this.installPath = installPath;
     }

engine/storage/src/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java

@@ -22,12 +22,12 @@ import java.util.List;
 
 import javax.inject.Inject;
 
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
+import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
 import org.springframework.stereotype.Component;
 
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
-import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
 import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager;
 
 import com.cloud.storage.DataStoreRole;

engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java

@@ -26,6 +26,7 @@ import java.util.Map;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
@@ -67,6 +68,7 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
     private SearchBuilder<TemplateDataStoreVO> storeTemplateDownloadStatusSearch;
     private SearchBuilder<TemplateDataStoreVO> downloadTemplateSearch;
     private SearchBuilder<TemplateDataStoreVO> uploadTemplateStateSearch;
+    private SearchBuilder<TemplateDataStoreVO> directDownloadTemplateSeach;
     private SearchBuilder<VMTemplateVO> templateOnlySearch;
     private static final String EXPIRE_DOWNLOAD_URLS_FOR_ZONE = "update template_store_ref set download_url_created=? where download_url_created is not null and store_id in (select id from image_store where data_center_id=?)";
 
@@ -143,6 +145,13 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
         downloadTemplateSearch.and("destroyed", downloadTemplateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
         downloadTemplateSearch.done();
 
+        directDownloadTemplateSeach = createSearchBuilder();
+        directDownloadTemplateSeach.and("template_id", directDownloadTemplateSeach.entity().getTemplateId(), Op.EQ);
+        directDownloadTemplateSeach.and("download_state", directDownloadTemplateSeach.entity().getDownloadState(), Op.EQ);
+        directDownloadTemplateSeach.and("store_id", directDownloadTemplateSeach.entity().getDataStoreId(), Op.NULL);
+        directDownloadTemplateSeach.and("state", directDownloadTemplateSeach.entity().getState(), Op.EQ);
+        directDownloadTemplateSeach.done();
+
         templateOnlySearch = _tmpltDao.createSearchBuilder();
         templateOnlySearch.and("states", templateOnlySearch.entity().getState(), SearchCriteria.Op.IN);
         uploadTemplateStateSearch = createSearchBuilder();
@@ -549,4 +558,34 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
         sc.setParameters("destroyed", false);
         return listIncludingRemovedBy(sc);
     }
+
+    @Override
+    public TemplateDataStoreVO createTemplateDirectDownloadEntry(long templateId, Long size) {
+        TemplateDataStoreVO templateDataStoreVO = new TemplateDataStoreVO();
+        templateDataStoreVO.setTemplateId(templateId);
+        templateDataStoreVO.setDataStoreRole(DataStoreRole.Image);
+        templateDataStoreVO.setState(State.Ready);
+        templateDataStoreVO.setDownloadState(Status.BYPASSED);
+        templateDataStoreVO.setSize(size == null ? 0l : size);
+        return templateDataStoreVO;
+    }
+
+    @Override
+    public TemplateDataStoreVO getReadyBypassedTemplate(long templateId) {
+        SearchCriteria<TemplateDataStoreVO> sc = directDownloadTemplateSeach.create();
+        sc.setParameters("template_id", templateId);
+        sc.setParameters("download_state", Status.BYPASSED);
+        sc.setParameters("state", State.Ready);
+        List<TemplateDataStoreVO> list = search(sc, null);
+        if (CollectionUtils.isEmpty(list) || list.size() > 1) {
+            return null;
+        }
+        return list.get(0);
+    }
+
+    @Override
+    public boolean isTemplateMarkedForDirectDownload(long templateId) {
+        TemplateDataStoreVO templateRef = getReadyBypassedTemplate(templateId);
+        return templateRef != null;
+    }
 }

framework/direct-download/pom.xml

@@ -0,0 +1,31 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>cloud-framework-direct-download</artifactId>
+    <name>Apache CloudStack Framework - Direct Download to Primary Storage</name>
+    <parent>
+        <artifactId>cloudstack-framework</artifactId>
+        <groupId>org.apache.cloudstack</groupId>
+        <version>4.11.0.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+</project>

framework/direct-download/src/org/apache/cloudstack/framework/agent/direct/download/DirectDownloadService.java

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.framework.agent.direct.download;
+
+public interface DirectDownloadService {
+
+    /**
+     * Download template/ISO into poolId bypassing secondary storage. Download performed by hostId
+     */
+    void downloadTemplate(long templateId, long poolId, long hostId);
+
+    /**
+     * Upload client certificate to each running host
+     */
+    boolean uploadCertificateToHosts(String certificateCer, String certificateName);
+}

framework/pom.xml

@@ -56,5 +56,6 @@
     <module>spring/lifecycle</module>
     <module>spring/module</module>
 	<module>security</module>
+    <module>direct-download</module>
   </modules>
 </project>

packaging/centos63/cloud.spec

@@ -130,6 +130,8 @@ Requires: perl
 Requires: libvirt-python
 Requires: qemu-img
 Requires: qemu-kvm
+Requires: epel-release
+Requires: aria2
 Provides: cloud-agent
 Obsoletes: cloud-agent < 4.1.0
 Obsoletes: cloud-agent-libs < 4.1.0

packaging/centos7/cloud.spec

@@ -111,6 +111,8 @@ Requires: perl
 Requires: libvirt-python
 Requires: qemu-img
 Requires: qemu-kvm
+Requires: epel-release
+Requires: aria2
 Provides: cloud-agent
 Group: System Environment/Libraries
 %description agent

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -2200,8 +2200,18 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
             KVMStoragePool pool = null;
             final DataTO data = volume.getData();
             if (volume.getType() == Volume.Type.ISO && data.getPath() != null) {
-                final NfsTO nfsStore = (NfsTO)data.getDataStore();
-                final String volPath = nfsStore.getUrl() + File.separator + data.getPath();
+                DataStoreTO dataStore = data.getDataStore();
+                String dataStoreUrl = null;
+                if (dataStore instanceof NfsTO) {
+                    NfsTO nfsStore = (NfsTO)data.getDataStore();
+                    dataStoreUrl = nfsStore.getUrl();
+                } else if (dataStore instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO) dataStore).getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
+                    //In order to support directly downloaded ISOs
+                    String psHost = ((PrimaryDataStoreTO) dataStore).getHost();
+                    String psPath = ((PrimaryDataStoreTO) dataStore).getPath();
+                    dataStoreUrl = "nfs://" + psHost + File.separator + psPath;
+                }
+                final String volPath = dataStoreUrl + File.separator + data.getPath();
                 final int index = volPath.lastIndexOf("/");
                 final String volDir = volPath.substring(0, index);
                 final String volName = volPath.substring(index + 1);

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java

@@ -0,0 +1,50 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+import com.cloud.utils.UriUtils;
+import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer;
+import org.apache.cloudstack.agent.directdownload.CheckUrlCommand;
+import org.apache.log4j.Logger;
+
+@ResourceWrapper(handles =  CheckUrlCommand.class)
+public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, CheckUrlAnswer, LibvirtComputingResource> {
+
+    private static final Logger s_logger = Logger.getLogger(LibvirtCheckUrlCommand.class);
+
+    @Override
+    public CheckUrlAnswer execute(CheckUrlCommand cmd, LibvirtComputingResource serverResource) {
+        final String url = cmd.getUrl();
+        s_logger.info("Checking URL: " + url);
+        boolean checkResult = true;
+        Long remoteSize = null;
+        try {
+            UriUtils.checkUrlExistence(url);
+            remoteSize = UriUtils.getRemoteSize(url);
+        }
+        catch (IllegalArgumentException e) {
+            s_logger.warn(e.getMessage());
+            checkResult = false;
+        }
+        return new CheckUrlAnswer(checkResult, remoteSize);
+    }
+}

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -36,6 +36,18 @@ import java.util.UUID;
 
 import javax.naming.ConfigurationException;
 
+import com.cloud.agent.direct.download.DirectTemplateDownloader;
+import com.cloud.agent.direct.download.DirectTemplateDownloader.DirectTemplateInformation;
+import com.cloud.agent.direct.download.HttpDirectTemplateDownloader;
+import com.cloud.agent.direct.download.MetalinkDirectTemplateDownloader;
+import com.cloud.agent.direct.download.NfsDirectTemplateDownloader;
+import com.cloud.agent.direct.download.HttpsDirectTemplateDownloader;
+import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
 import org.apache.cloudstack.storage.command.AttachAnswer;
 import org.apache.cloudstack.storage.command.AttachCommand;
 import org.apache.cloudstack.storage.command.CopyCmdAnswer;
@@ -894,13 +906,21 @@ public class KVMStorageProcessor implements StorageProcessor {
         final DiskTO disk = cmd.getDisk();
         final TemplateObjectTO isoTO = (TemplateObjectTO)disk.getData();
         final DataStoreTO store = isoTO.getDataStore();
-        if (!(store instanceof NfsTO)) {
+        String dataStoreUrl = null;
+        if (store instanceof NfsTO) {
+            NfsTO nfsStore = (NfsTO)store;
+            dataStoreUrl = nfsStore.getUrl();
+        } else if (store instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO) store).getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
+            //In order to support directly downloaded ISOs
+            String psHost = ((PrimaryDataStoreTO) store).getHost();
+            String psPath = ((PrimaryDataStoreTO) store).getPath();
+            dataStoreUrl = "nfs://" + psHost + File.separator + psPath;
+        } else {
             return new AttachAnswer("unsupported protocol");
         }
-        final NfsTO nfsStore = (NfsTO)store;
         try {
             final Connect conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName());
-            attachOrDetachISO(conn, cmd.getVmName(), nfsStore.getUrl() + File.separator + isoTO.getPath(), true);
+            attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), true);
         } catch (final LibvirtException e) {
             return new Answer(cmd, false, e.toString());
         } catch (final URISyntaxException e) {
@@ -1387,4 +1407,37 @@ public class KVMStorageProcessor implements StorageProcessor {
         return new Answer(cmd, false, "not implememented yet");
     }
 
+    @Override
+    public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
+        final PrimaryDataStoreTO pool = cmd.getDestPool();
+        if (!pool.getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
+            return new DirectDownloadAnswer(false, "Unsupported pool type " + pool.getPoolType().toString());
+        }
+        KVMStoragePool destPool = storagePoolMgr.getStoragePool(pool.getPoolType(), pool.getUuid());
+        DirectTemplateDownloader downloader;
+
+        if (cmd instanceof HttpDirectDownloadCommand) {
+            downloader = new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), ((HttpDirectDownloadCommand) cmd).getHeaders());
+        } else if (cmd instanceof HttpsDirectDownloadCommand) {
+            downloader = new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum());
+        } else if (cmd instanceof NfsDirectDownloadCommand) {
+            downloader = new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum());
+        } else if (cmd instanceof MetalinkDirectDownloadCommand) {
+            downloader = new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum());
+        } else {
+            return new DirectDownloadAnswer(false, "Unsupported protocol, please provide HTTP(S), NFS or a metalink");
+        }
+
+        if (!downloader.downloadTemplate()) {
+            return new DirectDownloadAnswer(false, "Could not download template " + cmd.getTemplateId() + " on " + destPool.getLocalPath());
+        }
+        if (!downloader.validateChecksum()) {
+            return new DirectDownloadAnswer(false, "Checksum validation failed for template " + cmd.getTemplateId());
+        }
+        if (!downloader.extractAndInstallDownloadedTemplate()) {
+            return new DirectDownloadAnswer(false, "Template downloaded but there was an error on installation");
+        }
+        DirectTemplateInformation info = downloader.getTemplateInformation();
+        return new DirectDownloadAnswer(true, info.getSize(), info.getInstallPath());
+    }
 }

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java

@@ -1341,4 +1341,5 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
     private void deleteDirVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtException {
         Script.runSimpleBashScript("rm -r --interactive=never " + vol.getPath());
     }
+
 }

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java

@@ -71,5 +71,4 @@ public interface StorageAdaptor {
     public boolean deleteStoragePool(KVMStoragePool pool);
 
     public boolean createFolder(String uuid, String path);
-
 }

plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java

@@ -18,14 +18,34 @@
  */
 package com.cloud.hypervisor.kvm.storage;
 
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
 import javax.naming.ConfigurationException;
 
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.mockito.Spy;
 
 public class KVMStorageProcessorTest {
+
+    @Mock
+    KVMStoragePoolManager storagePoolManager;
+    @Mock
+    LibvirtComputingResource resource;
+
+    private static final Long TEMPLATE_ID = 202l;
+    private static final String EXPECTED_DIRECT_DOWNLOAD_DIR = "template/2/202";
+
+    @Spy
+    @InjectMocks
+    private KVMStorageProcessor storageProcessor;
+
     @Before
     public void setUp() throws ConfigurationException {
+        MockitoAnnotations.initMocks(this);
+        storageProcessor = new KVMStorageProcessor(storagePoolManager, resource);
     }
 
     @Test

plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java

@@ -21,6 +21,7 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.UUID;
 
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
 import org.apache.cloudstack.storage.command.AttachAnswer;
 import org.apache.cloudstack.storage.command.AttachCommand;
 import org.apache.cloudstack.storage.command.CopyCmdAnswer;
@@ -820,6 +821,11 @@ public class Ovm3StorageProcessor implements StorageProcessor {
         return new ResignatureAnswer("Not implemented");
     }
 
+    @Override
+    public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
+        return null;
+    }
+
     /**
      * Attach disks
      * @param cmd

plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorStorageProcessor.java

@@ -22,6 +22,7 @@ package com.cloud.resource;
 import java.io.File;
 import java.util.UUID;
 
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.storage.command.AttachAnswer;
@@ -75,6 +76,11 @@ public class SimulatorStorageProcessor implements StorageProcessor {
         return new ResignatureAnswer();
     }
 
+    @Override
+    public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
+        return null;
+    }
+
     @Override
     public Answer copyTemplateToPrimaryStorage(CopyCommand cmd) {
         TemplateObjectTO template = new TemplateObjectTO();

plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java

@@ -34,6 +34,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.base.Strings;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 
@@ -2425,4 +2426,9 @@ public class VmwareStorageProcessor implements StorageProcessor {
         this._fullCloneFlag = value;
         s_logger.debug("VmwareProcessor instance - create full clone = " + (value ? "TRUE" : "FALSE"));
     }
+
+    @Override
+    public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
+        return null;
+    }
 }

plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java

@@ -44,6 +44,7 @@ import com.xensource.xenapi.Types.XenAPIException;
 import com.xensource.xenapi.VBD;
 import com.xensource.xenapi.VDI;
 import com.xensource.xenapi.VM;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
 import org.apache.cloudstack.storage.command.AttachAnswer;
 import org.apache.cloudstack.storage.command.AttachCommand;
 import org.apache.cloudstack.storage.command.CopyCmdAnswer;
@@ -195,6 +196,12 @@ public class XenServerStorageProcessor implements StorageProcessor {
         }
     }
 
+    @Override
+    public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
+        //Not implemented for Xen
+        return null;
+    }
+
     @Override
     public AttachAnswer attachIso(final AttachCommand cmd) {
         final DiskTO disk = cmd.getDisk();

scripts/vm/hypervisor/kvm/setup_agent.sh

@@ -224,5 +224,17 @@ then
     setenforce 0
 fi
 
+which aria2c
+if [ $? -gt 0 ]
+then
+    yum install epel-release -y
+    yum install aria2 -y
+    if [ $? -gt 0 ]
+    then
+        printf "failed to install aria2"
+        exit 1
+    fi
+fi
+
 cloudstack-setup-agent --host=$host --zone=$zone --pod=$pod --cluster=$cluster --guid=$guid $paramters -a > /dev/null
 #cloud_consoleP_setup $host $zone $pod

server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml

@@ -295,4 +295,5 @@
 
     <bean id="annotationService" class="org.apache.cloudstack.annotation.AnnotationManagerImpl" />
 
+    <bean id="directDownloadManager" class="org.apache.cloudstack.direct.download.DirectDownloadManagerImpl" />
 </beans>

server/src/com/cloud/api/query/dao/TemplateJoinDaoImpl.java

@@ -105,6 +105,8 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation<Templa
                 } else {
                     templateStatus = template.getDownloadPercent() + "% Downloaded";
                 }
+            } else if (template.getDownloadState() == Status.BYPASSED) {
+                templateStatus = "Bypassed Secondary Storage";
             }else if (template.getErrorString()==null){
                 templateStatus = template.getTemplateState().toString();
             }else {
@@ -197,6 +199,8 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation<Templa
             addTagInformation(template, templateResponse);
         }
 
+        templateResponse.setDirectDownload(template.isDirectDownload());
+
         templateResponse.setObjectName("template");
         return templateResponse;
     }
@@ -320,6 +324,8 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation<Templa
                     } else {
                         isoStatus = iso.getDownloadPercent() + "% Downloaded";
                     }
+                } else if (iso.getDownloadState() == Status.BYPASSED) {
+                    isoStatus = "Bypassed Secondary Storage";
                 } else {
                     isoStatus = iso.getErrorString();
                 }
@@ -348,6 +354,8 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation<Templa
             }
         }
 
+        isoResponse.setDirectDownload(iso.isDirectDownload());
+
         isoResponse.setObjectName("iso");
         return isoResponse;
 

server/src/com/cloud/api/query/vo/TemplateJoinVO.java

@@ -222,6 +222,9 @@ public class TemplateJoinVO extends BaseViewWithTagInformationVO implements Cont
     @Column(name = "temp_zone_pair")
     private String tempZonePair; // represent a distinct (templateId, data_center_id) pair
 
+    @Column(name = "direct_download")
+    private boolean directDownload;
+
     public TemplateJoinVO() {
     }
 
@@ -477,4 +480,7 @@ public class TemplateJoinVO extends BaseViewWithTagInformationVO implements Cont
         this.accountId = accountId;
     }
 
+    public boolean isDirectDownload() {
+        return directDownload;
+    }
 }

server/src/com/cloud/resource/ResourceManagerImpl.java

@@ -25,6 +25,7 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
@@ -2797,6 +2798,23 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
         return null;
     }
 
+    @Override
+    public HostVO findOneRandomRunningHostByHypervisor(HypervisorType type) {
+        final QueryBuilder<HostVO> sc = QueryBuilder.create(HostVO.class);
+        sc.and(sc.entity().getHypervisorType(), Op.EQ, type);
+        sc.and(sc.entity().getType(),Op.EQ, Type.Routing);
+        sc.and(sc.entity().getStatus(), Op.EQ, Status.Up);
+        sc.and(sc.entity().getResourceState(), Op.EQ, ResourceState.Enabled);
+        sc.and(sc.entity().getRemoved(), Op.NULL);
+        List<HostVO> hosts = sc.list();
+        if (CollectionUtils.isEmpty(hosts)) {
+            return null;
+        } else {
+            Collections.shuffle(hosts, new Random(System.currentTimeMillis()));
+            return hosts.get(0);
+        }
+    }
+
     @Override
     @DB
     @ActionEvent(eventType = EventTypes.EVENT_HOST_RESERVATION_RELEASE, eventDescription = "releasing host reservation", async = true)

server/src/com/cloud/server/ManagementServerImpl.java

@@ -65,6 +65,7 @@ import org.apache.cloudstack.api.command.admin.config.ListDeploymentPlannersCmd;
 import org.apache.cloudstack.api.command.admin.config.ListHypervisorCapabilitiesCmd;
 import org.apache.cloudstack.api.command.admin.config.UpdateCfgCmd;
 import org.apache.cloudstack.api.command.admin.config.UpdateHypervisorCapabilitiesCmd;
+import org.apache.cloudstack.api.command.admin.direct.download.UploadTemplateDirectDownloadCertificate;
 import org.apache.cloudstack.api.command.admin.domain.CreateDomainCmd;
 import org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd;
 import org.apache.cloudstack.api.command.admin.domain.ListDomainChildrenCmd;
@@ -3030,6 +3031,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         cmdList.add(ReleasePodIpCmdByAdmin.class);
         cmdList.add(CreateManagementNetworkIpRangeCmd.class);
         cmdList.add(DeleteManagementNetworkIpRangeCmd.class);
+        cmdList.add(UploadTemplateDirectDownloadCertificate.class);
 
         // Out-of-band management APIs for admins
         cmdList.add(EnableOutOfBandManagementForHostCmd.class);
@@ -3042,7 +3044,6 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         cmdList.add(IssueOutOfBandManagementPowerActionCmd.class);
         cmdList.add(ChangeOutOfBandManagementPasswordCmd.class);
         cmdList.add(GetUserKeysCmd.class);
-
         return cmdList;
     }
 

server/src/com/cloud/storage/TemplateProfile.java

@@ -51,6 +51,8 @@ public class TemplateProfile {
     Map details;
     Boolean isDynamicallyScalable;
     TemplateType templateType;
+    Boolean directDownload;
+    Long size;
 
     public TemplateProfile(Long templateId, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHvm, String url,
                            Boolean isPublic, Boolean featured, Boolean isExtractable, ImageFormat format, Long guestOsId, List<Long> zoneIdList, HypervisorType hypervisorType,
@@ -93,7 +95,7 @@ public class TemplateProfile {
             Boolean isPublic, Boolean featured, Boolean isExtractable, ImageFormat format, Long guestOsId, List<Long> zoneId,
 
             HypervisorType hypervisorType, String accountName, Long domainId, Long accountId, String chksum, Boolean bootable, String templateTag, Map details,
-            Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType) {
+            Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType, Boolean directDownload) {
         this(templateId,
             userId,
             name,
@@ -119,6 +121,7 @@ public class TemplateProfile {
         this.templateTag = templateTag;
         this.isDynamicallyScalable = isDynamicallyScalable;
         this.templateType = templateType;
+        this.directDownload = directDownload;
     }
 
     public Long getTemplateId() {
@@ -316,4 +319,16 @@ public class TemplateProfile {
     public void setTemplateType(TemplateType templateType) {
         this.templateType = templateType;
     }
+
+    public boolean isDirectDownload() {
+        return directDownload == null ? false : directDownload;
+    }
+
+    public Long getSize() {
+        return size;
+    }
+
+    public void setSize(Long size) {
+        this.size = size;
+    }
 }

server/src/com/cloud/template/HypervisorTemplateAdapter.java

@@ -16,6 +16,10 @@
 // under the License.
 package com.cloud.template;
 
+import com.cloud.agent.api.Answer;
+import com.cloud.host.HostVO;
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.resource.ResourceManager;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -29,6 +33,8 @@ import com.cloud.configuration.Config;
 import com.cloud.utils.db.Transaction;
 import com.cloud.utils.db.TransactionCallback;
 import com.cloud.utils.db.TransactionStatus;
+import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer;
+import org.apache.cloudstack.agent.directdownload.CheckUrlCommand;
 import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@@ -68,6 +74,9 @@ import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.org.Grouping;
 import com.cloud.server.StatsCollector;
+import com.cloud.template.VirtualMachineTemplate.State;
+import com.cloud.user.Account;
+import com.cloud.utils.Pair;
 import com.cloud.storage.ScopeType;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Storage.TemplateType;
@@ -77,9 +86,6 @@ import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.VMTemplateZoneVO;
 import com.cloud.storage.dao.VMTemplateZoneDao;
 import com.cloud.storage.download.DownloadMonitor;
-import com.cloud.template.VirtualMachineTemplate.State;
-import com.cloud.user.Account;
-import com.cloud.utils.Pair;
 import com.cloud.utils.UriUtils;
 import com.cloud.utils.db.DB;
 import com.cloud.utils.db.EntityManager;
@@ -113,17 +119,42 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
     DataCenterDao _dcDao;
     @Inject
     MessageBus _messageBus;
+    @Inject
+    ResourceManager resourceManager;
 
     @Override
     public String getName() {
         return TemplateAdapterType.Hypervisor.getName();
     }
 
+    /**
+     * Validate on random running KVM host that URL is reachable
+     * @param url url
+     */
+    private Long performDirectDownloadUrlValidation(final String url) {
+        HostVO host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM);
+        if (host == null) {
+            throw new CloudRuntimeException("Couldn't find a host to validate URL " + url);
+        }
+        CheckUrlCommand cmd = new CheckUrlCommand(url);
+        s_logger.debug("Performing URL " + url + " validation on host " + host.getId());
+        Answer answer = _agentMgr.easySend(host.getId(), cmd);
+        if (answer == null || !answer.getResult()) {
+            throw new CloudRuntimeException("URL: " + url + " validation failed on host id " + host.getId());
+        }
+        CheckUrlAnswer ans = (CheckUrlAnswer) answer;
+        return ans.getTemplateSize();
+    }
+
     @Override
     public TemplateProfile prepare(RegisterIsoCmd cmd) throws ResourceAllocationException {
         TemplateProfile profile = super.prepare(cmd);
         String url = profile.getUrl();
         UriUtils.validateUrl(ImageFormat.ISO.getFileExtension(), url);
+        if (cmd.isDirectDownload()) {
+            Long templateSize = performDirectDownloadUrlValidation(url);
+            profile.setSize(templateSize);
+        }
         profile.setUrl(url);
         // Check that the resource limit for secondary storage won't be exceeded
         _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
@@ -135,6 +166,10 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
         TemplateProfile profile = super.prepare(cmd);
         String url = profile.getUrl();
         UriUtils.validateUrl(cmd.getFormat(), url);
+        if (cmd.isDirectDownload()) {
+            Long templateSize = performDirectDownloadUrlValidation(url);
+            profile.setSize(templateSize);
+        }
         profile.setUrl(url);
         // Check that the resource limit for secondary storage won't be exceeded
         _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
@@ -150,6 +185,14 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
         return profile;
     }
 
+    /**
+     * Persist template marking it for direct download to Primary Storage, skipping Secondary Storage
+     */
+    private void persistDirectDownloadTemplate(long templateId, Long size) {
+        TemplateDataStoreVO directDownloadEntry = templateDataStoreDao.createTemplateDirectDownloadEntry(templateId, size);
+        templateDataStoreDao.persist(directDownloadEntry);
+    }
+
     @Override
     public VMTemplateVO create(TemplateProfile profile) {
         // persist entry in vm_template, vm_template_details and template_zone_ref tables, not that entry at template_store_ref is not created here, and created in createTemplateAsync.
@@ -159,6 +202,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
             throw new CloudRuntimeException("Unable to persist the template " + profile.getTemplate());
         }
 
+        if (!profile.isDirectDownload()) {
             List<Long> zones = profile.getZoneIdList();
 
             //zones is null when this template is to be registered to all zones
@@ -170,6 +214,11 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
                     createTemplateWithinZone(zId, profile, template);
                 }
             }
+        } else {
+            //KVM direct download templates bypassing Secondary Storage
+            persistDirectDownloadTemplate(template.getId(), profile.getSize());
+        }
+
         _resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template);
         return template;
     }

server/src/com/cloud/template/TemplateAdapter.java

@@ -71,11 +71,11 @@ public interface TemplateAdapter extends Adapter {
 
     public TemplateProfile prepare(boolean isIso, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url,
         Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List<Long> zoneId, HypervisorType hypervisorType, String accountName,
-        Long domainId, String chksum, Boolean bootable, Map details) throws ResourceAllocationException;
+        Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload) throws ResourceAllocationException;
 
     public TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url,
         Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List<Long> zoneId, HypervisorType hypervisorType, String chksum,
         Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshKeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable,
-        TemplateType templateType) throws ResourceAllocationException;
+        TemplateType templateType, boolean directDownload) throws ResourceAllocationException;
 
 }

server/src/com/cloud/template/TemplateAdapterBase.java

@@ -112,6 +112,8 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
     ConfigurationServer _configServer;
     @Inject
     ProjectManager _projectMgr;
+    @Inject
+    private TemplateDataStoreDao templateDataStoreDao;
 
     @Override
     public boolean stop() {
@@ -121,16 +123,16 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
     @Override
     public TemplateProfile prepare(boolean isIso, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url,
         Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List<Long> zoneId, HypervisorType hypervisorType, String accountName,
-        Long domainId, String chksum, Boolean bootable, Map details) throws ResourceAllocationException {
+        Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload) throws ResourceAllocationException {
         return prepare(isIso, userId, name, displayText, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, format, guestOSId, zoneId,
-            hypervisorType, chksum, bootable, null, null, details, false, null, false, TemplateType.USER);
+            hypervisorType, chksum, bootable, null, null, details, false, null, false, TemplateType.USER, directDownload);
     }
 
     @Override
     public TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url,
         Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List<Long> zoneIdList, HypervisorType hypervisorType, String chksum,
         Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshkeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable,
-        TemplateType templateType) throws ResourceAllocationException {
+        TemplateType templateType, boolean directDownload) throws ResourceAllocationException {
         //Long accountId = null;
         // parameters verification
 
@@ -249,7 +251,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
         CallContext.current().setEventDetails("Id: " + id + " name: " + name);
         return new TemplateProfile(id, userId, name, displayText, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, imgfmt, guestOSId, zoneIdList,
             hypervisorType, templateOwner.getAccountName(), templateOwner.getDomainId(), templateOwner.getAccountId(), chksum, bootable, templateTag, details,
-            sshkeyEnabled, null, isDynamicallyScalable, templateType);
+            sshkeyEnabled, null, isDynamicallyScalable, templateType, directDownload);
 
     }
 
@@ -277,7 +279,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
 
         return prepare(false, CallContext.current().getCallingUserId(), cmd.getTemplateName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(), cmd.getRequiresHvm(),
                 cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneId, hypervisorType, cmd.getChecksum(), true,
-                cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER);
+                cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, cmd.isDirectDownload());
 
     }
 
@@ -308,7 +310,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
         return prepare(false, CallContext.current().getCallingUserId(), cmd.getName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(),
                        cmd.getRequiresHvm(), null, cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneList,
                        hypervisorType, cmd.getChecksum(), true, cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null,
-                       cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER);
+                       cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, false);
 
     }
 
@@ -330,7 +332,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
 
         return prepare(true, CallContext.current().getCallingUserId(), cmd.getIsoName(), cmd.getDisplayText(), 64, false, true, cmd.getUrl(), cmd.isPublic(),
             cmd.isFeatured(), cmd.isExtractable(), ImageFormat.ISO.toString(), cmd.getOsTypeId(), zoneList, HypervisorType.None, cmd.getChecksum(), cmd.isBootable(), null,
-            owner, null, false, cmd.getImageStoreUuid(), cmd.isDynamicallyScalable(), TemplateType.USER);
+            owner, null, false, cmd.getImageStoreUuid(), cmd.isDynamicallyScalable(), TemplateType.USER, cmd.isDirectDownload());
     }
 
     protected VMTemplateVO persistTemplate(TemplateProfile profile, VirtualMachineTemplate.State initialState) {
@@ -339,9 +341,13 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat
             new VMTemplateVO(profile.getTemplateId(), profile.getName(), profile.getFormat(), profile.getIsPublic(), profile.getFeatured(), profile.getIsExtractable(),
                 profile.getTemplateType(), profile.getUrl(), profile.getRequiresHVM(), profile.getBits(), profile.getAccountId(), profile.getCheckSum(),
                 profile.getDisplayText(), profile.getPasswordEnabled(), profile.getGuestOsId(), profile.getBootable(), profile.getHypervisorType(),
-                profile.getTemplateTag(), profile.getDetails(), profile.getSshKeyEnabled(), profile.IsDynamicallyScalable());
+                profile.getTemplateTag(), profile.getDetails(), profile.getSshKeyEnabled(), profile.IsDynamicallyScalable(), profile.isDirectDownload());
         template.setState(initialState);
 
+        if (profile.isDirectDownload()) {
+            template.setSize(profile.getSize());
+        }
+
         if (zoneIdList == null) {
             List<DataCenterVO> dcs = _dcDao.listAll();
 

server/src/com/cloud/template/TemplateManagerImpl.java

@@ -32,6 +32,7 @@ import java.util.concurrent.Executors;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.deploy.DeployDestination;
 import com.cloud.storage.ImageStoreUploadMonitorImpl;
 import com.cloud.utils.StringUtils;
 import com.cloud.utils.EncryptionUtil;
@@ -551,10 +552,18 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
     }
 
     @Override
-    public void prepareIsoForVmProfile(VirtualMachineProfile profile) {
+    public void prepareIsoForVmProfile(VirtualMachineProfile profile, DeployDestination dest) {
         UserVmVO vm = _userVmDao.findById(profile.getId());
         if (vm.getIsoId() != null) {
-            TemplateInfo template = prepareIso(vm.getIsoId(), vm.getDataCenterId());
+            Map<Volume, StoragePool> storageForDisks = dest.getStorageForDisks();
+            Long poolId = null;
+            for (StoragePool storagePool : storageForDisks.values()) {
+                if (poolId != null && storagePool.getId() != poolId) {
+                    throw new CloudRuntimeException("Cannot determine where to download iso");
+                }
+                poolId = storagePool.getId();
+            }
+            TemplateInfo template = prepareIso(vm.getIsoId(), vm.getDataCenterId(), dest.getHost().getId(), poolId);
             if (template == null){
                 s_logger.error("Failed to prepare ISO on secondary or cache storage");
                 throw new CloudRuntimeException("Failed to prepare ISO on secondary or cache storage");
@@ -1156,14 +1165,22 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
 
     // for ISO, we need to consider whether to copy to cache storage or not if it is not on NFS, since our hypervisor resource always assumes that they are in NFS
     @Override
-    public TemplateInfo prepareIso(long isoId, long dcId) {
-        TemplateInfo tmplt = _tmplFactory.getTemplate(isoId, DataStoreRole.Image, dcId);
+    public TemplateInfo prepareIso(long isoId, long dcId, Long hostId, Long poolId) {
+        TemplateInfo tmplt;
+        boolean bypassed = false;
+        if (_tmplFactory.isTemplateMarkedForDirectDownload(isoId)) {
+            tmplt = _tmplFactory.getReadyBypassedTemplateOnPrimaryStore(isoId, poolId, hostId);
+            bypassed = true;
+        } else {
+            tmplt = _tmplFactory.getTemplate(isoId, DataStoreRole.Image, dcId);
+        }
+
         if (tmplt == null || tmplt.getFormat() != ImageFormat.ISO) {
             s_logger.warn("ISO: " + isoId + " does not exist in vm_template table");
             return null;
         }
 
-        if (tmplt.getDataStore() != null && !(tmplt.getDataStore().getTO() instanceof NfsTO)) {
+        if (!bypassed && tmplt.getDataStore() != null && !(tmplt.getDataStore().getTO() instanceof NfsTO)) {
             // if it is s3, need to download into cache storage first
             Scope destScope = new ZoneScope(dcId);
             TemplateInfo cacheData = (TemplateInfo)cacheMgr.createCacheObject(tmplt, destScope);
@@ -1187,7 +1204,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
         }
 
         // prepare ISO ready to mount on hypervisor resource level
-        TemplateInfo tmplt = prepareIso(isoId, vm.getDataCenterId());
+        TemplateInfo tmplt = prepareIso(isoId, vm.getDataCenterId(), vm.getHostId(), null);
 
         String vmName = vm.getInstanceName();
 
@@ -1804,7 +1821,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
         }
         privateTemplate = new VMTemplateVO(nextTemplateId, name, ImageFormat.RAW, isPublic, featured, isExtractable,
                 TemplateType.USER, null, requiresHvmValue, bitsValue, templateOwner.getId(), null, description,
-                passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), false, isDynamicScalingEnabled);
+                passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), false, isDynamicScalingEnabled, false);
 
         if (sourceTemplateId != null) {
             if (s_logger.isDebugEnabled()) {
@@ -1912,6 +1929,10 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
 
     @Override
     public Long getTemplateSize(long templateId, long zoneId) {
+        if (_tmplStoreDao.isTemplateMarkedForDirectDownload(templateId)) {
+            // check if template is marked for direct download
+            return _tmplStoreDao.getReadyBypassedTemplate(templateId).getSize();
+        }
         TemplateDataStoreVO templateStoreRef = _tmplStoreDao.findByTemplateZoneDownloadStatus(templateId, zoneId, VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
         if (templateStoreRef == null) {
             // check if it is ready on image cache stores

server/src/com/cloud/vm/UserVmManagerImpl.java

@@ -4055,7 +4055,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
 
 
 
-        _templateMgr.prepareIsoForVmProfile(profile);
+        _templateMgr.prepareIsoForVmProfile(profile, dest);
         return true;
     }
 

server/src/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java

@@ -0,0 +1,245 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.direct.download;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Answer;
+import com.cloud.host.Host;
+import com.cloud.host.HostVO;
+import com.cloud.host.Status;
+import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.VMTemplateStoragePoolVO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.storage.dao.VMTemplatePoolDao;
+import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.inject.Inject;
+
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand.DownloadProtocol;
+import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
+import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+import org.apache.commons.collections.MapUtils;
+import org.apache.log4j.Logger;
+
+public class DirectDownloadManagerImpl extends ManagerBase implements DirectDownloadManager {
+
+    private static final Logger s_logger = Logger.getLogger(DirectDownloadManagerImpl.class);
+    protected static final String httpHeaderDetailKey = "HTTP_HEADER";
+    protected static final String BEGIN_CERT = "-----BEGIN CERTIFICATE-----";
+    protected static final String END_CERT = "-----END CERTIFICATE-----";
+    protected final static String LINE_SEPARATOR = "\n";
+
+    @Inject
+    VMTemplateDao vmTemplateDao;
+    @Inject
+    PrimaryDataStoreDao primaryDataStoreDao;
+    @Inject
+    HostDao hostDao;
+    @Inject
+    AgentManager agentManager;
+    @Inject
+    VMTemplatePoolDao vmTemplatePoolDao;
+    @Inject
+    DataStoreManager dataStoreManager;
+
+    @Override
+    public List<Class<?>> getCommands() {
+        final List<Class<?>> cmdList = new ArrayList<Class<?>>();
+        return cmdList;
+    }
+
+    /**
+     * Return protocol to use from provided URL
+     * @param url
+     * @return
+     */
+    public static DownloadProtocol getProtocolFromUrl(String url) {
+        URI uri;
+        try {
+            uri = new URI(url);
+        } catch (URISyntaxException e) {
+            throw new CloudRuntimeException("URI is incorrect: " + url);
+        }
+        if ((uri != null) && (uri.getScheme() != null)) {
+            if (uri.getPath().endsWith(".metalink")) {
+                return DownloadProtocol.METALINK;
+            } else if (uri.getScheme().equalsIgnoreCase("http")) {
+                return DownloadProtocol.HTTP;
+            } else if (uri.getScheme().equalsIgnoreCase("https")) {
+                return DownloadProtocol.HTTPS;
+            } else if (uri.getScheme().equalsIgnoreCase("nfs")) {
+                return DownloadProtocol.NFS;
+            } else {
+                throw new CloudRuntimeException("Scheme is not supported " + url);
+            }
+        } else {
+            throw new CloudRuntimeException("URI is incorrect: " + url);
+        }
+    }
+
+    /**
+     * Return HTTP headers from template details
+     * @param templateDetails
+     * @return
+     */
+    protected Map<String, String> getHeadersFromDetails(Map<String, String> templateDetails) {
+        if (MapUtils.isEmpty(templateDetails)) {
+            return new HashMap<>();
+        }
+        Map<String, String> headers = new HashMap<>();
+        for (String key : templateDetails.keySet()) {
+            if (key.startsWith(httpHeaderDetailKey)) {
+                String header = key.split(":")[1];
+                String value = templateDetails.get(key);
+                headers.put(header, value);
+            }
+        }
+        return headers;
+    }
+
+    @Override
+    public void downloadTemplate(long templateId, long poolId, long hostId) {
+        VMTemplateVO template = vmTemplateDao.findById(templateId);
+        StoragePoolVO pool = primaryDataStoreDao.findById(poolId);
+        HostVO host = hostDao.findById(hostId);
+        if (pool == null) {
+            throw new CloudRuntimeException("Storage pool " + poolId + " could not be found");
+        }
+        if (template == null) {
+            throw new CloudRuntimeException("Template " + templateId + " could not be found");
+        }
+        if (host == null) {
+            throw new CloudRuntimeException("Host " + hostId + " could not be found");
+        }
+        if (!template.isDirectDownload()) {
+            throw new CloudRuntimeException("Template " + templateId + " is not marked for direct download");
+        }
+        Map<String, String> details = template.getDetails();
+        String url = template.getUrl();
+        String checksum = template.getChecksum();
+        Map<String, String> headers = getHeadersFromDetails(details);
+        DataStore store = dataStoreManager.getDataStore(poolId, DataStoreRole.Primary);
+        if (store == null) {
+            throw new CloudRuntimeException("Data store " + poolId + " could not be found");
+        }
+        PrimaryDataStore primaryDataStore = (PrimaryDataStore) store;
+        PrimaryDataStoreTO to = (PrimaryDataStoreTO) primaryDataStore.getTO();
+
+        DownloadProtocol protocol = getProtocolFromUrl(url);
+        DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers);
+        Answer answer = agentManager.easySend(hostId, cmd);
+        if (answer == null || !answer.getResult()) {
+            throw new CloudRuntimeException("Host " + hostId + " could not download template " +
+                    templateId + " on pool " + poolId);
+        }
+
+        VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, templateId);
+        if (sPoolRef == null) {
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("Not found (templateId:" + templateId + " poolId: " + poolId + ") in template_spool_ref, persisting it");
+            }
+            DirectDownloadAnswer ans = (DirectDownloadAnswer) answer;
+            sPoolRef = new VMTemplateStoragePoolVO(poolId, templateId);
+            sPoolRef.setDownloadPercent(100);
+            sPoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
+            sPoolRef.setState(ObjectInDataStoreStateMachine.State.Ready);
+            sPoolRef.setTemplateSize(ans.getTemplateSize());
+            sPoolRef.setLocalDownloadPath(ans.getInstallPath());
+            sPoolRef.setInstallPath(ans.getInstallPath());
+            vmTemplatePoolDao.persist(sPoolRef);
+        }
+    }
+
+    /**
+     * Return DirectDownloadCommand according to the protocol
+     * @param protocol
+     * @param url
+     * @param templateId
+     * @param destPool
+     * @return
+     */
+    private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool,
+                                                                       String checksum, Map<String, String> httpHeaders) {
+        if (protocol.equals(DownloadProtocol.HTTP)) {
+            return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
+        } else if (protocol.equals(DownloadProtocol.HTTPS)) {
+            return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
+        } else if (protocol.equals(DownloadProtocol.NFS)) {
+            return new NfsDirectDownloadCommand(url, templateId, destPool, checksum);
+        } else if (protocol.equals(DownloadProtocol.METALINK)) {
+            return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum);
+        } else {
+            return null;
+        }
+    }
+
+    @Override
+    public boolean uploadCertificateToHosts(String certificateCer, String certificateName) {
+        List<HostVO> hosts = hostDao.listAllHostsByType(Host.Type.Routing)
+                .stream()
+                .filter(x -> x.getStatus().equals(Status.Up) &&
+                            x.getHypervisorType().equals(Hypervisor.HypervisorType.KVM))
+                .collect(Collectors.toList());
+        for (HostVO host : hosts) {
+            if (!uploadCertificate(certificateCer, certificateName, host.getId())) {
+                throw new CloudRuntimeException("Uploading certificate " + certificateName + " failed on host: " + host.getId());
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Upload and import certificate to hostId on keystore
+     */
+    protected boolean uploadCertificate(String certificate, String certificateName, long hostId) {
+        String cert = certificate.replaceAll("(.{64})", "$1\n");
+        final String prettified_cert = BEGIN_CERT + LINE_SEPARATOR + cert + LINE_SEPARATOR + END_CERT;
+        SetupDirectDownloadCertificate cmd = new SetupDirectDownloadCertificate(prettified_cert, certificateName);
+        Answer answer = agentManager.easySend(hostId, cmd);
+        if (answer == null || !answer.getResult()) {
+            return false;
+        }
+        s_logger.info("Certificate " + certificateName + " successfully uploaded to host: " + hostId);
+        return true;
+    }
+}

server/test/com/cloud/resource/MockResourceManagerImpl.java

@@ -614,6 +614,12 @@ public class MockResourceManagerImpl extends ManagerBase implements ResourceMana
         return null;
     }
 
+    @Override
+    public HostVO findOneRandomRunningHostByHypervisor(HypervisorType type) {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
     @Override
     public boolean isHostGpuEnabled(final long hostId) {
         // TODO Auto-generated method stub

server/test/org/apache/cloudstack/direct/download/DirectDownloadManagerImplTest.java

@@ -0,0 +1,106 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.direct.download;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.host.dao.HostDao;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand.DownloadProtocol;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@RunWith(MockitoJUnitRunner.class)
+public class DirectDownloadManagerImplTest {
+
+    @Mock
+    HostDao hostDao;
+    @Mock
+    AgentManager agentManager;
+
+    @Spy
+    @InjectMocks
+    private DirectDownloadManagerImpl manager = new DirectDownloadManagerImpl();
+
+    private static final String HTTP_HEADER_1 = "Content-Type";
+    private static final String HTTP_VALUE_1 = "application/x-www-form-urlencoded";
+    private static final String HTTP_HEADER_2 = "Accept-Encoding";
+    private static final String HTTP_VALUE_2 = "gzip";
+
+    @Before
+    public void setUp() {
+    }
+
+    @Test
+    public void testGetProtocolMetalink() {
+        String url = "http://192.168.1.2/tmpl.metalink";
+        DownloadProtocol protocol = DirectDownloadManagerImpl.getProtocolFromUrl(url);
+        Assert.assertEquals(DownloadProtocol.METALINK, protocol);
+    }
+
+    @Test
+    public void testGetProtocolHttp() {
+        String url = "http://192.168.1.2/tmpl.qcow2";
+        DownloadProtocol protocol = DirectDownloadManagerImpl.getProtocolFromUrl(url);
+        Assert.assertEquals(DownloadProtocol.HTTP, protocol);
+    }
+
+    @Test
+    public void testGetProtocolHttps() {
+        String url = "https://192.168.1.2/tmpl.qcow2";
+        DownloadProtocol protocol = DirectDownloadManagerImpl.getProtocolFromUrl(url);
+        Assert.assertEquals(DownloadProtocol.HTTPS, protocol);
+    }
+
+    @Test
+    public void testGetProtocolNfs() {
+        String url = "nfs://192.168.1.2/tmpl.qcow2";
+        DownloadProtocol protocol = DirectDownloadManagerImpl.getProtocolFromUrl(url);
+        Assert.assertEquals(DownloadProtocol.NFS, protocol);
+    }
+
+    @Test
+    public void testGetHeadersFromDetailsHttpHeaders() {
+        Map<String, String> details = new HashMap<>();
+        details.put("Message.ReservedCapacityFreed.Flag", "false");
+        details.put(DirectDownloadManagerImpl.httpHeaderDetailKey + ":" + HTTP_HEADER_1, HTTP_VALUE_1);
+        details.put(DirectDownloadManagerImpl.httpHeaderDetailKey + ":" + HTTP_HEADER_2, HTTP_VALUE_2);
+        Map<String, String> headers = manager.getHeadersFromDetails(details);
+        Assert.assertEquals(2, headers.keySet().size());
+        Assert.assertTrue(headers.containsKey(HTTP_HEADER_1));
+        Assert.assertTrue(headers.containsKey(HTTP_HEADER_2));
+        Assert.assertEquals(HTTP_VALUE_1, headers.get(HTTP_HEADER_1));
+        Assert.assertEquals(HTTP_VALUE_2, headers.get(HTTP_HEADER_2));
+    }
+
+    @Test
+    public void testGetHeadersFromDetailsNonHttpHeaders() {
+        Map<String, String> details = new HashMap<>();
+        details.put("Message.ReservedCapacityFreed.Flag", "false");
+        Map<String, String> headers = manager.getHeadersFromDetails(details);
+        Assert.assertTrue(headers.isEmpty());
+    }
+}

services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java

@@ -38,6 +38,23 @@ import java.util.concurrent.Executors;
 
 import javax.naming.ConfigurationException;
 
+import com.cloud.storage.template.Processor;
+import com.cloud.storage.template.S3TemplateDownloader;
+import com.cloud.storage.template.TemplateDownloader;
+import com.cloud.storage.template.TemplateLocation;
+import com.cloud.storage.template.MetalinkTemplateDownloader;
+import com.cloud.storage.template.HttpTemplateDownloader;
+import com.cloud.storage.template.LocalTemplateDownloader;
+import com.cloud.storage.template.ScpTemplateDownloader;
+import com.cloud.storage.template.TemplateProp;
+import com.cloud.storage.template.OVAProcessor;
+import com.cloud.storage.template.IsoProcessor;
+import com.cloud.storage.template.QCOW2Processor;
+import com.cloud.storage.template.VmdkProcessor;
+import com.cloud.storage.template.RawImageProcessor;
+import com.cloud.storage.template.TARProcessor;
+import com.cloud.storage.template.VhdProcessor;
+import com.cloud.storage.template.TemplateConstants;
 import org.apache.cloudstack.storage.command.DownloadCommand;
 import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
 import org.apache.cloudstack.storage.command.DownloadProgressCommand;
@@ -56,25 +73,9 @@ import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.StorageLayer;
 import com.cloud.storage.VMTemplateHostVO;
 import com.cloud.storage.VMTemplateStorageResourceAssoc;
-import com.cloud.storage.template.HttpTemplateDownloader;
-import com.cloud.storage.template.IsoProcessor;
-import com.cloud.storage.template.LocalTemplateDownloader;
-import com.cloud.storage.template.OVAProcessor;
-import com.cloud.storage.template.Processor;
 import com.cloud.storage.template.Processor.FormatInfo;
-import com.cloud.storage.template.QCOW2Processor;
-import com.cloud.storage.template.RawImageProcessor;
-import com.cloud.storage.template.S3TemplateDownloader;
-import com.cloud.storage.template.ScpTemplateDownloader;
-import com.cloud.storage.template.TARProcessor;
-import com.cloud.storage.template.TemplateConstants;
-import com.cloud.storage.template.TemplateDownloader;
 import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback;
 import com.cloud.storage.template.TemplateDownloader.Status;
-import com.cloud.storage.template.TemplateLocation;
-import com.cloud.storage.template.TemplateProp;
-import com.cloud.storage.template.VhdProcessor;
-import com.cloud.storage.template.VmdkProcessor;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.StringUtils;
 import com.cloud.utils.component.ManagerBase;
@@ -560,7 +561,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
                     }
                     TemplateDownloader td;
                     if ((uri != null) && (uri.getScheme() != null)) {
-                        if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
+                        if (uri.getPath().endsWith(".metalink")) {
+                            td = new MetalinkTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes);
+                        } else if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
                             td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType);
                         } else if (uri.getScheme().equalsIgnoreCase("file")) {
                             td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId));

test/integration/smoke/test_templates.py

@@ -1197,3 +1197,163 @@ class TestCopyDeleteTemplate(cloudstackTestCase):
             "Removed state is not correct."
         )
         return
+
+class TestCreateTemplateWithDirectDownload(cloudstackTestCase):
+
+    @classmethod
+    def setUpClass(self):
+        self.testClient = super(TestCreateTemplateWithDirectDownload, self).getClsTestClient()
+        self.apiclient = self.testClient.getApiClient()
+        self.dbclient = self.testClient.getDbConnection()
+        self._cleanup = []
+        self.templates = []
+
+        self.services = self.testClient.getParsedTestDataConfig()
+        self.unsupportedHypervisor = False
+        self.hypervisor = self.testClient.getHypervisorInfo()
+        if self.hypervisor.lower() not in ['kvm']:
+            # Direct Download is only available for KVM hypervisor
+            self.unsupportedHypervisor = True
+            self.skipTest("Skipping test because unsupported hypervisor\
+                            %s" % self.hypervisor)
+            return
+
+        # Get Zone, Domain and templates
+        self.domain = get_domain(self.apiclient)
+        self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests())
+        self.services["mode"] = self.zone.networktype
+        self.services["virtual_machine"]["zoneid"] = self.zone.id
+        self.account = Account.create(
+            self.apiclient,
+            self.services["account"],
+            admin=True,
+            domainid=self.domain.id
+        )
+        self._cleanup.append(self.account)
+        self.user = Account.create(
+            self.apiclient,
+            self.services["account"],
+            domainid=self.domain.id
+        )
+        self._cleanup.append(self.user)
+        self.service_offering = ServiceOffering.create(
+            self.apiclient,
+            self.services["service_offerings"]["tiny"]
+        )
+        self._cleanup.append(self.service_offering)
+
+        self.template = {
+            "name": "tiny-kvm",
+            "displaytext": "tiny kvm",
+            "format": "QCOW2",
+            "url": "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-kvm.qcow2.bz2",
+            "requireshvm": "True",
+            "ispublic": "True",
+            "isextractable": "True",
+            "checksum": "{SHA-1}" + "6952e58f39b470bd166ace11ffd20bf479bed936",
+            "hypervisor": self.hypervisor,
+            "zoneid": self.zone.id,
+            "ostype": "Other Linux (64-bit)",
+            "directdownload": True
+        }
+
+        return
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            cleanup_resources(cls.apiclient, cls._cleanup)
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    def setUp(self):
+        self.apiclient = self.testClient.getApiClient()
+        self.dbclient = self.testClient.getDbConnection()
+        self.cleanup = []
+
+        if self.unsupportedHypervisor:
+            self.skipTest("Skipping test because unsupported hypervisor\
+                        %s" % self.hypervisor)
+        return
+
+    def tearDown(self):
+        try:
+            #Clean up, terminate the created templates
+            cleanup_resources(self.apiclient, self.cleanup)
+
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    @attr(tags=["advanced", "smoke"], required_hardware="true")
+    def test_01_register_template_direct_download_flag(self):
+        """
+        Register a template using Direct Download flag
+        """
+        self.bypassed_template = Template.register(self.apiclient, self.template, zoneid=self.zone.id, hypervisor=self.hypervisor, randomize_name=False)
+        self._cleanup.append(self.bypassed_template)
+        self.templates.append(self.bypassed_template)
+
+        tmplt = self.dbclient.execute("select id, direct_download from vm_template where uuid='%s';" % self.bypassed_template.id)
+        det = tmplt[0]
+
+        self.assertEqual(det[1],
+                         1,
+                         "Template should be marked as Direct Download"
+                         )
+        qresultset = self.dbclient.execute("select download_state, state from template_store_ref where template_id='%s' and store_id is NULL;"
+                                           % det[0])
+        ref = qresultset[0]
+        self.assertEqual(ref[0],
+                         "BYPASSED",
+                         "Template store ref download state should be marked as BYPASSED"
+                         )
+        self.assertEqual(ref[1],
+                         "Ready",
+                         "Template store ref state should be marked as Ready"
+                         )
+        return
+
+    @attr(tags=["advanced", "smoke"], required_hardware="true")
+    def test_02_deploy_vm_from_direct_download_template(self):
+        """
+        Deploy a VM from a Direct Download registered template
+        """
+        bp = self.templates[0]
+        virtual_machine = VirtualMachine.create(
+            self.apiclient,
+            self.services["virtual_machine"],
+            templateid=bp.id,
+            accountid=self.account.name,
+            domainid=self.account.domainid,
+            serviceofferingid=self.service_offering.id
+        )
+        self.cleanup.append(virtual_machine)
+        return
+
+    @attr(tags=["advanced", "smoke"], required_hardware="true")
+    def test_03_deploy_vm_wrong_checksum(self):
+        """
+        Deploy a VM from a Direct Download registered template with wrong checksum
+        """
+        self.template["checksum"]="{MD5}XXXXXXX"
+        tmpl = Template.register(self.apiclient, self.template, zoneid=self.zone.id, hypervisor=self.hypervisor, randomize_name=False)
+
+        try:
+            virtual_machine = VirtualMachine.create(
+                self.apiclient,
+                self.services["virtual_machine"],
+                templateid=tmpl.id,
+                accountid=self.account.name,
+                domainid=self.account.domainid,
+                serviceofferingid=self.service_offering.id
+            )
+            self.cleanup.append(tmpl)
+            self.fail("Expected to fail deployment")
+        except Exception as e:
+            self.debug("Expected exception")
+
+        self.cleanup.append(virtual_machine)
+        self.cleanup.append(tmpl)
+        return

tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh

@@ -68,7 +68,7 @@ function install_packages() {
     python-flask \
     haproxy \
     radvd \
-    sharutils genisoimage \
+    sharutils genisoimage aria2 \
     strongswan libcharon-extra-plugins libstrongswan-extra-plugins \
     virt-what open-vm-tools qemu-guest-agent hyperv-daemons
 

tools/marvin/marvin/lib/base.py

@@ -1302,6 +1302,10 @@ class Template:
         if details:
             cmd.details = details
 
+        if "directdownload" in services:
+            cmd.directdownload = services["directdownload"]
+
+
         # Register Template
         template = apiclient.registerTemplate(cmd)
 

ui/l10n/ar.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "الأجهزة",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "Direct Attached Public IP",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "الشبكة المشتركة IPs",
     "label.disable.autoscale": "Disable Autoscale",
     "label.disable.host": "Disable Host",

ui/l10n/ca.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "Devices",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "Direct Attached Public IP",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "Shared Network IPs",
     "label.disable.autoscale": "Disable Autoscale",
     "label.disable.host": "Disable Host",

ui/l10n/de_DE.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "Geräte",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "Direkt angeschlossene öffentliche IP",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "Gemeinsame Netzwerk-IPs",
     "label.disable.autoscale": "Automatische Skalierung deaktivieren",
     "label.disable.host": "Host deaktivieren",

ui/l10n/en.js

@@ -647,6 +647,7 @@ var dictionary = {"ICMP.code":"ICMP Code",
 "label.devices":"Devices",
 "label.dhcp":"DHCP",
 "label.direct.attached.public.ip":"Direct Attached Public IP",
+"label.direct.download":"Direct Download",
 "label.direct.ips":"Shared Network IPs",
 "label.disable.autoscale":"Disable Autoscale",
 "label.disable.host":"Disable Host",

ui/l10n/es.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "Dispositivos",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "IP Pública Conectada en forma Directa",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "IPs de la Red Compartida",
     "label.disable.autoscale": "Deshabilitar Escalado Automático",
     "label.disable.host": "Deshabitar Anfitrión",

ui/l10n/fr_FR.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "Machines",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "IP publique attachée directement",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "Adresses IP du réseau partagé",
     "label.disable.autoscale": "Désactiver Autoscale",
     "label.disable.host": "Désactiver Hôte",

ui/l10n/hu.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "Eszközök",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "Direct Attached Public IP",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "Osztott hálózati IP címek",
     "label.disable.autoscale": "Automatikus skálázás kikapcsolása",
     "label.disable.host": "Kiszolgáló kikapcsolása",

ui/l10n/it_IT.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "Device",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "Direct Attached Public IP",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "Indirizzi IP di Rete condivisi",
     "label.disable.autoscale": "Disable Autoscale",
     "label.disable.host": "Disable Host",

ui/l10n/ja_JP.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "デバイス",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "直接アタッチされているパブリック IP アドレス",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "共有ネットワークの IP アドレス",
     "label.disable.autoscale": "自動サイズ設定の無効化",
     "label.disable.host": "ホストの無効化",

ui/l10n/ko_KR.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "기기",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "Direct Attached Public IP",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "직접 IP 주소",
     "label.disable.autoscale": "Disable Autoscale",
     "label.disable.host": "Disable Host",

ui/l10n/nb_NO.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "Enheter",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "Direkte Tilknyttet Offentlig IP-adresse",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "Deltnettverk-IPadresser",
     "label.disable.autoscale": "Deaktiver autoskalering",
     "label.disable.host": "Deaktiver host",

ui/l10n/nl_NL.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "Apparaten",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "direct verbonden publieke IP",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "Shared Netwerk IPs",
     "label.disable.autoscale": "Autoscale uitschakelen",
     "label.disable.host": "schakel host uit",

ui/l10n/pl.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "Devices",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "Direct Attached Public IP",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "Shared Network IPs",
     "label.disable.autoscale": "Disable Autoscale",
     "label.disable.host": "Disable Host",

ui/l10n/pt_BR.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "Dispositivos",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "IP Público COnectado Diretamente",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "IPs Diretos",
     "label.disable.autoscale": "Desabilita Auto-escala",
     "label.disable.host": "Desabilita Host",

ui/l10n/ru_RU.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "Устройство",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "Выданные публичные IP",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "Прямые IP-адреса",
     "label.disable.autoscale": "Выключить автоматическое маштабирование",
     "label.disable.host": "Отключить хост",

ui/l10n/zh_CN.js

@@ -636,6 +636,7 @@ var dictionary = {
     "label.devices": "设备",
     "label.dhcp": "DHCP",
     "label.direct.attached.public.ip": "直连公用 IP",
+    "label.direct.download":"Direct Download",
     "label.direct.ips": "共享网络 IP",
     "label.disable.autoscale": "禁用自动缩放",
     "label.disable.host": "禁用主机",