Added QCOW2 virtual size checking for S3.

- Cleaned up S3TemplateDownloader
- Created static QCOW2 utils class.
- Reformatted some parts of DownloadManagerImpl

core/src/com/cloud/storage/template/S3TemplateDownloader.java

@@ -27,6 +27,8 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.Date;
 
+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
 import org.apache.commons.httpclient.ChunkedInputStream;
 import org.apache.commons.httpclient.Credentials;
 import org.apache.commons.httpclient.Header;
@@ -50,10 +52,6 @@ import com.amazonaws.services.s3.model.ProgressEvent;
 import com.amazonaws.services.s3.model.ProgressListener;
 import com.amazonaws.services.s3.model.PutObjectRequest;
 import com.amazonaws.services.s3.model.StorageClass;
-
-import org.apache.cloudstack.managed.context.ManagedContextRunnable;
-import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
-
 import com.cloud.agent.api.storage.Proxy;
 import com.cloud.agent.api.to.S3TO;
 import com.cloud.utils.Pair;
@@ -61,46 +59,48 @@ import com.cloud.utils.S3Utils;
 import com.cloud.utils.UriUtils;
 
 /**
- * Download a template file using HTTP
+ * Download a template file using HTTP(S)
- *
  */
 public class S3TemplateDownloader extends ManagedContextRunnable implements TemplateDownloader {
-    public static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName());
+    private static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName());
     private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
 
     private String downloadUrl;
     private String installPath;
     private String s3Key;
     private String fileName;
-    public TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
+    private String fileExtension;
-    public String errorString = " ";
+    private String errorString = " ";
-    private long remoteSize = 0;
+
-    public long downloadTime = 0;
+    private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
-    public long totalBytes;
+    private ResourceType resourceType = ResourceType.TEMPLATE;
     private final HttpClient client;
+    private final HttpMethodRetryHandler myretryhandler;
     private GetMethod request;
-    private boolean resume = false;
     private DownloadCompleteCallback completionCallback;
-    private S3TO s3;
+    private S3TO s3to;
+
+    private long remoteSize = 0;
+    private long downloadTime = 0;
+    private long totalBytes;
+    private long maxTemplateSizeInByte;
+
+    private boolean resume = false;
     private boolean inited = true;
 
-    private long maxTemplateSizeInByte;
+    public S3TemplateDownloader(S3TO s3to, String downloadUrl, String installPath, DownloadCompleteCallback callback,
-    private ResourceType resourceType = ResourceType.TEMPLATE;
+            long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) {
-    private final HttpMethodRetryHandler myretryhandler;
+        this.s3to = s3to;
-
-    public S3TemplateDownloader(S3TO storageLayer, String downloadUrl, String installPath, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user,
-            String password, Proxy proxy, ResourceType resourceType) {
-        s3 = storageLayer;
         this.downloadUrl = downloadUrl;
         this.installPath = installPath;
-        status = TemplateDownloader.Status.NOT_STARTED;
+        this.status = TemplateDownloader.Status.NOT_STARTED;
         this.resourceType = resourceType;
-        maxTemplateSizeInByte = maxTemplateSizeInBytes;
+        this.maxTemplateSizeInByte = maxTemplateSizeInBytes;
 
-        totalBytes = 0;
+        this.totalBytes = 0;
-        client = new HttpClient(s_httpClientManager);
+        this.client = new HttpClient(s_httpClientManager);
 
-        myretryhandler = new HttpMethodRetryHandler() {
+        this.myretryhandler = new HttpMethodRetryHandler() {
             @Override
             public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
                 if (executionCount >= 2) {
@@ -128,6 +128,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
 
             Pair<String, Integer> hostAndPort = UriUtils.validateUrl(downloadUrl);
             fileName = StringUtils.substringAfterLast(downloadUrl, "/");
+            fileExtension = StringUtils.substringAfterLast(fileName, ".");
 
             if (proxy != null) {
                 client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort());
@@ -139,8 +140,10 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
             if ((user != null) && (password != null)) {
                 client.getParams().setAuthenticationPreemptive(true);
                 Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
-                client.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
+                client.getState().setCredentials(
-                s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second());
+                        new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
+                s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first()
+                        + ":" + hostAndPort.second());
             } else {
                 s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second());
             }
@@ -160,11 +163,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
     @Override
     public long download(boolean resume, DownloadCompleteCallback callback) {
         switch (status) {
-            case ABORTED:
+        case ABORTED:
-            case UNRECOVERABLE_ERROR:
+        case UNRECOVERABLE_ERROR:
-            case DOWNLOAD_FINISHED:
+        case DOWNLOAD_FINISHED:
-                return 0;
+            return 0;
-            default:
+        default:
 
         }
 
@@ -215,10 +218,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
                 contentType = contentTypeHeader.getValue();
             }
 
-            InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream()) : new ChunkedInputStream(request.getResponseBodyAsStream());
+            InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream())
+                    : new ChunkedInputStream(request.getResponseBodyAsStream());
 
-            s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3.getBucketName() + " remoteSize=" + remoteSize + " , max size=" +
+            s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3to.getBucketName()
-                maxTemplateSizeInByte);
+                    + " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInByte);
 
             Date start = new Date();
             // compute s3 key
@@ -230,9 +234,9 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
             if (contentType != null) {
                 metadata.setContentType(contentType);
             }
-            PutObjectRequest putObjectRequest = new PutObjectRequest(s3.getBucketName(), s3Key, in, metadata);
+            PutObjectRequest putObjectRequest = new PutObjectRequest(s3to.getBucketName(), s3Key, in, metadata);
             // check if RRS is enabled
-            if (s3.getEnableRRS()) {
+            if (s3to.getEnableRRS()) {
                 putObjectRequest = putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
             }
             // register progress listenser
@@ -257,14 +261,15 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
 
             });
 
-            if (!s3.getSingleUpload(remoteSize)) {
+            if (!s3to.getSingleUpload(remoteSize)) {
                 // use TransferManager to do multipart upload
-                S3Utils.mputObject(s3, putObjectRequest);
+                S3Utils.mputObject(s3to, putObjectRequest);
             } else {
                 // single part upload, with 5GB limit in Amazon
-                S3Utils.putObject(s3, putObjectRequest);
+                S3Utils.putObject(s3to, putObjectRequest);
-                while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED && status != TemplateDownloader.Status.UNRECOVERABLE_ERROR &&
+                while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED
-                    status != TemplateDownloader.Status.ABORTED) {
+                        && status != TemplateDownloader.Status.UNRECOVERABLE_ERROR
+                        && status != TemplateDownloader.Status.ABORTED) {
                     // wait for completion
                 }
             }
@@ -324,32 +329,59 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
         return totalBytes;
     }
 
+    /**
+     * Returns an InputStream only when the status is DOWNLOAD_FINISHED.
+     *
+     * The caller of this method must close the InputStream to prevent resource leaks!
+     *
+     * @return S3ObjectInputStream of the object.
+     */
+    public InputStream getS3ObjectInputStream() {
+        // Check if the download is finished
+        if (status != Status.DOWNLOAD_FINISHED) {
+            return null;
+        }
+
+        return S3Utils.getObjectStream(s3to, s3to.getBucketName(), s3Key);
+    }
+
+    public void cleanupAfterError() {
+        if (status != Status.UNRECOVERABLE_ERROR) {
+            s_logger.debug("S3Template downloader does not have state UNRECOVERABLE_ERROR, no cleanup neccesarry.");
+            return;
+        }
+
+        s_logger.info("Cleanup after UNRECOVERABLE_ERROR, trying to remove object: " + s3Key);
+
+        S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
+    }
+
     @Override
     @SuppressWarnings("fallthrough")
     public boolean stopDownload() {
         switch (getStatus()) {
-            case IN_PROGRESS:
+        case IN_PROGRESS:
-                if (request != null) {
+            if (request != null) {
-                    request.abort();
+                request.abort();
-                }
+            }
-                status = TemplateDownloader.Status.ABORTED;
+            status = TemplateDownloader.Status.ABORTED;
-                return true;
+            return true;
-            case UNKNOWN:
+        case UNKNOWN:
-            case NOT_STARTED:
+        case NOT_STARTED:
-            case RECOVERABLE_ERROR:
+        case RECOVERABLE_ERROR:
-            case UNRECOVERABLE_ERROR:
+        case UNRECOVERABLE_ERROR:
-            case ABORTED:
+        case ABORTED:
-                status = TemplateDownloader.Status.ABORTED;
+            status = TemplateDownloader.Status.ABORTED;
-            case DOWNLOAD_FINISHED:
+        case DOWNLOAD_FINISHED:
-                try {
+            try {
-                    S3Utils.deleteObject(s3, s3.getBucketName(), s3Key);
+                S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
-                } catch (Exception ex) {
+            } catch (Exception ex) {
-                    // ignore delete exception if it is not there
+                // ignore delete exception if it is not there
-                }
+            }
-                return true;
+            return true;
 
-            default:
+        default:
-                return true;
+            return true;
         }
     }
 
@@ -359,7 +391,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
             return 0;
         }
 
-        return (int)(100.0 * totalBytes / remoteSize);
+        return (int) (100.0 * totalBytes / remoteSize);
     }
 
     @Override
@@ -417,4 +449,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
         return resourceType;
     }
 
-}
+    public long getTotalBytes() {
+        return totalBytes;
+    }
+
+    public String getFileExtension() {
+        return fileExtension;
+    }
+}

services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java

@@ -41,13 +41,12 @@ import java.util.concurrent.Executors;
 import javax.ejb.Local;
 import javax.naming.ConfigurationException;
 
-import org.apache.log4j.Logger;
-
 import org.apache.cloudstack.storage.command.DownloadCommand;
 import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
 import org.apache.cloudstack.storage.command.DownloadProgressCommand;
 import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType;
 import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
+import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.storage.DownloadAnswer;
 import com.cloud.agent.api.storage.Proxy;
@@ -67,9 +66,9 @@ import com.cloud.storage.template.Processor;
 import com.cloud.storage.template.Processor.FormatInfo;
 import com.cloud.storage.template.QCOW2Processor;
 import com.cloud.storage.template.RawImageProcessor;
-import com.cloud.storage.template.TARProcessor;
 import com.cloud.storage.template.S3TemplateDownloader;
 import com.cloud.storage.template.ScpTemplateDownloader;
+import com.cloud.storage.template.TARProcessor;
 import com.cloud.storage.template.TemplateConstants;
 import com.cloud.storage.template.TemplateDownloader;
 import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback;
@@ -83,6 +82,7 @@ import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
+import com.cloud.utils.storage.QCOW2Utils;
 
 @Local(value = DownloadManager.class)
 public class DownloadManagerImpl extends ManagerBase implements DownloadManager {
@@ -129,10 +129,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
             this.tmpltName = tmpltName;
             this.format = format;
             this.hvm = hvm;
-            description = descr;
+            this.description = descr;
-            checksum = cksum;
+            this.checksum = cksum;
             this.installPathPrefix = installPathPrefix;
-            templatesize = 0;
+            this.templatesize = 0;
             this.id = id;
             this.resourceType = resourceType;
         }
@@ -276,11 +276,27 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
             threadPool.execute(td);
             break;
         case DOWNLOAD_FINISHED:
-            if (!(td instanceof S3TemplateDownloader)) {
+            if(td instanceof S3TemplateDownloader) {
-                // we currently only create template.properties for NFS by
+                // For S3 and Swift, which are considered "remote",
-                // running some post download script
+                // as in the file cannot be accessed locally,
+                // we run the postRemoteDownload() method.
                 td.setDownloadError("Download success, starting install ");
-                String result = postDownload(jobId);
+                String result = postRemoteDownload(jobId);
+                if (result != null) {
+                    s_logger.error("Failed post download install: " + result);
+                    td.setStatus(Status.UNRECOVERABLE_ERROR);
+                    td.setDownloadError("Failed post download install: " + result);
+                    ((S3TemplateDownloader) td).cleanupAfterError();
+                } else {
+                    td.setStatus(Status.POST_DOWNLOAD_FINISHED);
+                    td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date()));
+                }
+            }
+            else {
+                // For other TemplateDownloaders where files are locally available,
+                // we run the postLocalDownload() method.
+                td.setDownloadError("Download success, starting install ");
+                String result = postLocalDownload(jobId);
                 if (result != null) {
                     s_logger.error("Failed post download script: " + result);
                     td.setStatus(Status.UNRECOVERABLE_ERROR);
@@ -289,17 +305,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
                     td.setStatus(Status.POST_DOWNLOAD_FINISHED);
                     td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date()));
                 }
-            } else {
-                // for s3 and swift, we skip post download step and just set
-                // status to trigger callback.
-                td.setStatus(Status.POST_DOWNLOAD_FINISHED);
-                // set template size for S3
-                S3TemplateDownloader std = (S3TemplateDownloader)td;
-                long size = std.totalBytes;
-                DownloadJob dnld = jobs.get(jobId);
-                dnld.setTemplatesize(size);
-                dnld.setTemplatePhysicalSize(size);
-                dnld.setTmpltPath(std.getDownloadLocalPath()); // update template path to include file name.
             }
             dj.cleanup();
             break;
@@ -339,12 +344,48 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
     }
 
     /**
-     * Post download activity (install and cleanup). Executed in context of
+     * Post remote download activity (install and cleanup). Executed in context of the downloader thread.
+     */
+    private String postRemoteDownload(String jobId) {
+        String result = null;
+        DownloadJob dnld = jobs.get(jobId);
+        S3TemplateDownloader td = (S3TemplateDownloader)dnld.getTemplateDownloader();
+
+        if (td.getFileExtension().equalsIgnoreCase("QCOW2")) {
+            // The QCOW2 is the only format with a header,
+            // and as such can be easily read.
+
+            try {
+                InputStream inputStream = td.getS3ObjectInputStream();
+
+                dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream));
+
+                inputStream.close();
+            }
+            catch (IOException e) {
+                result = "Couldn't read QCOW2 virtual size. Error: " + e.getMessage();
+            }
+
+        }
+        else {
+            // For the other formats, both the virtual
+            // and actual file size are set the same.
+            dnld.setTemplatesize(td.getTotalBytes());
+        }
+
+        dnld.setTemplatePhysicalSize(td.getTotalBytes());
+        dnld.setTmpltPath(td.getDownloadLocalPath());
+
+        return result;
+    }
+
+    /**
+     * Post local download activity (install and cleanup). Executed in context of
      * downloader thread
      *
      * @throws IOException
      */
-    private String postDownload(String jobId) {
+    private String postLocalDownload(String jobId) {
         DownloadJob dnld = jobs.get(jobId);
         TemplateDownloader td = dnld.getTemplateDownloader();
         String resourcePath = dnld.getInstallPathPrefix(); // path with mount

utils/src/main/java/com/cloud/utils/S3Utils.java

@@ -62,6 +62,7 @@ import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.PutObjectRequest;
 import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.transfer.TransferManager;
 import com.amazonaws.services.s3.transfer.Upload;
@@ -256,6 +257,21 @@ public final class S3Utils {
 
     }
 
+    // Note that whenever S3Object is returned, client code needs to close the internal stream to avoid resource leak.
+    public static S3ObjectInputStream getObjectStream(final ClientOptions clientOptions, final String bucketName, final String key) {
+
+        assert clientOptions != null;
+        assert !isBlank(bucketName);
+        assert !isBlank(key);
+
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(format("Get S3 object %1$s in " + "bucket %2$s", key, bucketName));
+        }
+
+        return acquireClient(clientOptions).getObject(bucketName, key).getObjectContent();
+
+    }
+
     @SuppressWarnings("unchecked")
     public static File getFile(final ClientOptions clientOptions, final String bucketName, final String key, final File targetDirectory,
         final FileNamingStrategy namingStrategy) {

utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java

@@ -0,0 +1,60 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.utils.storage;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import com.cloud.utils.NumbersUtil;
+
+public final class QCOW2Utils {
+    private static final int VIRTUALSIZE_HEADER_LOCATION = 24;
+    private static final int VIRTUALSIZE_HEADER_LENGTH = 8;
+
+    /**
+     * Private constructor ->  This utility class cannot be instantiated.
+     */
+    private QCOW2Utils() {}
+
+    /**
+     * @return the header location of the virtual size field.
+     */
+    public static int getVirtualSizeHeaderLocation() {
+        return VIRTUALSIZE_HEADER_LOCATION;
+    }
+
+    /**
+     * @param inputStream The QCOW2 object in stream format.
+     * @return The virtual size of the QCOW2 object.
+     */
+    public static long getVirtualSize(InputStream inputStream) throws IOException {
+        byte[] bytes = new byte[VIRTUALSIZE_HEADER_LENGTH];
+
+        if (inputStream.skip(VIRTUALSIZE_HEADER_LOCATION) != VIRTUALSIZE_HEADER_LOCATION) {
+            throw new IOException("Unable to skip to the virtual size header");
+        }
+
+        if (inputStream.read(bytes) != VIRTUALSIZE_HEADER_LENGTH) {
+            throw new IOException("Unable to properly read the size");
+        }
+
+        return NumbersUtil.bytesToLong(bytes);
+    }
+}