Added QCOW2 virtual size checking for S3.

- Cleaned up S3TemplateDownloader - Created static QCOW2 utils class. - Reformatted some parts of DownloadManagerImpl
2025-10-26 08:42:29 +01:00 · 2015-09-09 17:53:35 +02:00 · 2015-09-09 17:53:35 +02:00 · 1c6378ec00
commit 1c6378ec00
parent 1a02773b55
4 changed files with 245 additions and 89 deletions
--- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
@ -27,6 +27,8 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.Date;

+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
 import org.apache.commons.httpclient.ChunkedInputStream;
 import org.apache.commons.httpclient.Credentials;
 import org.apache.commons.httpclient.Header;
@ -50,10 +52,6 @@ import com.amazonaws.services.s3.model.ProgressEvent;
 import com.amazonaws.services.s3.model.ProgressListener;
 import com.amazonaws.services.s3.model.PutObjectRequest;
 import com.amazonaws.services.s3.model.StorageClass;
-
-import org.apache.cloudstack.managed.context.ManagedContextRunnable;
-import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
-
 import com.cloud.agent.api.storage.Proxy;
 import com.cloud.agent.api.to.S3TO;
 import com.cloud.utils.Pair;
@ -61,46 +59,48 @@ import com.cloud.utils.S3Utils;
 import com.cloud.utils.UriUtils;

 /**
- * Download a template file using HTTP
- *
+ * Download a template file using HTTP(S)
 */
 public class S3TemplateDownloader extends ManagedContextRunnable implements TemplateDownloader {
-    public static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName());
+    private static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName());
    private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();

    private String downloadUrl;
    private String installPath;
    private String s3Key;
    private String fileName;
-    public TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
-    public String errorString = " ";
-    private long remoteSize = 0;
-    public long downloadTime = 0;
-    public long totalBytes;
+    private String fileExtension;
+    private String errorString = " ";
+
+    private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
+    private ResourceType resourceType = ResourceType.TEMPLATE;
    private final HttpClient client;
+    private final HttpMethodRetryHandler myretryhandler;
    private GetMethod request;
-    private boolean resume = false;
    private DownloadCompleteCallback completionCallback;
-    private S3TO s3;
+    private S3TO s3to;
+
+    private long remoteSize = 0;
+    private long downloadTime = 0;
+    private long totalBytes;
+    private long maxTemplateSizeInByte;
+
+    private boolean resume = false;
    private boolean inited = true;

-    private long maxTemplateSizeInByte;
-    private ResourceType resourceType = ResourceType.TEMPLATE;
-    private final HttpMethodRetryHandler myretryhandler;
-
-    public S3TemplateDownloader(S3TO storageLayer, String downloadUrl, String installPath, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user,
-            String password, Proxy proxy, ResourceType resourceType) {
-        s3 = storageLayer;
+    public S3TemplateDownloader(S3TO s3to, String downloadUrl, String installPath, DownloadCompleteCallback callback,
+            long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) {
+        this.s3to = s3to;
        this.downloadUrl = downloadUrl;
        this.installPath = installPath;
-        status = TemplateDownloader.Status.NOT_STARTED;
+        this.status = TemplateDownloader.Status.NOT_STARTED;
        this.resourceType = resourceType;
-        maxTemplateSizeInByte = maxTemplateSizeInBytes;
+        this.maxTemplateSizeInByte = maxTemplateSizeInBytes;

-        totalBytes = 0;
-        client = new HttpClient(s_httpClientManager);
+        this.totalBytes = 0;
+        this.client = new HttpClient(s_httpClientManager);

-        myretryhandler = new HttpMethodRetryHandler() {
+        this.myretryhandler = new HttpMethodRetryHandler() {
            @Override
            public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
                if (executionCount >= 2) {
@ -128,6 +128,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp

            Pair<String, Integer> hostAndPort = UriUtils.validateUrl(downloadUrl);
            fileName = StringUtils.substringAfterLast(downloadUrl, "/");
+            fileExtension = StringUtils.substringAfterLast(fileName, ".");

            if (proxy != null) {
                client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort());
@ -139,8 +140,10 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
            if ((user != null) && (password != null)) {
                client.getParams().setAuthenticationPreemptive(true);
                Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
-                client.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
-                s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second());
+                client.getState().setCredentials(
+                        new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
+                s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first()
+                        + ":" + hostAndPort.second());
            } else {
                s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second());
            }
@ -160,11 +163,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
    @Override
    public long download(boolean resume, DownloadCompleteCallback callback) {
        switch (status) {
-            case ABORTED:
-            case UNRECOVERABLE_ERROR:
-            case DOWNLOAD_FINISHED:
-                return 0;
-            default:
+        case ABORTED:
+        case UNRECOVERABLE_ERROR:
+        case DOWNLOAD_FINISHED:
+            return 0;
+        default:

        }

@ -215,10 +218,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
                contentType = contentTypeHeader.getValue();
            }

-            InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream()) : new ChunkedInputStream(request.getResponseBodyAsStream());
+            InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream())
+                    : new ChunkedInputStream(request.getResponseBodyAsStream());

-            s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3.getBucketName() + " remoteSize=" + remoteSize + " , max size=" +
-                maxTemplateSizeInByte);
+            s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3to.getBucketName()
+                    + " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInByte);

            Date start = new Date();
            // compute s3 key
@ -230,9 +234,9 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
            if (contentType != null) {
                metadata.setContentType(contentType);
            }
-            PutObjectRequest putObjectRequest = new PutObjectRequest(s3.getBucketName(), s3Key, in, metadata);
+            PutObjectRequest putObjectRequest = new PutObjectRequest(s3to.getBucketName(), s3Key, in, metadata);
            // check if RRS is enabled
-            if (s3.getEnableRRS()) {
+            if (s3to.getEnableRRS()) {
                putObjectRequest = putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
            }
            // register progress listenser
@ -257,14 +261,15 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp

            });

-            if (!s3.getSingleUpload(remoteSize)) {
+            if (!s3to.getSingleUpload(remoteSize)) {
                // use TransferManager to do multipart upload
-                S3Utils.mputObject(s3, putObjectRequest);
+                S3Utils.mputObject(s3to, putObjectRequest);
            } else {
                // single part upload, with 5GB limit in Amazon
-                S3Utils.putObject(s3, putObjectRequest);
-                while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED && status != TemplateDownloader.Status.UNRECOVERABLE_ERROR &&
-                    status != TemplateDownloader.Status.ABORTED) {
+                S3Utils.putObject(s3to, putObjectRequest);
+                while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED
+                        && status != TemplateDownloader.Status.UNRECOVERABLE_ERROR
+                        && status != TemplateDownloader.Status.ABORTED) {
                    // wait for completion
                }
            }
@ -324,32 +329,59 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
        return totalBytes;
    }

+    /**
+     * Returns an InputStream only when the status is DOWNLOAD_FINISHED.
+     *
+     * The caller of this method must close the InputStream to prevent resource leaks!
+     *
+     * @return S3ObjectInputStream of the object.
+     */
+    public InputStream getS3ObjectInputStream() {
+        // Check if the download is finished
+        if (status != Status.DOWNLOAD_FINISHED) {
+            return null;
+        }
+
+        return S3Utils.getObjectStream(s3to, s3to.getBucketName(), s3Key);
+    }
+
+    public void cleanupAfterError() {
+        if (status != Status.UNRECOVERABLE_ERROR) {
+            s_logger.debug("S3Template downloader does not have state UNRECOVERABLE_ERROR, no cleanup neccesarry.");
+            return;
+        }
+
+        s_logger.info("Cleanup after UNRECOVERABLE_ERROR, trying to remove object: " + s3Key);
+
+        S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
+    }
+
    @Override
    @SuppressWarnings("fallthrough")
    public boolean stopDownload() {
        switch (getStatus()) {
-            case IN_PROGRESS:
-                if (request != null) {
-                    request.abort();
-                }
-                status = TemplateDownloader.Status.ABORTED;
-                return true;
-            case UNKNOWN:
-            case NOT_STARTED:
-            case RECOVERABLE_ERROR:
-            case UNRECOVERABLE_ERROR:
-            case ABORTED:
-                status = TemplateDownloader.Status.ABORTED;
-            case DOWNLOAD_FINISHED:
-                try {
-                    S3Utils.deleteObject(s3, s3.getBucketName(), s3Key);
-                } catch (Exception ex) {
-                    // ignore delete exception if it is not there
-                }
-                return true;
+        case IN_PROGRESS:
+            if (request != null) {
+                request.abort();
+            }
+            status = TemplateDownloader.Status.ABORTED;
+            return true;
+        case UNKNOWN:
+        case NOT_STARTED:
+        case RECOVERABLE_ERROR:
+        case UNRECOVERABLE_ERROR:
+        case ABORTED:
+            status = TemplateDownloader.Status.ABORTED;
+        case DOWNLOAD_FINISHED:
+            try {
+                S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
+            } catch (Exception ex) {
+                // ignore delete exception if it is not there
+            }
+            return true;

-            default:
-                return true;
+        default:
+            return true;
        }
    }

@ -359,7 +391,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
            return 0;
        }

-        return (int)(100.0 * totalBytes / remoteSize);
+        return (int) (100.0 * totalBytes / remoteSize);
    }

    @Override
@ -417,4 +449,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
        return resourceType;
    }

-}
+    public long getTotalBytes() {
+        return totalBytes;
+    }
+
+    public String getFileExtension() {
+        return fileExtension;
+    }
+}
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@ -41,13 +41,12 @@ import java.util.concurrent.Executors;
 import javax.ejb.Local;
 import javax.naming.ConfigurationException;

-import org.apache.log4j.Logger;
-
 import org.apache.cloudstack.storage.command.DownloadCommand;
 import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
 import org.apache.cloudstack.storage.command.DownloadProgressCommand;
 import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType;
 import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
+import org.apache.log4j.Logger;

 import com.cloud.agent.api.storage.DownloadAnswer;
 import com.cloud.agent.api.storage.Proxy;
@ -67,9 +66,9 @@ import com.cloud.storage.template.Processor;
 import com.cloud.storage.template.Processor.FormatInfo;
 import com.cloud.storage.template.QCOW2Processor;
 import com.cloud.storage.template.RawImageProcessor;
-import com.cloud.storage.template.TARProcessor;
 import com.cloud.storage.template.S3TemplateDownloader;
 import com.cloud.storage.template.ScpTemplateDownloader;
+import com.cloud.storage.template.TARProcessor;
 import com.cloud.storage.template.TemplateConstants;
 import com.cloud.storage.template.TemplateDownloader;
 import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback;
@ -83,6 +82,7 @@ import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
+import com.cloud.utils.storage.QCOW2Utils;

@Local(value = DownloadManager.class)
 public class DownloadManagerImpl extends ManagerBase implements DownloadManager {
@ -129,10 +129,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
            this.tmpltName = tmpltName;
            this.format = format;
            this.hvm = hvm;
-            description = descr;
-            checksum = cksum;
+            this.description = descr;
+            this.checksum = cksum;
            this.installPathPrefix = installPathPrefix;
-            templatesize = 0;
+            this.templatesize = 0;
            this.id = id;
            this.resourceType = resourceType;
        }
@ -276,11 +276,27 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
            threadPool.execute(td);
            break;
        case DOWNLOAD_FINISHED:
-            if (!(td instanceof S3TemplateDownloader)) {
-                // we currently only create template.properties for NFS by
-                // running some post download script
+            if(td instanceof S3TemplateDownloader) {
+                // For S3 and Swift, which are considered "remote",
+                // as in the file cannot be accessed locally,
+                // we run the postRemoteDownload() method.
                td.setDownloadError("Download success, starting install ");
-                String result = postDownload(jobId);
+                String result = postRemoteDownload(jobId);
+                if (result != null) {
+                    s_logger.error("Failed post download install: " + result);
+                    td.setStatus(Status.UNRECOVERABLE_ERROR);
+                    td.setDownloadError("Failed post download install: " + result);
+                    ((S3TemplateDownloader) td).cleanupAfterError();
+                } else {
+                    td.setStatus(Status.POST_DOWNLOAD_FINISHED);
+                    td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date()));
+                }
+            }
+            else {
+                // For other TemplateDownloaders where files are locally available,
+                // we run the postLocalDownload() method.
+                td.setDownloadError("Download success, starting install ");
+                String result = postLocalDownload(jobId);
                if (result != null) {
                    s_logger.error("Failed post download script: " + result);
                    td.setStatus(Status.UNRECOVERABLE_ERROR);
@ -289,17 +305,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
                    td.setStatus(Status.POST_DOWNLOAD_FINISHED);
                    td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date()));
                }
-            } else {
-                // for s3 and swift, we skip post download step and just set
-                // status to trigger callback.
-                td.setStatus(Status.POST_DOWNLOAD_FINISHED);
-                // set template size for S3
-                S3TemplateDownloader std = (S3TemplateDownloader)td;
-                long size = std.totalBytes;
-                DownloadJob dnld = jobs.get(jobId);
-                dnld.setTemplatesize(size);
-                dnld.setTemplatePhysicalSize(size);
-                dnld.setTmpltPath(std.getDownloadLocalPath()); // update template path to include file name.
            }
            dj.cleanup();
            break;
@ -339,12 +344,48 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
    }

    /**
-     * Post download activity (install and cleanup). Executed in context of
+     * Post remote download activity (install and cleanup). Executed in context of the downloader thread.
+     */
+    private String postRemoteDownload(String jobId) {
+        String result = null;
+        DownloadJob dnld = jobs.get(jobId);
+        S3TemplateDownloader td = (S3TemplateDownloader)dnld.getTemplateDownloader();
+
+        if (td.getFileExtension().equalsIgnoreCase("QCOW2")) {
+            // The QCOW2 is the only format with a header,
+            // and as such can be easily read.
+
+            try {
+                InputStream inputStream = td.getS3ObjectInputStream();
+
+                dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream));
+
+                inputStream.close();
+            }
+            catch (IOException e) {
+                result = "Couldn't read QCOW2 virtual size. Error: " + e.getMessage();
+            }
+
+        }
+        else {
+            // For the other formats, both the virtual
+            // and actual file size are set the same.
+            dnld.setTemplatesize(td.getTotalBytes());
+        }
+
+        dnld.setTemplatePhysicalSize(td.getTotalBytes());
+        dnld.setTmpltPath(td.getDownloadLocalPath());
+
+        return result;
+    }
+
+    /**
+     * Post local download activity (install and cleanup). Executed in context of
     * downloader thread
     *
     * @throws IOException
     */
-    private String postDownload(String jobId) {
+    private String postLocalDownload(String jobId) {
        DownloadJob dnld = jobs.get(jobId);
        TemplateDownloader td = dnld.getTemplateDownloader();
        String resourcePath = dnld.getInstallPathPrefix(); // path with mount
--- a/utils/src/main/java/com/cloud/utils/S3Utils.java
+++ b/utils/src/main/java/com/cloud/utils/S3Utils.java
@ -62,6 +62,7 @@ import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.PutObjectRequest;
 import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.transfer.TransferManager;
 import com.amazonaws.services.s3.transfer.Upload;
@ -256,6 +257,21 @@ public final class S3Utils {

    }

+    // Note that whenever S3Object is returned, client code needs to close the internal stream to avoid resource leak.
+    public static S3ObjectInputStream getObjectStream(final ClientOptions clientOptions, final String bucketName, final String key) {
+
+        assert clientOptions != null;
+        assert !isBlank(bucketName);
+        assert !isBlank(key);
+
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(format("Get S3 object %1$s in " + "bucket %2$s", key, bucketName));
+        }
+
+        return acquireClient(clientOptions).getObject(bucketName, key).getObjectContent();
+
+    }
+
    @SuppressWarnings("unchecked")
    public static File getFile(final ClientOptions clientOptions, final String bucketName, final String key, final File targetDirectory,
        final FileNamingStrategy namingStrategy) {
--- a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
+++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
@ -0,0 +1,60 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.utils.storage;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import com.cloud.utils.NumbersUtil;
+
+public final class QCOW2Utils {
+    private static final int VIRTUALSIZE_HEADER_LOCATION = 24;
+    private static final int VIRTUALSIZE_HEADER_LENGTH = 8;
+
+    /**
+     * Private constructor ->  This utility class cannot be instantiated.
+     */
+    private QCOW2Utils() {}
+
+    /**
+     * @return the header location of the virtual size field.
+     */
+    public static int getVirtualSizeHeaderLocation() {
+        return VIRTUALSIZE_HEADER_LOCATION;
+    }
+
+    /**
+     * @param inputStream The QCOW2 object in stream format.
+     * @return The virtual size of the QCOW2 object.
+     */
+    public static long getVirtualSize(InputStream inputStream) throws IOException {
+        byte[] bytes = new byte[VIRTUALSIZE_HEADER_LENGTH];
+
+        if (inputStream.skip(VIRTUALSIZE_HEADER_LOCATION) != VIRTUALSIZE_HEADER_LOCATION) {
+            throw new IOException("Unable to skip to the virtual size header");
+        }
+
+        if (inputStream.read(bytes) != VIRTUALSIZE_HEADER_LENGTH) {
+            throw new IOException("Unable to properly read the size");
+        }
+
+        return NumbersUtil.bytesToLong(bytes);
+    }
+}