From 45b35aaeef9c9f938f57c9d0dbd65c67c4c3e406 Mon Sep 17 00:00:00 2001 From: Edison Su Date: Mon, 19 Aug 2013 17:51:06 -0700 Subject: [PATCH] CLOUDSTACK-4398: get physical size of template if the template is stored in S3 --- .../cloud/storage/template/IsoProcessor.java | 5 ++++ .../com/cloud/storage/template/Processor.java | 5 ++++ .../storage/template/QCOW2Processor.java | 23 ++++++++++++++ .../storage/template/RawImageProcessor.java | 5 ++++ .../cloud/storage/template/VhdProcessor.java | 28 +++++++++++++++++ .../cloud/storage/template/VmdkProcessor.java | 11 +++++++ .../resource/NfsSecondaryStorageResource.java | 30 ++++++++++++++++++- 7 files changed, 106 insertions(+), 1 deletion(-) diff --git a/core/src/com/cloud/storage/template/IsoProcessor.java b/core/src/com/cloud/storage/template/IsoProcessor.java index c8cde65738d..3e009d69cf0 100644 --- a/core/src/com/cloud/storage/template/IsoProcessor.java +++ b/core/src/com/cloud/storage/template/IsoProcessor.java @@ -57,6 +57,11 @@ public class IsoProcessor extends AdapterBase implements Processor { return info; } + @Override + public Long getVirtualSize(File file) { + return file.length(); + } + @Override public boolean configure(String name, Map params) throws ConfigurationException { _storage = (StorageLayer)params.get(StorageLayer.InstanceConfigKey); diff --git a/core/src/com/cloud/storage/template/Processor.java b/core/src/com/cloud/storage/template/Processor.java index 2ec359318ff..77c1c65c213 100644 --- a/core/src/com/cloud/storage/template/Processor.java +++ b/core/src/com/cloud/storage/template/Processor.java @@ -20,6 +20,8 @@ import com.cloud.exception.InternalErrorException; import com.cloud.storage.Storage.ImageFormat; import com.cloud.utils.component.Adapter; +import java.io.File; + /** * Generic interface to process different types of image formats * for templates downloaded and for conversion from one format @@ -45,4 +47,7 @@ public interface Processor extends Adapter { public String filename; public boolean isCorrupted; } + + Long getVirtualSize(File file); + } diff --git a/core/src/com/cloud/storage/template/QCOW2Processor.java b/core/src/com/cloud/storage/template/QCOW2Processor.java index 09dcfe2ea1c..8d7853330ed 100644 --- a/core/src/com/cloud/storage/template/QCOW2Processor.java +++ b/core/src/com/cloud/storage/template/QCOW2Processor.java @@ -82,6 +82,29 @@ public class QCOW2Processor extends AdapterBase implements Processor { return info; } + public Long getVirtualSize(File file) { + FileInputStream strm = null; + byte[] b = new byte[8]; + try { + strm = new FileInputStream(file); + strm.skip(24); + strm.read(b); + } catch (Exception e) { + s_logger.warn("Unable to read qcow2 file " + file, e); + return null; + } finally { + if (strm != null) { + try { + strm.close(); + } catch (IOException e) { + } + } + } + + long templateSize = NumbersUtil.bytesToLong(b); + return templateSize; + } + @Override public boolean configure(String name, Map params) throws ConfigurationException { diff --git a/core/src/com/cloud/storage/template/RawImageProcessor.java b/core/src/com/cloud/storage/template/RawImageProcessor.java index a002df5c9b2..0e4c8c1822a 100644 --- a/core/src/com/cloud/storage/template/RawImageProcessor.java +++ b/core/src/com/cloud/storage/template/RawImageProcessor.java @@ -68,4 +68,9 @@ public class RawImageProcessor extends AdapterBase implements Processor { return info; } + @Override + public Long getVirtualSize(File file) { + return file.length(); + } + } diff --git a/core/src/com/cloud/storage/template/VhdProcessor.java b/core/src/com/cloud/storage/template/VhdProcessor.java index cabc74b40a6..f7d65288944 100644 --- a/core/src/com/cloud/storage/template/VhdProcessor.java +++ b/core/src/com/cloud/storage/template/VhdProcessor.java @@ -25,6 +25,7 @@ import java.util.Map; import javax.ejb.Local; import javax.naming.ConfigurationException; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.log4j.Logger; import com.cloud.exception.InternalErrorException; @@ -100,6 +101,33 @@ public class VhdProcessor extends AdapterBase implements Processor { return info; } + @Override + public Long getVirtualSize(File file) { + FileInputStream strm = null; + byte[] currentSize = new byte[8]; + byte[] creatorApp = new byte[4]; + try { + strm = new FileInputStream(file); + strm.skip(file.length() - vhd_footer_size + vhd_footer_creator_app_offset); + strm.read(creatorApp); + strm.skip(vhd_footer_current_size_offset - vhd_footer_creator_ver_offset); + strm.read(currentSize); + } catch (Exception e) { + s_logger.warn("Unable to read vhd file " + file.getAbsolutePath(), e); + throw new CloudRuntimeException("Unable to read vhd file " + file.getAbsolutePath() + ": " + e); + } finally { + if (strm != null) { + try { + strm.close(); + } catch (IOException e) { + } + } + } + + long templateSize = NumbersUtil.bytesToLong(currentSize); + return templateSize; + } + @Override public boolean configure(String name, Map params) throws ConfigurationException { _name = name; diff --git a/core/src/com/cloud/storage/template/VmdkProcessor.java b/core/src/com/cloud/storage/template/VmdkProcessor.java index 43650c68c0f..861e98d6954 100644 --- a/core/src/com/cloud/storage/template/VmdkProcessor.java +++ b/core/src/com/cloud/storage/template/VmdkProcessor.java @@ -81,6 +81,17 @@ public class VmdkProcessor extends AdapterBase implements Processor { return info; } + @Override + public Long getVirtualSize(File file) { + try { + long size = getTemplateVirtualSize(file.getParent(), file.getName()); + return size; + } catch (Exception e) { + + } + return file.length(); + } + public long getTemplateVirtualSize(String templatePath, String templateName) throws InternalErrorException { // get the virtual size from the OVF file meta data long virtualSize=0; diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 30d0ff78f1a..f79bae8f1f5 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -270,6 +270,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S newTemplTO.setPath(finalDownloadPath); newTemplTO.setName(finalFileName); newTemplTO.setSize(size); + newTemplTO.setPhysicalSize(size); newDestTO = newTemplTO; } else { VolumeObjectTO newVolTO = new VolumeObjectTO(); @@ -505,6 +506,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S TemplateObjectTO template = new TemplateObjectTO(); template.setPath(swiftPath); template.setSize(templateFile.length()); + template.setPhysicalSize(template.getSize()); SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData; template.setFormat(snapshot.getVolume().getFormat()); return new CopyCmdAnswer(template); @@ -718,6 +720,31 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } + protected Long getVirtualSize(File file, ImageFormat format) { + Processor processor = null; + try { + if (format == null) { + return file.length(); + } else if (format == ImageFormat.QCOW2) { + processor = new QCOW2Processor(); + } else if (format == ImageFormat.OVA) { + processor = new VmdkProcessor(); + } else if (format == ImageFormat.VHD) { + processor = new VhdProcessor(); + } + + if (processor == null) { + return file.length(); + } + + processor.configure("template processor", new HashMap()); + return processor.getVirtualSize(file); + } catch (Exception e) { + s_logger.debug("Failed to get virtual size:" ,e); + } + return file.length(); + } + protected Answer copyFromNfsToS3(CopyCommand cmd) { final DataTO srcData = cmd.getSrcTO(); final DataTO destData = cmd.getDestTO(); @@ -759,7 +786,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S if (destData.getObjectType() == DataObjectType.TEMPLATE) { TemplateObjectTO newTemplate = new TemplateObjectTO(); newTemplate.setPath(key); - newTemplate.setSize(srcFile.length()); + newTemplate.setSize(getVirtualSize(srcFile, format)); + newTemplate.setPhysicalSize(srcFile.length()); newTemplate.setFormat(format); retObj = newTemplate; } else if (destData.getObjectType() == DataObjectType.VOLUME) {