[CLOUDSTACK-9423] Add ability to get virtual size of compressed VHDs

core/pom.xml

@@ -47,6 +47,11 @@
       <groupId>commons-codec</groupId>
       <artifactId>commons-codec</artifactId>
     </dependency>
+      <dependency>
+          <groupId>org.apache.commons</groupId>
+          <artifactId>commons-compress</artifactId>
+          <version>1.10</version>
+      </dependency>
   </dependencies>
 
   <build>

core/src/com/cloud/storage/template/VhdProcessor.java

@@ -19,20 +19,24 @@
 
 package com.cloud.storage.template;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Map;
-
-import javax.naming.ConfigurationException;
-
 import com.cloud.exception.InternalErrorException;
-import org.apache.log4j.Logger;
-
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.StorageLayer;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.component.AdapterBase;
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
+import org.apache.log4j.Logger;
+
+import javax.naming.ConfigurationException;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
 
 /**
  * VhdProcessor processes the downloaded template for VHD.  It
@@ -45,10 +49,12 @@ public class VhdProcessor extends AdapterBase implements Processor {
     private static final Logger s_logger = Logger.getLogger(VhdProcessor.class);
     StorageLayer _storage;
     private int vhdFooterSize = 512;
+    private int vhdCookieOffset = 8;
     private int vhdFooterCreatorAppOffset = 28;
     private int vhdFooterCreatorVerOffset = 32;
     private int vhdFooterCurrentSizeOffset = 48;
     private byte[][] citrixCreatorApp = { {0x74, 0x61, 0x70, 0x00}, {0x43, 0x54, 0x58, 0x53}}; /*"tap ", and "CTXS"*/
+    private String vhdIdentifierCookie = "conectix";
 
     @Override
     public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException {
@@ -93,10 +99,32 @@ public class VhdProcessor extends AdapterBase implements Processor {
 
     protected long getTemplateVirtualSize(File file) throws IOException {
         byte[] currentSize = new byte[8];
+        byte[] cookie = new byte[8];
         byte[] creatorApp = new byte[4];
 
-        try (FileInputStream strm = new FileInputStream(file)) {
-            long skipped = strm.skip(file.length() - vhdFooterSize + vhdFooterCreatorAppOffset);
+
+        BufferedInputStream fileStream = new BufferedInputStream(new FileInputStream(file));
+        InputStream strm = fileStream;
+
+        boolean isCompressed = checkCompressed(file.getAbsolutePath());
+
+        if ( isCompressed ) {
+            try {
+                strm = new CompressorStreamFactory().createCompressorInputStream(fileStream);
+            } catch (CompressorException e) {
+                s_logger.info("error opening compressed VHD file " + file.getName());
+                return file.length();
+            }
+        } try {
+
+            //read the backup footer present at the top of the VHD file
+            strm.read(cookie);
+            if (! new String(cookie).equals(vhdIdentifierCookie)) {
+                strm.close();
+                return  file.length();
+            }
+
+            long skipped = strm.skip(vhdFooterCreatorAppOffset - vhdCookieOffset);
             if (skipped == -1) {
                 throw new IOException("Unexpected end-of-file");
             }
@@ -104,7 +132,7 @@ public class VhdProcessor extends AdapterBase implements Processor {
             if (read == -1) {
                 throw new IOException("Unexpected end-of-file");
             }
-            skipped = strm.skip(vhdFooterCurrentSizeOffset - vhdFooterCreatorVerOffset);
+            skipped = strm.skip(vhdFooterCurrentSizeOffset - vhdFooterCreatorVerOffset - vhdCookieOffset);
             if (skipped == -1) {
                 throw new IOException("Unexpected end-of-file");
             }
@@ -112,6 +140,13 @@ public class VhdProcessor extends AdapterBase implements Processor {
             if (read == -1) {
                 throw new IOException("Unexpected end-of-file");
             }
+        } catch (IOException e) {
+            s_logger.warn("Error reading virtual size from VHD file " + e.getMessage() + " VHD: " + file.getName());
+            return file.length();
+        } finally {
+            if (strm != null) {
+                strm.close();
+            }
         }
 
         return NumbersUtil.bytesToLong(currentSize);
@@ -128,4 +163,30 @@ public class VhdProcessor extends AdapterBase implements Processor {
         return true;
     }
 
+    private boolean checkCompressed(String fileName) throws IOException {
+
+        FileInputStream fin = null;
+        BufferedInputStream bin = null;
+        CompressorInputStream cin = null;
+
+        try {
+            fin = new FileInputStream(fileName);
+            bin = new BufferedInputStream(fin);
+            cin = new CompressorStreamFactory().createCompressorInputStream(bin);
+
+        } catch (CompressorException e) {
+            s_logger.warn(e.getMessage());
+            return false;
+
+        } catch (FileNotFoundException e) {
+            s_logger.warn(e.getMessage());
+            return false;
+        } finally {
+            if (cin != null)
+                cin.close();
+            else if (bin != null)
+                bin.close();
+        }
+        return true;
+    }
 }

core/test/com/cloud/storage/template/VhdProcessorTest.java

@@ -32,6 +32,8 @@ import org.mockito.runners.MockitoJUnitRunner;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URLDecoder;
+import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -107,4 +109,36 @@ public class VhdProcessorTest {
         Assert.assertEquals(virtualSize, processor.getVirtualSize(mockFile));
         Mockito.verify(mockFile,Mockito.times(0)).length();
     }
-}
+
+    @Test
+    public void testVhdGetVirtualSize() throws Exception {
+        String vhdPath = URLDecoder.decode(getClass().getResource("/vhds/test.vhd").getFile(), Charset.defaultCharset().name());
+        long expectedVirtualSizeBytes = 104857600;
+        long actualVirtualSizeBytes = processor.getVirtualSize(new File(vhdPath));
+        Assert.assertEquals(expectedVirtualSizeBytes, actualVirtualSizeBytes);
+    }
+
+    @Test
+    public void testGzipVhdGetVirtualSize() throws Exception {
+        String gzipVhdPath = URLDecoder.decode(getClass().getResource("/vhds/test.vhd.gz").getFile(), Charset.defaultCharset().name());
+        long expectedVirtualSizeBytes = 104857600;
+        long actualVirtualSizeBytes = processor.getVirtualSize(new File(gzipVhdPath));
+        Assert.assertEquals(expectedVirtualSizeBytes, actualVirtualSizeBytes);
+    }
+
+    @Test
+    public void testBzip2VhdGetVirtualSize() throws Exception {
+        String bzipVhdPath = URLDecoder.decode(getClass().getResource("/vhds/test.vhd.bz2").getFile(), Charset.defaultCharset().name());
+        long expectedVirtualSizeBytes = 104857600;
+        long actualVirtualSizeBytes = processor.getVirtualSize(new File(bzipVhdPath));
+        Assert.assertEquals(expectedVirtualSizeBytes, actualVirtualSizeBytes);
+    }
+
+    @Test
+    public void testZipVhdGetVirtualSize() throws Exception {
+        String zipVhdPath = URLDecoder.decode(getClass().getResource("/vhds/test.vhd.zip").getFile(), Charset.defaultCharset().name());
+        long expectedVirtualSizeBytes = 341; //Zip is not a supported format, virtual size should return the filesize as a fallback
+        long actualVirtualSizeBytes = processor.getVirtualSize(new File(zipVhdPath));
+        Assert.assertEquals(expectedVirtualSizeBytes, actualVirtualSizeBytes);
+    }
+}