From b0247b53f99ff97fd05d4d8528115ed6e7d497c0 Mon Sep 17 00:00:00 2001 From: Syed Date: Mon, 27 Jun 2016 16:11:14 -0400 Subject: [PATCH] [CLOUDSTACK-9423] Add ability to get virtual size of compressed VHDs --- core/pom.xml | 5 ++ .../cloud/storage/template/VhdProcessor.java | 85 +++++++++++++++--- .../storage/template/VhdProcessorTest.java | 36 +++++++- core/test/resources/vhds/test.vhd | Bin 0 -> 3584 bytes core/test/resources/vhds/test.vhd.bz2 | Bin 0 -> 212 bytes core/test/resources/vhds/test.vhd.gz | Bin 0 -> 163 bytes core/test/resources/vhds/test.vhd.zip | Bin 0 -> 341 bytes 7 files changed, 113 insertions(+), 13 deletions(-) create mode 100644 core/test/resources/vhds/test.vhd create mode 100644 core/test/resources/vhds/test.vhd.bz2 create mode 100644 core/test/resources/vhds/test.vhd.gz create mode 100644 core/test/resources/vhds/test.vhd.zip diff --git a/core/pom.xml b/core/pom.xml index 60dc02ff576..f50f8af7f9b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -47,6 +47,11 @@ commons-codec commons-codec + + org.apache.commons + commons-compress + 1.10 + diff --git a/core/src/com/cloud/storage/template/VhdProcessor.java b/core/src/com/cloud/storage/template/VhdProcessor.java index 605d2a376e1..cb13d06687b 100644 --- a/core/src/com/cloud/storage/template/VhdProcessor.java +++ b/core/src/com/cloud/storage/template/VhdProcessor.java @@ -19,20 +19,24 @@ package com.cloud.storage.template; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.Map; - -import javax.naming.ConfigurationException; - import com.cloud.exception.InternalErrorException; -import org.apache.log4j.Logger; - import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; import com.cloud.utils.NumbersUtil; import com.cloud.utils.component.AdapterBase; +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorInputStream; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.log4j.Logger; + +import javax.naming.ConfigurationException; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; /** * VhdProcessor processes the downloaded template for VHD. It @@ -45,10 +49,12 @@ public class VhdProcessor extends AdapterBase implements Processor { private static final Logger s_logger = Logger.getLogger(VhdProcessor.class); StorageLayer _storage; private int vhdFooterSize = 512; + private int vhdCookieOffset = 8; private int vhdFooterCreatorAppOffset = 28; private int vhdFooterCreatorVerOffset = 32; private int vhdFooterCurrentSizeOffset = 48; private byte[][] citrixCreatorApp = { {0x74, 0x61, 0x70, 0x00}, {0x43, 0x54, 0x58, 0x53}}; /*"tap ", and "CTXS"*/ + private String vhdIdentifierCookie = "conectix"; @Override public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { @@ -93,10 +99,32 @@ public class VhdProcessor extends AdapterBase implements Processor { protected long getTemplateVirtualSize(File file) throws IOException { byte[] currentSize = new byte[8]; + byte[] cookie = new byte[8]; byte[] creatorApp = new byte[4]; - try (FileInputStream strm = new FileInputStream(file)) { - long skipped = strm.skip(file.length() - vhdFooterSize + vhdFooterCreatorAppOffset); + + BufferedInputStream fileStream = new BufferedInputStream(new FileInputStream(file)); + InputStream strm = fileStream; + + boolean isCompressed = checkCompressed(file.getAbsolutePath()); + + if ( isCompressed ) { + try { + strm = new CompressorStreamFactory().createCompressorInputStream(fileStream); + } catch (CompressorException e) { + s_logger.info("error opening compressed VHD file " + file.getName()); + return file.length(); + } + } try { + + //read the backup footer present at the top of the VHD file + strm.read(cookie); + if (! new String(cookie).equals(vhdIdentifierCookie)) { + strm.close(); + return file.length(); + } + + long skipped = strm.skip(vhdFooterCreatorAppOffset - vhdCookieOffset); if (skipped == -1) { throw new IOException("Unexpected end-of-file"); } @@ -104,7 +132,7 @@ public class VhdProcessor extends AdapterBase implements Processor { if (read == -1) { throw new IOException("Unexpected end-of-file"); } - skipped = strm.skip(vhdFooterCurrentSizeOffset - vhdFooterCreatorVerOffset); + skipped = strm.skip(vhdFooterCurrentSizeOffset - vhdFooterCreatorVerOffset - vhdCookieOffset); if (skipped == -1) { throw new IOException("Unexpected end-of-file"); } @@ -112,6 +140,13 @@ public class VhdProcessor extends AdapterBase implements Processor { if (read == -1) { throw new IOException("Unexpected end-of-file"); } + } catch (IOException e) { + s_logger.warn("Error reading virtual size from VHD file " + e.getMessage() + " VHD: " + file.getName()); + return file.length(); + } finally { + if (strm != null) { + strm.close(); + } } return NumbersUtil.bytesToLong(currentSize); @@ -128,4 +163,30 @@ public class VhdProcessor extends AdapterBase implements Processor { return true; } + private boolean checkCompressed(String fileName) throws IOException { + + FileInputStream fin = null; + BufferedInputStream bin = null; + CompressorInputStream cin = null; + + try { + fin = new FileInputStream(fileName); + bin = new BufferedInputStream(fin); + cin = new CompressorStreamFactory().createCompressorInputStream(bin); + + } catch (CompressorException e) { + s_logger.warn(e.getMessage()); + return false; + + } catch (FileNotFoundException e) { + s_logger.warn(e.getMessage()); + return false; + } finally { + if (cin != null) + cin.close(); + else if (bin != null) + bin.close(); + } + return true; + } } diff --git a/core/test/com/cloud/storage/template/VhdProcessorTest.java b/core/test/com/cloud/storage/template/VhdProcessorTest.java index b8b362a017b..3c695d7c250 100644 --- a/core/test/com/cloud/storage/template/VhdProcessorTest.java +++ b/core/test/com/cloud/storage/template/VhdProcessorTest.java @@ -32,6 +32,8 @@ import org.mockito.runners.MockitoJUnitRunner; import java.io.File; import java.io.IOException; +import java.net.URLDecoder; +import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; @@ -107,4 +109,36 @@ public class VhdProcessorTest { Assert.assertEquals(virtualSize, processor.getVirtualSize(mockFile)); Mockito.verify(mockFile,Mockito.times(0)).length(); } -} \ No newline at end of file + + @Test + public void testVhdGetVirtualSize() throws Exception { + String vhdPath = URLDecoder.decode(getClass().getResource("/vhds/test.vhd").getFile(), Charset.defaultCharset().name()); + long expectedVirtualSizeBytes = 104857600; + long actualVirtualSizeBytes = processor.getVirtualSize(new File(vhdPath)); + Assert.assertEquals(expectedVirtualSizeBytes, actualVirtualSizeBytes); + } + + @Test + public void testGzipVhdGetVirtualSize() throws Exception { + String gzipVhdPath = URLDecoder.decode(getClass().getResource("/vhds/test.vhd.gz").getFile(), Charset.defaultCharset().name()); + long expectedVirtualSizeBytes = 104857600; + long actualVirtualSizeBytes = processor.getVirtualSize(new File(gzipVhdPath)); + Assert.assertEquals(expectedVirtualSizeBytes, actualVirtualSizeBytes); + } + + @Test + public void testBzip2VhdGetVirtualSize() throws Exception { + String bzipVhdPath = URLDecoder.decode(getClass().getResource("/vhds/test.vhd.bz2").getFile(), Charset.defaultCharset().name()); + long expectedVirtualSizeBytes = 104857600; + long actualVirtualSizeBytes = processor.getVirtualSize(new File(bzipVhdPath)); + Assert.assertEquals(expectedVirtualSizeBytes, actualVirtualSizeBytes); + } + + @Test + public void testZipVhdGetVirtualSize() throws Exception { + String zipVhdPath = URLDecoder.decode(getClass().getResource("/vhds/test.vhd.zip").getFile(), Charset.defaultCharset().name()); + long expectedVirtualSizeBytes = 341; //Zip is not a supported format, virtual size should return the filesize as a fallback + long actualVirtualSizeBytes = processor.getVirtualSize(new File(zipVhdPath)); + Assert.assertEquals(expectedVirtualSizeBytes, actualVirtualSizeBytes); + } +} diff --git a/core/test/resources/vhds/test.vhd b/core/test/resources/vhds/test.vhd new file mode 100644 index 0000000000000000000000000000000000000000..87d5f274a9fd815a1d15fa26b6eff9239b31002c GIT binary patch literal 3584 zcmYe!&r3}%$*f>tU|?ck1d?FD#30XNv#cbsfB`7R3=?H@fUrO`^J^YKAdmU~|Mx+2 zb~W7pveo^`G%>xb6GvVuLR5{CLm>o`D~byei;7eKLjlCiY@m1t0V4(l28RFtzqo>V zqr_+kjE2Cl3ISNYAL{h5iWE9}rX(dPu_PB%(SraNh+qUlCQy0+aYn&t2#kin@C||F N(f0rFja(vp1^^@x1!w>O literal 0 HcmV?d00001 diff --git a/core/test/resources/vhds/test.vhd.bz2 b/core/test/resources/vhds/test.vhd.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..02821b63ec207ec04c50b9dcfdd628abe8365ea7 GIT binary patch literal 212 zcmV;_04x7OT4*^jL0KkKS!Mb!CIA3rfB*dyApqb35(Gkk5CA?i+(1A80R#ac0uTg1 z0Dz1rgb1(zve4Bu0GeP3WWrz!fK8y0Ng4piXaE2J2AT;{X+03r1Z2cuhJXR$nny$; zj_Il*sX{y4AQ&qy0|`bTk=Ghl1_-?69j1Y%1Pky$q#8u5CMnVyoS5cDryhrdmkt1Z zLB-|}*Kb?k2&qPhivcqQ1wle-R^uf?y$M-~@b*@07;7n1;MwoIM? literal 0 HcmV?d00001 diff --git a/core/test/resources/vhds/test.vhd.gz b/core/test/resources/vhds/test.vhd.gz new file mode 100644 index 0000000000000000000000000000000000000000..e48bf2bf0d17168408b886372156a098e347485a GIT binary patch literal 163 zcmV;U09^kciwFQ_fN@s<153`&OHD4xtYBbZU}9iofC454c@~>xC5Z(LKrv>RD4PR< zW`4~h2xKz<|NlN{&aQ_0U$(kGnI@)}b>hfNMTSu@WI%F7aY15Haq534fQYg&K+}Q| zg8~D?|NmcH2hN3~U=)l3T7mz=1PpXI4pmT+l9X7I3##ZrfD1%00wL3An2v%`F#JLC RX#0Oe0{{tJ^Y{P`0017ULeKyJ literal 0 HcmV?d00001 diff --git a/core/test/resources/vhds/test.vhd.zip b/core/test/resources/vhds/test.vhd.zip new file mode 100644 index 0000000000000000000000000000000000000000..4857d5a131c17bf6c3799675c70df14f223fc7a4 GIT binary patch literal 341 zcmWIWW@gc4U|`^2SXy)2gEQvykN;4>$RNU?%g7+Wz`zHT2ZD(ZioyHbd2e66lWP)y zASofGfk}<6QfzjrY9}j?gJIeXjoHjiX?16nIZ}*(CfQ}2Ti5nGe%mhlnQF_Xmp(e- z?b;S-km69}TIiS*Soo@5qoHGyp~K}S!Dc4r!~g%s$FQGkU7R4&!xYu{|1b+ri3MM1 zz@$l2A|`RKdg0i>?P9>BcsXN6UsIsLAIDD_zw2F?83Mc+ndF#JeF1hX(BX_wM}w#( zjUZAtz?+Rt2gCvbh&BitrW8i&LxdTQKxv>pb`bM3ln+uD;LXYg;xYmuNIgU?0B_D& AD*ylh literal 0 HcmV?d00001