diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 02e5c08fa33..e8924ecf5eb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -79,6 +79,7 @@ import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageLayer; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import com.cloud.utils.storage.TemplateDownloaderUtil; public class LibvirtStorageAdaptor implements StorageAdaptor { protected Logger logger = LogManager.getLogger(getClass()); @@ -172,37 +173,11 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { return null; } - /** - * Checks if downloaded template is extractable - * @return true if it should be extracted, false if not - */ - public static boolean isTemplateExtractable(String templatePath) { - String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'"); - return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); - } - - /** - * Return extract command to execute given downloaded file - * @param downloadedTemplateFile - * @param templateUuid - */ - public static String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateUuid) { - if (downloadedTemplateFile.endsWith(".zip")) { - return "unzip -p " + downloadedTemplateFile + " | cat > " + templateUuid; - } else if (downloadedTemplateFile.endsWith(".bz2")) { - return "bunzip2 -c " + downloadedTemplateFile + " > " + templateUuid; - } else if (downloadedTemplateFile.endsWith(".gz")) { - return "gunzip -c " + downloadedTemplateFile + " > " + templateUuid; - } else { - throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile); - } - } - /** * Extract downloaded template into installPath, remove compressed file */ public static void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, String destinationFile) { - String extractCommand = getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile); + String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile); Script.runSimpleBashScript(extractCommand); Script.runSimpleBashScript("rm -f " + downloadedTemplateFile); } @@ -221,7 +196,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { if (destPool.getType() == StoragePoolType.NetworkFilesystem || destPool.getType() == StoragePoolType.Filesystem || destPool.getType() == StoragePoolType.SharedMountPoint) { - if (!Storage.ImageFormat.ISO.equals(format) && isTemplateExtractable(templateFilePath)) { + if (!Storage.ImageFormat.ISO.equals(format) && TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) { extractDownloadedTemplate(templateFilePath, destPool, destinationFile); } else { Script.runSimpleBashScript("mv " + templateFilePath + " " + destinationFile); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java index 7ba29ffc26e..514e412559c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java @@ -423,24 +423,6 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { throw new UnsupportedOperationException("Unimplemented method 'createPhysicalDisk'"); } - boolean isTemplateExtractable(String templatePath) { - ScriptResult result = runScript("file", 5000L, templatePath, "| awk -F' ' '{print $2}'"); - String type = result.getResult(); - return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); - } - - String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) { - if (downloadedTemplateFile.endsWith(".zip")) { - return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile; - } else if (downloadedTemplateFile.endsWith(".bz2")) { - return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile; - } else if (downloadedTemplateFile.endsWith(".gz")) { - return "gunzip -c " + downloadedTemplateFile + " > " + templateFile; - } else { - throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile); - } - } - boolean waitForDiskToBecomeAvailable(AddressInfo address, KVMStoragePool pool, long waitTimeInSec) { LOGGER.debug("Waiting for the volume with id: " + address.getPath() + " of the storage pool: " + pool.getUuid() + " to become available for " + waitTimeInSec + " secs"); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index efa8024a34b..e336e0e5a54 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -53,6 +53,8 @@ import com.cloud.utils.Ternary; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; +import com.cloud.utils.storage.TemplateDownloaderUtil; + import org.apache.commons.lang3.StringUtils; public class ScaleIOStorageAdaptor implements StorageAdaptor { @@ -572,10 +574,10 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { throw new CloudRuntimeException("Failed to find the disk: " + destTemplatePath + " of the storage pool: " + destPool.getUuid()); } - if (isTemplateExtractable(templateFilePath)) { + if (TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) { srcTemplateFilePath = sourceFile.getParent() + "/" + UUID.randomUUID().toString(); logger.debug("Extract the downloaded template " + templateFilePath + " to " + srcTemplateFilePath); - String extractCommand = getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath); + String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath); Script.runSimpleBashScript(extractCommand); Script.runSimpleBashScript("rm -f " + templateFilePath); } @@ -611,23 +613,6 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { return destDisk; } - private boolean isTemplateExtractable(String templatePath) { - String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'"); - return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); - } - - private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) { - if (downloadedTemplateFile.endsWith(".zip")) { - return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile; - } else if (downloadedTemplateFile.endsWith(".bz2")) { - return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile; - } else if (downloadedTemplateFile.endsWith(".gz")) { - return "gunzip -c " + downloadedTemplateFile + " > " + templateFile; - } else { - throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile); - } - } - public void resizeQcow2ToVolume(String volumePath, QemuImageOptions options, List objects, Integer timeout) throws QemuImgException, LibvirtException { long rawSizeBytes = getPhysicalDiskSize(volumePath); long usableSizeBytes = getUsableBytesFromRawBytes(rawSizeBytes); diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index a664e7ed03b..a1ba41b1d6e 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -38,6 +38,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.libvirt.LibvirtException; +import com.cloud.utils.storage.TemplateDownloaderUtil; import com.linbit.linstor.api.ApiClient; import com.linbit.linstor.api.ApiConsts; import com.linbit.linstor.api.ApiException; @@ -694,7 +695,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor { private String getFinalDirectDownloadPath(String templateFilePath, KVMStoragePool destPool) { String finalSourcePath = templateFilePath; - if (LibvirtStorageAdaptor.isTemplateExtractable(templateFilePath)) { + if (TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) { finalSourcePath = templateFilePath.substring(0, templateFilePath.lastIndexOf('.')); LibvirtStorageAdaptor.extractDownloadedTemplate(templateFilePath, destPool, finalSourcePath); } diff --git a/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java b/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java index 4dcc67f3f06..545f7b33c5f 100644 --- a/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java +++ b/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java @@ -26,6 +26,7 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; +import com.cloud.utils.storage.TemplateDownloaderUtil; import com.google.gson.Gson; import com.google.gson.JsonObject; import com.google.gson.JsonParser; @@ -528,32 +529,15 @@ public class StorPoolStorageAdaptor implements StorageAdaptor { private String extractTemplate(String templateFilePath, File sourceFile, String srcTemplateFilePath, String templateName) { - if (isTemplateExtractable(templateFilePath)) { + if (TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) { srcTemplateFilePath = sourceFile.getParent() + "/" + templateName; - String extractCommand = getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath); + String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath); Script.runSimpleBashScript(extractCommand); Script.runSimpleBashScript("rm -f " + templateFilePath); } return srcTemplateFilePath; } - private boolean isTemplateExtractable(String templatePath) { - String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'"); - return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); - } - - private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) { - if (downloadedTemplateFile.endsWith(".zip")) { - return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile; - } else if (downloadedTemplateFile.endsWith(".bz2")) { - return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile; - } else if (downloadedTemplateFile.endsWith(".gz")) { - return "gunzip -c " + downloadedTemplateFile + " > " + templateFile; - } else { - throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile); - } - } - private String getNameFromResponse(String resp, boolean tildeNeeded, boolean isSnapshot) { JsonParser jsonParser = new JsonParser(); JsonObject respObj = (JsonObject) jsonParser.parse(resp); diff --git a/scripts/installer/createtmplt.sh b/scripts/installer/createtmplt.sh index db3cfd71ee1..b9b403a94bb 100755 --- a/scripts/installer/createtmplt.sh +++ b/scripts/installer/createtmplt.sh @@ -87,6 +87,8 @@ uncompress() { ;; [zZ][iI][pP]) unzip -p $1 | cat > $tmpfile ;; + XZ) xz -d -c $1 > $tmpfile + ;; *) printf "$1" return 0 ;; diff --git a/scripts/installer/createvolume.sh b/scripts/installer/createvolume.sh index 52792e9fc3a..4726404b76a 100755 --- a/scripts/installer/createvolume.sh +++ b/scripts/installer/createvolume.sh @@ -88,6 +88,8 @@ uncompress() { ;; ZIP) unzip -p $1 | cat > $tmpfile ;; + XZ) xz -d -c $1 > $tmpfile + ;; *) printf "$1" return 0 ;; diff --git a/scripts/storage/qcow2/createtmplt.sh b/scripts/storage/qcow2/createtmplt.sh index 4bb955e130c..de7010b0a25 100755 --- a/scripts/storage/qcow2/createtmplt.sh +++ b/scripts/storage/qcow2/createtmplt.sh @@ -65,6 +65,8 @@ uncompress() { ;; [zZ][iI][pP]) unzip -p $1 | cat > $tmpfile ;; + XZ) xz -d -c $1 > $tmpfile + ;; *) printf "$1" return 0 ;; diff --git a/scripts/storage/qcow2/createvolume.sh b/scripts/storage/qcow2/createvolume.sh index ff04ee3fe5e..b2ce249a3d1 100755 --- a/scripts/storage/qcow2/createvolume.sh +++ b/scripts/storage/qcow2/createvolume.sh @@ -66,6 +66,8 @@ uncompress() { ;; ZIP) unzip -p $1 | cat > $tmpfile ;; + XZ) xz -d -c $1 > $tmpfile + ;; *) printf "$1" return 0 ;; diff --git a/scripts/storage/secondary/createtmplt.sh b/scripts/storage/secondary/createtmplt.sh index ed9bbe869e1..cfc4be28a01 100755 --- a/scripts/storage/secondary/createtmplt.sh +++ b/scripts/storage/secondary/createtmplt.sh @@ -63,6 +63,8 @@ is_compressed() { ;; [zZ][iI][pP]) ctype="zip" ;; + XZ) ctype="xz" + ;; *) echo "File $1 does not appear to be compressed" >&2 return 1 ;; @@ -82,6 +84,8 @@ uncompress() { ;; [zZ][iI][pP]) unzip -q -p $1 | cat > $tmpfile ;; + XZ) xz -d -c $1 > $tmpfile + ;; *) printf "$1" return 0 ;; diff --git a/scripts/storage/secondary/createvolume.sh b/scripts/storage/secondary/createvolume.sh index ac69ebe88ab..a47d1fdc2d3 100755 --- a/scripts/storage/secondary/createvolume.sh +++ b/scripts/storage/secondary/createvolume.sh @@ -85,6 +85,8 @@ is_compressed() { ;; ZIP) ctype="zip" ;; + XZ) ctype="xz" + ;; *) echo "File $1 does not appear to be compressed" >&2 return 1 ;; @@ -104,6 +106,8 @@ uncompress() { ;; ZIP) unzip -q -p $1 | cat > $tmpfile ;; + XZ) xz -d -c $1 > $tmpfile + ;; *) printf "$1" return 0 ;; diff --git a/tools/devcloud4/common/development-installation/files/default/createtmplt.sh b/tools/devcloud4/common/development-installation/files/default/createtmplt.sh index 2f8f3421350..ca5022bae4f 100755 --- a/tools/devcloud4/common/development-installation/files/default/createtmplt.sh +++ b/tools/devcloud4/common/development-installation/files/default/createtmplt.sh @@ -75,6 +75,8 @@ is_compressed() { ;; ZIP) ctype="zip" ;; + XZ) ctype="xz" + ;; *) echo "File $1 does not appear to be compressed" >&2 return 1 ;; @@ -94,6 +96,8 @@ uncompress() { ;; ZIP) unzip -q -p $1 | cat > $tmpfile ;; + XZ) xz -d -c $1 > $tmpfile + ;; *) printf "$1" return 0 ;; diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index 961c121597f..42603237888 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -431,7 +431,7 @@ public class UriUtils { return urls; } - public static final Set COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz"); + public static final Set COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz", "xz"); public static final Set buildExtensionSet(boolean metalink, String... baseExtensions) { final ImmutableSet.Builder builder = ImmutableSet.builder(); diff --git a/utils/src/main/java/com/cloud/utils/storage/TemplateDownloaderUtil.java b/utils/src/main/java/com/cloud/utils/storage/TemplateDownloaderUtil.java new file mode 100644 index 00000000000..0a729478468 --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/storage/TemplateDownloaderUtil.java @@ -0,0 +1,61 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.storage; + +import org.apache.commons.io.FilenameUtils; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; + +public class TemplateDownloaderUtil { + + private TemplateDownloaderUtil() {} + + /** + * Checks if downloaded template is extractable + * @return true if it should be extracted, false if not + */ + public static boolean isTemplateExtractable(String templatePath) { + String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'"); + return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") + || type.equalsIgnoreCase("zip") || type.equalsIgnoreCase("xz"); + } + + /** + * Return extract command to execute given downloaded file + * @param downloadedTemplateFile + * @param templateFile + */ + public static String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) { + String extension = FilenameUtils.getExtension(downloadedTemplateFile).toLowerCase(); + switch (extension) { + case "zip": + return String.format("unzip -p '%s' | cat > '%s'", downloadedTemplateFile, templateFile); + case "bz2": + return String.format("bunzip2 -c '%s' > '%s'", downloadedTemplateFile, templateFile); + case "gz": + return String.format("gunzip -c '%s' > '%s'", downloadedTemplateFile, templateFile); + case "xz": + return String.format("xz -d -c '%s' > '%s'", downloadedTemplateFile, templateFile); + default: + throw new CloudRuntimeException("Unable to extract template: " + downloadedTemplateFile + " (unsupported format: ." + extension + ")"); + } + } +} diff --git a/utils/src/test/java/com/cloud/utils/UriUtilsParametrizedTest.java b/utils/src/test/java/com/cloud/utils/UriUtilsParametrizedTest.java index 3f1e707c607..2a4c5c34102 100644 --- a/utils/src/test/java/com/cloud/utils/UriUtilsParametrizedTest.java +++ b/utils/src/test/java/com/cloud/utils/UriUtilsParametrizedTest.java @@ -44,7 +44,7 @@ public class UriUtilsParametrizedTest { } private static final Set COMMPRESSION_FORMATS = ImmutableSet.of("",".zip", ".bz2", ".gz"); - private static final Set ILLEGAL_COMMPRESSION_FORMATS = ImmutableSet.of(".7z", ".xz"); + private static final Set ILLEGAL_COMMPRESSION_FORMATS = ImmutableSet.of(".7z"); private final static Set FORMATS = ImmutableSet.of( "vhd", "vhdx", diff --git a/utils/src/test/java/com/cloud/utils/UriUtilsTest.java b/utils/src/test/java/com/cloud/utils/UriUtilsTest.java index 47fd389e3ae..8e6408f7783 100644 --- a/utils/src/test/java/com/cloud/utils/UriUtilsTest.java +++ b/utils/src/test/java/com/cloud/utils/UriUtilsTest.java @@ -271,6 +271,7 @@ public class UriUtilsTest { Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.bz2")); Assert.assertTrue(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.zip")); Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.gz")); + Assert.assertTrue(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.xz")); Assert.assertFalse(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.qcow2")); }