Support xz format for template registration (#11786)

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java

@@ -79,6 +79,7 @@ import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StorageLayer;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.Script;
+import com.cloud.utils.storage.TemplateDownloaderUtil;
 
 public class LibvirtStorageAdaptor implements StorageAdaptor {
     protected Logger logger = LogManager.getLogger(getClass());
@@ -172,37 +173,11 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
         return null;
     }
 
-    /**
-     * Checks if downloaded template is extractable
-     * @return true if it should be extracted, false if not
-     */
-    public static boolean isTemplateExtractable(String templatePath) {
-        String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'");
-        return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
-    }
-
-    /**
-     * Return extract command to execute given downloaded file
-     * @param downloadedTemplateFile
-     * @param templateUuid
-     */
-    public static String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateUuid) {
-        if (downloadedTemplateFile.endsWith(".zip")) {
-            return "unzip -p " + downloadedTemplateFile + " | cat > " + templateUuid;
-        } else if (downloadedTemplateFile.endsWith(".bz2")) {
-            return "bunzip2 -c " + downloadedTemplateFile + " > " + templateUuid;
-        } else if (downloadedTemplateFile.endsWith(".gz")) {
-            return "gunzip -c " + downloadedTemplateFile + " > " + templateUuid;
-        } else {
-            throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile);
-        }
-    }
-
     /**
      * Extract downloaded template into installPath, remove compressed file
      */
     public static void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, String destinationFile) {
-        String extractCommand = getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile);
+        String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile);
         Script.runSimpleBashScript(extractCommand);
         Script.runSimpleBashScript("rm -f " + downloadedTemplateFile);
     }
@@ -221,7 +196,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
 
         if (destPool.getType() == StoragePoolType.NetworkFilesystem || destPool.getType() == StoragePoolType.Filesystem
             || destPool.getType() == StoragePoolType.SharedMountPoint) {
-            if (!Storage.ImageFormat.ISO.equals(format) && isTemplateExtractable(templateFilePath)) {
+            if (!Storage.ImageFormat.ISO.equals(format) && TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) {
                 extractDownloadedTemplate(templateFilePath, destPool, destinationFile);
             } else {
                 Script.runSimpleBashScript("mv " + templateFilePath + " " + destinationFile);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java

@@ -423,24 +423,6 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor {
         throw new UnsupportedOperationException("Unimplemented method 'createPhysicalDisk'");
     }
 
-    boolean isTemplateExtractable(String templatePath) {
-        ScriptResult result = runScript("file", 5000L, templatePath, "| awk -F' ' '{print $2}'");
-        String type = result.getResult();
-        return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
-    }
-
-    String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) {
-        if (downloadedTemplateFile.endsWith(".zip")) {
-            return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile;
-        } else if (downloadedTemplateFile.endsWith(".bz2")) {
-            return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile;
-        } else if (downloadedTemplateFile.endsWith(".gz")) {
-            return "gunzip -c " + downloadedTemplateFile + " > " + templateFile;
-        } else {
-            throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile);
-        }
-    }
-
     boolean waitForDiskToBecomeAvailable(AddressInfo address, KVMStoragePool pool, long waitTimeInSec) {
         LOGGER.debug("Waiting for the volume with id: " + address.getPath() + " of the storage pool: " + pool.getUuid() + " to become available for " + waitTimeInSec + " secs");
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java

@@ -53,6 +53,8 @@ import com.cloud.utils.Ternary;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
+import com.cloud.utils.storage.TemplateDownloaderUtil;
+
 import org.apache.commons.lang3.StringUtils;
 
 public class ScaleIOStorageAdaptor implements StorageAdaptor {
@@ -572,10 +574,10 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
                 throw new CloudRuntimeException("Failed to find the disk: " + destTemplatePath + " of the storage pool: " + destPool.getUuid());
             }
 
-            if (isTemplateExtractable(templateFilePath)) {
+            if (TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) {
                 srcTemplateFilePath = sourceFile.getParent() + "/" + UUID.randomUUID().toString();
                 logger.debug("Extract the downloaded template " + templateFilePath + " to " + srcTemplateFilePath);
-                String extractCommand = getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath);
+                String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath);
                 Script.runSimpleBashScript(extractCommand);
                 Script.runSimpleBashScript("rm -f " + templateFilePath);
             }
@@ -611,23 +613,6 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
         return destDisk;
     }
 
-    private boolean isTemplateExtractable(String templatePath) {
-        String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'");
-        return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
-    }
-
-    private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) {
-        if (downloadedTemplateFile.endsWith(".zip")) {
-            return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile;
-        } else if (downloadedTemplateFile.endsWith(".bz2")) {
-            return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile;
-        } else if (downloadedTemplateFile.endsWith(".gz")) {
-            return "gunzip -c " + downloadedTemplateFile + " > " + templateFile;
-        } else {
-            throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile);
-        }
-    }
-
     public void resizeQcow2ToVolume(String volumePath, QemuImageOptions options, List<QemuObject> objects, Integer timeout) throws QemuImgException, LibvirtException {
         long rawSizeBytes = getPhysicalDiskSize(volumePath);
         long usableSizeBytes = getUsableBytesFromRawBytes(rawSizeBytes);

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java

@@ -38,6 +38,7 @@ import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 import org.libvirt.LibvirtException;
 
+import com.cloud.utils.storage.TemplateDownloaderUtil;
 import com.linbit.linstor.api.ApiClient;
 import com.linbit.linstor.api.ApiConsts;
 import com.linbit.linstor.api.ApiException;
@@ -694,7 +695,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor {
 
     private String getFinalDirectDownloadPath(String templateFilePath, KVMStoragePool destPool) {
         String finalSourcePath = templateFilePath;
-        if (LibvirtStorageAdaptor.isTemplateExtractable(templateFilePath)) {
+        if (TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) {
             finalSourcePath = templateFilePath.substring(0, templateFilePath.lastIndexOf('.'));
             LibvirtStorageAdaptor.extractDownloadedTemplate(templateFilePath, destPool, finalSourcePath);
         }

plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java

@@ -26,6 +26,7 @@ import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
 
+import com.cloud.utils.storage.TemplateDownloaderUtil;
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
@@ -528,32 +529,15 @@ public class StorPoolStorageAdaptor implements StorageAdaptor {
 
     private String extractTemplate(String templateFilePath, File sourceFile, String srcTemplateFilePath,
                                    String templateName) {
-        if (isTemplateExtractable(templateFilePath)) {
+        if (TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) {
             srcTemplateFilePath = sourceFile.getParent() + "/" + templateName;
-            String extractCommand = getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath);
+            String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath);
             Script.runSimpleBashScript(extractCommand);
             Script.runSimpleBashScript("rm -f " + templateFilePath);
         }
         return srcTemplateFilePath;
     }
 
-    private boolean isTemplateExtractable(String templatePath) {
-        String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'");
-        return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
-    }
-
-    private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) {
-        if (downloadedTemplateFile.endsWith(".zip")) {
-            return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile;
-        } else if (downloadedTemplateFile.endsWith(".bz2")) {
-            return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile;
-        } else if (downloadedTemplateFile.endsWith(".gz")) {
-            return "gunzip -c " + downloadedTemplateFile + " > " + templateFile;
-        } else {
-            throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile);
-        }
-    }
-
     private String getNameFromResponse(String resp, boolean tildeNeeded, boolean isSnapshot) {
         JsonParser jsonParser = new JsonParser();
         JsonObject respObj = (JsonObject) jsonParser.parse(resp);

scripts/installer/createtmplt.sh

@@ -87,6 +87,8 @@ uncompress() {
          ;;
   [zZ][iI][pP])  unzip -p $1 | cat > $tmpfile
         ;;
+  XZ)   xz -d -c $1 > $tmpfile
+        ;;
   *)	printf "$1"
         return 0
 	;;

scripts/installer/createvolume.sh

@@ -88,6 +88,8 @@ uncompress() {
          ;;
   ZIP)  unzip -p $1 | cat > $tmpfile
         ;;
+  XZ)   xz -d -c $1 > $tmpfile
+        ;;
   *)	printf "$1"
         return 0
 	;;

scripts/storage/qcow2/createtmplt.sh

@@ -65,6 +65,8 @@ uncompress() {
          ;;
   [zZ][iI][pP])  unzip -p $1 | cat > $tmpfile
         ;;
+  XZ)   xz -d -c $1 > $tmpfile
+        ;;
   *)	printf "$1"
         return 0
 	;;

scripts/storage/qcow2/createvolume.sh

@@ -66,6 +66,8 @@ uncompress() {
          ;;
   ZIP)  unzip -p $1 | cat > $tmpfile
         ;;
+  XZ)   xz -d -c $1 > $tmpfile
+        ;;
   *)	printf "$1"
         return 0
 	;;

scripts/storage/secondary/createtmplt.sh

@@ -63,6 +63,8 @@ is_compressed() {
          ;;
   [zZ][iI][pP])  ctype="zip"
         ;;
+  XZ)   ctype="xz"
+        ;;
     *) echo "File $1 does not appear to be compressed" >&2
         return 1
 	;;
@@ -82,6 +84,8 @@ uncompress() {
          ;;
   [zZ][iI][pP])  unzip -q -p $1 | cat > $tmpfile
         ;;
+  XZ)   xz -d -c $1 > $tmpfile
+        ;;
     *) printf "$1"
        return 0
 	;;

scripts/storage/secondary/createvolume.sh

@@ -85,6 +85,8 @@ is_compressed() {
          ;;
   ZIP)  ctype="zip"
         ;;
+  XZ)   ctype="xz"
+        ;;
     *) echo "File $1 does not appear to be compressed" >&2
         return 1
 	;;
@@ -104,6 +106,8 @@ uncompress() {
          ;;
   ZIP)  unzip -q -p $1 | cat > $tmpfile
         ;;
+  XZ)   xz -d -c $1 > $tmpfile
+        ;;
     *) printf "$1"
        return 0
 	;;

tools/devcloud4/common/development-installation/files/default/createtmplt.sh

@@ -75,6 +75,8 @@ is_compressed() {
          ;;
   ZIP)  ctype="zip"
         ;;
+  XZ)  ctype="xz"
+        ;;
     *) echo "File $1 does not appear to be compressed" >&2
         return 1
 	;;
@@ -94,6 +96,8 @@ uncompress() {
          ;;
   ZIP)  unzip -q -p $1 | cat > $tmpfile
         ;;
+  XZ)   xz -d -c $1 > $tmpfile
+        ;;
     *) printf "$1"
        return 0
 	;;

utils/src/main/java/com/cloud/utils/UriUtils.java

@@ -431,7 +431,7 @@ public class UriUtils {
         return urls;
     }
 
-    public static final Set<String> COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz");
+    public static final Set<String> COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz", "xz");
 
     public static final Set<String> buildExtensionSet(boolean metalink, String... baseExtensions) {
         final ImmutableSet.Builder<String> builder = ImmutableSet.builder();

utils/src/main/java/com/cloud/utils/storage/TemplateDownloaderUtil.java

@@ -0,0 +1,61 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.utils.storage;
+
+import org.apache.commons.io.FilenameUtils;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+
+public class TemplateDownloaderUtil {
+
+    private TemplateDownloaderUtil() {}
+
+    /**
+     * Checks if downloaded template is extractable
+     * @return true if it should be extracted, false if not
+     */
+    public static boolean isTemplateExtractable(String templatePath) {
+        String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'");
+        return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip")
+                || type.equalsIgnoreCase("zip") || type.equalsIgnoreCase("xz");
+    }
+
+    /**
+     * Return extract command to execute given downloaded file
+     * @param downloadedTemplateFile
+     * @param templateFile
+     */
+    public static String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) {
+        String extension = FilenameUtils.getExtension(downloadedTemplateFile).toLowerCase();
+        switch (extension) {
+            case "zip":
+                return String.format("unzip -p '%s' | cat > '%s'", downloadedTemplateFile, templateFile);
+            case "bz2":
+                return String.format("bunzip2 -c '%s' > '%s'", downloadedTemplateFile, templateFile);
+            case "gz":
+                return String.format("gunzip -c '%s' > '%s'", downloadedTemplateFile, templateFile);
+            case "xz":
+                return String.format("xz -d -c '%s' > '%s'", downloadedTemplateFile, templateFile);
+            default:
+                throw new CloudRuntimeException("Unable to extract template: " + downloadedTemplateFile + " (unsupported format: ." + extension + ")");
+        }
+    }
+}

utils/src/test/java/com/cloud/utils/UriUtilsParametrizedTest.java

@@ -44,7 +44,7 @@ public class UriUtilsParametrizedTest {
     }
 
     private static final Set<String> COMMPRESSION_FORMATS = ImmutableSet.of("",".zip", ".bz2", ".gz");
-    private static final Set<String> ILLEGAL_COMMPRESSION_FORMATS = ImmutableSet.of(".7z", ".xz");
+    private static final Set<String> ILLEGAL_COMMPRESSION_FORMATS = ImmutableSet.of(".7z");
     private final static Set<String> FORMATS = ImmutableSet.of(
             "vhd",
             "vhdx",

utils/src/test/java/com/cloud/utils/UriUtilsTest.java

@@ -271,6 +271,7 @@ public class UriUtilsTest {
         Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.bz2"));
         Assert.assertTrue(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.zip"));
         Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.gz"));
+        Assert.assertTrue(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.xz"));
         Assert.assertFalse(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.qcow2"));
     }