apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-25 17:22:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

Support xz format for template registration (#11786)

Browse Source
This commit is contained in:
Manoj Kumar 2025-10-14 12:43:12 +05:30 committed by GitHub
parent dfcbd2e977
commit 9e535e35d2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 96 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java
View File

@ -79,6 +79,7 @@ import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.storage.StorageLayer;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import com.cloud.utils.storage.TemplateDownloaderUtil;
public class LibvirtStorageAdaptor implements StorageAdaptor {
protected Logger logger = LogManager.getLogger(getClass());
@ -172,37 +173,11 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
return null;
}
/**
* Checks if downloaded template is extractable
* @return true if it should be extracted, false if not
*/
public static boolean isTemplateExtractable(String templatePath) {
String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'");
return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
}
/**
* Return extract command to execute given downloaded file
* @param downloadedTemplateFile
* @param templateUuid
*/
public static String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateUuid) {
if (downloadedTemplateFile.endsWith(".zip")) {
return "unzip -p " + downloadedTemplateFile + " | cat > " + templateUuid;
} else if (downloadedTemplateFile.endsWith(".bz2")) {
return "bunzip2 -c " + downloadedTemplateFile + " > " + templateUuid;
} else if (downloadedTemplateFile.endsWith(".gz")) {
return "gunzip -c " + downloadedTemplateFile + " > " + templateUuid;
} else {
throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile);
}
}
/**
* Extract downloaded template into installPath, remove compressed file
*/
public static void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, String destinationFile) {
String extractCommand = getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile);
String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile);
Script.runSimpleBashScript(extractCommand);
Script.runSimpleBashScript("rm -f " + downloadedTemplateFile);
}
@ -221,7 +196,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
if (destPool.getType() == StoragePoolType.NetworkFilesystem || destPool.getType() == StoragePoolType.Filesystem
|| destPool.getType() == StoragePoolType.SharedMountPoint) {
if (!Storage.ImageFormat.ISO.equals(format) && isTemplateExtractable(templateFilePath)) {
if (!Storage.ImageFormat.ISO.equals(format) && TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) {
extractDownloadedTemplate(templateFilePath, destPool, destinationFile);
} else {
Script.runSimpleBashScript("mv " + templateFilePath + " " + destinationFile);

18
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java
View File

@ -423,24 +423,6 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor {
throw new UnsupportedOperationException("Unimplemented method 'createPhysicalDisk'");
}
boolean isTemplateExtractable(String templatePath) {
ScriptResult result = runScript("file", 5000L, templatePath, "| awk -F' ' '{print $2}'");
String type = result.getResult();
return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
}
String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) {
if (downloadedTemplateFile.endsWith(".zip")) {
return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile;
} else if (downloadedTemplateFile.endsWith(".bz2")) {
return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile;
} else if (downloadedTemplateFile.endsWith(".gz")) {
return "gunzip -c " + downloadedTemplateFile + " > " + templateFile;
} else {
throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile);
}
}
boolean waitForDiskToBecomeAvailable(AddressInfo address, KVMStoragePool pool, long waitTimeInSec) {
LOGGER.debug("Waiting for the volume with id: " + address.getPath() + " of the storage pool: " + pool.getUuid() + " to become available for " + waitTimeInSec + " secs");

23
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
View File

@ -53,6 +53,8 @@ import com.cloud.utils.Ternary;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
import com.cloud.utils.storage.TemplateDownloaderUtil;
import org.apache.commons.lang3.StringUtils;
public class ScaleIOStorageAdaptor implements StorageAdaptor {
@ -572,10 +574,10 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
throw new CloudRuntimeException("Failed to find the disk: " + destTemplatePath + " of the storage pool: " + destPool.getUuid());
}
if (isTemplateExtractable(templateFilePath)) {
if (TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) {
srcTemplateFilePath = sourceFile.getParent() + "/" + UUID.randomUUID().toString();
logger.debug("Extract the downloaded template " + templateFilePath + " to " + srcTemplateFilePath);
String extractCommand = getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath);
String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath);
Script.runSimpleBashScript(extractCommand);
Script.runSimpleBashScript("rm -f " + templateFilePath);
}
@ -611,23 +613,6 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
return destDisk;
}
private boolean isTemplateExtractable(String templatePath) {
String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'");
return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
}
private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) {
if (downloadedTemplateFile.endsWith(".zip")) {
return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile;
} else if (downloadedTemplateFile.endsWith(".bz2")) {
return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile;
} else if (downloadedTemplateFile.endsWith(".gz")) {
return "gunzip -c " + downloadedTemplateFile + " > " + templateFile;
} else {
throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile);
}
}
public void resizeQcow2ToVolume(String volumePath, QemuImageOptions options, List<QemuObject> objects, Integer timeout) throws QemuImgException, LibvirtException {
long rawSizeBytes = getPhysicalDiskSize(volumePath);
long usableSizeBytes = getUsableBytesFromRawBytes(rawSizeBytes);

3
plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java
View File

@ -38,6 +38,7 @@ import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.libvirt.LibvirtException;
import com.cloud.utils.storage.TemplateDownloaderUtil;
import com.linbit.linstor.api.ApiClient;
import com.linbit.linstor.api.ApiConsts;
import com.linbit.linstor.api.ApiException;
@ -694,7 +695,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor {
private String getFinalDirectDownloadPath(String templateFilePath, KVMStoragePool destPool) {
String finalSourcePath = templateFilePath;
if (LibvirtStorageAdaptor.isTemplateExtractable(templateFilePath)) {
if (TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) {
finalSourcePath = templateFilePath.substring(0, templateFilePath.lastIndexOf('.'));
LibvirtStorageAdaptor.extractDownloadedTemplate(templateFilePath, destPool, finalSourcePath);
}

22
plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java
View File

@ -26,6 +26,7 @@ import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
import com.cloud.utils.storage.TemplateDownloaderUtil;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
@ -528,32 +529,15 @@ public class StorPoolStorageAdaptor implements StorageAdaptor {
private String extractTemplate(String templateFilePath, File sourceFile, String srcTemplateFilePath,
String templateName) {
if (isTemplateExtractable(templateFilePath)) {
if (TemplateDownloaderUtil.isTemplateExtractable(templateFilePath)) {
srcTemplateFilePath = sourceFile.getParent() + "/" + templateName;
String extractCommand = getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath);
String extractCommand = TemplateDownloaderUtil.getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath);
Script.runSimpleBashScript(extractCommand);
Script.runSimpleBashScript("rm -f " + templateFilePath);
}
return srcTemplateFilePath;
}
private boolean isTemplateExtractable(String templatePath) {
String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'");
return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
}
private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) {
if (downloadedTemplateFile.endsWith(".zip")) {
return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile;
} else if (downloadedTemplateFile.endsWith(".bz2")) {
return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile;
} else if (downloadedTemplateFile.endsWith(".gz")) {
return "gunzip -c " + downloadedTemplateFile + " > " + templateFile;
} else {
throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile);
}
}
private String getNameFromResponse(String resp, boolean tildeNeeded, boolean isSnapshot) {
JsonParser jsonParser = new JsonParser();
JsonObject respObj = (JsonObject) jsonParser.parse(resp);

2
scripts/installer/createtmplt.sh
View File

@ -87,6 +87,8 @@ uncompress() {
;;
[zZ][iI][pP]) unzip -p $1 | cat > $tmpfile
;;
XZ) xz -d -c $1 > $tmpfile
;;
*) printf "$1"
return 0
;;

2
scripts/installer/createvolume.sh
View File

@ -88,6 +88,8 @@ uncompress() {
;;
ZIP) unzip -p $1 | cat > $tmpfile
;;
XZ) xz -d -c $1 > $tmpfile
;;
*) printf "$1"
return 0
;;

2
scripts/storage/qcow2/createtmplt.sh
View File

@ -65,6 +65,8 @@ uncompress() {
;;
[zZ][iI][pP]) unzip -p $1 | cat > $tmpfile
;;
XZ) xz -d -c $1 > $tmpfile
;;
*) printf "$1"
return 0
;;

2
scripts/storage/qcow2/createvolume.sh
View File

@ -66,6 +66,8 @@ uncompress() {
;;
ZIP) unzip -p $1 | cat > $tmpfile
;;
XZ) xz -d -c $1 > $tmpfile
;;
*) printf "$1"
return 0
;;

4
scripts/storage/secondary/createtmplt.sh
View File

@ -63,6 +63,8 @@ is_compressed() {
;;
[zZ][iI][pP]) ctype="zip"
;;
XZ) ctype="xz"
;;
*) echo "File $1 does not appear to be compressed" >&2
return 1
;;
@ -82,6 +84,8 @@ uncompress() {
;;
[zZ][iI][pP]) unzip -q -p $1 | cat > $tmpfile
;;
XZ) xz -d -c $1 > $tmpfile
;;
*) printf "$1"
return 0
;;

4
scripts/storage/secondary/createvolume.sh
View File

@ -85,6 +85,8 @@ is_compressed() {
;;
ZIP) ctype="zip"
;;
XZ) ctype="xz"
;;
*) echo "File $1 does not appear to be compressed" >&2
return 1
;;
@ -104,6 +106,8 @@ uncompress() {
;;
ZIP) unzip -q -p $1 | cat > $tmpfile
;;
XZ) xz -d -c $1 > $tmpfile
;;
*) printf "$1"
return 0
;;

4
tools/devcloud4/common/development-installation/files/default/createtmplt.sh
View File

@ -75,6 +75,8 @@ is_compressed() {
;;
ZIP) ctype="zip"
;;
XZ) ctype="xz"
;;
*) echo "File $1 does not appear to be compressed" >&2
return 1
;;
@ -94,6 +96,8 @@ uncompress() {
;;
ZIP) unzip -q -p $1 | cat > $tmpfile
;;
XZ) xz -d -c $1 > $tmpfile
;;
*) printf "$1"
return 0
;;

2
utils/src/main/java/com/cloud/utils/UriUtils.java
View File

@ -431,7 +431,7 @@ public class UriUtils {
return urls;
}
public static final Set<String> COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz");
public static final Set<String> COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz", "xz");
public static final Set<String> buildExtensionSet(boolean metalink, String... baseExtensions) {
final ImmutableSet.Builder<String> builder = ImmutableSet.builder();

61
utils/src/main/java/com/cloud/utils/storage/TemplateDownloaderUtil.java Normal file
View File

@ -0,0 +1,61 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.utils.storage;
import org.apache.commons.io.FilenameUtils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
public class TemplateDownloaderUtil {
private TemplateDownloaderUtil() {}
/**
* Checks if downloaded template is extractable
* @return true if it should be extracted, false if not
*/
public static boolean isTemplateExtractable(String templatePath) {
String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'");
return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip")
|| type.equalsIgnoreCase("zip") || type.equalsIgnoreCase("xz");
}
/**
* Return extract command to execute given downloaded file
* @param downloadedTemplateFile
* @param templateFile
*/
public static String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) {
String extension = FilenameUtils.getExtension(downloadedTemplateFile).toLowerCase();
switch (extension) {
case "zip":
return String.format("unzip -p '%s' | cat > '%s'", downloadedTemplateFile, templateFile);
case "bz2":
return String.format("bunzip2 -c '%s' > '%s'", downloadedTemplateFile, templateFile);
case "gz":
return String.format("gunzip -c '%s' > '%s'", downloadedTemplateFile, templateFile);
case "xz":
return String.format("xz -d -c '%s' > '%s'", downloadedTemplateFile, templateFile);
default:
throw new CloudRuntimeException("Unable to extract template: " + downloadedTemplateFile + " (unsupported format: ." + extension + ")");
}
}
}

2
utils/src/test/java/com/cloud/utils/UriUtilsParametrizedTest.java
View File

@ -44,7 +44,7 @@ public class UriUtilsParametrizedTest {
}
private static final Set<String> COMMPRESSION_FORMATS = ImmutableSet.of("",".zip", ".bz2", ".gz");
private static final Set<String> ILLEGAL_COMMPRESSION_FORMATS = ImmutableSet.of(".7z", ".xz");
private static final Set<String> ILLEGAL_COMMPRESSION_FORMATS = ImmutableSet.of(".7z");
private final static Set<String> FORMATS = ImmutableSet.of(
"vhd",
"vhdx",

1
utils/src/test/java/com/cloud/utils/UriUtilsTest.java
View File

@ -271,6 +271,7 @@ public class UriUtilsTest {
Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.bz2"));
Assert.assertTrue(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.zip"));
Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.gz"));
Assert.assertTrue(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.xz"));
Assert.assertFalse(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.qcow2"));
}