From 685ee9e78f63a41ec27fe04f75fc26639debe2a5 Mon Sep 17 00:00:00 2001 From: slavkap <51903378+slavkap@users.noreply.github.com> Date: Sat, 14 Jun 2025 12:19:37 +0300 Subject: [PATCH] StorPool: support for direct download (#9833) --- .../kvm/storage/StorPoolStorageAdaptor.java | 222 ++++++++++++++++-- 1 file changed, 203 insertions(+), 19 deletions(-) diff --git a/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java b/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java index d68e9cbd110..f6cc3c1e216 100644 --- a/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java +++ b/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java @@ -17,6 +17,29 @@ package com.cloud.hypervisor.kvm.storage; +import com.cloud.agent.api.to.DiskTO; +import com.cloud.storage.Storage; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Storage.ProvisioningType; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import org.apache.cloudstack.storage.datastore.util.StorPoolUtil; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.libvirt.LibvirtException; + import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; @@ -26,19 +49,7 @@ import java.util.Calendar; import java.util.HashMap; import java.util.List; import java.util.Map; - -import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; - -import com.cloud.agent.api.to.DiskTO; -import com.cloud.storage.Storage; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.Storage.ProvisioningType; -import com.cloud.storage.Storage.StoragePoolType; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.OutputInterpreter; -import com.cloud.utils.script.Script; +import java.util.UUID; public class StorPoolStorageAdaptor implements StorageAdaptor { public static void SP_LOG(String fmt, Object... args) { @@ -149,6 +160,10 @@ public class StorPoolStorageAdaptor implements StorageAdaptor { } public static boolean attachOrDetachVolume(String command, String type, String volumeUuid) { + if (volumeUuid == null) { + LOGGER.debug("Could not attach volume. The volume ID is null"); + return false; + } final String name = getVolumeNameFromPath(volumeUuid, true); if (name == null) { return false; @@ -345,11 +360,85 @@ public class StorPoolStorageAdaptor implements StorageAdaptor { throw new UnsupportedOperationException("A folder cannot be created in this configuration."); } - public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, - KVMStoragePool destPool, ImageFormat format, int timeout) { + @Override + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, + PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { return null; } + @Override + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, + KVMStoragePool destPool, ImageFormat format, int timeout) { + if (StringUtils.isEmpty(templateFilePath) || destPool == null) { + throw new CloudRuntimeException( + "Unable to create template from direct download template file due to insufficient data"); + } + + File sourceFile = new File(templateFilePath); + if (!sourceFile.exists()) { + throw new CloudRuntimeException( + "Direct download template file " + templateFilePath + " does not exist on this host"); + } + + if (!StoragePoolType.StorPool.equals(destPool.getType())) { + throw new CloudRuntimeException("Unsupported storage pool type: " + destPool.getType().toString()); + } + + if (!Storage.ImageFormat.QCOW2.equals(format)) { + throw new CloudRuntimeException("Unsupported template format: " + format.toString()); + } + + String srcTemplateFilePath = templateFilePath; + KVMPhysicalDisk destDisk = null; + QemuImgFile srcFile = null; + QemuImgFile destFile = null; + String templateName = UUID.randomUUID().toString(); + String volume = null; + try { + + srcTemplateFilePath = extractTemplate(templateFilePath, sourceFile, srcTemplateFilePath, templateName); + + QemuImg.PhysicalDiskFormat srcFileFormat = QemuImg.PhysicalDiskFormat.QCOW2; + + srcFile = new QemuImgFile(srcTemplateFilePath, srcFileFormat); + + String spTemplate = destPool.getUuid().split(";")[0]; + + QemuImg qemu = new QemuImg(timeout); + OutputInterpreter.AllLinesParser parser = createStorPoolVolume(destPool, srcFile, qemu, spTemplate); + + String response = parser.getLines(); + + LOGGER.debug(response); + volume = StorPoolUtil.devPath(getNameFromResponse(response, false, false)); + attachOrDetachVolume("attach", "volume", volume); + destDisk = destPool.getPhysicalDisk(volume); + if (destDisk == null) { + throw new CloudRuntimeException( + "Failed to find the disk: " + volume + " of the storage pool: " + destPool.getUuid()); + } + + destFile = new QemuImgFile(destDisk.getPath(), QemuImg.PhysicalDiskFormat.RAW); + + qemu.convert(srcFile, destFile); + parser = volumeSnapshot(StorPoolStorageAdaptor.getVolumeNameFromPath(volume, true), spTemplate); + response = parser.getLines(); + LOGGER.debug(response); + String newPath = StorPoolUtil.devPath(getNameFromResponse(response, false, true)); + destDisk = destPool.getPhysicalDisk(newPath); + } catch (QemuImgException | LibvirtException e) { + destDisk = null; + } finally { + if (volume != null) { + attachOrDetachVolume("detach", "volume", volume); + volumeDelete(StorPoolStorageAdaptor.getVolumeNameFromPath(volume, true)); + } + Script.runSimpleBashScript("rm -f " + srcTemplateFilePath); + } + + return destDisk; + } + @Override public boolean createFolder(String uuid, String path, String localPath) { return false; @@ -367,9 +456,104 @@ public class StorPoolStorageAdaptor implements StorageAdaptor { return null; } - @Override - public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, - PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { - return null; + private OutputInterpreter.AllLinesParser createStorPoolVolume(KVMStoragePool destPool, QemuImgFile srcFile, + QemuImg qemu, String templateUuid) throws QemuImgException, LibvirtException { + Map info = qemu.info(srcFile); + Map reqParams = new HashMap<>(); + reqParams.put("template", templateUuid); + reqParams.put("size", info.get("virtual_size")); + Map tags = new HashMap<>(); + tags.put("cs", "template"); + reqParams.put("tags", tags); + Gson gson = new Gson(); + String js = gson.toJson(reqParams); + + Script sc = createStorPoolRequest(js, "VolumeCreate", null,true); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + + String res = sc.execute(parser); + if (res != null) { + throw new CloudRuntimeException("Could not create volume due to: " + res); + } + return parser; + } + + private OutputInterpreter.AllLinesParser volumeSnapshot(String volumeName, String templateUuid) { + Map reqParams = new HashMap<>(); + reqParams.put("template", templateUuid); + Gson gson = new Gson(); + String js = gson.toJson(reqParams); + + Script sc = createStorPoolRequest(js, "VolumeSnapshot", volumeName,true); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + + String res = sc.execute(parser); + if (res != null) { + throw new CloudRuntimeException("Could not snapshot volume due to: " + res); + } + return parser; + } + + private OutputInterpreter.AllLinesParser volumeDelete(String volumeName) { + Script sc = createStorPoolRequest(null, "VolumeDelete", volumeName, false); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + + String res = sc.execute(parser); + if (res != null) { + throw new CloudRuntimeException("Could not delete volume due to: " + res); + } + return parser; + } + @NotNull + private static Script createStorPoolRequest(String js, String apiCall, String param, boolean jsonRequired) { + Script sc = new Script("storpool_req", 0, LOGGER); + sc.add("-P"); + sc.add("-M"); + if (jsonRequired) { + sc.add("--json"); + sc.add(js); + } + sc.add(apiCall); + if (param != null) { + sc.add(param); + } + return sc; + } + + private String extractTemplate(String templateFilePath, File sourceFile, String srcTemplateFilePath, + String templateName) { + if (isTemplateExtractable(templateFilePath)) { + srcTemplateFilePath = sourceFile.getParent() + "/" + templateName; + String extractCommand = getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath); + Script.runSimpleBashScript(extractCommand); + Script.runSimpleBashScript("rm -f " + templateFilePath); + } + return srcTemplateFilePath; + } + + private boolean isTemplateExtractable(String templatePath) { + String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'"); + return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); + } + + private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) { + if (downloadedTemplateFile.endsWith(".zip")) { + return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile; + } else if (downloadedTemplateFile.endsWith(".bz2")) { + return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile; + } else if (downloadedTemplateFile.endsWith(".gz")) { + return "gunzip -c " + downloadedTemplateFile + " > " + templateFile; + } else { + throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile); + } + } + + private String getNameFromResponse(String resp, boolean tildeNeeded, boolean isSnapshot) { + JsonParser jsonParser = new JsonParser(); + JsonObject respObj = (JsonObject) jsonParser.parse(resp); + JsonPrimitive data = isSnapshot ? respObj.getAsJsonPrimitive("snapshotGlobalId") : respObj.getAsJsonPrimitive("globalId"); + String name = data !=null ? data.getAsString() : null; + name = name != null ? name.startsWith("~") && !tildeNeeded ? name.split("~")[1] : name : name; + return name; } }