Fix volume snapshot of encrypted NFS/StorPool volume (#8873)

* Fix volume snapshot of encrypted NFS/StorPool volume

* remove comments

* removed invoking the real qemu convert command

* fix UnsatisfiedLink error in unit tests

* addressed comments

extracted method
This commit is contained in:
slavkap 2024-06-24 10:39:21 +03:00 committed by GitHub
parent 3e30283500
commit 8b07b66f14
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 190 additions and 128 deletions

View File

@ -18,73 +18,6 @@
*/
package com.cloud.hypervisor.kvm.storage;
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
import static com.cloud.utils.storage.S3.S3Utils.putFile;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.naming.ConfigurationException;
import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.direct.download.DirectDownloadHelper;
import org.apache.cloudstack.direct.download.DirectTemplateDownloader;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.CreateObjectAnswer;
import org.apache.cloudstack.storage.command.CreateObjectCommand;
import org.apache.cloudstack.storage.command.DeleteCommand;
import org.apache.cloudstack.storage.command.DettachAnswer;
import org.apache.cloudstack.storage.command.DettachCommand;
import org.apache.cloudstack.storage.command.ForgetObjectCmd;
import org.apache.cloudstack.storage.command.IntroduceObjectCmd;
import org.apache.cloudstack.storage.command.ResignatureAnswer;
import org.apache.cloudstack.storage.command.ResignatureCommand;
import org.apache.cloudstack.storage.command.SnapshotAndCopyAnswer;
import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand;
import org.apache.cloudstack.storage.command.SyncVolumePathCommand;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.cloudstack.utils.qemu.QemuImg;
import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
import org.apache.cloudstack.utils.qemu.QemuImgException;
import org.apache.cloudstack.utils.qemu.QemuImgFile;
import org.apache.cloudstack.utils.qemu.QemuObject;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import org.apache.log4j.Logger;
import org.libvirt.Connect;
import org.libvirt.Domain;
import org.libvirt.DomainInfo;
import org.libvirt.DomainSnapshot;
import org.libvirt.LibvirtException;
import com.ceph.rados.IoCTX;
import com.ceph.rados.Rados;
import com.ceph.rados.exceptions.ErrorCode;
@ -133,6 +66,75 @@ import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
import com.cloud.utils.storage.S3.S3Utils;
import com.cloud.vm.VmDetailConstants;
import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.direct.download.DirectDownloadHelper;
import org.apache.cloudstack.direct.download.DirectTemplateDownloader;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.CreateObjectAnswer;
import org.apache.cloudstack.storage.command.CreateObjectCommand;
import org.apache.cloudstack.storage.command.DeleteCommand;
import org.apache.cloudstack.storage.command.DettachAnswer;
import org.apache.cloudstack.storage.command.DettachCommand;
import org.apache.cloudstack.storage.command.ForgetObjectCmd;
import org.apache.cloudstack.storage.command.IntroduceObjectCmd;
import org.apache.cloudstack.storage.command.ResignatureAnswer;
import org.apache.cloudstack.storage.command.ResignatureCommand;
import org.apache.cloudstack.storage.command.SnapshotAndCopyAnswer;
import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand;
import org.apache.cloudstack.storage.command.SyncVolumePathCommand;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.cloudstack.utils.cryptsetup.KeyFile;
import org.apache.cloudstack.utils.qemu.QemuImageOptions;
import org.apache.cloudstack.utils.qemu.QemuImg;
import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
import org.apache.cloudstack.utils.qemu.QemuImgException;
import org.apache.cloudstack.utils.qemu.QemuImgFile;
import org.apache.cloudstack.utils.qemu.QemuObject;
import org.apache.cloudstack.utils.qemu.QemuObject.EncryptFormat;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import org.apache.log4j.Logger;
import org.libvirt.Connect;
import org.libvirt.Domain;
import org.libvirt.DomainInfo;
import org.libvirt.DomainSnapshot;
import org.libvirt.LibvirtException;
import javax.naming.ConfigurationException;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
import static com.cloud.utils.storage.S3.S3Utils.putFile;
public class KVMStorageProcessor implements StorageProcessor {
private static final Logger s_logger = Logger.getLogger(KVMStorageProcessor.class);
@ -1744,7 +1746,7 @@ public class KVMStorageProcessor implements StorageProcessor {
snapshotPath = getSnapshotPathInPrimaryStorage(primaryPool.getLocalPath(), snapshotName);
String diskLabel = takeVolumeSnapshot(resource.getDisks(conn, vmName), snapshotName, diskPath, vm);
String convertResult = convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPool, diskPath, snapshotPath, volume, cmd.getWait());
String convertResult = convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPool, disk, snapshotPath, volume, cmd.getWait());
mergeSnapshotIntoBaseFile(vm, diskLabel, diskPath, snapshotName, volume, conn);
@ -1813,7 +1815,7 @@ public class KVMStorageProcessor implements StorageProcessor {
}
} else {
snapshotPath = getSnapshotPathInPrimaryStorage(primaryPool.getLocalPath(), snapshotName);
String convertResult = convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPool, diskPath, snapshotPath, volume, cmd.getWait());
String convertResult = convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPool, disk, snapshotPath, volume, cmd.getWait());
validateConvertResult(convertResult, snapshotPath);
}
}
@ -1936,26 +1938,43 @@ public class KVMStorageProcessor implements StorageProcessor {
* @param snapshotPath Path to convert the base file;
* @return null if the conversion occurs successfully or an error message that must be handled.
*/
protected String convertBaseFileToSnapshotFileInPrimaryStorageDir(KVMStoragePool primaryPool, String baseFile, String snapshotPath, VolumeObjectTO volume, int wait) {
try {
s_logger.debug(String.format("Trying to convert volume [%s] (%s) to snapshot [%s].", volume, baseFile, snapshotPath));
protected String convertBaseFileToSnapshotFileInPrimaryStorageDir(KVMStoragePool primaryPool,
KVMPhysicalDisk baseFile, String snapshotPath, VolumeObjectTO volume, int wait) {
try (KeyFile srcKey = new KeyFile(volume.getPassphrase())) {
s_logger.debug(
String.format("Trying to convert volume [%s] (%s) to snapshot [%s].", volume, baseFile, snapshotPath));
primaryPool.createFolder(TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR);
QemuImgFile srcFile = new QemuImgFile(baseFile);
srcFile.setFormat(PhysicalDiskFormat.QCOW2);
QemuImgFile destFile = new QemuImgFile(snapshotPath);
destFile.setFormat(PhysicalDiskFormat.QCOW2);
QemuImg q = new QemuImg(wait);
q.convert(srcFile, destFile);
s_logger.debug(String.format("Converted volume [%s] (from path \"%s\") to snapshot [%s].", volume, baseFile, snapshotPath));
return null;
} catch (QemuImgException | LibvirtException ex) {
return String.format("Failed to convert %s snapshot of volume [%s] to [%s] due to [%s].", volume, baseFile, snapshotPath, ex.getMessage());
convertTheBaseFileToSnapshot(baseFile, snapshotPath, wait, srcKey);
} catch (QemuImgException | LibvirtException | IOException ex) {
return String.format("Failed to convert %s snapshot of volume [%s] to [%s] due to [%s].", volume, baseFile,
snapshotPath, ex.getMessage());
}
s_logger.debug(String.format("Converted volume [%s] (from path \"%s\") to snapshot [%s].", volume, baseFile,
snapshotPath));
return null;
}
private void convertTheBaseFileToSnapshot(KVMPhysicalDisk baseFile, String snapshotPath, int wait, KeyFile srcKey)
throws LibvirtException, QemuImgException {
List<QemuObject> qemuObjects = new ArrayList<>();
Map<String, String> options = new HashMap<>();
QemuImageOptions qemuImageOpts = new QemuImageOptions(baseFile.getPath());
if (srcKey.isSet()) {
String srcKeyName = "sec0";
qemuObjects.add(QemuObject.prepareSecretForQemuImg(baseFile.getFormat(), EncryptFormat.LUKS,
srcKey.toString(), srcKeyName, options));
qemuImageOpts = new QemuImageOptions(baseFile.getFormat(), baseFile.getPath(), srcKeyName);
}
QemuImgFile srcFile = new QemuImgFile(baseFile.getPath());
srcFile.setFormat(PhysicalDiskFormat.QCOW2);
QemuImgFile destFile = new QemuImgFile(snapshotPath);
destFile.setFormat(PhysicalDiskFormat.QCOW2);
QemuImg q = new QemuImg(wait);
q.convert(srcFile, destFile, options, qemuObjects, qemuImageOpts, null, true);
}
/**

View File

@ -26,19 +26,10 @@ import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtUtilitiesHelper;
import com.cloud.storage.template.TemplateConstants;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import javax.naming.ConfigurationException;
import com.cloud.utils.script.Script;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.cloudstack.utils.qemu.QemuImageOptions;
import org.apache.cloudstack.utils.qemu.QemuImg;
import org.apache.cloudstack.utils.qemu.QemuImgException;
import org.apache.cloudstack.utils.qemu.QemuImgFile;
@ -59,6 +50,17 @@ import org.mockito.MockitoAnnotations;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import javax.naming.ConfigurationException;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@RunWith(MockitoJUnitRunner.class)
public class KVMStorageProcessorTest {
@ -259,40 +261,48 @@ public class KVMStorageProcessorTest {
}
@Test
public void convertBaseFileToSnapshotFileInPrimaryStorageDirTestFailToConvertWithQemuImgExceptionReturnErrorMessage() throws Exception {
String baseFile = "baseFile";
String snapshotPath = "snapshotPath";
public void convertBaseFileToSnapshotFileInPrimaryStorageDirTestFailToConvertWithQemuImgExceptionReturnErrorMessage() throws QemuImgException {
KVMPhysicalDisk baseFile = Mockito.mock(KVMPhysicalDisk.class);
String errorMessage = "error";
String expectedResult = String.format("Failed to convert %s snapshot of volume [%s] to [%s] due to [%s].", volumeObjectToMock, baseFile, snapshotPath, errorMessage);
KVMStoragePool primaryPoolMock = Mockito.mock(KVMStoragePool.class);
KVMPhysicalDisk baseFileMock = Mockito.mock(KVMPhysicalDisk.class);
VolumeObjectTO volumeMock = Mockito.mock(VolumeObjectTO.class);
QemuImgFile srcFileMock = Mockito.mock(QemuImgFile.class);
QemuImgFile destFileMock = Mockito.mock(QemuImgFile.class);
QemuImg qemuImgMock = Mockito.mock(QemuImg.class);
Mockito.doReturn(true).when(kvmStoragePoolMock).createFolder(Mockito.anyString());
try (MockedConstruction<QemuImg> ignored = Mockito.mockConstruction(QemuImg.class, (mock,context) -> {
Mockito.doThrow(new QemuImgException(errorMessage)).when(mock).convert(Mockito.any(QemuImgFile.class), Mockito.any(QemuImgFile.class));
})) {
String result = storageProcessorSpy.convertBaseFileToSnapshotFileInPrimaryStorageDir(kvmStoragePoolMock, baseFile, snapshotPath, volumeObjectToMock, 1);
Assert.assertEquals(expectedResult, result);
Mockito.when(baseFileMock.getPath()).thenReturn("/path/to/baseFile");
Mockito.when(primaryPoolMock.createFolder(Mockito.anyString())).thenReturn(true);
try (MockedConstruction<Script> scr = Mockito.mockConstruction(Script.class, ((mock, context) -> {
Mockito.doReturn("").when(mock).execute();
}));
MockedConstruction<QemuImg> qemu = Mockito.mockConstruction(QemuImg.class, ((mock, context) -> {
Mockito.lenient().doThrow(new QemuImgException(errorMessage)).when(mock).convert(Mockito.any(QemuImgFile.class), Mockito.any(QemuImgFile.class), Mockito.any(Map.class),
Mockito.any(List.class), Mockito.any(QemuImageOptions.class),Mockito.nullable(String.class), Mockito.any(Boolean.class));
}))) {
String test = storageProcessor.convertBaseFileToSnapshotFileInPrimaryStorageDir(primaryPoolMock, baseFileMock, "/path/to/snapshot", volumeMock, 0);
Assert.assertNotNull(test);
}
}
@Test
public void convertBaseFileToSnapshotFileInPrimaryStorageDirTestFailToConvertWithLibvirtExceptionReturnErrorMessage() throws Exception {
String baseFile = "baseFile";
KVMPhysicalDisk baseFile = Mockito.mock(KVMPhysicalDisk.class);
String snapshotPath = "snapshotPath";
String errorMessage = "null";
String expectedResult = String.format("Failed to convert %s snapshot of volume [%s] to [%s] due to [%s].", volumeObjectToMock, baseFile, snapshotPath, errorMessage);
QemuImg qemuImg = Mockito.mock(QemuImg.class);
Mockito.doReturn(true).when(kvmStoragePoolMock).createFolder(Mockito.anyString());
try (MockedConstruction<QemuImg> ignored = Mockito.mockConstruction(QemuImg.class, (mock,context) -> {
Mockito.doThrow(LibvirtException.class).when(mock).convert(Mockito.any(QemuImgFile.class), Mockito.any(QemuImgFile.class));
try (MockedConstruction<QemuImg> ignored = Mockito.mockConstructionWithAnswer(QemuImg.class, invocation -> {
throw Mockito.mock(LibvirtException.class);
})) {
String result = storageProcessorSpy.convertBaseFileToSnapshotFileInPrimaryStorageDir(kvmStoragePoolMock, baseFile, snapshotPath, volumeObjectToMock, 1);
Assert.assertEquals(expectedResult, result);
Assert.assertNotNull(result);
}
}
@Test
public void convertBaseFileToSnapshotFileInPrimaryStorageDirTestConvertSuccessReturnNull() throws Exception {
String baseFile = "baseFile";
KVMPhysicalDisk baseFile = Mockito.mock(KVMPhysicalDisk.class);
String snapshotPath = "snapshotPath";
Mockito.doReturn(true).when(kvmStoragePoolMock).createFolder(Mockito.anyString());

View File

@ -22,12 +22,22 @@ package com.cloud.hypervisor.kvm.resource.wrapper;
import static com.cloud.hypervisor.kvm.storage.StorPoolStorageAdaptor.SP_LOG;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.utils.cryptsetup.KeyFile;
import org.apache.cloudstack.utils.qemu.QemuImageOptions;
import org.apache.cloudstack.utils.qemu.QemuImg;
import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
import org.apache.cloudstack.utils.qemu.QemuImgException;
import org.apache.cloudstack.utils.qemu.QemuImgFile;
import org.apache.cloudstack.utils.qemu.QemuObject;
import org.apache.cloudstack.utils.qemu.QemuObject.EncryptFormat;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
@ -59,28 +69,24 @@ public final class StorPoolBackupSnapshotCommandWrapper extends CommandWrapper<S
SP_LOG("StorpoolBackupSnapshotCommandWrapper.execute: src=" + src.getPath() + "dst=" + dst.getPath());
StorPoolStorageAdaptor.attachOrDetachVolume("attach", "snapshot", src.getPath());
srcPath = src.getPath();
final QemuImgFile srcFile = new QemuImgFile(srcPath, PhysicalDiskFormat.RAW);
long size = 0;
String srcKeyName = "sec0";
String destKeyName = "sec1";
List<QemuObject> qemuObjects = new ArrayList<>();
Map<String, String> options = new HashMap<>();
QemuImageOptions qemuImageOpts = new QemuImageOptions(srcPath);
final QemuImg qemu = new QemuImg(cmd.getWaitInMillSeconds());
final DataStoreTO dstDataStore = dst.getDataStore();
if (!(dstDataStore instanceof NfsTO)) {
return new CopyCmdAnswer("Backup Storpool snapshot: Only NFS secondary supported at present!");
}
secondaryPool = storagePoolMgr.getStoragePoolByURI(dstDataStore.getUrl());
try (KeyFile srcKey = new KeyFile(src.getVolume().getPassphrase())) {
final String dstDir = secondaryPool.getLocalPath() + File.separator + dst.getPath();
FileUtils.forceMkdir(new File(dstDir));
final String dstPath = dstDir + File.separator + dst.getName();
final QemuImgFile dstFile = new QemuImgFile(dstPath, PhysicalDiskFormat.QCOW2);
final QemuImg qemu = new QemuImg(cmd.getWaitInMillSeconds());
qemu.convert(srcFile, dstFile);
SP_LOG("StorpoolBackupSnapshotCommandWrapper srcFileFormat=%s, dstFileFormat=%s", srcFile.getFormat(), dstFile.getFormat());
final File snapFile = new File(dstPath);
final long size = snapFile.exists() ? snapFile.length() : 0;
size = convertSnapshot(srcPath, secondaryPool, dst, srcKeyName, qemuObjects, options, qemuImageOpts,
qemu, srcKey);
}
final SnapshotObjectTO snapshot = new SnapshotObjectTO();
snapshot.setPath(dst.getPath() + File.separator + dst.getName());
@ -106,4 +112,31 @@ public final class StorPoolBackupSnapshotCommandWrapper extends CommandWrapper<S
}
}
}
private long convertSnapshot(String srcPath, KVMStoragePool secondaryPool, final SnapshotObjectTO dst,
String srcKeyName, List<QemuObject> qemuObjects, Map<String, String> options,
QemuImageOptions qemuImageOpts, final QemuImg qemu, KeyFile srcKey) throws IOException, QemuImgException {
long size;
final QemuImgFile srcFile = new QemuImgFile(srcPath, PhysicalDiskFormat.RAW);
final String dstDir = secondaryPool.getLocalPath() + File.separator + dst.getPath();
FileUtils.forceMkdir(new File(dstDir));
final String dstPath = dstDir + File.separator + dst.getName();
final QemuImgFile dstFile = new QemuImgFile(dstPath, PhysicalDiskFormat.QCOW2);
if (srcKey.isSet()) {
qemuObjects.add(QemuObject.prepareSecretForQemuImg(PhysicalDiskFormat.RAW, EncryptFormat.LUKS,
srcKey.toString(), srcKeyName, options));
qemuImageOpts = new QemuImageOptions(PhysicalDiskFormat.RAW, srcPath, srcKeyName);
dstFile.setFormat(PhysicalDiskFormat.LUKS);
}
qemuImageOpts.setImageOptsFlag(true);
qemu.convert(srcFile, dstFile, options, qemuObjects, qemuImageOpts, null, true);
SP_LOG("StorpoolBackupSnapshotCommandWrapper srcFileFormat=%s, dstFileFormat=%s", srcFile.getFormat(), dstFile.getFormat());
final File snapFile = new File(dstPath);
size = snapFile.exists() ? snapFile.length() : 0;
return size;
}
}