apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Support local storage live migration for direct download templates (#7453)

Browse Source 
Co-authored-by: Marcus Sorensen <mls@apple.com>
This commit is contained in:
Marcus Sorensen 2023-05-04 14:37:58 -06:00 committed by GitHub
parent d147f1cc3b
commit ec0f8bddf6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 124 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
core/src/main/java/com/cloud/agent/api/MigrateCommand.java
View File

@ -171,6 +171,7 @@ public class MigrateCommand extends Command {
private final DriverType driverType;
private final Source source;
private final String sourceText;
private final String backingStoreText;
private boolean isSourceDiskOnStorageFileSystem;
public MigrateDiskInfo(final String serialNumber, final DiskType diskType, final DriverType driverType, final Source source, final String sourceText) {
@ -179,6 +180,16 @@ public class MigrateCommand extends Command {
this.driverType = driverType;
this.source = source;
this.sourceText = sourceText;
this.backingStoreText = null;
}
public MigrateDiskInfo(final String serialNumber, final DiskType diskType, final DriverType driverType, final Source source, final String sourceText, final String backingStoreText) {
this.serialNumber = serialNumber;
this.diskType = diskType;
this.driverType = driverType;
this.source = source;
this.sourceText = sourceText;
this.backingStoreText = backingStoreText;
}
public String getSerialNumber() {
@ -201,6 +212,8 @@ public class MigrateCommand extends Command {
return sourceText;
}
public String getBackingStoreText() { return backingStoreText; }
public boolean isSourceDiskOnStorageFileSystem() {
return isSourceDiskOnStorageFileSystem;
}

1
core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java
View File

@ -86,6 +86,7 @@ public class TemplateObjectTO implements DataTO {
this.hypervisorType = template.getHypervisorType();
this.deployAsIs = template.isDeployAsIs();
this.deployAsIsConfiguration = template.getDeployAsIsConfiguration();
this.directDownload = template.isDirectDownload();
}
@Override

21
engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java
View File

@ -149,9 +149,9 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot
* Configures a {@link MigrateDiskInfo} object configured for migrating a File System volume and calls rootImageProvisioning.
*/
@Override
protected MigrateCommand.MigrateDiskInfo configureMigrateDiskInfo(VolumeInfo srcVolumeInfo, String destPath) {
return new MigrateCommand.MigrateDiskInfo(srcVolumeInfo.getPath(), MigrateCommand.MigrateDiskInfo.DiskType.FILE, MigrateCommand.MigrateDiskInfo.DriverType.QCOW2,
MigrateCommand.MigrateDiskInfo.Source.FILE, destPath);
protected MigrateCommand.MigrateDiskInfo configureMigrateDiskInfo(VolumeInfo srcVolumeInfo, String destPath, String backingPath) {
return new MigrateCommand.MigrateDiskInfo(srcVolumeInfo.getPath(), MigrateCommand.MigrateDiskInfo.DiskType.FILE, MigrateCommand.MigrateDiskInfo.DriverType.QCOW2,
MigrateCommand.MigrateDiskInfo.Source.FILE, destPath, backingPath);
}
/**
@ -163,6 +163,15 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot
return new File(destStoragePool.getPath(), destVolumeInfo.getUuid()).getAbsolutePath();
}
@Override
protected String generateBackingPath(StoragePoolVO destStoragePool, VolumeInfo destVolumeInfo) {
String templateInstallPath = getVolumeBackingFile(destVolumeInfo);
if (templateInstallPath == null) {
return null;
}
return new File(destStoragePool.getPath(), templateInstallPath).getAbsolutePath();
}
/**
* Returns the template UUID with the given id. If the template ID is null, it returns null.
*/
@ -201,6 +210,12 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot
return;
}
TemplateInfo directDownloadTemplateInfo = templateDataFactory.getReadyBypassedTemplateOnPrimaryStore(srcVolumeInfo.getTemplateId(), destDataStore.getId(), destHost.getId());
if (directDownloadTemplateInfo != null) {
LOGGER.debug(String.format("Template %s was of direct download type and successfully staged to primary store %s", directDownloadTemplateInfo.getId(), directDownloadTemplateInfo.getDataStore().getId()));
return;
}
VMTemplateStoragePoolVO sourceVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(destStoragePool.getId(), srcVolumeInfo.getTemplateId(), null);
if (sourceVolumeTemplateStoragePoolVO == null && (isStoragePoolTypeInList(destStoragePool.getPoolType(), StoragePoolType.Filesystem, StoragePoolType.SharedMountPoint))) {
DataStore sourceTemplateDataStore = dataStoreManagerImpl.getRandomImageStore(srcVolumeInfo.getDataCenterId());

13
engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java
View File

@ -1871,7 +1871,8 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy {
MigrateCommand.MigrateDiskInfo.Source.FILE,
connectHostToVolume(destHost, destVolumeInfo.getPoolId(), volumeIdentifier));
} else {
migrateDiskInfo = configureMigrateDiskInfo(srcVolumeInfo, destPath);
String backingPath = generateBackingPath(destStoragePool, destVolumeInfo);
migrateDiskInfo = configureMigrateDiskInfo(srcVolumeInfo, destPath, backingPath);
migrateDiskInfo.setSourceDiskOnStorageFileSystem(isStoragePoolTypeOfFile(sourceStoragePool));
migrateDiskInfoList.add(migrateDiskInfo);
prepareDiskWithSecretConsumerDetail(vmTO, srcVolumeInfo, destVolumeInfo.getPath());
@ -1994,14 +1995,18 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy {
return connectHostToVolume(destHost, destVolumeInfo.getPoolId(), destVolumeInfo.get_iScsiName());
}
protected String generateBackingPath(StoragePoolVO destStoragePool, VolumeInfo destVolumeInfo) {
return null;
}
/**
* Configures a {@link MigrateDiskInfo} object with disk type of BLOCK, Driver type RAW and Source DEV
*/
protected MigrateCommand.MigrateDiskInfo configureMigrateDiskInfo(VolumeInfo srcVolumeInfo, String destPath) {
protected MigrateCommand.MigrateDiskInfo configureMigrateDiskInfo(VolumeInfo srcVolumeInfo, String destPath, String backingPath) {
return new MigrateCommand.MigrateDiskInfo(srcVolumeInfo.getPath(),
MigrateCommand.MigrateDiskInfo.DiskType.BLOCK,
MigrateCommand.MigrateDiskInfo.DriverType.RAW,
MigrateCommand.MigrateDiskInfo.Source.DEV, destPath);
MigrateCommand.MigrateDiskInfo.Source.DEV, destPath, backingPath);
}
/**
@ -2023,7 +2028,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy {
/*
* Return backing file for volume (if any), only for KVM volumes
*/
private String getVolumeBackingFile(VolumeInfo srcVolumeInfo) {
String getVolumeBackingFile(VolumeInfo srcVolumeInfo) {
if (srcVolumeInfo.getHypervisorType() == HypervisorType.KVM &&
srcVolumeInfo.getTemplateId() != null && srcVolumeInfo.getPoolId() != null) {
VMTemplateVO template = _vmTemplateDao.findById(srcVolumeInfo.getTemplateId());

15
engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java
View File

@ -240,7 +240,7 @@ public class KvmNonManagedStorageSystemDataMotionTest {
public void configureMigrateDiskInfoTest() {
VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
Mockito.doReturn("volume path").when(srcVolumeInfo).getPath();
MigrateCommand.MigrateDiskInfo migrateDiskInfo = kvmNonManagedStorageDataMotionStrategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath");
MigrateCommand.MigrateDiskInfo migrateDiskInfo = kvmNonManagedStorageDataMotionStrategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath", null);
Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DiskType.FILE, migrateDiskInfo.getDiskType());
Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DriverType.QCOW2, migrateDiskInfo.getDriverType());
Assert.assertEquals(MigrateCommand.MigrateDiskInfo.Source.FILE, migrateDiskInfo.getSource());
@ -248,6 +248,19 @@ public class KvmNonManagedStorageSystemDataMotionTest {
Assert.assertEquals("volume path", migrateDiskInfo.getSerialNumber());
}
@Test
public void configureMigrateDiskInfoWithBackingTest() {
VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
Mockito.doReturn("volume path").when(srcVolumeInfo).getPath();
MigrateCommand.MigrateDiskInfo migrateDiskInfo = kvmNonManagedStorageDataMotionStrategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath", "backingPath");
Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DiskType.FILE, migrateDiskInfo.getDiskType());
Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DriverType.QCOW2, migrateDiskInfo.getDriverType());
Assert.assertEquals(MigrateCommand.MigrateDiskInfo.Source.FILE, migrateDiskInfo.getSource());
Assert.assertEquals("destPath", migrateDiskInfo.getSourceText());
Assert.assertEquals("volume path", migrateDiskInfo.getSerialNumber());
Assert.assertEquals("backingPath", migrateDiskInfo.getBackingStoreText());
}
@Test
public void shouldMigrateVolumeTest() {
StoragePoolVO sourceStoragePool = Mockito.spy(new StoragePoolVO());

15
engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategyTest.java
View File

@ -192,7 +192,7 @@ public class StorageSystemDataMotionStrategyTest {
public void configureMigrateDiskInfoTest() {
VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
Mockito.doReturn("volume path").when(srcVolumeInfo).getPath();
MigrateCommand.MigrateDiskInfo migrateDiskInfo = strategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath");
MigrateCommand.MigrateDiskInfo migrateDiskInfo = strategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath", null);
Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DiskType.BLOCK, migrateDiskInfo.getDiskType());
Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DriverType.RAW, migrateDiskInfo.getDriverType());
Assert.assertEquals(MigrateCommand.MigrateDiskInfo.Source.DEV, migrateDiskInfo.getSource());
@ -200,6 +200,19 @@ public class StorageSystemDataMotionStrategyTest {
Assert.assertEquals("volume path", migrateDiskInfo.getSerialNumber());
}
@Test
public void configureMigrateDiskInfoWithBackingTest() {
VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject());
Mockito.doReturn("volume path").when(srcVolumeInfo).getPath();
MigrateCommand.MigrateDiskInfo migrateDiskInfo = strategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath", "backingPath");
Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DiskType.BLOCK, migrateDiskInfo.getDiskType());
Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DriverType.RAW, migrateDiskInfo.getDriverType());
Assert.assertEquals(MigrateCommand.MigrateDiskInfo.Source.DEV, migrateDiskInfo.getSource());
Assert.assertEquals("destPath", migrateDiskInfo.getSourceText());
Assert.assertEquals("volume path", migrateDiskInfo.getSerialNumber());
Assert.assertEquals("backingPath", migrateDiskInfo.getBackingStoreText());
}
@Test
public void setVolumePathTest() {
VolumeVO volume = new VolumeVO("name", 0l, 0l, 0l, 0l, 0l, "folder", "path", Storage.ProvisioningType.THIN, 0l, Volume.Type.ROOT);

10
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java
View File

@ -575,6 +575,16 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
diskNode.appendChild(newChildSourceNode);
} else if (migrateStorageManaged && "auth".equals(diskChildNode.getNodeName())) {
diskNode.removeChild(diskChildNode);
} else if ("backingStore".equals(diskChildNode.getNodeName()) && migrateDiskInfo.getBackingStoreText() != null) {
for (int b = 0; b < diskChildNode.getChildNodes().getLength(); b++) {
Node backingChild = diskChildNode.getChildNodes().item(b);
if ("source".equals(backingChild.getNodeName())) {
diskChildNode.removeChild(backingChild);
Element newChildBackingElement = doc.createElement("source");
newChildBackingElement.setAttribute(migrateDiskInfo.getSource().toString(), migrateDiskInfo.getBackingStoreText());
diskChildNode.appendChild(newChildBackingElement);
}
}
} else if ("encryption".equals(diskChildNode.getNodeName())) {
for (int s = 0; s < diskChildNode.getChildNodes().getLength(); s++) {
Node encryptionChild = diskChildNode.getChildNodes().item(s);

45
plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java
View File

@ -794,6 +794,51 @@ public class LibvirtMigrateCommandWrapperTest {
assertXpath(doc, "/domain/devices/disk/encryption/secret/@uuid", expectedSecretUuid);
}
@Test
public void testReplaceBackingStore() throws Exception {
Map<String, MigrateDiskInfo> mapMigrateStorage = new HashMap<String, MigrateDiskInfo>();
final String xmlDesc =
"<domain type='kvm' id='3'>" +
" <devices>" +
" <disk type='file' device='disk'>\n" +
" <driver name='qemu' type='qcow2' cache='none'/>\n" +
" <source file='/mnt/07eb495b-5590-3877-9fb7-23c6e9a40d40/bf8621b3-027c-497d-963b-06319650f048'/>\n" +
" <target dev='vdb' bus='virtio'/>\n" +
" <serial>bf8621b3027c497d963b</serial>\n" +
" <alias name='virtio-disk1'/>\n" +
" <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>\n" +
" <backingStore type='file' index='1'>\n" +
" <format type='raw'/>\n" +
" <source file='/mnt/07eb495b-5590-3877-9fb7-23c6e9a40d40/bb4d4df4-c004-11e5-94ed-5254001daa61'/>\n" +
" <backingStore/>\n" +
" </backingStore>\n" +
" <encryption format='luks'>\n" +
" <secret type='passphrase' uuid='5644d664-a238-3a9b-811c-961f609d29f4'/>\n" +
" </encryption>\n" +
" </disk>\n" +
" </devices>" +
"</domain>";
final String volumeFile = "3530f749-82fd-458e-9485-a357e6e541db";
final String backingFile = "0bc745b6-f3d7-44a9-ad8e-68904b77e2ab";
String newDiskPath = "/mnt/2d0435e1-99e0-4f1d-94c0-bee1f6f8b99e/" + volumeFile;
String newBackingStorePath = "/mnt/2d0435e1-99e0-4f1d-94c0-bee1f6f8b99e/" + backingFile;
MigrateDiskInfo diskInfo = new MigrateDiskInfo("123456", DiskType.BLOCK, DriverType.RAW, Source.FILE, newDiskPath, newBackingStorePath);
mapMigrateStorage.put("/mnt/07eb495b-5590-3877-9fb7-23c6e9a40d40/bf8621b3-027c-497d-963b-06319650f048", diskInfo);
final String result = libvirtMigrateCmdWrapper.replaceStorage(xmlDesc, mapMigrateStorage, false);
InputStream in = IOUtils.toInputStream(result);
DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
Document doc = docBuilder.parse(in);
assertXpath(doc, "/domain/devices/disk/backingStore/source/@file", newBackingStorePath);
assertXpath(doc, "/domain/devices/disk/source/@file", newDiskPath);
assertXpath(doc, "/domain/devices/disk/serial", "bf8621b3027c497d963b");
final String expectedSecretUuid = LibvirtComputingResource.generateSecretUUIDFromString(volumeFile);
assertXpath(doc, "/domain/devices/disk/encryption/secret/@uuid", expectedSecretUuid);
}
public void testReplaceStorageXmlDiskNotManagedStorage() throws ParserConfigurationException, TransformerException, SAXException, IOException {
final LibvirtMigrateCommandWrapper lw = new LibvirtMigrateCommandWrapper();
String destDisk1FileName = "XXXXXXXXXXXXXX";