apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

ScaleIO volume live migration - use usable bytes from source disk to format the destination disk (#9174)

Browse Source
This commit is contained in:
Suresh Kumar Anaparti 2024-06-12 13:58:10 +05:30 committed by GitHub
parent 1577218999
commit 4ec0f823cf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 317 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
core/src/main/java/com/cloud/agent/api/GetVolumeStatAnswer.java Normal file
View File

@ -0,0 +1,85 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.api;
import com.cloud.agent.api.LogLevel.Log4jLevel;
import com.cloud.storage.Storage.StoragePoolType;
@LogLevel(Log4jLevel.Trace)
public class GetVolumeStatAnswer extends Answer {
String poolUuid;
StoragePoolType poolType;
String volumePath;
long size = 0;
long virtualSize = 0;
public GetVolumeStatAnswer(GetVolumeStatCommand cmd, long size, long virtualSize) {
super(cmd, true, "");
this.poolUuid = cmd.getPoolUuid();
this.poolType = cmd.getPoolType();
this.volumePath = cmd.getVolumePath();
this.size = size;
this.virtualSize = virtualSize;
}
public GetVolumeStatAnswer(GetVolumeStatCommand cmd, boolean success, String details) {
super(cmd, success, details);
}
protected GetVolumeStatAnswer() {
//no-args constructor for json serialization-deserialization
}
public String getPoolUuid() {
return poolUuid;
}
public void setPoolUuid(String poolUuid) {
this.poolUuid = poolUuid;
}
public StoragePoolType getPoolType() {
return poolType;
}
public void setPoolType(StoragePoolType poolType) {
this.poolType = poolType;
}
public long getSize() {
return size;
}
public void setSize(long size) {
this.size = size;
}
public long getVirtualSize() {
return virtualSize;
}
public void setVirtualSize(long virtualSize) {
this.virtualSize = virtualSize;
}
public String getString() {
return "GetVolumeStatAnswer [poolUuid=" + poolUuid + ", poolType=" + poolType + ", volumePath=" + volumePath + ", size=" + size + ", virtualSize=" + virtualSize + "]";
}
}

72
core/src/main/java/com/cloud/agent/api/GetVolumeStatCommand.java Normal file
View File

@ -0,0 +1,72 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.api;
import com.cloud.agent.api.LogLevel.Log4jLevel;
import com.cloud.storage.Storage.StoragePoolType;
@LogLevel(Log4jLevel.Trace)
public class GetVolumeStatCommand extends Command {
String volumePath;
StoragePoolType poolType;
String poolUuid;
protected GetVolumeStatCommand() {
}
public GetVolumeStatCommand(String volumePath, StoragePoolType poolType, String poolUuid) {
this.volumePath = volumePath;
this.poolType = poolType;
this.poolUuid = poolUuid;
}
public String getVolumePath() {
return volumePath;
}
public void setVolumePath(String volumePath) {
this.volumePath = volumePath;
}
public StoragePoolType getPoolType() {
return poolType;
}
public void setPoolType(StoragePoolType poolType) {
this.poolType = poolType;
}
public String getPoolUuid() {
return poolUuid;
}
public void setPoolUuid(String storeUuid) {
this.poolUuid = storeUuid;
}
@Override
public boolean executeInSequence() {
return false;
}
public String getString() {
return "GetVolumeStatCommand [volumePath=" + volumePath + ", poolType=" + poolType + ", poolUuid=" + poolUuid + "]";
}
}

9
core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
View File

@ -39,6 +39,7 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
private DataStoreTO dataStore;
private String name;
private Long size;
private Long usableSize;
private String path;
private Long volumeId;
private String vmName;
@ -161,6 +162,10 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
return size;
}
public Long getUsableSize() {
return usableSize;
}
@Override
public DataObjectType getObjectType() {
return DataObjectType.VOLUME;
@ -178,6 +183,10 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
this.size = size;
}
public void setUsableSize(Long usableSize) {
this.usableSize = usableSize;
}
public void setPath(String path) {
this.path = path;
}

66
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumeStatCommandWrapper.java Normal file
View File

@ -0,0 +1,66 @@
//
//Licensed to the Apache Software Foundation (ASF) under one
//or more contributor license agreements. See the NOTICE file
//distributed with this work for additional information
//regarding copyright ownership. The ASF licenses this file
//to you under the Apache License, Version 2.0 (the
//"License"); you may not use this file except in compliance
//with the License. You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing,
//software distributed under the License is distributed on an
//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
//KIND, either express or implied. See the License for the
//specific language governing permissions and limitations
//under the License.
//
package com.cloud.hypervisor.kvm.resource.wrapper;
import org.apache.log4j.Logger;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.GetVolumeStatAnswer;
import com.cloud.agent.api.GetVolumeStatCommand;
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
import com.cloud.resource.CommandWrapper;
import com.cloud.resource.ResourceWrapper;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.utils.exception.CloudRuntimeException;
@ResourceWrapper(handles = GetVolumeStatCommand.class)
public final class LibvirtGetVolumeStatCommandWrapper extends CommandWrapper<GetVolumeStatCommand, Answer, LibvirtComputingResource> {
private static final Logger s_logger = Logger.getLogger(LibvirtGetVolumeStatCommandWrapper.class);
@Override
public Answer execute(final GetVolumeStatCommand cmd, final LibvirtComputingResource libvirtComputingResource) {
try {
String volumePath = cmd.getVolumePath();
StoragePoolType poolType = cmd.getPoolType();
String poolUuid = cmd.getPoolUuid();
KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(poolType, poolUuid);
if (primaryPool == null) {
String msg = "Can't get volume stats as pool details unavailable for volume: " + volumePath + " on the storage pool: " + poolUuid;
return new GetVolumeStatAnswer(cmd, false, msg);
}
KVMPhysicalDisk disk = primaryPool.getPhysicalDisk(volumePath);
if (disk == null) {
String msg = "Can't get volume stats as disk details unavailable for volume: " + volumePath + " on the storage pool: " + poolUuid;
return new GetVolumeStatAnswer(cmd, false, msg);
}
return new GetVolumeStatAnswer(cmd, disk.getSize(), disk.getVirtualSize());
} catch (CloudRuntimeException e) {
s_logger.error("Can't get volume stats, due to: " + e.getMessage(), e);
return new GetVolumeStatAnswer(cmd, false, "Can't get volume stats, due to: " + e.getMessage());
}
}
}

23
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java
View File

@ -142,7 +142,6 @@ public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<MigrateVo
LOGGER.info(String.format("Block copy has started for the volume %s : %s ", destDiskLabel, srcPath));
return checkBlockJobStatus(command, dm, destDiskLabel, srcPath, destPath, libvirtComputingResource, conn, srcSecretUUID);
} catch (Exception e) {
String msg = "Migrate volume failed due to " + e.toString();
LOGGER.warn(msg, e);
@ -168,17 +167,27 @@ public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<MigrateVo
protected MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath, LibvirtComputingResource libvirtComputingResource, Connect conn, String srcSecretUUID) throws LibvirtException {
int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found
int waitTimeInSec = command.getWait();
double blockCopyProgress = 0;
while (waitTimeInSec > 0) {
DomainBlockJobInfo blockJobInfo = dm.getBlockJobInfo(diskLabel, 0);
if (blockJobInfo != null) {
LOGGER.debug(String.format("Volume %s : %s block copy progress: %s%% current value:%s end value:%s", diskLabel, srcPath, (blockJobInfo.end == 0)? 0 : 100*(blockJobInfo.cur / (double) blockJobInfo.end), blockJobInfo.cur, blockJobInfo.end));
blockCopyProgress = (blockJobInfo.end == 0)? blockCopyProgress : 100 * (blockJobInfo.cur / (double) blockJobInfo.end);
LOGGER.debug(String.format("Volume %s : %s, block copy progress: %s%%, current value: %s end value: %s, job info - type: %s, bandwidth: %s",
diskLabel, srcPath, blockCopyProgress, blockJobInfo.cur, blockJobInfo.end, blockJobInfo.type, blockJobInfo.bandwidth));
if (blockJobInfo.cur == blockJobInfo.end) {
LOGGER.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath));
dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT);
if (StringUtils.isNotEmpty(srcSecretUUID)) {
libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID);
if (blockJobInfo.end > 0) {
LOGGER.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath));
dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT);
if (StringUtils.isNotEmpty(srcSecretUUID)) {
libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID);
}
break;
} else {
// cur = 0, end = 0 - at this point, disk does not have an active block job (so, no need to abort job)
String msg = String.format("No active block copy job for the volume %s : %s - job stopped at %s progress", diskLabel, srcPath, blockCopyProgress);
LOGGER.warn(msg);
return new MigrateVolumeAnswer(command, false, msg, null);
}
break;
}
} else {
LOGGER.info("Failed to get the block copy status, trying to abort the job");

3
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java
View File

@ -37,6 +37,9 @@ public interface KVMStoragePool {
public static final long HeartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP);
public static final long HeartBeatCheckerTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_CHECKER_TIMEOUT);
public default KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) {
return createPhysicalDisk(volumeUuid, format, provisioningType, size, passphrase);
}
public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase);

2
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View File

@ -1629,7 +1629,7 @@ public class KVMStorageProcessor implements StorageProcessor {
}
} else {
vol = primaryPool.createPhysicalDisk(volume.getUuid(), format,
volume.getProvisioningType(), disksize, volume.getPassphrase());
volume.getProvisioningType(), disksize, volume.getUsableSize(), volume.getPassphrase());
}
final VolumeObjectTO newVol = new VolumeObjectTO();

15
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
View File

@ -151,6 +151,11 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
return MapStorageUuidToStoragePool.remove(uuid) != null;
}
@Override
public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
return createPhysicalDisk(name, pool, format, provisioningType, size, null, passphrase);
}
/**
* ScaleIO doesn't need to communicate with the hypervisor normally to create a volume. This is used only to prepare a ScaleIO data disk for encryption.
* Thin encrypted volumes are provisioned in QCOW2 format, which insulates the guest from zeroes/unallocated blocks in the block device that would
@ -160,11 +165,12 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
* @param format disk format
* @param provisioningType provisioning type
* @param size disk size
* @param usableSize usage disk size
* @param passphrase passphrase
* @return the disk object
*/
@Override
public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) {
if (passphrase == null || passphrase.length == 0) {
return null;
}
@ -182,7 +188,12 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
QemuImg qemuImg = new QemuImg(0, true, false);
Map<String, String> options = new HashMap<>();
List<QemuObject> qemuObjects = new ArrayList<>();
long formattedSize = getUsableBytesFromRawBytes(disk.getSize());
long formattedSize;
if (usableSize != null && usableSize > 0) {
formattedSize = usableSize;
} else {
formattedSize = getUsableBytesFromRawBytes(disk.getSize());
}
options.put("preallocation", QemuImg.PreallocationType.Metadata.toString());
qemuObjects.add(QemuObject.prepareSecretForQemuImg(disk.getFormat(), disk.getQemuEncryptFormat(), keyFile.toString(), "sec0", options));

5
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java
View File

@ -72,6 +72,11 @@ public class ScaleIOStoragePool implements KVMStoragePool {
}
}
@Override
public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) {
return this.storageAdaptor.createPhysicalDisk(volumeUuid, this, format, provisioningType, size, usableSize, passphrase);
}
@Override
public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
return this.storageAdaptor.createPhysicalDisk(volumeUuid, this, format, provisioningType, size, passphrase);

5
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java
View File

@ -39,6 +39,11 @@ public interface StorageAdaptor {
public boolean deleteStoragePool(String uuid);
public default KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool,
PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) {
return createPhysicalDisk(name, pool, format, provisioningType, size, passphrase);
}
public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool,
PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase);

23
plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
View File

@ -65,6 +65,8 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.GetVolumeStatAnswer;
import com.cloud.agent.api.GetVolumeStatCommand;
import com.cloud.agent.api.storage.MigrateVolumeCommand;
import com.cloud.agent.api.storage.ResizeVolumeCommand;
import com.cloud.agent.api.to.DataObjectType;
@ -490,10 +492,10 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
}
public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId) {
return createVolume(volumeInfo, storagePoolId, false);
return createVolume(volumeInfo, storagePoolId, false, null);
}
public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId, boolean migrationInvolved) {
public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId, boolean migrationInvolved, Long usageSize) {
LOGGER.debug("Creating PowerFlex volume");
StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId);
@ -543,6 +545,9 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
VolumeObjectTO prepVolume = (VolumeObjectTO) createdObject.getTO();
prepVolume.setPath(volumePath);
prepVolume.setUuid(volumePath);
if (usageSize != null) {
prepVolume.setUsableSize(usageSize);
}
CreateObjectCommand cmd = new CreateObjectCommand(prepVolume);
EndPoint ep = selector.select(volumeInfo, true);
if (ep == null) {
@ -845,7 +850,8 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
// Volume migration across different PowerFlex/ScaleIO clusters
final long srcVolumeId = srcData.getId();
DataStore srcStore = srcData.getDataStore();
Map<String, String> srcDetails = getVolumeDetails((VolumeInfo) srcData, srcStore);
VolumeInfo srcVolumeInfo = (VolumeInfo) srcData;
Map<String, String> srcDetails = getVolumeDetails(srcVolumeInfo, srcStore);
DataStore destStore = destData.getDataStore();
final long destPoolId = destStore.getId();
@ -857,8 +863,17 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
EndPoint ep = RemoteHostEndPoint.getHypervisorHostEndPoint(host);
Answer answer = null;
Long srcVolumeUsableSize = null;
try {
CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId(), true);
GetVolumeStatCommand statCmd = new GetVolumeStatCommand(srcVolumeInfo.getPath(), srcVolumeInfo.getStoragePoolType(), srcStore.getUuid());
GetVolumeStatAnswer statAnswer = (GetVolumeStatAnswer) ep.sendMessage(statCmd);
if (!statAnswer.getResult() ) {
LOGGER.warn(String.format("Unable to get volume %s stats", srcVolumeInfo.getId()));
} else if (statAnswer.getVirtualSize() > 0) {
srcVolumeUsableSize = statAnswer.getVirtualSize();
}
CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId(), true, srcVolumeUsableSize);
destVolumePath = createAnswer.getData().getPath();
destVolTO.setPath(destVolumePath);

31
plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriverTest.java
View File

@ -20,7 +20,10 @@
package org.apache.cloudstack.storage.datastore.driver;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.GetVolumeStatAnswer;
import com.cloud.agent.api.GetVolumeStatCommand;
import com.cloud.agent.api.storage.MigrateVolumeAnswer;
import com.cloud.agent.api.storage.MigrateVolumeCommand;
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.configuration.Config;
@ -196,16 +199,23 @@ public class ScaleIOPrimaryDataStoreDriverTest {
RemoteHostEndPoint ep = Mockito.mock(RemoteHostEndPoint.class);
remoteHostEndPointMock.when(() -> RemoteHostEndPoint.getHypervisorHostEndPoint(host)).thenReturn(ep);
long volumeVirtualSize = 68673077248L;
DataTO dataTO = Mockito.mock(DataTO.class);
CreateObjectAnswer createAnswer = new CreateObjectAnswer(dataTO);
doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true);
doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true, volumeVirtualSize);
when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee");
doReturn(true).when(scaleIOPrimaryDataStoreDriver)
.grantAccess(any(), any(), any());
when(configDao.getValue(Config.MigrateWait.key())).thenReturn("3600");
GetVolumeStatAnswer getVolumeStatAnswer = Mockito.mock(GetVolumeStatAnswer.class);
when(ep.sendMessage(any(GetVolumeStatCommand.class))).thenReturn(getVolumeStatAnswer);
when(getVolumeStatAnswer.getResult()).thenReturn(true);
when(getVolumeStatAnswer.getVirtualSize()).thenReturn(volumeVirtualSize);
MigrateVolumeAnswer migrateVolumeAnswer = Mockito.mock(MigrateVolumeAnswer.class);
when(ep.sendMessage(any())).thenReturn(migrateVolumeAnswer);
when(ep.sendMessage(any(MigrateVolumeCommand.class))).thenReturn(migrateVolumeAnswer);
when(migrateVolumeAnswer.getResult()).thenReturn(true);
Mockito.doNothing().when(scaleIOPrimaryDataStoreDriver)
@ -242,16 +252,21 @@ public class ScaleIOPrimaryDataStoreDriverTest {
DataTO dataTO = Mockito.mock(DataTO.class);
CreateObjectAnswer createAnswer = new CreateObjectAnswer(dataTO);
doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true);
when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee");
doReturn(true).when(scaleIOPrimaryDataStoreDriver)
Mockito.lenient().doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true, null);
Mockito.lenient().when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee");
Mockito.lenient().doReturn(true).when(scaleIOPrimaryDataStoreDriver)
.grantAccess(any(), any(), any());
when(configDao.getValue(Config.MigrateWait.key())).thenReturn("3600");
Mockito.lenient().when(configDao.getValue(Config.MigrateWait.key())).thenReturn("3600");
GetVolumeStatAnswer getVolumeStatAnswer = Mockito.mock(GetVolumeStatAnswer.class);
Mockito.lenient().when(ep.sendMessage(any(GetVolumeStatCommand.class))).thenReturn(getVolumeStatAnswer);
Mockito.lenient().when(getVolumeStatAnswer.getResult()).thenReturn(false);
MigrateVolumeAnswer migrateVolumeAnswer = Mockito.mock(MigrateVolumeAnswer.class);
when(ep.sendMessage(any())).thenReturn(migrateVolumeAnswer);
when(migrateVolumeAnswer.getResult()).thenReturn(false);
Mockito.doNothing().when(scaleIOPrimaryDataStoreDriver)
Mockito.lenient().when(migrateVolumeAnswer.getResult()).thenReturn(false);
Mockito.lenient().doNothing().when(scaleIOPrimaryDataStoreDriver)
.revertBlockCopyVolumeOperations(any(), any(), any(), any());
Answer answer = scaleIOPrimaryDataStoreDriver.liveMigrateVolume(srcData, destData);