ScaleIO volume live migration - use usable bytes from source disk to format the destination disk (#9174)

core/src/main/java/com/cloud/agent/api/GetVolumeStatAnswer.java

@@ -0,0 +1,85 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.LogLevel.Log4jLevel;
+import com.cloud.storage.Storage.StoragePoolType;
+
+@LogLevel(Log4jLevel.Trace)
+public class GetVolumeStatAnswer extends Answer {
+    String poolUuid;
+    StoragePoolType poolType;
+    String volumePath;
+    long size = 0;
+    long virtualSize = 0;
+
+    public GetVolumeStatAnswer(GetVolumeStatCommand cmd, long size, long virtualSize) {
+        super(cmd, true, "");
+        this.poolUuid = cmd.getPoolUuid();
+        this.poolType = cmd.getPoolType();
+        this.volumePath = cmd.getVolumePath();
+        this.size = size;
+        this.virtualSize = virtualSize;
+    }
+
+    public GetVolumeStatAnswer(GetVolumeStatCommand cmd, boolean success, String details) {
+        super(cmd, success, details);
+    }
+
+    protected GetVolumeStatAnswer() {
+        //no-args constructor for json serialization-deserialization
+    }
+
+    public String getPoolUuid() {
+        return poolUuid;
+    }
+
+    public void setPoolUuid(String poolUuid) {
+        this.poolUuid = poolUuid;
+    }
+
+    public StoragePoolType getPoolType() {
+        return poolType;
+    }
+
+    public void setPoolType(StoragePoolType poolType) {
+        this.poolType = poolType;
+    }
+
+    public long getSize() {
+        return size;
+    }
+
+    public void setSize(long size) {
+        this.size = size;
+    }
+
+    public long getVirtualSize() {
+        return virtualSize;
+    }
+
+    public void setVirtualSize(long virtualSize) {
+        this.virtualSize = virtualSize;
+    }
+
+    public String getString() {
+        return "GetVolumeStatAnswer [poolUuid=" + poolUuid + ", poolType=" + poolType + ", volumePath=" + volumePath + ", size=" + size + ", virtualSize=" + virtualSize + "]";
+    }
+}

core/src/main/java/com/cloud/agent/api/GetVolumeStatCommand.java

@@ -0,0 +1,72 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.LogLevel.Log4jLevel;
+import com.cloud.storage.Storage.StoragePoolType;
+
+@LogLevel(Log4jLevel.Trace)
+public class GetVolumeStatCommand extends Command {
+    String volumePath;
+    StoragePoolType poolType;
+    String poolUuid;
+
+    protected GetVolumeStatCommand() {
+    }
+
+    public GetVolumeStatCommand(String volumePath, StoragePoolType poolType, String poolUuid) {
+        this.volumePath = volumePath;
+        this.poolType = poolType;
+        this.poolUuid = poolUuid;
+    }
+
+    public String getVolumePath() {
+        return volumePath;
+    }
+
+    public void setVolumePath(String volumePath) {
+        this.volumePath = volumePath;
+    }
+
+    public StoragePoolType getPoolType() {
+        return poolType;
+    }
+
+    public void setPoolType(StoragePoolType poolType) {
+        this.poolType = poolType;
+    }
+
+    public String getPoolUuid() {
+        return poolUuid;
+    }
+
+    public void setPoolUuid(String storeUuid) {
+        this.poolUuid = storeUuid;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+
+    public String getString() {
+        return "GetVolumeStatCommand [volumePath=" + volumePath + ", poolType=" + poolType + ", poolUuid=" + poolUuid + "]";
+    }
+}

core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java

@@ -39,6 +39,7 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
     private DataStoreTO dataStore;
     private String name;
     private Long size;
+    private Long usableSize;
     private String path;
     private Long volumeId;
     private String vmName;
@@ -161,6 +162,10 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
         return size;
     }
 
+    public Long getUsableSize() {
+        return usableSize;
+    }
+
     @Override
     public DataObjectType getObjectType() {
         return DataObjectType.VOLUME;
@@ -178,6 +183,10 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO {
         this.size = size;
     }
 
+    public void setUsableSize(Long usableSize) {
+        this.usableSize = usableSize;
+    }
+
     public void setPath(String path) {
         this.path = path;
     }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumeStatCommandWrapper.java

@@ -0,0 +1,66 @@
+//
+//Licensed to the Apache Software Foundation (ASF) under one
+//or more contributor license agreements.  See the NOTICE file
+//distributed with this work for additional information
+//regarding copyright ownership.  The ASF licenses this file
+//to you under the Apache License, Version 2.0 (the
+//"License"); you may not use this file except in compliance
+//with the License.  You may obtain a copy of the License at
+//
+//http://www.apache.org/licenses/LICENSE-2.0
+//
+//Unless required by applicable law or agreed to in writing,
+//software distributed under the License is distributed on an
+//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+//KIND, either express or implied.  See the License for the
+//specific language governing permissions and limitations
+//under the License.
+//
+
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.GetVolumeStatAnswer;
+import com.cloud.agent.api.GetVolumeStatCommand;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
+import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+import com.cloud.storage.Storage.StoragePoolType;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+@ResourceWrapper(handles = GetVolumeStatCommand.class)
+public final class LibvirtGetVolumeStatCommandWrapper extends CommandWrapper<GetVolumeStatCommand, Answer, LibvirtComputingResource> {
+    private static final Logger s_logger = Logger.getLogger(LibvirtGetVolumeStatCommandWrapper.class);
+
+    @Override
+    public Answer execute(final GetVolumeStatCommand cmd, final LibvirtComputingResource libvirtComputingResource) {
+        try {
+            String volumePath = cmd.getVolumePath();
+            StoragePoolType poolType = cmd.getPoolType();
+            String poolUuid = cmd.getPoolUuid();
+
+            KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
+            KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(poolType, poolUuid);
+            if (primaryPool == null) {
+                String msg = "Can't get volume stats as pool details unavailable for volume: " + volumePath + " on the storage pool: " + poolUuid;
+                return new GetVolumeStatAnswer(cmd, false, msg);
+            }
+
+            KVMPhysicalDisk disk = primaryPool.getPhysicalDisk(volumePath);
+            if (disk == null) {
+                String msg = "Can't get volume stats as disk details unavailable for volume: " + volumePath + " on the storage pool: " + poolUuid;
+                return new GetVolumeStatAnswer(cmd, false, msg);
+            }
+
+            return new GetVolumeStatAnswer(cmd, disk.getSize(), disk.getVirtualSize());
+        } catch (CloudRuntimeException e) {
+            s_logger.error("Can't get volume stats, due to: " + e.getMessage(), e);
+            return new GetVolumeStatAnswer(cmd, false, "Can't get volume stats, due to: " + e.getMessage());
+        }
+    }
+}

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java

@@ -142,7 +142,6 @@ public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<MigrateVo
             LOGGER.info(String.format("Block copy has started for the volume %s : %s ", destDiskLabel, srcPath));
 
             return checkBlockJobStatus(command, dm, destDiskLabel, srcPath, destPath, libvirtComputingResource, conn, srcSecretUUID);
-
         } catch (Exception e) {
             String msg = "Migrate volume failed due to " + e.toString();
             LOGGER.warn(msg, e);
@@ -168,17 +167,27 @@ public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper<MigrateVo
     protected MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath, LibvirtComputingResource libvirtComputingResource, Connect conn, String srcSecretUUID) throws LibvirtException {
         int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found
         int waitTimeInSec = command.getWait();
+        double blockCopyProgress = 0;
         while (waitTimeInSec > 0) {
             DomainBlockJobInfo blockJobInfo = dm.getBlockJobInfo(diskLabel, 0);
             if (blockJobInfo != null) {
-                LOGGER.debug(String.format("Volume %s : %s block copy progress: %s%% current value:%s end value:%s", diskLabel, srcPath, (blockJobInfo.end == 0)? 0 : 100*(blockJobInfo.cur / (double) blockJobInfo.end), blockJobInfo.cur, blockJobInfo.end));
+                blockCopyProgress = (blockJobInfo.end == 0)? blockCopyProgress : 100 * (blockJobInfo.cur / (double) blockJobInfo.end);
+                LOGGER.debug(String.format("Volume %s : %s, block copy progress: %s%%, current value: %s end value: %s, job info - type: %s, bandwidth: %s",
+                        diskLabel, srcPath, blockCopyProgress, blockJobInfo.cur, blockJobInfo.end, blockJobInfo.type, blockJobInfo.bandwidth));
                 if (blockJobInfo.cur == blockJobInfo.end) {
-                    LOGGER.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath));
+                    if (blockJobInfo.end > 0) {
-                    dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT);
+                        LOGGER.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath));
-                    if (StringUtils.isNotEmpty(srcSecretUUID)) {
+                        dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT);
-                        libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID);
+                        if (StringUtils.isNotEmpty(srcSecretUUID)) {
+                            libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID);
+                        }
+                        break;
+                    } else {
+                        // cur = 0, end = 0 - at this point, disk does not have an active block job (so, no need to abort job)
+                        String msg = String.format("No active block copy job for the volume %s : %s - job stopped at %s progress", diskLabel, srcPath, blockCopyProgress);
+                        LOGGER.warn(msg);
+                        return new MigrateVolumeAnswer(command, false, msg, null);
                     }
-                    break;
                 }
             } else {
                 LOGGER.info("Failed to get the block copy status, trying to abort the job");

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java

@@ -37,6 +37,9 @@ public interface KVMStoragePool {
     public static final long HeartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP);
     public static final long HeartBeatCheckerTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_CHECKER_TIMEOUT);
 
+    public default KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) {
+        return createPhysicalDisk(volumeUuid, format, provisioningType, size, passphrase);
+    }
 
     public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase);
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -1629,7 +1629,7 @@ public class KVMStorageProcessor implements StorageProcessor {
                 }
             } else {
                 vol = primaryPool.createPhysicalDisk(volume.getUuid(), format,
-                        volume.getProvisioningType(), disksize, volume.getPassphrase());
+                        volume.getProvisioningType(), disksize, volume.getUsableSize(), volume.getPassphrase());
             }
 
             final VolumeObjectTO newVol = new VolumeObjectTO();

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java

@@ -151,6 +151,11 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
         return MapStorageUuidToStoragePool.remove(uuid) != null;
     }
 
+    @Override
+    public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
+        return createPhysicalDisk(name, pool, format, provisioningType, size, null, passphrase);
+    }
+
     /**
      * ScaleIO doesn't need to communicate with the hypervisor normally to create a volume. This is used only to prepare a ScaleIO data disk for encryption.
      * Thin encrypted volumes are provisioned in QCOW2 format, which insulates the guest from zeroes/unallocated blocks in the block device that would
@@ -160,11 +165,12 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
      * @param format disk format
      * @param provisioningType provisioning type
      * @param size disk size
+     * @param usableSize usage disk size
      * @param passphrase passphrase
      * @return the disk object
      */
     @Override
-    public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
+    public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) {
         if (passphrase == null || passphrase.length == 0) {
             return null;
         }
@@ -182,7 +188,12 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor {
                 QemuImg qemuImg = new QemuImg(0, true, false);
                 Map<String, String> options = new HashMap<>();
                 List<QemuObject> qemuObjects = new ArrayList<>();
-                long formattedSize = getUsableBytesFromRawBytes(disk.getSize());
+                long formattedSize;
+                if (usableSize != null && usableSize > 0) {
+                    formattedSize = usableSize;
+                } else {
+                    formattedSize = getUsableBytesFromRawBytes(disk.getSize());
+                }
 
                 options.put("preallocation", QemuImg.PreallocationType.Metadata.toString());
                 qemuObjects.add(QemuObject.prepareSecretForQemuImg(disk.getFormat(), disk.getQemuEncryptFormat(), keyFile.toString(), "sec0", options));

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java

@@ -72,6 +72,11 @@ public class ScaleIOStoragePool implements KVMStoragePool {
         }
     }
 
+    @Override
+    public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) {
+        return this.storageAdaptor.createPhysicalDisk(volumeUuid, this, format, provisioningType, size, usableSize, passphrase);
+    }
+
     @Override
     public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, QemuImg.PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) {
         return this.storageAdaptor.createPhysicalDisk(volumeUuid, this, format, provisioningType, size, passphrase);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java

@@ -39,6 +39,11 @@ public interface StorageAdaptor {
 
     public boolean deleteStoragePool(String uuid);
 
+    public default KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool,
+                                                      PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) {
+        return createPhysicalDisk(name, pool, format, provisioningType, size, passphrase);
+    }
+
     public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool,
             PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase);
 

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java

@@ -65,6 +65,8 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.GetVolumeStatAnswer;
+import com.cloud.agent.api.GetVolumeStatCommand;
 import com.cloud.agent.api.storage.MigrateVolumeCommand;
 import com.cloud.agent.api.storage.ResizeVolumeCommand;
 import com.cloud.agent.api.to.DataObjectType;
@@ -490,10 +492,10 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
     }
 
     public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId) {
-        return createVolume(volumeInfo, storagePoolId, false);
+        return createVolume(volumeInfo, storagePoolId, false, null);
     }
 
-    public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId, boolean migrationInvolved) {
+    public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId, boolean migrationInvolved, Long usageSize) {
         LOGGER.debug("Creating PowerFlex volume");
 
         StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId);
@@ -543,6 +545,9 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
                 VolumeObjectTO prepVolume = (VolumeObjectTO) createdObject.getTO();
                 prepVolume.setPath(volumePath);
                 prepVolume.setUuid(volumePath);
+                if (usageSize != null) {
+                    prepVolume.setUsableSize(usageSize);
+                }
                 CreateObjectCommand cmd = new CreateObjectCommand(prepVolume);
                 EndPoint ep = selector.select(volumeInfo, true);
                 if (ep == null) {
@@ -845,7 +850,8 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
         // Volume migration across different PowerFlex/ScaleIO clusters
         final long srcVolumeId = srcData.getId();
         DataStore srcStore = srcData.getDataStore();
-        Map<String, String> srcDetails = getVolumeDetails((VolumeInfo) srcData, srcStore);
+        VolumeInfo srcVolumeInfo = (VolumeInfo) srcData;
+        Map<String, String> srcDetails = getVolumeDetails(srcVolumeInfo, srcStore);
 
         DataStore destStore = destData.getDataStore();
         final long destPoolId = destStore.getId();
@@ -857,8 +863,17 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
         EndPoint ep = RemoteHostEndPoint.getHypervisorHostEndPoint(host);
 
         Answer answer = null;
+        Long srcVolumeUsableSize = null;
         try {
-            CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId(), true);
+            GetVolumeStatCommand statCmd = new GetVolumeStatCommand(srcVolumeInfo.getPath(), srcVolumeInfo.getStoragePoolType(), srcStore.getUuid());
+            GetVolumeStatAnswer statAnswer = (GetVolumeStatAnswer) ep.sendMessage(statCmd);
+            if (!statAnswer.getResult() ) {
+                LOGGER.warn(String.format("Unable to get volume %s stats", srcVolumeInfo.getId()));
+            } else if (statAnswer.getVirtualSize() > 0) {
+                srcVolumeUsableSize = statAnswer.getVirtualSize();
+            }
+
+            CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId(), true, srcVolumeUsableSize);
             destVolumePath = createAnswer.getData().getPath();
             destVolTO.setPath(destVolumePath);
 

plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriverTest.java

@@ -20,7 +20,10 @@
 package org.apache.cloudstack.storage.datastore.driver;
 
 import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.GetVolumeStatAnswer;
+import com.cloud.agent.api.GetVolumeStatCommand;
 import com.cloud.agent.api.storage.MigrateVolumeAnswer;
+import com.cloud.agent.api.storage.MigrateVolumeCommand;
 import com.cloud.agent.api.to.DataTO;
 import com.cloud.agent.api.to.DiskTO;
 import com.cloud.configuration.Config;
@@ -196,16 +199,23 @@ public class ScaleIOPrimaryDataStoreDriverTest {
         RemoteHostEndPoint ep = Mockito.mock(RemoteHostEndPoint.class);
         remoteHostEndPointMock.when(() -> RemoteHostEndPoint.getHypervisorHostEndPoint(host)).thenReturn(ep);
 
+        long volumeVirtualSize = 68673077248L;
         DataTO dataTO = Mockito.mock(DataTO.class);
         CreateObjectAnswer createAnswer = new CreateObjectAnswer(dataTO);
-        doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true);
+        doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true, volumeVirtualSize);
         when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee");
         doReturn(true).when(scaleIOPrimaryDataStoreDriver)
                 .grantAccess(any(), any(), any());
 
         when(configDao.getValue(Config.MigrateWait.key())).thenReturn("3600");
+
+        GetVolumeStatAnswer getVolumeStatAnswer = Mockito.mock(GetVolumeStatAnswer.class);
+        when(ep.sendMessage(any(GetVolumeStatCommand.class))).thenReturn(getVolumeStatAnswer);
+        when(getVolumeStatAnswer.getResult()).thenReturn(true);
+        when(getVolumeStatAnswer.getVirtualSize()).thenReturn(volumeVirtualSize);
+
         MigrateVolumeAnswer migrateVolumeAnswer = Mockito.mock(MigrateVolumeAnswer.class);
-        when(ep.sendMessage(any())).thenReturn(migrateVolumeAnswer);
+        when(ep.sendMessage(any(MigrateVolumeCommand.class))).thenReturn(migrateVolumeAnswer);
         when(migrateVolumeAnswer.getResult()).thenReturn(true);
 
         Mockito.doNothing().when(scaleIOPrimaryDataStoreDriver)
@@ -242,16 +252,21 @@ public class ScaleIOPrimaryDataStoreDriverTest {
 
         DataTO dataTO = Mockito.mock(DataTO.class);
         CreateObjectAnswer createAnswer = new CreateObjectAnswer(dataTO);
-        doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true);
+        Mockito.lenient().doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true, null);
-        when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee");
+        Mockito.lenient().when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee");
-        doReturn(true).when(scaleIOPrimaryDataStoreDriver)
+        Mockito.lenient().doReturn(true).when(scaleIOPrimaryDataStoreDriver)
                 .grantAccess(any(), any(), any());
 
-        when(configDao.getValue(Config.MigrateWait.key())).thenReturn("3600");
+        Mockito.lenient().when(configDao.getValue(Config.MigrateWait.key())).thenReturn("3600");
+
+        GetVolumeStatAnswer getVolumeStatAnswer = Mockito.mock(GetVolumeStatAnswer.class);
+        Mockito.lenient().when(ep.sendMessage(any(GetVolumeStatCommand.class))).thenReturn(getVolumeStatAnswer);
+        Mockito.lenient().when(getVolumeStatAnswer.getResult()).thenReturn(false);
+
         MigrateVolumeAnswer migrateVolumeAnswer = Mockito.mock(MigrateVolumeAnswer.class);
         when(ep.sendMessage(any())).thenReturn(migrateVolumeAnswer);
-        when(migrateVolumeAnswer.getResult()).thenReturn(false);
+        Mockito.lenient().when(migrateVolumeAnswer.getResult()).thenReturn(false);
-        Mockito.doNothing().when(scaleIOPrimaryDataStoreDriver)
+        Mockito.lenient().doNothing().when(scaleIOPrimaryDataStoreDriver)
                 .revertBlockCopyVolumeOperations(any(), any(), any(), any());
 
         Answer answer = scaleIOPrimaryDataStoreDriver.liveMigrateVolume(srcData, destData);