CLOUDSTACK-3335:[Object_Store_Refactor][Automation]CopyTemplate failed

with Unexpected exception.

engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java

@@ -54,6 +54,7 @@ import org.apache.cloudstack.storage.datastore.DataObjectManager;
 import org.apache.cloudstack.storage.datastore.ObjectInDataStoreManager;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
+import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
 import org.apache.cloudstack.storage.image.store.TemplateObject;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.log4j.Logger;
@@ -63,6 +64,8 @@ import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.storage.ListTemplateAnswer;
 import com.cloud.agent.api.storage.ListTemplateCommand;
 import com.cloud.alert.AlertManager;
+import com.cloud.configuration.Config;
+import com.cloud.configuration.dao.ConfigurationDao;
 import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.dao.ClusterDao;
 import com.cloud.dc.dao.DataCenterDao;
@@ -82,6 +85,7 @@ import com.cloud.template.TemplateManager;
 import com.cloud.user.AccountManager;
 import com.cloud.user.ResourceLimitService;
 import com.cloud.utils.UriUtils;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 @Component
 public class TemplateServiceImpl implements TemplateService {
@@ -118,6 +122,8 @@ public class TemplateServiceImpl implements TemplateService {
     EndPointSelector _epSelector;
     @Inject
     TemplateManager _tmpltMgr;
+    @Inject
+    ConfigurationDao _configDao;
 
     class TemplateOpContext<T> extends AsyncRpcContext<T> {
         final TemplateObject template;
@@ -187,7 +193,7 @@ public class TemplateServiceImpl implements TemplateService {
         /* Baremetal need not to download any template */
         availHypers.remove(HypervisorType.BareMetal);
         availHypers.add(HypervisorType.None); // bug 9809: resume ISO
-                                              // download.
+        // download.
 
         for (VMTemplateVO template : toBeDownloaded) {
             if (availHypers.contains(template.getHypervisorType())) {
@@ -377,10 +383,10 @@ public class TemplateServiceImpl implements TemplateService {
             /* Baremetal need not to download any template */
             availHypers.remove(HypervisorType.BareMetal);
             availHypers.add(HypervisorType.None); // bug 9809: resume ISO
-                                                  // download.
+            // download.
             for (VMTemplateVO tmplt : toBeDownloaded) {
                 if (tmplt.getUrl() == null) { // If url is null we can't
-                                              // initiate the download
+                    // initiate the download
                     continue;
                 }
 
@@ -555,9 +561,66 @@ public class TemplateServiceImpl implements TemplateService {
 
     @Override
     public AsyncCallFuture<TemplateApiResult> copyTemplate(TemplateInfo srcTemplate, DataStore destStore) {
-        return copyAsync(srcTemplate, srcTemplate, destStore);
+        // generate a URL from source template ssvm to download to destination data store
+        String url = generateCopyUrl(srcTemplate);
+        if (url == null) {
+            s_logger.warn("Unable to start/resume copy of template " + srcTemplate.getUniqueName() + " to " + destStore.getName() + ", no secondary storage vm in running state in source zone");
+            throw new CloudRuntimeException("No secondary VM in running state in source template zone ");
+        }
+
+        TemplateObject tmplForCopy = (TemplateObject)_templateFactory.getTemplate(srcTemplate, destStore);
+        tmplForCopy.setUrl(url);
+
+        AsyncCallFuture<TemplateApiResult> future = new AsyncCallFuture<TemplateApiResult>();
+        DataObject templateOnStore = destStore.create(tmplForCopy);
+        templateOnStore.processEvent(Event.CreateOnlyRequested);
+
+        TemplateOpContext<TemplateApiResult> context = new TemplateOpContext<TemplateApiResult>(null,
+                (TemplateObject) templateOnStore, future);
+        AsyncCallbackDispatcher<TemplateServiceImpl, CopyCommandResult> caller = AsyncCallbackDispatcher.create(this);
+        caller.setCallback(caller.getTarget().copyTemplateCallBack(null, null)).setContext(context);
+        destStore.getDriver().createAsync(destStore, templateOnStore, caller);
+        return future;
     }
 
+
+    private String generateCopyUrl(String ipAddress, String dir, String path){
+        String hostname = ipAddress;
+        String scheme = "http";
+        boolean _sslCopy = false;
+        String sslCfg = _configDao.getValue(Config.SecStorageEncryptCopy.toString());
+        if ( sslCfg != null ){
+            _sslCopy = Boolean.parseBoolean(sslCfg);
+        }
+        if (_sslCopy) {
+            hostname = ipAddress.replace(".", "-");
+            hostname = hostname + ".realhostip.com";
+            scheme = "https";
+        }
+        return scheme + "://" + hostname + "/copy/SecStorage/" + dir + "/" + path;
+    }
+
+    private String generateCopyUrl(TemplateInfo  srcTemplate) {
+        DataStore srcStore = srcTemplate.getDataStore();
+        EndPoint ep = _epSelector.select(srcTemplate);
+        if ( ep != null ){
+            if (ep.getPublicAddr() == null) {
+                s_logger.warn("A running secondary storage vm has a null public ip?");
+                return null;
+            }
+            return generateCopyUrl(ep.getPublicAddr(), ((ImageStoreEntity) srcStore).getMountPoint(), srcTemplate.getInstallPath());
+        }
+
+        VMTemplateVO tmplt = _templateDao.findById(srcTemplate.getId());
+        HypervisorType hyperType = tmplt.getHypervisorType();
+        /*No secondary storage vm yet*/
+        if (hyperType != null && hyperType == HypervisorType.KVM) {
+            return "file://" + ((ImageStoreEntity) srcStore).getMountPoint() + "/" + srcTemplate.getInstallPath();
+        }
+        return null;
+    }
+
+
     @Override
     public AsyncCallFuture<TemplateApiResult> prepareTemplateOnPrimary(TemplateInfo srcTemplate, StoragePool pool) {
         return copyAsync(srcTemplate, srcTemplate, (DataStore) pool);
@@ -598,7 +661,7 @@ public class TemplateServiceImpl implements TemplateService {
                         tmplt.getUrl());
                 tmpltStore.setSize(0L);
                 tmpltStore.setPhysicalSize(0); // no size information for
-                                               // pre-seeded system vm templates
+                // pre-seeded system vm templates
                 tmpltStore.setDataStoreRole(store.getRole());
                 _vmTemplateStoreDao.persist(tmpltStore);
             }

engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java

@@ -56,6 +56,7 @@ public class TemplateObject implements TemplateInfo {
     private static final Logger s_logger = Logger.getLogger(TemplateObject.class);
     private VMTemplateVO imageVO;
     private DataStore dataStore;
+    private String url;
     @Inject
     VMTemplateDao imageDao;
     @Inject
@@ -109,6 +110,9 @@ public class TemplateObject implements TemplateInfo {
 
     @Override
     public String getUri() {
+        if ( url != null ){
+            return url;
+        }
         VMTemplateVO image = imageDao.findById(this.imageVO.getId());
 
         return image.getUrl();
@@ -364,9 +368,16 @@ public class TemplateObject implements TemplateInfo {
 
     @Override
     public String getUrl() {
+        if (url != null ){
+            return url;
+        }
         return this.imageVO.getUrl();
     }
 
+    public void setUrl(String url){
+        this.url = url;
+    }
+
     @Override
     public String getChecksum() {
         return this.imageVO.getChecksum();

server/src/com/cloud/template/TemplateManagerImpl.java

@@ -16,11 +16,7 @@
 // under the License.
 package com.cloud.template;
 
-import java.net.Inet6Address;
-import java.net.InetAddress;
-import java.net.URI;
 import java.net.URISyntaxException;
-import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
@@ -69,7 +65,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
 import org.apache.cloudstack.framework.async.AsyncCallFuture;
 import org.apache.cloudstack.storage.command.AttachCommand;
 import org.apache.cloudstack.storage.command.CommandResult;
-import org.apache.cloudstack.storage.command.CopyCommand;
 import org.apache.cloudstack.storage.command.DettachCommand;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
@@ -80,7 +75,6 @@ import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
-import com.amazonaws.services.s3.model.CannedAccessControlList;
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
@@ -90,12 +84,10 @@ import com.cloud.agent.api.storage.DestroyCommand;
 import com.cloud.agent.api.to.DataTO;
 import com.cloud.agent.api.to.DiskTO;
 import com.cloud.agent.api.to.NfsTO;
-import com.cloud.agent.api.to.S3TO;
 import com.cloud.api.ApiDBUtils;
 import com.cloud.api.query.dao.UserVmJoinDao;
 import com.cloud.api.query.vo.UserVmJoinVO;
 import com.cloud.async.AsyncJobManager;
-import com.cloud.async.AsyncJobVO;
 import com.cloud.configuration.Config;
 import com.cloud.configuration.Resource.ResourceType;
 import com.cloud.configuration.dao.ConfigurationDao;
@@ -114,7 +106,6 @@ import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.PermissionDeniedException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.StorageUnavailableException;
-import com.cloud.exception.UnsupportedServiceException;
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor;
@@ -141,10 +132,8 @@ import com.cloud.storage.StoragePoolHostVO;
 import com.cloud.storage.StoragePoolStatus;
 import com.cloud.storage.TemplateProfile;
 import com.cloud.storage.Upload;
-import com.cloud.storage.Upload.Type;
 import com.cloud.storage.VMTemplateZoneVO;
 
-import com.cloud.storage.UploadVO;
 import com.cloud.storage.VMTemplateHostVO;
 import com.cloud.storage.VMTemplateStoragePoolVO;
 import com.cloud.storage.VMTemplateStorageResourceAssoc;
@@ -184,7 +173,6 @@ import com.cloud.uservm.UserVm;
 import com.cloud.utils.EnumUtils;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.Pair;
-import com.cloud.utils.S3Utils;
 import com.cloud.utils.component.AdapterBase;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
@@ -423,8 +411,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
     public VirtualMachineTemplate prepareTemplate(long templateId, long zoneId) {
 
         VMTemplateVO vmTemplate = _tmpltDao.findById(templateId);
-        if (vmTemplate == null)
+        if (vmTemplate == null) {
             throw new InvalidParameterValueException("Unable to find template id=" + templateId);
+        }
 
         _accountMgr.checkAccess(UserContext.current().getCaller(), AccessType.ModifyEntry, true, vmTemplate);
 
@@ -737,6 +726,11 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
         boolean success = copy(userId, template, srcSecStore, dstZone);
 
         if (success) {
+            // increase resource count
+            long accountId = template.getAccountId();
+            if (template.getSize() != null) {
+                _resourceLimitMgr.incrementResourceCount(accountId, ResourceType.secondary_storage, template.getSize());
+            }
             return template;
         } else {
             throw new CloudRuntimeException("Failed to copy template");