Handle ListTemplateCommand in NfsSecondaryStorageResource for various

data store provider.

api/src/com/cloud/agent/api/DeleteTemplateFromS3Command.java

@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package com.cloud.agent.api;
-
-import com.cloud.agent.api.to.S3TO;
-
-public class DeleteTemplateFromS3Command extends Command {
-
-    private S3TO s3;
-    private Long templateId;
-    private Long accountId;
-
-    protected DeleteTemplateFromS3Command() {
-        super();
-    }
-
-    public DeleteTemplateFromS3Command(final S3TO s3, final Long accountId,
-            final Long templateId) {
-
-        super();
-
-        this.s3 = s3;
-        this.accountId = accountId;
-        this.templateId = templateId;
-
-    }
-
-    @Override
-    public int hashCode() {
-        final int prime = 31;
-        int result = 1;
-        result = prime * result
-                + ((accountId == null) ? 0 : accountId.hashCode());
-        result = prime * result + ((s3 == null) ? 0 : s3.hashCode());
-        result = prime * result
-                + ((templateId == null) ? 0 : templateId.hashCode());
-        return result;
-    }
-
-    @Override
-    public boolean equals(Object thatObject) {
-
-        if (this == thatObject) {
-            return true;
-        }
-
-        if (thatObject == null) {
-            return false;
-        }
-
-        if (getClass() != thatObject.getClass()) {
-            return false;
-        }
-
-        final DeleteTemplateFromS3Command thatCommand = (DeleteTemplateFromS3Command) thatObject;
-
-        if (!(accountId == thatCommand.accountId)
-                || (this.accountId != null && this.accountId
-                        .equals(thatCommand.accountId))) {
-            return false;
-        }
-
-        if (!(templateId == thatCommand.templateId)
-                || (this.templateId != null && this.templateId
-                        .equals(thatCommand.templateId))) {
-            return false;
-        }
-
-        return true;
-
-    }
-
-    public S3TO getS3() {
-        return s3;
-    }
-
-    public Long getTemplateId() {
-        return templateId;
-    }
-
-    public Long getAccountId() {
-        return accountId;
-    }
-
-    @Override
-    public boolean executeInSequence() {
-        return true;
-    }
-
-}

api/src/com/cloud/agent/api/storage/DeleteTemplateCommand.java

@@ -30,8 +30,7 @@ public class DeleteTemplateCommand extends ssCommand {
 	}
 
 
-	public DeleteTemplateCommand(DataStoreTO store, String secUrl, String templatePath, Long templateId, Long accountId) {
+	public DeleteTemplateCommand(DataStoreTO store, String templatePath, Long templateId, Long accountId) {
-	    this.setSecUrl(secUrl);
     	this.templatePath = templatePath;
     	this.templateId = templateId;
     	this.accountId = accountId;

api/src/com/cloud/agent/api/storage/DownloadSystemTemplateCommand.java

@@ -51,6 +51,7 @@ public class DownloadSystemTemplateCommand extends Command {
     private Long accountId;
     private String url;
     private Long maxDownloadSizeInBytes;
+    private String name;
 
 	protected DownloadSystemTemplateCommand() {
 	}
@@ -63,6 +64,7 @@ public class DownloadSystemTemplateCommand extends Command {
 	    this.url = secUrl;
 	    this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
 	    this.resourceId = template.getId();
+	    this.name = template.getUniqueName();
 	}
 
 
@@ -73,6 +75,7 @@ public class DownloadSystemTemplateCommand extends Command {
         this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
         this.resourceId = template.getId();
 		auth = new PasswordAuth(user, passwd);
+		this.name = template.getUniqueName();
 	}
 
 
@@ -144,6 +147,18 @@ public class DownloadSystemTemplateCommand extends Command {
 
 
 
+    public String getName() {
+        return name;
+    }
+
+
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+
+
     @Override
     public boolean executeInSequence() {
         // TODO Auto-generated method stub

api/src/com/cloud/agent/api/storage/ListTemplateCommand.java

@@ -18,37 +18,27 @@ package com.cloud.agent.api.storage;
 
 import com.cloud.agent.api.LogLevel;
 import com.cloud.agent.api.LogLevel.Log4jLevel;
-import com.cloud.agent.api.to.SwiftTO;
+import com.cloud.agent.api.to.DataStoreTO;
 
 public class ListTemplateCommand extends StorageCommand {
-    private String secUrl;
     @LogLevel(Log4jLevel.Off)
-    private SwiftTO swift;
+    private DataStoreTO store;
 
     public ListTemplateCommand() {
     }
 
-	public ListTemplateCommand(String secUrl) {
+	public ListTemplateCommand(DataStoreTO store) {
-	    this.secUrl = secUrl;
+        this.store = store;
-        this.swift = null;
 	}
 
-    public ListTemplateCommand(SwiftTO swift) {
-        this.secUrl = null;
-        this.swift = swift;
-    }
-
     @Override
     public boolean executeInSequence() {
         return true;
     }
 
-    public String getSecUrl() {
-        return secUrl;
-    }
 
-    public SwiftTO getSwift() {
+    public DataStoreTO getDataStore() {
-        return swift;
+        return store;
     }
 
 }

core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java

@@ -9,16 +9,20 @@ import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.util.List;
 
 import org.springframework.stereotype.Component;
 
+import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
+import com.cloud.agent.api.storage.DownloadAnswer;
 import com.cloud.agent.api.storage.DownloadSystemTemplateCommand;
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.NfsTO;
 import com.cloud.agent.api.to.S3TO;
 import com.cloud.agent.api.to.SwiftTO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.utils.S3Utils;
 import com.cloud.utils.UriUtils;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -62,10 +66,19 @@ public class LocalNfsSecondaryStorageResource extends
             }
 
             final String bucket = s3.getBucketName();
-            String key = join(asList(determineS3TemplateDirectory(cmd.getAccountId(), cmd.getResourceId()), urlObj.getFile()), S3Utils.SEPARATOR);
+            // convention is no / in the end for install path based on S3Utils implementation.
+            String path = determineS3TemplateDirectory(cmd.getAccountId(), cmd.getResourceId(), cmd.getName());
+            // template key is
+            // TEMPLATE_ROOT_DIR/account_id/template_id/template_name
+            String key = join(asList(path, urlObj.getFile()), S3Utils.SEPARATOR);
             S3Utils.putObject(s3, in, bucket, key);
-            return new Answer(cmd, true, format("Uploaded the contents of input stream from %1$s for template id %2$s to S3 bucket %3$s", url,
+            List<S3ObjectSummary> s3Obj = S3Utils.getDirectory(s3, bucket, path);
-                    cmd.getResourceId(), bucket));
+            if (s3Obj == null || s3Obj.size() == 0) {
+                return new Answer(cmd, false, "Failed to download to S3 bucket: " + bucket + " with key: " + key);
+            } else {
+                return new DownloadAnswer(null, 100, null, Status.DOWNLOADED, path, path, s3Obj.get(0).getSize(), s3Obj.get(0).getSize(), s3Obj.get(0)
+                        .getETag());
+            }
         }
         else if ( dstore instanceof NfsTO ){
             return new Answer(cmd, false, "Nfs needs to be pre-installed with system vm templates");

core/src/com/cloud/storage/resource/LocalSecondaryStorageResource.java

@@ -50,6 +50,7 @@ import com.cloud.storage.template.DownloadManager;
 import com.cloud.storage.template.DownloadManagerImpl;
 import com.cloud.storage.template.TemplateProp;
 import com.cloud.utils.component.ComponentContext;
+import com.cloud.agent.api.to.NfsTO;
 
 public class LocalSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
     private static final Logger s_logger = Logger.getLogger(LocalSecondaryStorageResource.class);
@@ -94,7 +95,7 @@ public class LocalSecondaryStorageResource extends ServerResourceBase implements
         } else if (cmd instanceof ReadyCommand) {
             return new ReadyAnswer((ReadyCommand)cmd);
         } else if (cmd instanceof ListTemplateCommand){
-            return execute((ListTemplateCommand)cmd);   
+            return execute((ListTemplateCommand)cmd);
         } else if (cmd instanceof ComputeChecksumCommand){
             return execute((ComputeChecksumCommand)cmd);
         } else {
@@ -103,14 +104,14 @@ public class LocalSecondaryStorageResource extends ServerResourceBase implements
     }
 
     private Answer execute(ComputeChecksumCommand cmd) {
-        return new Answer(cmd, false, null);   
+        return new Answer(cmd, false, null);
     }
 
 
     private Answer execute(ListTemplateCommand cmd) {
         String root = getRootDir();
         Map<String, TemplateProp> templateInfos = _dlMgr.gatherTemplateInfo(root);
-        return new ListTemplateAnswer(cmd.getSecUrl(), templateInfos);
+        return new ListTemplateAnswer(((NfsTO)cmd.getDataStore()).getUrl(), templateInfos);
     }
 
     @Override
@@ -213,14 +214,14 @@ public class LocalSecondaryStorageResource extends ServerResourceBase implements
 	@Override
 	public void setName(String name) {
 		// TODO Auto-generated method stub
-		
+
 	}
 
 
 	@Override
 	public void setConfigParams(Map<String, Object> params) {
 		// TODO Auto-generated method stub
-		
+
 	}
 
 
@@ -241,6 +242,6 @@ public class LocalSecondaryStorageResource extends ServerResourceBase implements
 	@Override
 	public void setRunLevel(int level) {
 		// TODO Auto-generated method stub
-		
+
 	}
 }

engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java

@@ -414,7 +414,7 @@ public class TemplateServiceImpl implements TemplateService {
 
 
     private Map<String, TemplateProp> listTemplate(DataStore ssStore) {
-        ListTemplateCommand cmd = new ListTemplateCommand(ssStore.getUri());
+        ListTemplateCommand cmd = new ListTemplateCommand(ssStore.getTO());
         EndPoint ep = _epSelector.select(ssStore);
         Answer answer = ep.sendMessage(cmd);
         if (answer != null && answer.getResult()) {

plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java

@@ -292,7 +292,7 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver {
             TemplateDataStoreVO tmplStore = _templateStoreDao.findByStoreTemplate(storeId, templateId);
             String installPath = tmplStore.getInstallPath();
             if (installPath != null) {
-                DeleteTemplateCommand cmd = new DeleteTemplateCommand(store.getTO(), store.getUri(), installPath, template.getId(), template.getAccountId());
+                DeleteTemplateCommand cmd = new DeleteTemplateCommand(store.getTO(), installPath, template.getId(), template.getAccountId());
                 EndPoint ep = _epSelector.select(templateObj);
                 Answer answer = ep.sendMessage(cmd);
 

plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java

@@ -249,7 +249,7 @@ public class S3ImageStoreDriverImpl implements ImageStoreDriver {
             TemplateDataStoreVO tmplStore = _templateStoreDao.findByStoreTemplate(storeId, templateId);
             String installPath = tmplStore.getInstallPath();
             if (installPath != null) {
-                DeleteTemplateCommand cmd = new DeleteTemplateCommand(store.getTO(), store.getUri(), installPath, template.getId(), template.getAccountId());
+                DeleteTemplateCommand cmd = new DeleteTemplateCommand(store.getTO(), installPath, template.getId(), template.getAccountId());
                 EndPoint ep = _epSelector.select(templateObj);
                 Answer answer = ep.sendMessage(cmd);
 

plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java

@@ -243,7 +243,7 @@ public class SwiftImageStoreDriverImpl implements ImageStoreDriver {
             TemplateDataStoreVO tmplStore = _templateStoreDao.findByStoreTemplate(storeId, templateId);
             String installPath = tmplStore.getInstallPath();
             if (installPath != null) {
-                DeleteTemplateCommand cmd = new DeleteTemplateCommand(store.getTO(), store.getUri(), installPath, template.getId(), template.getAccountId());
+                DeleteTemplateCommand cmd = new DeleteTemplateCommand(store.getTO(), installPath, template.getId(), template.getAccountId());
                 EndPoint ep = _epSelector.select(templateObj);
                 Answer answer = ep.sendMessage(cmd);
 

plugins/storage/volume/sample/src/org/apache/cloudstack/storage/datastore/driver/SamplePrimaryDataStoreDriverImpl.java

@@ -93,7 +93,7 @@ public class SamplePrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
         CreateCmdResult result = null;
         CreateObjectAnswer volAnswer = (CreateObjectAnswer) callback.getResult();
         if (volAnswer.getResult()) {
-            result = new CreateCmdResult(volAnswer.getPath(), volAnswer.getSize());
+            result = new CreateCmdResult(volAnswer.getPath(), volAnswer);
         } else {
             result = new CreateCmdResult("", null);
             result.setResult(volAnswer.getDetails());

server/src/com/cloud/storage/StorageManagerImpl.java

@@ -1245,7 +1245,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
                         if (installPath != null) {
                             EndPoint ep = _epSelector.select(store);
-                            Command cmd = new DeleteTemplateCommand(store.getTO(), store.getUri(), destroyedTemplateStoreVO.getInstallPath(),
+                            Command cmd = new DeleteTemplateCommand(store.getTO(), destroyedTemplateStoreVO.getInstallPath(),
                                     destroyedTemplate.getId(), destroyedTemplate.getAccountId());
                             Answer answer = ep.sendMessage(cmd);
 

server/src/com/cloud/storage/s3/S3Manager.java

@@ -49,7 +49,7 @@ public interface S3Manager extends Manager {
 
     boolean isTemplateInstalled(Long templateId);
 
-    void deleteTemplate(final Long accountId, final Long templateId);
+    //void deleteTemplate(final Long accountId, final Long templateId);
 
     String downloadTemplateFromS3ToSecondaryStorage(final long dcId,
             final long templateId, final int primaryStorageDownloadWait);

server/src/com/cloud/storage/s3/S3ManagerImpl.java

@@ -41,7 +41,6 @@ import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.Callable;
 
-import javax.annotation.PostConstruct;
 import javax.ejb.Local;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
@@ -54,7 +53,6 @@ import org.springframework.stereotype.Component;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.DeleteTemplateFromS3Command;
 import com.cloud.agent.api.DownloadTemplateFromS3ToSecondaryStorageCommand;
 import com.cloud.agent.api.UploadTemplateToS3FromSecondaryStorageCommand;
 import com.cloud.agent.api.to.S3TO;
@@ -282,70 +280,7 @@ public class S3ManagerImpl extends ManagerBase implements S3Manager {
                         + "been implemented");
     }
 
-    @Override
-    public void deleteTemplate(final Long templateId, final Long accountId) {
 
-        final S3TO s3 = getS3TO();
-
-        if (s3 == null) {
-            final String errorMessage = "Delete Template Failed: No S3 configuration defined.";
-            LOGGER.error(errorMessage);
-            throw new CloudRuntimeException(errorMessage);
-        }
-
-        final VMTemplateS3VO vmTemplateS3VO = vmTemplateS3Dao
-                .findOneByS3Template(s3.getId(), templateId);
-        if (vmTemplateS3VO == null) {
-            final String errorMessage = format(
-                    "Delete Template Failed: Unable to find Template %1$s in S3.",
-                    templateId);
-            LOGGER.error(errorMessage);
-            throw new CloudRuntimeException(errorMessage);
-        }
-
-        try {
-
-            executeWithNoWaitLock(determineLockId(accountId, templateId),
-                    new Callable<Void>() {
-
-                @Override
-                public Void call() throws Exception {
-
-                    final Answer answer = agentManager.sendToSSVM(null,
-                            new DeleteTemplateFromS3Command(s3,
-                                    accountId, templateId));
-                    if (answer == null || !answer.getResult()) {
-                        final String errorMessage = format(
-                                "Delete Template Failed: Unable to delete template id %1$s from S3 due to following error: %2$s",
-                                templateId,
-                                ((answer == null) ? "answer is null"
-                                        : answer.getDetails()));
-                        LOGGER.error(errorMessage);
-                        throw new CloudRuntimeException(errorMessage);
-                    }
-
-                    vmTemplateS3Dao.remove(vmTemplateS3VO.getId());
-                    LOGGER.debug(format(
-                            "Deleted template %1$s from S3.",
-                            templateId));
-
-                    return null;
-
-                }
-
-            });
-
-        } catch (Exception e) {
-
-            final String errorMessage = format(
-                    "Delete Template Failed: Unable to delete template id %1$s from S3 due to the following error: %2$s.",
-                    templateId, e.getMessage());
-            LOGGER.error(errorMessage);
-            throw new CloudRuntimeException(errorMessage, e);
-
-        }
-
-    }
 
     @SuppressWarnings("unchecked")
     @Override

utils/src/com/cloud/utils/S3Utils.java

@@ -54,6 +54,7 @@ import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.Bucket;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.cloud.utils.exception.CloudRuntimeException;
 
@@ -155,6 +156,22 @@ public final class S3Utils {
 
     }
 
+    // Note that whenever S3Object is returned, client code needs to close the internal stream to avoid resource leak.
+    public static S3Object getObject(final ClientOptions clientOptions,
+            final String bucketName, final String key) {
+
+        assert clientOptions != null;
+        assert !isBlank(bucketName);
+        assert !isBlank(key);
+
+        if (LOGGER.isDebugEnabled()) {
+            LOGGER.debug(format("Get S3 object %1$s in "
+                    + "bucket %2$s", key, bucketName));
+        }
+
+        return acquireClient(clientOptions).getObject(bucketName, key);
+
+    }
 
     @SuppressWarnings("unchecked")
     public static File getFile(final ClientOptions clientOptions,
@@ -243,6 +260,18 @@ public final class S3Utils {
 
     }
 
+    public static List<S3ObjectSummary> getDirectory(final ClientOptions clientOptions,
+            final String bucketName, final String sourcePath){
+        assert clientOptions != null;
+        assert isNotBlank(bucketName);
+        assert isNotBlank(sourcePath);
+
+        final AmazonS3 connection = acquireClient(clientOptions);
+
+        // List the objects in the source directory on S3
+        return listDirectory(bucketName, sourcePath, connection);
+    }
+
     private static List<S3ObjectSummary> listDirectory(final String bucketName,
             final String directory, final AmazonS3 client) {