Merge branch 'master' into ui-plugins

.gitignore

@@ -67,8 +67,10 @@ awsapi/modules/*
 .settings/
 db.properties.override
 awsapi/overlays/
+tools/marvin/marvin/cloudstackAPI/*
 *.egg-info/
 docs/tmp
 docs/publish
 docs/runbook/tmp
 docs/runbook/publish
+Gemfile.lock

api/src/com/cloud/agent/api/BackupSnapshotCommand.java

@@ -17,6 +17,7 @@
 package com.cloud.agent.api;
 
 import com.cloud.agent.api.LogLevel.Log4jLevel;
+import com.cloud.agent.api.to.S3TO;
 import com.cloud.agent.api.to.StorageFilerTO;
 import com.cloud.agent.api.to.SwiftTO;
 import com.cloud.storage.StoragePool;
@@ -32,6 +33,7 @@ public class BackupSnapshotCommand extends SnapshotCommand {
     private Long snapshotId;
     @LogLevel(Log4jLevel.Off)
     private SwiftTO swift;
+    private S3TO s3;
     StorageFilerTO pool;
 
     protected BackupSnapshotCommand() {
@@ -48,8 +50,7 @@ public class BackupSnapshotCommand extends SnapshotCommand {
      * @param isFirstSnapshotOfRootVolume true if this is the first snapshot of a root volume. Set the parent of the backup to null.
      * @param isVolumeInactive         True if the volume belongs to a VM that is not running or is detached. 
      */
-    public BackupSnapshotCommand(String primaryStoragePoolNameLabel,
-                                 String secondaryStoragePoolURL,
+    public BackupSnapshotCommand(String secondaryStoragePoolURL,
                                  Long   dcId,
                                  Long   accountId,
                                  Long   volumeId,
@@ -64,13 +65,12 @@ public class BackupSnapshotCommand extends SnapshotCommand {
                                  String vmName,
                                  int wait) 
     {
-        super(primaryStoragePoolNameLabel, secondaryStoragePoolURL, snapshotUuid, snapshotName, dcId, accountId, volumeId);
+        super(pool, secondaryStoragePoolURL, snapshotUuid, snapshotName, dcId, accountId, volumeId);
         this.snapshotId = snapshotId;
         this.prevSnapshotUuid = prevSnapshotUuid;
         this.prevBackupUuid = prevBackupUuid;
         this.isVolumeInactive = isVolumeInactive;
         this.vmName = vmName;
-        this.pool = new StorageFilerTO(pool);
         setVolumePath(volumePath);
         setWait(wait);
     }
@@ -99,11 +99,16 @@ public class BackupSnapshotCommand extends SnapshotCommand {
         this.swift = swift;
     }
 
+    public S3TO getS3() {
+        return s3;
+    }
+
+    public void setS3(S3TO s3) {
+        this.s3 = s3;
+    }
+
     public Long getSnapshotId() {
         return snapshotId;
     }
 
-    public StorageFilerTO getPool() {
-        return pool;
-    }
 }

api/src/com/cloud/agent/api/CreatePrivateTemplateFromSnapshotCommand.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.agent.api;
 
+import com.cloud.storage.StoragePool;
+
 /**
  * This currently assumes that both primary and secondary storage are mounted on the XenServer.  
  */
@@ -40,7 +42,8 @@ public class CreatePrivateTemplateFromSnapshotCommand extends SnapshotCommand {
      *                                 It may not be the UUID of the base copy of the snapshot, if no data was written since last snapshot.
      * @param origTemplateInstallPath  The install path of the original template VHD on the secondary                                 
      */
-    public CreatePrivateTemplateFromSnapshotCommand(String primaryStoragePoolNameLabel,
+
+    public CreatePrivateTemplateFromSnapshotCommand(StoragePool pool,
                                                     String secondaryStoragePoolURL,
                                                     Long   dcId,
                                                     Long   accountId,
@@ -52,7 +55,7 @@ public class CreatePrivateTemplateFromSnapshotCommand extends SnapshotCommand {
                                                     String templateName,
                                                     int wait) 
     {
-        super(primaryStoragePoolNameLabel, secondaryStoragePoolURL, backedUpSnapshotUuid, backedUpSnapshotName, dcId, accountId, volumeId);
+        super(pool, secondaryStoragePoolURL, backedUpSnapshotUuid, backedUpSnapshotName, dcId, accountId, volumeId);
         this.origTemplateInstallPath = origTemplateInstallPath;
         this.newTemplateId = newTemplateId;
         this.templateName = templateName;

api/src/com/cloud/agent/api/CreatePrivateTemplateFromVolumeCommand.java

@@ -16,6 +16,9 @@
 // under the License.
 package com.cloud.agent.api;
 
+import com.cloud.agent.api.to.StorageFilerTO;
+import com.cloud.storage.StoragePool;
+
 public class CreatePrivateTemplateFromVolumeCommand extends SnapshotCommand {
     private String _vmName;
     private String _volumePath;
@@ -23,13 +26,14 @@ public class CreatePrivateTemplateFromVolumeCommand extends SnapshotCommand {
     private String _uniqueName;
     private long _templateId;
     private long _accountId;
+    StorageFilerTO _primaryPool;
     // For XenServer
     private String _secondaryStorageUrl;
 
     public CreatePrivateTemplateFromVolumeCommand() {
     }
 
-    public CreatePrivateTemplateFromVolumeCommand(String StoragePoolUUID, String secondaryStorageUrl, long templateId, long accountId, String userSpecifiedName, String uniqueName, String volumePath, String vmName, int wait) {
+    public CreatePrivateTemplateFromVolumeCommand(StoragePool pool, String secondaryStorageUrl, long templateId, long accountId, String userSpecifiedName, String uniqueName, String volumePath, String vmName, int wait) {
         _secondaryStorageUrl = secondaryStorageUrl;
         _templateId = templateId;
         _accountId = accountId;
@@ -37,7 +41,8 @@ public class CreatePrivateTemplateFromVolumeCommand extends SnapshotCommand {
         _uniqueName = uniqueName;
         _volumePath = volumePath;
         _vmName = vmName;
-        primaryStoragePoolNameLabel = StoragePoolUUID;
+        primaryStoragePoolNameLabel = pool.getUuid();
+        _primaryPool = new StorageFilerTO(pool);
         setWait(wait);
     }
 
@@ -46,6 +51,10 @@ public class CreatePrivateTemplateFromVolumeCommand extends SnapshotCommand {
         return false;
     }
 
+    public StorageFilerTO getPool() {
+        return _primaryPool;
+    }
+
     public String getSecondaryStorageUrl() {
         return _secondaryStorageUrl;
     }

api/src/com/cloud/agent/api/CreateVolumeFromSnapshotCommand.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.agent.api;
 
+import com.cloud.storage.StoragePool;
+
 /**
  * This currently assumes that both primary and secondary storage are mounted on the XenServer.  
  */
@@ -39,7 +41,8 @@ public class CreateVolumeFromSnapshotCommand extends SnapshotCommand {
      *                                 It may not be the UUID of the base copy of the snapshot, if no data was written since last snapshot.
      * @param templatePath             The install path of the template VHD on the secondary, if this a root volume                                 
      */
-    public CreateVolumeFromSnapshotCommand(String primaryStoragePoolNameLabel,
+
+    public CreateVolumeFromSnapshotCommand(StoragePool pool,
                                            String secondaryStoragePoolURL,
                                            Long   dcId,
                                            Long   accountId,
@@ -48,7 +51,7 @@ public class CreateVolumeFromSnapshotCommand extends SnapshotCommand {
                                            String backedUpSnapshotName,
                                            int wait)
     {
-        super(primaryStoragePoolNameLabel, secondaryStoragePoolURL, backedUpSnapshotUuid, backedUpSnapshotName, dcId, accountId, volumeId);
+        super(pool, secondaryStoragePoolURL, backedUpSnapshotUuid, backedUpSnapshotName, dcId, accountId, volumeId);
         setWait(wait);
     }
 }

api/src/com/cloud/agent/api/DeleteSnapshotBackupCommand.java

@@ -17,6 +17,7 @@
 package com.cloud.agent.api;
 
 import com.cloud.agent.api.LogLevel.Log4jLevel;
+import com.cloud.agent.api.to.S3TO;
 import com.cloud.agent.api.to.SwiftTO;
 
 /**
@@ -26,6 +27,7 @@ import com.cloud.agent.api.to.SwiftTO;
 public class DeleteSnapshotBackupCommand extends SnapshotCommand {
     @LogLevel(Log4jLevel.Off)
     private SwiftTO swift;
+    private S3TO s3;
     private Boolean all;
 
     public SwiftTO getSwift() {
@@ -44,6 +46,10 @@ public class DeleteSnapshotBackupCommand extends SnapshotCommand {
         this.swift = swift;
     }
 
+    public S3TO getS3() {
+        return s3;
+    }
+
     protected DeleteSnapshotBackupCommand() {
     }
     
@@ -73,6 +79,7 @@ public class DeleteSnapshotBackupCommand extends SnapshotCommand {
      * @param childUUID                   The child VHD file of the backup whose parent is reset to its grandparent.  
      */
     public DeleteSnapshotBackupCommand(SwiftTO swift,
+                                       S3TO s3,
                                        String secondaryStoragePoolURL,
                                        Long   dcId,
                                        Long   accountId,
@@ -81,6 +88,7 @@ public class DeleteSnapshotBackupCommand extends SnapshotCommand {
     {
         super(null, secondaryStoragePoolURL, backupUUID, null, dcId, accountId, volumeId);
         setSwift(swift);
+        this.s3 = s3;
         setAll(all);
     }
 }

api/src/com/cloud/agent/api/DeleteTemplateFromS3Command.java

@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.to.S3TO;
+
+public class DeleteTemplateFromS3Command extends Command {
+
+    private S3TO s3;
+    private Long templateId;
+    private Long accountId;
+
+    protected DeleteTemplateFromS3Command() {
+        super();
+    }
+
+    public DeleteTemplateFromS3Command(final S3TO s3, final Long accountId,
+            final Long templateId) {
+
+        super();
+
+        this.s3 = s3;
+        this.accountId = accountId;
+        this.templateId = templateId;
+
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result
+                + ((accountId == null) ? 0 : accountId.hashCode());
+        result = prime * result + ((s3 == null) ? 0 : s3.hashCode());
+        result = prime * result
+                + ((templateId == null) ? 0 : templateId.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object thatObject) {
+
+        if (this == thatObject) {
+            return true;
+        }
+
+        if (thatObject == null) {
+            return false;
+        }
+
+        if (getClass() != thatObject.getClass()) {
+            return false;
+        }
+
+        final DeleteTemplateFromS3Command thatCommand = (DeleteTemplateFromS3Command) thatObject;
+
+        if (!(accountId == thatCommand.accountId)
+                || (this.accountId != null && this.accountId
+                        .equals(thatCommand.accountId))) {
+            return false;
+        }
+
+        if (!(templateId == thatCommand.templateId)
+                || (this.templateId != null && this.templateId
+                        .equals(thatCommand.templateId))) {
+            return false;
+        }
+
+        return true;
+
+    }
+
+    public S3TO getS3() {
+        return s3;
+    }
+
+    public Long getTemplateId() {
+        return templateId;
+    }
+
+    public Long getAccountId() {
+        return accountId;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return true;
+    }
+
+}

api/src/com/cloud/agent/api/DownloadSnapshotFromS3Command.java

@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.to.S3TO;
+
+public class DownloadSnapshotFromS3Command extends SnapshotCommand {
+
+    private S3TO s3;
+    private String parent;
+
+    protected DownloadSnapshotFromS3Command() {
+        super();
+    }
+
+    public DownloadSnapshotFromS3Command(S3TO s3, String parent,
+            String secondaryStorageUrl, Long dcId, Long accountId,
+            Long volumeId, String backupUuid, int wait) {
+
+        super(null, secondaryStorageUrl, backupUuid, "", dcId, accountId,
+                volumeId);
+
+        this.s3 = s3;
+        this.parent = parent;
+        setWait(wait);
+
+    }
+
+    public S3TO getS3() {
+        return s3;
+    }
+
+    public void setS3(S3TO s3) {
+        this.s3 = s3;
+    }
+
+    public String getParent() {
+        return parent;
+    }
+
+    public void setParent(String parent) {
+        this.parent = parent;
+    }
+
+}

api/src/com/cloud/agent/api/DownloadTemplateFromS3ToSecondaryStorageCommand.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.to.S3TO;
+
+public final class DownloadTemplateFromS3ToSecondaryStorageCommand extends Command {
+
+    private final S3TO s3;
+    private final Long accountId;
+    private final Long templateId;
+    private final String storagePath;
+
+    public DownloadTemplateFromS3ToSecondaryStorageCommand(final S3TO s3,
+        final Long accountId, final Long templateId,
+        final String storagePath, final int wait) {
+
+        super();
+
+        this.s3 = s3;
+        this.accountId = accountId;
+        this.templateId = templateId;
+        this.storagePath = storagePath;
+
+        setWait(wait);
+
+    }
+
+    public S3TO getS3() {
+        return this.s3;
+    }
+
+    public Long getAccountId() {
+        return this.accountId;
+    }
+
+    public Long getTemplateId() {
+        return this.templateId;
+    }
+
+    public String getStoragePath() {
+        return this.storagePath;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return true;
+    }
+
+}

api/src/com/cloud/agent/api/SnapshotCommand.java

@@ -16,12 +16,16 @@
 // under the License.
 package com.cloud.agent.api;
 
+import com.cloud.agent.api.to.StorageFilerTO;
+import com.cloud.storage.StoragePool;
+
 /**
  * This currently assumes that both primary and secondary storage are mounted on
  * the XenServer.
  */
 public class SnapshotCommand extends Command {
     protected String primaryStoragePoolNameLabel;
+    StorageFilerTO primaryPool;
     private String snapshotUuid;
     private String snapshotName;
     private String secondaryStorageUrl;
@@ -46,10 +50,11 @@ public class SnapshotCommand extends Command {
      *            is the value of that field If you have better ideas on how to
      *            get it, you are welcome.
      */
-    public SnapshotCommand(String primaryStoragePoolNameLabel,
+    public SnapshotCommand(StoragePool pool,
             String secondaryStorageUrl, String snapshotUuid,
             String snapshotName, Long dcId, Long accountId, Long volumeId) {
-        this.primaryStoragePoolNameLabel = primaryStoragePoolNameLabel;
+        this.primaryStoragePoolNameLabel = pool.getUuid();
+        this.primaryPool = new StorageFilerTO(pool);
         this.snapshotUuid = snapshotUuid;
         this.secondaryStorageUrl = secondaryStorageUrl;
         this.dcId = dcId;
@@ -65,6 +70,13 @@ public class SnapshotCommand extends Command {
         return primaryStoragePoolNameLabel;
     }
 
+    /**
+     * @return the primaryPool
+     */
+    public StorageFilerTO getPool() {
+        return primaryPool;
+    }
+
     /**
      * @return the snapshotUuid
      */

api/src/com/cloud/agent/api/UpgradeSnapshotCommand.java

@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 package com.cloud.agent.api;
+import com.cloud.storage.StoragePool;
+
 public class UpgradeSnapshotCommand extends SnapshotCommand {
     private String version;
     private Long templateId;
@@ -30,7 +32,7 @@ public class UpgradeSnapshotCommand extends SnapshotCommand {
      * @param snapshotUuid             The UUID of the snapshot which is going to be upgraded
      * @param _version          version for this snapshot                                 
      */
-    public UpgradeSnapshotCommand(String primaryStoragePoolNameLabel,
+    public UpgradeSnapshotCommand(StoragePool pool,
                                  String secondaryStoragePoolURL,
                                  Long   dcId,
                                  Long   accountId,
@@ -42,7 +44,7 @@ public class UpgradeSnapshotCommand extends SnapshotCommand {
                                  String snapshotName,
                                  String version)
     {
-        super(primaryStoragePoolNameLabel, secondaryStoragePoolURL, snapshotUuid, snapshotName, dcId, accountId, volumeId);
+        super(pool, secondaryStoragePoolURL, snapshotUuid, snapshotName, dcId, accountId, volumeId);
         this.version = version;
         this.templateId = templateId;
         this.tmpltAccountId = tmpltAccountId;

api/src/com/cloud/agent/api/UploadTemplateToS3FromSecondaryStorageCommand.java

@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.to.S3TO;
+
+public class UploadTemplateToS3FromSecondaryStorageCommand extends Command {
+
+    private final S3TO s3;
+    private final String storagePath;
+    private final Long dataCenterId;
+    private final Long accountId;
+    private final Long templateId;
+
+    public UploadTemplateToS3FromSecondaryStorageCommand(final S3TO s3,
+        final String storagePath, final Long dataCenterId, final Long accountId,
+        final Long templateId) {
+
+        super();
+
+        this.s3 = s3;
+        this.storagePath = storagePath;
+        this.dataCenterId = dataCenterId;
+        this.accountId = accountId;
+        this.templateId = templateId;
+
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+
+    @Override
+    public boolean equals(final Object thatObject) {
+
+        if (this == thatObject) {
+            return true;
+        }
+
+        if (thatObject == null || getClass() != thatObject.getClass()) {
+            return false;
+        }
+
+        final UploadTemplateToS3FromSecondaryStorageCommand thatCommand =
+                (UploadTemplateToS3FromSecondaryStorageCommand) thatObject;
+
+        if (this.accountId != null ? !this.accountId.equals(thatCommand
+                .accountId) : thatCommand.accountId != null) {
+            return false;
+        }
+
+        if (this.dataCenterId != null ? !this.dataCenterId.equals(thatCommand
+                .dataCenterId) : thatCommand.dataCenterId != null) {
+            return false;
+        }
+
+        if (this.s3 != null ? !this.s3.equals(thatCommand.s3) : thatCommand.s3 != null) {
+            return false;
+        }
+
+        if (this.storagePath != null ? !this.storagePath.equals(thatCommand
+                .storagePath) : thatCommand.storagePath != null) {
+            return false;
+        }
+
+        if (this.templateId != null ? !this.templateId.equals(thatCommand.templateId) :
+                thatCommand.templateId != null) {
+            return false;
+        }
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = this.s3 != null ? this.s3.hashCode() : 0;
+        result = 31 * result + (this.storagePath != null ? this.storagePath.hashCode() : 0);
+        result = 31 * result + (this.dataCenterId != null ? this.dataCenterId.hashCode() : 0);
+        result = 31 * result + (this.accountId != null ? this.accountId.hashCode() : 0);
+        result = 31 * result + (this.templateId != null ? this.templateId.hashCode() : 0);
+        return result;
+    }
+
+    public S3TO getS3() {
+        return this.s3;
+    }
+
+    public String getStoragePath() {
+        return this.storagePath;
+    }
+
+    public Long getDataCenterId() {
+        return this.dataCenterId;
+    }
+
+    public Long getAccountId() {
+        return this.accountId;
+    }
+
+    public Long getTemplateId() {
+        return this.templateId;
+    }
+
+}

api/src/com/cloud/agent/api/downloadSnapshotFromSwiftCommand.java

@@ -34,7 +34,7 @@ public class downloadSnapshotFromSwiftCommand extends SnapshotCommand {
    
     public downloadSnapshotFromSwiftCommand(SwiftTO swift, String secondaryStorageUrl, Long dcId, Long accountId, Long volumeId, String parent, String BackupUuid, int wait) {
 
-        super("", secondaryStorageUrl, BackupUuid, "", dcId, accountId, volumeId);
+        super(null, secondaryStorageUrl, BackupUuid, "", dcId, accountId, volumeId);
         setParent(parent);
         setSwift(swift);
         setWait(wait);

api/src/com/cloud/agent/api/storage/PrimaryStorageDownloadCommand.java

@@ -17,7 +17,8 @@
 package com.cloud.agent.api.storage;
 
 import com.cloud.storage.Storage.ImageFormat;
-
+import com.cloud.agent.api.to.StorageFilerTO;
+import com.cloud.storage.StoragePool;
 
 /**
  *
@@ -26,6 +27,7 @@ public class PrimaryStorageDownloadCommand extends AbstractDownloadCommand {
 	String localPath;
 	String poolUuid;
 	long poolId;
+        StorageFilerTO primaryPool;
 	
 	String secondaryStorageUrl;
 	String primaryStorageUrl;
@@ -33,10 +35,11 @@ public class PrimaryStorageDownloadCommand extends AbstractDownloadCommand {
     protected PrimaryStorageDownloadCommand() {
 	}
 
-    public PrimaryStorageDownloadCommand(String name, String url, ImageFormat format, long accountId, long poolId, String poolUuid, int wait) {
+    public PrimaryStorageDownloadCommand(String name, String url, ImageFormat format, long accountId, StoragePool pool, int wait) {
         super(name, url, format, accountId);
-        this.poolId = poolId;
-        this.poolUuid = poolUuid;
+        this.poolId = pool.getId();
+        this.poolUuid = pool.getUuid();
+        this.primaryPool = new StorageFilerTO(pool);
         setWait(wait);
     }
    
@@ -48,6 +51,10 @@ public class PrimaryStorageDownloadCommand extends AbstractDownloadCommand {
         return poolId;
     }
 
+    public StorageFilerTO getPool() {
+        return primaryPool;
+    }
+    
     public void setLocalPath(String path) {
     	this.localPath = path;
     }

api/src/com/cloud/agent/api/to/S3TO.java

@@ -0,0 +1,252 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+import com.cloud.utils.S3Utils;
+
+import java.util.Date;
+
+public final class S3TO implements S3Utils.ClientOptions {
+
+    private Long id;
+    private String uuid;
+    private String accessKey;
+    private String secretKey;
+    private String endPoint;
+    private String bucketName;
+    private Boolean httpsFlag;
+    private Integer connectionTimeout;
+    private Integer maxErrorRetry;
+    private Integer socketTimeout;
+    private Date created;
+
+    public S3TO() {
+
+        super();
+
+    }
+
+    public S3TO(final Long id, final String uuid, final String accessKey,
+            final String secretKey, final String endPoint,
+            final String bucketName, final Boolean httpsFlag,
+            final Integer connectionTimeout, final Integer maxErrorRetry,
+            final Integer socketTimeout, final Date created) {
+
+        super();
+
+        this.id = id;
+        this.uuid = uuid;
+        this.accessKey = accessKey;
+        this.secretKey = secretKey;
+        this.endPoint = endPoint;
+        this.bucketName = bucketName;
+        this.httpsFlag = httpsFlag;
+        this.connectionTimeout = connectionTimeout;
+        this.maxErrorRetry = maxErrorRetry;
+        this.socketTimeout = socketTimeout;
+        this.created = created;
+
+    }
+
+    @Override
+    public boolean equals(final Object thatObject) {
+
+        if (this == thatObject)
+            return true;
+        if (thatObject == null || getClass() != thatObject.getClass())
+            return false;
+
+        final S3TO thatS3TO = (S3TO) thatObject;
+
+        if (httpsFlag != null ? !httpsFlag.equals(thatS3TO.httpsFlag)
+                : thatS3TO.httpsFlag != null) {
+            return false;
+        }
+
+        if (accessKey != null ? !accessKey.equals(thatS3TO.accessKey)
+                : thatS3TO.accessKey != null) {
+            return false;
+        }
+
+        if (connectionTimeout != null ? !connectionTimeout
+                .equals(thatS3TO.connectionTimeout)
+                : thatS3TO.connectionTimeout != null) {
+            return false;
+        }
+
+        if (endPoint != null ? !endPoint.equals(thatS3TO.endPoint)
+                : thatS3TO.endPoint != null) {
+            return false;
+        }
+
+        if (id != null ? !id.equals(thatS3TO.id) : thatS3TO.id != null) {
+            return false;
+        }
+
+        if (uuid != null ? !uuid.equals(thatS3TO.uuid) : thatS3TO.uuid != null) {
+            return false;
+        }
+
+        if (maxErrorRetry != null ? !maxErrorRetry
+                .equals(thatS3TO.maxErrorRetry)
+                : thatS3TO.maxErrorRetry != null) {
+            return false;
+        }
+
+        if (secretKey != null ? !secretKey.equals(thatS3TO.secretKey)
+                : thatS3TO.secretKey != null) {
+            return false;
+        }
+
+        if (socketTimeout != null ? !socketTimeout
+                .equals(thatS3TO.socketTimeout)
+                : thatS3TO.socketTimeout != null) {
+            return false;
+        }
+
+        if (bucketName != null ? !bucketName.equals(thatS3TO.bucketName)
+                : thatS3TO.bucketName != null) {
+            return false;
+        }
+
+        if (created != null ? !created.equals(thatS3TO.created)
+                : thatS3TO.created != null) {
+            return false;
+        }
+
+        return true;
+
+    }
+
+    @Override
+    public int hashCode() {
+
+        int result = id != null ? id.hashCode() : 0;
+
+        result = 31 * result + (accessKey != null ? accessKey.hashCode() : 0);
+        result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
+        result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
+        result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
+        result = 31 * result + (httpsFlag ? 1 : 0);
+        result = 31
+                * result
+                + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
+        result = 31 * result
+                + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
+        result = 31 * result
+                + (socketTimeout != null ? socketTimeout.hashCode() : 0);
+
+        return result;
+
+    }
+
+    public Long getId() {
+        return this.id;
+    }
+
+    public void setId(final Long id) {
+        this.id = id;
+    }
+
+    public String getUuid() {
+        return this.uuid;
+    }
+
+    public void setUuid(final String uuid) {
+        this.uuid = uuid;
+    }
+
+    @Override
+    public String getAccessKey() {
+        return this.accessKey;
+    }
+
+    public void setAccessKey(final String accessKey) {
+        this.accessKey = accessKey;
+    }
+
+    @Override
+    public String getSecretKey() {
+        return this.secretKey;
+    }
+
+    public void setSecretKey(final String secretKey) {
+        this.secretKey = secretKey;
+    }
+
+    @Override
+    public String getEndPoint() {
+        return this.endPoint;
+    }
+
+    public void setEndPoint(final String endPoint) {
+        this.endPoint = endPoint;
+    }
+
+    public String getBucketName() {
+        return this.bucketName;
+    }
+
+    public void setBucketName(final String bucketName) {
+        this.bucketName = bucketName;
+    }
+
+    @Override
+    public Boolean isHttps() {
+        return this.httpsFlag;
+    }
+
+    public void setHttps(final Boolean httpsFlag) {
+        this.httpsFlag = httpsFlag;
+    }
+
+    @Override
+    public Integer getConnectionTimeout() {
+        return connectionTimeout;
+    }
+
+    public void setConnectionTimeout(final Integer connectionTimeout) {
+        this.connectionTimeout = connectionTimeout;
+    }
+
+    @Override
+    public Integer getMaxErrorRetry() {
+        return maxErrorRetry;
+    }
+
+    public void setMaxErrorRetry(final Integer maxErrorRetry) {
+        this.maxErrorRetry = maxErrorRetry;
+    }
+
+    @Override
+    public Integer getSocketTimeout() {
+        return socketTimeout;
+    }
+
+    public void setSocketTimeout(final Integer socketTimeout) {
+        this.socketTimeout = socketTimeout;
+    }
+
+    public Date getCreated() {
+        return this.created;
+    }
+
+    public void setCreated(final Date created) {
+        this.created = created;
+    }
+
+}

api/src/com/cloud/api/ApiConstants.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.api;
 
+import org.omg.CORBA.PUBLIC_MEMBER;
+
 public class ApiConstants {
     public static final String ACCOUNT = "account";
     public static final String ACCOUNTS = "accounts";
@@ -383,6 +385,16 @@ public class ApiConstants {
     public static final String NICIRA_NVP_TRANSPORT_ZONE_UUID = "transportzoneuuid";
     public static final String NICIRA_NVP_DEVICE_NAME = "niciradevicename";
 	public static final String NICIRA_NVP_GATEWAYSERVICE_UUID = "l3gatewayserviceuuid";
+    public static final String S3_ACCESS_KEY = "accesskey";
+    public static final String S3_SECRET_KEY = "secretkey";
+    public static final String S3_END_POINT = "endpoint";
+    public static final String S3_BUCKET_NAME = "bucket";
+    public static final String S3_HTTPS_FLAG = "usehttps";
+    public static final String S3_CONNECTION_TIMEOUT = "connectiontimeout";
+    public static final String S3_MAX_ERROR_RETRY = "maxerrorretry";
+    public static final String S3_SOCKET_TIMEOUT = "sockettimeout";
+    public static final String INCL_ZONES = "includezones";
+    public static final String EXCL_ZONES = "excludezones";
     
     public static final String SOURCE = "source";
     public static final String COUNTER_ID = "counterid";

api/src/com/cloud/api/ResponseGenerator.java

@@ -64,6 +64,7 @@ import com.cloud.api.response.RemoteAccessVpnResponse;
 import com.cloud.api.response.ResourceCountResponse;
 import com.cloud.api.response.ResourceLimitResponse;
 import com.cloud.api.response.ResourceTagResponse;
+import com.cloud.api.response.S3Response;
 import com.cloud.api.response.SecurityGroupResponse;
 import com.cloud.api.response.ServiceOfferingResponse;
 import com.cloud.api.response.ServiceResponse;
@@ -141,6 +142,7 @@ import com.cloud.projects.Project;
 import com.cloud.projects.ProjectAccount;
 import com.cloud.projects.ProjectInvitation;
 import com.cloud.server.ResourceTag;
+import com.cloud.storage.S3;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.Swift;
@@ -287,6 +289,8 @@ public interface ResponseGenerator {
 
     SwiftResponse createSwiftResponse(Swift swift);
 
+    S3Response createS3Response(S3 result);
+
     PhysicalNetworkResponse createPhysicalNetworkResponse(PhysicalNetwork result);
 
     ServiceResponse createNetworkServiceResponse(Service service);
@@ -361,4 +365,5 @@ public interface ResponseGenerator {
     AutoScaleVmProfileResponse createAutoScaleVmProfileResponse(AutoScaleVmProfile profile);
 
     AutoScaleVmGroupResponse createAutoScaleVmGroupResponse(AutoScaleVmGroup vmGroup);
+
 }

api/src/com/cloud/api/commands/AddS3Cmd.java

@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.api.commands;
+
+import static com.cloud.api.ApiConstants.S3_ACCESS_KEY;
+import static com.cloud.api.ApiConstants.S3_CONNECTION_TIMEOUT;
+import static com.cloud.api.ApiConstants.S3_END_POINT;
+import static com.cloud.api.ApiConstants.S3_HTTPS_FLAG;
+import static com.cloud.api.ApiConstants.S3_MAX_ERROR_RETRY;
+import static com.cloud.api.ApiConstants.S3_SECRET_KEY;
+import static com.cloud.api.ApiConstants.S3_SOCKET_TIMEOUT;
+import static com.cloud.api.ApiConstants.S3_BUCKET_NAME;
+import static com.cloud.api.BaseCmd.CommandType.INTEGER;
+import static com.cloud.api.BaseCmd.CommandType.STRING;
+import static com.cloud.api.BaseCmd.CommandType.BOOLEAN;
+import static com.cloud.user.Account.ACCOUNT_ID_SYSTEM;
+
+import com.cloud.api.BaseCmd;
+import com.cloud.api.Implementation;
+import com.cloud.api.Parameter;
+import com.cloud.api.ServerApiException;
+import com.cloud.api.response.S3Response;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.DiscoveryException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.storage.S3;
+
+@Implementation(description = "Adds S3", responseObject = S3Response.class, since = "4.0.0")
+public final class AddS3Cmd extends BaseCmd {
+
+    private static String COMMAND_NAME = "adds3response";
+
+    @Parameter(name = S3_ACCESS_KEY, type = STRING, required = true,
+            description = "S3 access key")
+    private String accessKey;
+
+    @Parameter(name = S3_SECRET_KEY, type = STRING, required = true,
+            description = "S3 secret key")
+    private String secretKey;
+
+    @Parameter(name = S3_END_POINT, type = STRING, required = false,
+            description = "S3 host name")
+    private String endPoint = null;
+
+    @Parameter(name = S3_BUCKET_NAME, type = STRING, required = true,
+            description = "name of the template storage bucket")
+    private String bucketName;
+
+    @Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, required = false,
+            description = "connect to the S3 endpoint via HTTPS?")
+    private Boolean httpsFlag = null;
+
+    @Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, required = false,
+            description = "connection timeout (milliseconds)")
+    private Integer connectionTimeout = null;
+
+    @Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, required = false,
+            description = "maximum number of times to retry on error")
+    private Integer maxErrorRetry = null;
+
+    @Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, required = false,
+            description = "socket timeout (milliseconds)")
+    private Integer socketTimeout = null;
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException,
+            ServerApiException, ConcurrentOperationException, ResourceAllocationException,
+            NetworkRuleConflictException {
+
+        final S3 result;
+
+        try {
+
+            result = _resourceService.discoverS3(this);
+
+            if (result == null) {
+                throw new ServerApiException(INTERNAL_ERROR, "Failed to add S3.");
+            }
+
+        } catch (DiscoveryException e) {
+
+            throw new ServerApiException(INTERNAL_ERROR, "Failed to add S3 due to " + e.getMessage());
+
+        }
+
+        final S3Response response = _responseGenerator.createS3Response(result);
+        response.setResponseName(this.getCommandName());
+        this.setResponseObject(response);
+
+    }
+
+    @Override
+    public boolean equals(final Object thatObject) {
+
+        if (this == thatObject) {
+            return true;
+        }
+
+        if (thatObject == null || this.getClass() != thatObject.getClass()) {
+            return false;
+        }
+
+        final AddS3Cmd thatAddS3Cmd = (AddS3Cmd) thatObject;
+
+        if (this.httpsFlag != null ? !this.httpsFlag.equals(thatAddS3Cmd.httpsFlag) : thatAddS3Cmd.httpsFlag != null) {
+            return false;
+        }
+
+        if (this.accessKey != null ? !this.accessKey.equals(thatAddS3Cmd.accessKey) : thatAddS3Cmd.accessKey != null) {
+            return false;
+        }
+
+        if (this.connectionTimeout != null ? !this.connectionTimeout.equals(thatAddS3Cmd.connectionTimeout) : thatAddS3Cmd.connectionTimeout != null) {
+            return false;
+        }
+
+        if (this.endPoint != null ? !this.endPoint.equals(thatAddS3Cmd.endPoint) : thatAddS3Cmd.endPoint != null) {
+            return false;
+        }
+
+        if (this.maxErrorRetry != null ? !this.maxErrorRetry.equals(thatAddS3Cmd.maxErrorRetry) : thatAddS3Cmd.maxErrorRetry != null) {
+            return false;
+        }
+
+        if (this.secretKey != null ? !this.secretKey.equals(thatAddS3Cmd.secretKey) : thatAddS3Cmd.secretKey != null) {
+            return false;
+        }
+
+        if (this.socketTimeout != null ? !this.socketTimeout.equals(thatAddS3Cmd.socketTimeout) : thatAddS3Cmd.socketTimeout != null) {
+            return false;
+        }
+
+        if (this.bucketName != null ? !this.bucketName.equals(thatAddS3Cmd.bucketName) : thatAddS3Cmd.bucketName != null) {
+            return false;
+        }
+
+        return true;
+
+    }
+
+    @Override
+    public int hashCode() {
+
+        int result = this.accessKey != null ? this.accessKey.hashCode() : 0;
+        result = 31 * result + (this.secretKey != null ? this.secretKey.hashCode() : 0);
+        result = 31 * result + (this.endPoint != null ? this.endPoint.hashCode() : 0);
+        result = 31 * result + (this.bucketName != null ? this.bucketName.hashCode() : 0);
+        result = 31 * result + (this.httpsFlag != null && this.httpsFlag == true ? 1 : 0);
+        result = 31 * result + (this.connectionTimeout != null ? this.connectionTimeout.hashCode() : 0);
+        result = 31 * result + (this.maxErrorRetry != null ? this.maxErrorRetry.hashCode() : 0);
+        result = 31 * result + (this.socketTimeout != null ? this.socketTimeout.hashCode() : 0);
+
+        return result;
+
+    }
+
+    @Override
+    public String getCommandName() {
+        return COMMAND_NAME;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return ACCOUNT_ID_SYSTEM;
+    }
+
+    public String getAccessKey() {
+        return this.accessKey;
+    }
+
+    public String getSecretKey() {
+        return this.secretKey;
+    }
+
+    public String getEndPoint() {
+        return this.endPoint;
+    }
+
+    public String getBucketName() {
+        return this.bucketName;
+    }
+
+    public Boolean getHttpsFlag() {
+        return this.httpsFlag;
+    }
+
+    public Integer getConnectionTimeout() {
+        return this.connectionTimeout;
+    }
+
+    public Integer getMaxErrorRetry() {
+        return this.maxErrorRetry;
+    }
+
+    public Integer getSocketTimeout() {
+        return this.socketTimeout;
+    }
+
+}

api/src/com/cloud/api/commands/ListS3sCmd.java

@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.api.commands;
+
+import static com.cloud.api.ApiConstants.ID;
+import static com.cloud.api.BaseCmd.CommandType.LONG;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.cloud.api.BaseListCmd;
+import com.cloud.api.Implementation;
+import com.cloud.api.Parameter;
+import com.cloud.api.ServerApiException;
+import com.cloud.api.response.ListResponse;
+import com.cloud.api.response.S3Response;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.storage.S3;
+
+@Implementation(description = "Lists S3s", responseObject = S3Response.class, since = "4.0.0")
+public final class ListS3sCmd extends BaseListCmd {
+
+    private static final String COMMAND_NAME = "lists3sresponse";
+
+    @Parameter(name = ID, type = LONG, required = true, description = "The ID of the S3")
+    private Long id;
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException,
+            ServerApiException, ConcurrentOperationException, ResourceAllocationException,
+            NetworkRuleConflictException {
+
+        final List<? extends S3> result = _resourceService.listS3s(this);
+        final ListResponse<S3Response> response = new ListResponse<S3Response>();
+        final List<S3Response> s3Responses = new ArrayList<S3Response>();
+
+        if (result != null) {
+
+            for (S3 s3 : result) {
+
+                S3Response s3Response = _responseGenerator.createS3Response(s3);
+                s3Response.setResponseName(this.getCommandName());
+                s3Response.setObjectName("s3");
+                s3Responses.add(s3Response);
+
+            }
+
+        }
+
+        response.setResponses(s3Responses);
+        response.setResponseName(this.getCommandName());
+
+        this.setResponseObject(response);
+
+    }
+
+    @Override
+    public boolean equals(final Object thatObject) {
+
+        if (this == thatObject) {
+            return true;
+        }
+
+        if (thatObject == null || getClass() != thatObject.getClass()) {
+            return false;
+        }
+
+        final ListS3sCmd thatListS3sCmd = (ListS3sCmd) thatObject;
+
+        if (this.id != null ? !this.id.equals(thatListS3sCmd.id) : thatListS3sCmd.id != null) {
+            return false;
+        }
+
+        return true;
+
+    }
+
+    @Override
+    public int hashCode() {
+        return this.id != null ? this.id.hashCode() : 0;
+    }
+
+    @Override
+    public String getCommandName() {
+        return COMMAND_NAME;
+    }
+
+    public Long getId() {
+        return this.id;
+    }
+
+}

api/src/com/cloud/api/response/S3Response.java

@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.api.response;
+
+import com.cloud.serializer.Param;
+import com.cloud.utils.IdentityProxy;
+import com.google.gson.annotations.SerializedName;
+
+import static com.cloud.api.ApiConstants.*;
+
+public class S3Response extends BaseResponse {
+
+    @SerializedName(ID)
+    @Param(description = "The ID of the S3 configuration")
+    private IdentityProxy id = new IdentityProxy("s3");
+
+    @SerializedName(S3_ACCESS_KEY)
+    @Param(description = "The S3 access key")
+    private String accessKey;
+
+    @SerializedName(S3_SECRET_KEY)
+    @Param(description = "The S3 secret key")
+    private String secretKey;
+
+    @SerializedName(S3_END_POINT)
+    @Param(description = "The S3 end point")
+    private String endPoint;
+
+    @SerializedName(S3_BUCKET_NAME)
+    @Param(description = "The name of the template storage bucket")
+    private String bucketName;
+
+    @SerializedName(S3_HTTPS_FLAG)
+    @Param(description = "Connect to S3 using HTTPS?")
+    private Integer httpsFlag;
+
+    @SerializedName(S3_CONNECTION_TIMEOUT)
+    @Param(description = "The connection timeout (milliseconds)")
+    private Integer connectionTimeout;
+
+    @SerializedName(S3_MAX_ERROR_RETRY)
+    @Param(description = "The maximum number of time to retry a connection on error.")
+    private Integer maxErrorRetry;
+
+    @SerializedName(S3_SOCKET_TIMEOUT)
+    @Param(description = "The connection socket (milliseconds)")
+    private Integer socketTimeout;
+
+    @Override
+    public boolean equals(final Object thatObject) {
+
+        if (this == thatObject) {
+            return true;
+        }
+
+        if (thatObject == null || this.getClass() != thatObject.getClass()) {
+            return false;
+        }
+
+        final S3Response thatS3Response = (S3Response) thatObject;
+
+        if (this.httpsFlag != null ? !this.httpsFlag.equals(thatS3Response.httpsFlag) : thatS3Response.httpsFlag != null) {
+            return false;
+        }
+
+        if (this.accessKey != null ? !this.accessKey.equals(thatS3Response.accessKey) : thatS3Response.accessKey != null) {
+            return false;
+        }
+
+        if (this.connectionTimeout != null ? !this.connectionTimeout.equals(thatS3Response.connectionTimeout) : thatS3Response.connectionTimeout != null) {
+            return false;
+        }
+
+        if (this.endPoint != null ? !this.endPoint.equals(thatS3Response.endPoint) : thatS3Response.endPoint != null) {
+            return false;
+        }
+
+        if (this.id != null ? !this.id.equals(thatS3Response.id) : thatS3Response.id != null) {
+            return false;
+        }
+
+        if (this.maxErrorRetry != null ? !this.maxErrorRetry.equals(thatS3Response.maxErrorRetry) : thatS3Response.maxErrorRetry != null) {
+            return false;
+        }
+
+        if (this.secretKey != null ? !this.secretKey.equals(thatS3Response.secretKey) : thatS3Response.secretKey != null) {
+            return false;
+        }
+
+        if (this.socketTimeout != null ? !this.socketTimeout.equals(thatS3Response.socketTimeout) : thatS3Response.socketTimeout != null) {
+            return false;
+        }
+
+        if (this.bucketName != null ? !this.bucketName.equals(thatS3Response.bucketName) : thatS3Response.bucketName != null) {
+            return false;
+        }
+
+        return true;
+
+    }
+
+    @Override
+    public int hashCode() {
+
+        int result = this.id != null ? this.id.hashCode() : 0;
+        result = 31 * result + (this.accessKey != null ? this.accessKey.hashCode() : 0);
+        result = 31 * result + (this.secretKey != null ? this.secretKey.hashCode() : 0);
+        result = 31 * result + (this.endPoint != null ? this.endPoint.hashCode() : 0);
+        result = 31 * result + (this.bucketName != null ? this.bucketName.hashCode() : 0);
+        result = 31 * result + (this.httpsFlag != null ? this.httpsFlag : 0);
+        result = 31 * result + (this.connectionTimeout != null ? this.connectionTimeout.hashCode() : 0);
+        result = 31 * result + (this.maxErrorRetry != null ? this.maxErrorRetry.hashCode() : 0);
+        result = 31 * result + (this.socketTimeout != null ? this.socketTimeout.hashCode() : 0);
+
+        return result;
+
+    }
+
+    @Override
+    public Long getObjectId() {
+        return this.id.getValue();
+    }
+
+    public void setObjectId(Long id) {
+        this.id.setValue(id);
+    }
+
+    public String getAccessKey() {
+        return this.accessKey;
+    }
+
+    public void setAccessKey(final String accessKey) {
+        this.accessKey = accessKey;
+    }
+
+    public String getSecretKey() {
+        return this.secretKey;
+    }
+
+    public void setSecretKey(final String secretKey) {
+        this.secretKey = secretKey;
+    }
+
+    public String getEndPoint() {
+        return this.endPoint;
+    }
+
+    public void setEndPoint(final String endPoint) {
+        this.endPoint = endPoint;
+    }
+
+
+    public String getTemplateBucketName() {
+        return this.bucketName;
+    }
+
+    public void setTemplateBucketName(final String templateBucketName) {
+        this.bucketName = templateBucketName;
+    }
+
+    public Integer getHttpsFlag() {
+        return this.httpsFlag;
+    }
+
+    public void setHttpsFlag(final Integer httpsFlag) {
+        this.httpsFlag = httpsFlag;
+    }
+
+    public Integer getConnectionTimeout() {
+        return this.connectionTimeout;
+    }
+
+    public void setConnectionTimeout(final Integer connectionTimeout) {
+        this.connectionTimeout = connectionTimeout;
+    }
+
+    public Integer getMaxErrorRetry() {
+        return this.maxErrorRetry;
+    }
+
+    public void setMaxErrorRetry(final Integer maxErrorRetry) {
+        this.maxErrorRetry = maxErrorRetry;
+    }
+
+    public Integer getSocketTimeout() {
+        return this.socketTimeout;
+    }
+
+    public void setSocketTimeout(final Integer socketTimeout) {
+        this.socketTimeout = socketTimeout;
+    }
+
+}

api/src/com/cloud/resource/ResourceService.java

@@ -20,10 +20,12 @@ import java.util.List;
 
 import com.cloud.api.commands.AddClusterCmd;
 import com.cloud.api.commands.AddHostCmd;
+import com.cloud.api.commands.AddS3Cmd;
 import com.cloud.api.commands.AddSecondaryStorageCmd;
 import com.cloud.api.commands.AddSwiftCmd;
 import com.cloud.api.commands.CancelMaintenanceCmd;
 import com.cloud.api.commands.DeleteClusterCmd;
+import com.cloud.api.commands.ListS3sCmd;
 import com.cloud.api.commands.ListSwiftsCmd;
 import com.cloud.api.commands.PrepareForMaintenanceCmd;
 import com.cloud.api.commands.ReconnectHostCmd;
@@ -35,6 +37,7 @@ import com.cloud.exception.ResourceInUseException;
 import com.cloud.host.Host;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.org.Cluster;
+import com.cloud.storage.S3;
 import com.cloud.storage.Swift;
 import com.cloud.utils.fsm.NoTransitionException;
 
@@ -94,7 +97,12 @@ public interface ResourceService {
 
     Swift discoverSwift(AddSwiftCmd addSwiftCmd) throws DiscoveryException;
 
+    S3 discoverS3(AddS3Cmd cmd) throws DiscoveryException;
+    
     List<HypervisorType> getSupportedHypervisorTypes(long zoneId, boolean forVirtualRouter, Long podId);
 
     List<? extends Swift> listSwifts(ListSwiftsCmd cmd);
+
+    List<? extends S3> listS3s(ListS3sCmd cmd);
+
 }

api/src/com/cloud/storage/S3.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage;
+
+import com.cloud.agent.api.to.S3TO;
+
+import java.util.Date;
+
+public interface S3 {
+
+    long getId();
+
+    String getUuid();
+
+    String getAccessKey();
+
+    String getSecretKey();
+
+    String getEndPoint();
+
+    String getBucketName();
+
+    Integer getHttpsFlag();
+
+    Integer getConnectionTimeout();
+
+    Integer getMaxErrorRetry();
+
+    Integer getSocketTimeout();
+
+    Date getCreated();
+
+    S3TO toS3TO();
+
+}

api/test/src/com/cloud/agent/api/test/BackupSnapshotAnswerTest.java

@@ -36,7 +36,7 @@ public class BackupSnapshotAnswerTest {
 
         StoragePool pool = Mockito.mock(StoragePool.class);
 
-        bsc = new BackupSnapshotCommand("primaryStoragePoolNameLabel",
+        bsc = new BackupSnapshotCommand(
                 "secondaryStoragePoolURL", 101L, 102L, 103L, 104L,
                 "volumePath", pool, "snapshotUuid", "snapshotName",
                 "prevSnapshotUuid", "prevBackupUuid", false, "vmName", 5);

api/test/src/com/cloud/agent/api/test/BackupSnapshotCommandTest.java

@@ -116,14 +116,14 @@ public class BackupSnapshotCommandTest {
     };
 
     BackupSnapshotCommand bsc = new BackupSnapshotCommand(
-            "primaryStoragePoolNameLabel", "http://secondary.Storage.Url",
+            "http://secondary.Storage.Url",
             101L, 102L, 103L, 104L, "vPath", pool,
             "420fa39c-4ef1-a83c-fd93-46dc1ff515ae", "sName",
             "9012793e-0657-11e2-bebc-0050568b0057",
             "7167e0b2-f5b0-11e1-8414-0050568b0057", false, "vmName", 5);
 
     BackupSnapshotCommand bsc1 = new BackupSnapshotCommand(
-            "primaryStoragePoolNameLabel", "http://secondary.Storage.Url",
+            "http://secondary.Storage.Url",
             101L, 102L, 103L, 104L, "vPath", pool,
             "420fa39c-4ef1-a83c-fd93-46dc1ff515ae", "sName",
             "9012793e-0657-11e2-bebc-0050568b0057",
@@ -132,7 +132,7 @@ public class BackupSnapshotCommandTest {
     @Test
     public void testGetPrimaryStoragePoolNameLabel() {
         String label = bsc.getPrimaryStoragePoolNameLabel();
-        assertTrue(label.equals("primaryStoragePoolNameLabel"));
+        assertTrue(label.equals("bed9f83e-cac3-11e1-ac8a-0050568b007e"));
     }
 
     @Test

api/test/src/com/cloud/agent/api/test/SnapshotCommandTest.java

@@ -23,19 +23,109 @@ import static org.junit.Assert.assertTrue;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
 import com.cloud.agent.api.SnapshotCommand;
+import com.cloud.storage.Storage.StoragePoolType;
+import com.cloud.storage.StoragePool;
+import com.cloud.storage.StoragePoolStatus;
 
 public class SnapshotCommandTest {
-    SnapshotCommand ssc = new SnapshotCommand("primaryStoragePoolNameLabel",
+
+    public StoragePool pool = new StoragePool() {
+        public long getId() {
+            return 1L;
+        };
+
+        public String getName() {
+            return "name";
+        };
+
+        public String getUuid() {
+            return "bed9f83e-cac3-11e1-ac8a-0050568b007e";
+        };
+
+        public StoragePoolType getPoolType() {
+            return StoragePoolType.Filesystem;
+        };
+
+        public Date getCreated() {
+            Date date = null;
+            try {
+                date = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss")
+                        .parse("01/01/1970 12:12:12");
+            } catch (ParseException e) {
+                e.printStackTrace();
+            }
+            return date;
+        }
+
+        public Date getUpdateTime() {
+            return new Date();
+        };
+
+        public long getDataCenterId() {
+            return 0L;
+        };
+
+        public long getCapacityBytes() {
+            return 0L;
+        };
+
+        public long getAvailableBytes() {
+            return 0L;
+        };
+
+        public Long getClusterId() {
+            return 0L;
+        };
+
+        public String getHostAddress() {
+            return "hostAddress";
+        };
+
+        public String getPath() {
+            return "path";
+        };
+
+        public String getUserInfo() {
+            return "userInfo";
+        };
+
+        public boolean isShared() {
+            return false;
+        };
+
+        public boolean isLocal() {
+            return false;
+        };
+
+        public StoragePoolStatus getStatus() {
+            return StoragePoolStatus.Up;
+        };
+
+        public int getPort() {
+            return 25;
+        };
+
+        public Long getPodId() {
+            return 0L;
+        };
+    };
+
+    SnapshotCommand ssc = new SnapshotCommand(pool,
             "http://secondary.Storage.Url",
             "420fa39c-4ef1-a83c-fd93-46dc1ff515ae", "snapshotName", 101L, 102L,
             103L);
 
     SnapshotCommand ssc1;
 
+
     @Before
     public void setUp() {
-        ssc1 = new SnapshotCommand("primaryStoragePoolNameLabel",
+        ssc1 = new SnapshotCommand(pool,
                 "secondaryStorageUrl", "snapshotUuid", "snapshotName", 101L,
                 102L, 103L);
     }
@@ -43,7 +133,7 @@ public class SnapshotCommandTest {
     @Test
     public void testGetPrimaryStoragePoolNameLabel() {
         String label = ssc.getPrimaryStoragePoolNameLabel();
-        assertTrue(label.equals("primaryStoragePoolNameLabel"));
+        assertTrue(label.equals("bed9f83e-cac3-11e1-ac8a-0050568b007e"));
     }
 
     @Test

client/WEB-INF/classes/resources/messages.properties

@@ -32,6 +32,17 @@ label.destroy=Destroy
 label.restore=Restore
 label.isolation.uri=Isolation URI
 label.broadcast.uri=Broadcast URI
+label.enable.s3=Enable S3-backed Secondary Storage
+confirm.enable.s3=Please fill in the following information to enable support for S3-backed Secondary Storage
+message.after.enable.s3=S3-backed Secondary Storage configured. Note: When you leave this page, you will not be able to re-configure S3 again.
+label.s3.access_key=Access Key
+label.s3.secret_key=Secret Key
+label.s3.bucket=Bucket
+label.s3.endpoint=Endpoint
+label.s3.use_https=Use HTTPS
+label.s3.connection_timeout=Connection Timeout
+label.s3.max_error_retry=Max Error Retry
+label.s3.socket_timeout=Socket Timeout
 #new labels (end) ************************************************************************************************
 
 
@@ -305,8 +316,6 @@ label.installWizard.subtitle=This tour will aid you in setting up your CloudStac
 label.continue=Continue
 label.installWizard.title=Hello and Welcome to CloudStack&#8482
 label.agree=Agree
-label.license.agreement=License Agreement
-label.license.agreement.subtitle=Please accept the CloudStack&#8482 EULA before installing.
 label.manage.resources=Manage Resources
 label.port.forwarding.policies=Port forwarding policies
 label.load.balancing.policies=Load balancing policies

client/WEB-INF/classes/resources/messages_fr_FR.properties

@@ -292,8 +292,6 @@ label.installWizard.subtitle=Ce tutorial vous aidera
 label.continue=Continuer
 label.installWizard.title=Bonjour et bienvenue dans CloudStack
 label.agree=Accepter
-label.license.agreement=Accord de licence
-label.license.agreement.subtitle=Merci d\'accepter l\'EULA CloudStack avant d\'installer.
 label.manage.resources=Gérer les ressources
 label.port.forwarding.policies=Règles de transfert de port
 label.load.balancing.policies=Règles de répartition de charge

client/WEB-INF/classes/resources/messages_ja.properties

@@ -295,8 +295,6 @@ label.installWizard.subtitle=このガイド ツアーは CloudStack™ 環
 label.continue=続行
 label.installWizard.title=CloudStack™ へようこそ
 label.agree=同意する
-label.license.agreement=ライセンス契約
-label.license.agreement.subtitle=インストールの前に CloudStack™ エンド ユーザー ライセンス契約に同意してください。
 label.manage.resources=リソースの管理
 label.port.forwarding.policies=ポート転送ポリシー
 label.load.balancing.policies=負荷分散ポリシー

client/WEB-INF/classes/resources/messages_pt_BR.properties

@@ -292,8 +292,6 @@ label.installWizard.subtitle=Este tour vai auxiliar voc
 label.continue=Continuar
 label.installWizard.title=Olá, seja bem vindo ao CloudStack&\#8482
 label.agree=Concordo
-label.license.agreement=Acordo de Licença
-label.license.agreement.subtitle=Por favor aceite a EULA do CloudStack&\#8482 antes de instalar.
 label.manage.resources=Gerenciar Recursos
 label.port.forwarding.policies=Políticas de redirecionamento de portas
 label.load.balancing.policies=Políticas de balanceamento de carga

client/WEB-INF/classes/resources/messages_ru_RU.properties

@@ -292,8 +292,6 @@ label.installWizard.subtitle=Это руководство настроит ва
 label.continue=Продолжить
 label.installWizard.title=Здравствуйте и добро пожаловать в CloudStack\!
 label.agree=Согласен
-label.license.agreement=Лицензионное соглашение
-label.license.agreement.subtitle=Для продолжения установки согласитесь с лицензионным соглашением.
 label.manage.resources=Управление ресурсами
 label.port.forwarding.policies=Политики проброса портов
 label.load.balancing.policies=Политики балансировки нагрузки

client/WEB-INF/classes/resources/messages_zh_CN.properties

@@ -295,8 +295,6 @@ label.installWizard.subtitle=此教程将帮助您设置 CloudStack&#8482 安装
 label.continue=继续
 label.installWizard.title=您好，欢迎使用 CloudStack&#8482
 label.agree=同意
-label.license.agreement=许可协议
-label.license.agreement.subtitle=必须先接受 CloudStack&#8482 EULA 才能进行安装。
 label.manage.resources=管理资源
 label.port.forwarding.policies=端口转发策略
 label.load.balancing.policies=负载平衡策略

client/tomcatconf/commands.properties.in

@@ -227,6 +227,9 @@ listCapacity=com.cloud.api.commands.ListCapacityCmd;3
 addSwift=com.cloud.api.commands.AddSwiftCmd;1
 listSwifts=com.cloud.api.commands.ListSwiftsCmd;1
 
+#### s3 commands
+addS3=com.cloud.api.commands.AddS3Cmd;1
+listS3s=com.cloud.api.commands.ListS3sCmd;1
 
 #### host commands
 addHost=com.cloud.api.commands.AddHostCmd;3

client/tomcatconf/db-enc.properties.in

@@ -37,9 +37,6 @@ db.cloud.testOnBorrow=true
 db.cloud.testWhileIdle=true
 db.cloud.timeBetweenEvictionRunsMillis=40000
 db.cloud.minEvictableIdleTimeMillis=240000
-db.cloud.removeAbandoned=false
-db.cloud.removeAbandonedTimeout=300
-db.cloud.logAbandoned=true
 db.cloud.poolPreparedStatements=false
 db.cloud.url.params=prepStmtCacheSize=517&cachePrepStmts=true
 

client/tomcatconf/db.properties.in

@@ -37,9 +37,6 @@ db.cloud.testOnBorrow=true
 db.cloud.testWhileIdle=true
 db.cloud.timeBetweenEvictionRunsMillis=40000
 db.cloud.minEvictableIdleTimeMillis=240000
-db.cloud.removeAbandoned=false
-db.cloud.removeAbandonedTimeout=300
-db.cloud.logAbandoned=true
 db.cloud.poolPreparedStatements=false
 db.cloud.url.params=prepStmtCacheSize=517&cachePrepStmts=true
 

core/pom.xml

@@ -43,6 +43,11 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+      <version>${cs.codec.version}</version>
+    </dependency>
   </dependencies>
   <build>
     <defaultGoal>install</defaultGoal>

core/src/com/cloud/agent/api/DirectNetworkUsageCommand.java

@@ -24,11 +24,15 @@ public class DirectNetworkUsageCommand extends Command {
     private List<String> publicIps;
     private Date start;
     private Date end;
+    private String includeZones;
+    private String excludeZones;
 
-	public DirectNetworkUsageCommand(List<String> publicIps, Date start, Date end) {
+	public DirectNetworkUsageCommand(List<String> publicIps, Date start, Date end, String includeZones, String excludeZones) {
 	    this.setPublicIps(publicIps);
 	    this.setStart(start);
 	    this.setEnd(end);
+	    this.setIncludeZones(includeZones);
+	    this.setExcludeZones(excludeZones);
     }
 	
 	@Override
@@ -59,4 +63,21 @@ public class DirectNetworkUsageCommand extends Command {
     public Date getEnd() {
         return end;
     }
+
+	public String getIncludeZones() {
+		return includeZones;
+	}
+
+	public void setIncludeZones(String includeZones) {
+		this.includeZones = includeZones;
+	}
+
+	public String getExcludeZones() {
+		return excludeZones;
+	}
+
+	public void setExcludeZones(String excludeZones) {
+		this.excludeZones = excludeZones;
+	}
+    
 }

core/src/com/cloud/network/resource/TrafficSentinelResource.java

@@ -49,7 +49,6 @@ import com.cloud.agent.api.StartupCommand;
 import com.cloud.agent.api.StartupTrafficMonitorCommand;
 import com.cloud.host.Host;
 import com.cloud.resource.ServerResource;
-import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.exception.ExecutionException;
 
 public class TrafficSentinelResource implements ServerResource {
@@ -59,8 +58,8 @@ public class TrafficSentinelResource implements ServerResource {
     private String _ip;
     private String _guid;
     private String _url;
-    private static Integer _numRetries;
-    private static Integer _timeoutInSeconds;
+    private String _inclZones;
+    private String _exclZones;
 	
 	
 	private static final Logger s_logger = Logger.getLogger(TrafficSentinelResource.class);
@@ -91,9 +90,8 @@ public class TrafficSentinelResource implements ServerResource {
                 throw new ConfigurationException("Unable to find url");
             }
             
-            _numRetries = NumbersUtil.parseInt((String) params.get("numRetries"), 1);
-            
-            _timeoutInSeconds = NumbersUtil.parseInt((String) params.get("timeoutInSeconds"), 300);
+            _inclZones = (String)params.get("inclZones");
+            _exclZones = (String)params.get("exclZones");
 
     		return true;
     	} catch (Exception e) {
@@ -197,6 +195,15 @@ public class TrafficSentinelResource implements ServerResource {
 		try {
 		  //Direct Network Usage
             URL trafficSentinel;
+            //Use Global include/exclude zones if there are no per TS zones 
+            if(_inclZones == null){
+            	_inclZones = cmd.getIncludeZones();
+            }
+            
+            if(_exclZones == null){
+            	_exclZones = cmd.getExcludeZones();
+            }
+            
             try {
                 //Query traffic Sentinel
                 trafficSentinel = new URL(_url+"/inmsf/Query?script="+URLEncoder.encode(getScript(cmd.getPublicIps(), cmd.getStart(), cmd.getEnd()),"UTF-8")
@@ -247,12 +254,28 @@ public class TrafficSentinelResource implements ServerResource {
 	            IpAddresses += ",";
 	        }
 	    }
+	    String destZoneCondition = "";
+	    if(_inclZones !=null && !_inclZones.isEmpty()){
+	    	destZoneCondition = " & destinationzone = "+_inclZones; 
+	    }
+	    if(_exclZones !=null && !_exclZones.isEmpty()){
+	    	destZoneCondition += " & destinationzone != "+_exclZones; 
+	    }
+	    
+	    String srcZoneCondition = "";
+	    if(_inclZones !=null && !_inclZones.isEmpty()){
+	    	srcZoneCondition = " & sourcezone = "+_inclZones; 
+	    }
+	    if(_exclZones !=null && !_exclZones.isEmpty()){
+	    	srcZoneCondition += " & sourcezone != "+_exclZones; 
+	    }
+	    
 	    String startDate = getDateString(start);
 	    String endtDate = getDateString(end);
 	    StringBuffer sb = new StringBuffer();
 	    sb.append("var q = Query.topN(\"historytrmx\",");
 	    sb.append("                 \"ipsource,bytes\",");
-        sb.append("                 \"ipsource = "+IpAddresses+" & destinationzone = EXTERNAL\",");
+        sb.append("                 \"ipsource = "+IpAddresses+destZoneCondition+"\",");
         sb.append("                 \""+startDate+", "+endtDate+"\",");
 	    sb.append("                 \"bytes\",");
 	    sb.append("                 100000);");
@@ -265,7 +288,7 @@ public class TrafficSentinelResource implements ServerResource {
 	    sb.append("  });");
         sb.append("var q = Query.topN(\"historytrmx\",");
         sb.append("                 \"ipdestination,bytes\",");
-        sb.append("                 \"ipdestination = "+IpAddresses+" & sourcezone = EXTERNAL\",");
+        sb.append("                 \"ipdestination = "+IpAddresses+srcZoneCondition+"\",");
         sb.append("                 \""+startDate+", "+endtDate+"\",");
         sb.append("                 \"bytes\",");
         sb.append("                 100000);");	    

core/src/com/cloud/storage/S3VO.java

@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage;
+
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.api.Identity;
+import com.cloud.utils.db.GenericDao;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import java.util.Date;
+
+@Entity
+@Table(name = "s3")
+public class S3VO implements S3, Identity {
+
+    public static final String ID_COLUMN_NAME = "id";
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Column(name = ID_COLUMN_NAME)
+    private long id;
+
+    @Column(name = "uuid")
+    private String uuid;
+
+    @Column(name = "access_key")
+    private String accessKey;
+
+    @Column(name = "secret_key")
+    private String secretKey;
+
+    @Column(name = "end_point")
+    private String endPoint;
+
+    @Column(name = "bucket")
+    private String bucketName;
+
+    @Column(name = "https")
+    private Integer httpsFlag;
+
+    @Column(name = "connection_timeout")
+    private Integer connectionTimeout;
+
+    @Column(name = "max_error_retry")
+    private Integer maxErrorRetry;
+
+    @Column(name = "socket_timeout")
+    private Integer socketTimeout;
+
+    @Column(name = GenericDao.CREATED_COLUMN)
+    private Date created;
+
+    public S3VO() {
+        super();
+    }
+
+    public S3VO(final String uuid, final String accessKey,
+            final String secretKey, final String endPoint,
+            final String bucketName, final Boolean httpsFlag,
+            final Integer connectionTimeout, final Integer maxErrorRetry,
+            final Integer socketTimeout, final Date created) {
+
+        super();
+
+        this.uuid = uuid;
+        this.accessKey = accessKey;
+        this.secretKey = secretKey;
+        this.endPoint = endPoint;
+        this.bucketName = bucketName;
+
+        Integer value = null;
+        if (httpsFlag != null) {
+            value = httpsFlag == false ? 0 : 1;
+        }
+        this.httpsFlag = value;
+
+        this.connectionTimeout = connectionTimeout;
+        this.maxErrorRetry = maxErrorRetry;
+        this.socketTimeout = socketTimeout;
+        this.created = created;
+
+    }
+
+    @Override
+    public S3TO toS3TO() {
+
+        Boolean httpsFlag = null;
+        if (this.httpsFlag != null) {
+            httpsFlag = this.httpsFlag == 0 ? false : true;
+        }
+
+        return new S3TO(this.id, this.uuid, this.accessKey, this.secretKey,
+                this.endPoint, this.bucketName, httpsFlag,
+                this.connectionTimeout, this.maxErrorRetry, this.socketTimeout,
+                this.created);
+
+    }
+
+    public long getId() {
+        return this.id;
+    }
+
+    public void setId(final long id) {
+        this.id = id;
+    }
+
+    public String getUuid() {
+        return this.uuid;
+    }
+
+    public void setUuid(final String uuid) {
+        this.uuid = uuid;
+    }
+
+    public String getAccessKey() {
+        return this.accessKey;
+    }
+
+    public void setAccessKey(final String accessKey) {
+        this.accessKey = accessKey;
+    }
+
+    public String getSecretKey() {
+        return this.secretKey;
+    }
+
+    public void setSecretKey(final String secretKey) {
+        this.secretKey = secretKey;
+    }
+
+    public String getEndPoint() {
+        return this.endPoint;
+    }
+
+    public void setEndPoint(final String endPoint) {
+        this.endPoint = endPoint;
+    }
+
+    public String getBucketName() {
+        return this.bucketName;
+    }
+
+    public void setBucketName(final String bucketName) {
+        this.bucketName = bucketName;
+    }
+
+    public Integer getHttpsFlag() {
+        return this.httpsFlag;
+    }
+
+    public void setHttpsFlag(final Integer httpsFlag) {
+        this.httpsFlag = httpsFlag;
+    }
+
+    public Integer getConnectionTimeout() {
+        return this.connectionTimeout;
+    }
+
+    public void setConnectionTimeout(final int connectionTimeout) {
+        this.connectionTimeout = connectionTimeout;
+    }
+
+    public Integer getMaxErrorRetry() {
+        return this.maxErrorRetry;
+    }
+
+    public void setMaxErrorRetry(final int maxErrorRetry) {
+        this.maxErrorRetry = maxErrorRetry;
+    }
+
+    public Integer getSocketTimeout() {
+        return this.socketTimeout;
+    }
+
+    public void setSocketTimeout(final int socketTimeout) {
+        this.socketTimeout = socketTimeout;
+    }
+
+    public Date getCreated() {
+        return this.created;
+    }
+
+    public void setCreated(final Date created) {
+        this.created = created;
+    }
+
+}

core/src/com/cloud/storage/SnapshotVO.java

@@ -91,6 +91,9 @@ public class SnapshotVO implements Snapshot, Identity {
     @Column(name="swift_id")
     Long swiftId;
     
+    @Column(name="s3_id")
+    Long s3Id;
+
     @Column(name="sechost_id")
     Long secHostId;
 
@@ -289,4 +292,13 @@ public class SnapshotVO implements Snapshot, Identity {
     public void setUuid(String uuid) {
     	this.uuid = uuid;
     }
+
+    public Long getS3Id() {
+        return s3Id;
+    }
+
+    public void setS3Id(Long s3Id) {
+        this.s3Id = s3Id;
+    }
+
 }

core/src/com/cloud/storage/VMTemplateS3VO.java

@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage;
+
+import com.cloud.utils.db.GenericDaoBase;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+import java.text.DateFormat;
+import java.util.Date;
+
+@Entity
+@Table(name = "template_s3_ref")
+public class VMTemplateS3VO {
+
+    public static final String S3_ID_COLUMN_NAME = "s3_id";
+
+    public static final String TEMPLATE_ID_COLUMN_NAME = "template_id";
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private long id;
+
+    @Column(name = S3_ID_COLUMN_NAME)
+    private long s3Id;
+
+    @Column(name = TEMPLATE_ID_COLUMN_NAME)
+    private long templateId;
+
+    @Column(name = GenericDaoBase.CREATED_COLUMN)
+    private Date created;
+
+    @Column(name = "size")
+    private Long size;
+
+    @Column(name = "physical_size")
+    private Long physicalSize;
+
+    public VMTemplateS3VO() {
+        super();
+    }
+
+    public VMTemplateS3VO(final long s3Id, final long templateId,
+            final Date created, final Long size, final Long physicalSize) {
+
+        super();
+
+        this.s3Id = s3Id;
+        this.templateId = templateId;
+        this.created = created;
+        this.size = size;
+        this.physicalSize = physicalSize;
+
+    }
+
+    @Override
+    public boolean equals(final Object thatObject) {
+
+        if (this == thatObject) {
+            return true;
+        }
+
+        if (thatObject == null || getClass() != thatObject.getClass()) {
+            return false;
+        }
+
+        final VMTemplateS3VO thatVMTemplateS3VO = (VMTemplateS3VO) thatObject;
+
+        if (this.id != thatVMTemplateS3VO.id) {
+            return false;
+        }
+
+        if (this.s3Id != thatVMTemplateS3VO.s3Id) {
+            return false;
+        }
+
+        if (this.templateId != thatVMTemplateS3VO.templateId) {
+            return false;
+        }
+
+        if (this.created != null ? !created.equals(thatVMTemplateS3VO.created)
+                : thatVMTemplateS3VO.created != null) {
+            return false;
+        }
+
+        if (this.physicalSize != null ? !physicalSize
+                .equals(thatVMTemplateS3VO.physicalSize)
+                : thatVMTemplateS3VO.physicalSize != null) {
+            return false;
+        }
+
+        if (this.size != null ? !size.equals(thatVMTemplateS3VO.size)
+                : thatVMTemplateS3VO.size != null) {
+            return false;
+        }
+
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+
+        int result = (int) (this.id ^ (this.id >>> 32));
+
+        result = 31 * result + (int) (this.s3Id ^ (this.s3Id >>> 32));
+        result = 31 * result
+                + (int) (this.templateId ^ (this.templateId >>> 32));
+        result = 31 * result
+                + (this.created != null ? this.created.hashCode() : 0);
+        result = 31 * result + (this.size != null ? this.size.hashCode() : 0);
+        result = 31
+                * result
+                + (this.physicalSize != null ? this.physicalSize.hashCode() : 0);
+
+        return result;
+
+    }
+
+    public long getId() {
+        return this.id;
+    }
+
+    public void setId(final long id) {
+        this.id = id;
+    }
+
+    public long getS3Id() {
+        return this.s3Id;
+    }
+
+    public void setS3Id(final long s3Id) {
+        this.s3Id = s3Id;
+    }
+
+    public long getTemplateId() {
+        return this.templateId;
+    }
+
+    public void setTemplateId(final long templateId) {
+        this.templateId = templateId;
+    }
+
+    public Date getCreated() {
+        return this.created;
+    }
+
+    public void setCreated(final Date created) {
+        this.created = created;
+    }
+
+    public Long getSize() {
+        return this.size;
+    }
+
+    public void setSize(final Long size) {
+        this.size = size;
+    }
+
+    public Long getPhysicalSize() {
+        return this.physicalSize;
+    }
+
+    public void setPhysicalSize(final Long physicalSize) {
+        this.physicalSize = physicalSize;
+    }
+
+    @Override
+    public String toString() {
+
+        final StringBuilder stringBuilder = new StringBuilder(
+                "VMTemplateS3VO [ id: ").append(id).append(", created: ")
+                .append(DateFormat.getDateTimeInstance().format(created))
+                .append(", physicalSize: ").append(physicalSize)
+                .append(", size: ").append(size).append(", templateId: ")
+                .append(templateId).append(", s3Id: ").append(s3Id)
+                .append(" ]");
+
+        return stringBuilder.toString();
+
+    }
+
+}

core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java

@@ -16,10 +16,20 @@
 // under the License.
 package com.cloud.storage.resource;
 
+import static com.cloud.utils.S3Utils.deleteDirectory;
+import static com.cloud.utils.S3Utils.getDirectory;
+import static com.cloud.utils.S3Utils.putDirectory;
+import static com.cloud.utils.StringUtils.join;
+import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock;
+import static java.lang.String.format;
+import static java.util.Arrays.asList;
+import static org.apache.commons.lang.StringUtils.substringAfterLast;
+
 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileWriter;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigInteger;
@@ -32,6 +42,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.Callable;
 
 import javax.naming.ConfigurationException;
 
@@ -46,6 +57,9 @@ import com.cloud.agent.api.ComputeChecksumCommand;
 import com.cloud.agent.api.DeleteObjectFromSwiftCommand;
 import com.cloud.agent.api.DeleteSnapshotBackupCommand;
 import com.cloud.agent.api.DeleteSnapshotsDirCommand;
+import com.cloud.agent.api.DeleteTemplateFromS3Command;
+import com.cloud.agent.api.DownloadSnapshotFromS3Command;
+import com.cloud.agent.api.DownloadTemplateFromS3ToSecondaryStorageCommand;
 import com.cloud.agent.api.GetStorageStatsAnswer;
 import com.cloud.agent.api.GetStorageStatsCommand;
 import com.cloud.agent.api.PingCommand;
@@ -60,6 +74,8 @@ import com.cloud.agent.api.SecStorageSetupCommand.Certificates;
 import com.cloud.agent.api.StartupSecondaryStorageCommand;
 import com.cloud.agent.api.SecStorageVMSetupCommand;
 import com.cloud.agent.api.StartupCommand;
+import com.cloud.agent.api.StartupSecondaryStorageCommand;
+import com.cloud.agent.api.UploadTemplateToS3FromSecondaryStorageCommand;
 import com.cloud.agent.api.downloadSnapshotFromSwiftCommand;
 import com.cloud.agent.api.downloadTemplateFromSwiftToSecondaryStorageCommand;
 import com.cloud.agent.api.uploadTemplateToSwiftFromSecondaryStorageCommand;
@@ -75,6 +91,7 @@ import com.cloud.agent.api.storage.ListVolumeAnswer;
 import com.cloud.agent.api.storage.ListVolumeCommand;
 import com.cloud.agent.api.storage.UploadCommand;
 import com.cloud.agent.api.storage.ssCommand;
+import com.cloud.agent.api.to.S3TO;
 import com.cloud.agent.api.to.SwiftTO;
 import com.cloud.api.commands.DeleteVolumeCmd;
 import com.cloud.exception.InternalErrorException;
@@ -90,6 +107,9 @@ import com.cloud.storage.template.TemplateLocation;
 import com.cloud.storage.template.UploadManager;
 import com.cloud.storage.template.UploadManagerImpl;
 import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.S3Utils;
+import com.cloud.utils.S3Utils.FileNamingStrategy;
+import com.cloud.utils.S3Utils.ObjectNamingStrategy;
 import com.cloud.utils.component.ComponentLocator;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.net.NetUtils;
@@ -97,8 +117,15 @@ import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
 import com.cloud.vm.SecondaryStorageVm;
 
-public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
-    private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
+public class NfsSecondaryStorageResource extends ServerResourceBase implements
+        SecondaryStorageResource {
+
+    private static final Logger s_logger = Logger
+            .getLogger(NfsSecondaryStorageResource.class);
+
+    private static final String TEMPLATE_ROOT_DIR = "template/tmpl";
+    private static final String SNAPSHOT_ROOT_DIR = "snapshots";
+
     int _timeout;
     
     String _instance;  
@@ -168,16 +195,24 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
             return execute((ListVolumeCommand)cmd);
         }else if (cmd instanceof downloadSnapshotFromSwiftCommand){
             return execute((downloadSnapshotFromSwiftCommand)cmd);
+        } else if (cmd instanceof DownloadSnapshotFromS3Command) {
+            return execute((DownloadSnapshotFromS3Command) cmd);
         } else if (cmd instanceof DeleteSnapshotBackupCommand){
             return execute((DeleteSnapshotBackupCommand)cmd);
         } else if (cmd instanceof DeleteSnapshotsDirCommand){
             return execute((DeleteSnapshotsDirCommand)cmd);
         } else if (cmd instanceof downloadTemplateFromSwiftToSecondaryStorageCommand) {
             return execute((downloadTemplateFromSwiftToSecondaryStorageCommand) cmd);
+        } else if (cmd instanceof DownloadTemplateFromS3ToSecondaryStorageCommand) {
+            return execute((DownloadTemplateFromS3ToSecondaryStorageCommand) cmd);
         } else if (cmd instanceof uploadTemplateToSwiftFromSecondaryStorageCommand) {
             return execute((uploadTemplateToSwiftFromSecondaryStorageCommand) cmd);
+        } else if (cmd instanceof UploadTemplateToS3FromSecondaryStorageCommand) {
+            return execute((UploadTemplateToS3FromSecondaryStorageCommand) cmd);
         } else if (cmd instanceof DeleteObjectFromSwiftCommand) {
             return execute((DeleteObjectFromSwiftCommand) cmd);
+        } else if (cmd instanceof DeleteTemplateFromS3Command) {
+            return execute((DeleteTemplateFromS3Command) cmd);
         } else if (cmd instanceof CleanupSnapshotBackupCommand){
             return execute((CleanupSnapshotBackupCommand)cmd);
         } else {
@@ -185,6 +220,69 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         }
     }
     
+    @SuppressWarnings("unchecked")
+    private String determineS3TemplateDirectory(final Long accountId,
+            final Long templateId) {
+        return join(asList(TEMPLATE_ROOT_DIR, accountId, templateId),
+                S3Utils.SEPARATOR);
+    }
+
+    @SuppressWarnings("unchecked")
+    private String determineStorageTemplatePath(final String storagePath,
+            final Long accountId, final Long templateId) {
+        return join(
+                asList(getRootDir(storagePath), TEMPLATE_ROOT_DIR, accountId,
+                        templateId), File.separator);
+    }
+
+    private Answer execute(
+            final DownloadTemplateFromS3ToSecondaryStorageCommand cmd) {
+
+        final S3TO s3 = cmd.getS3();
+        final String storagePath = cmd.getStoragePath();
+        final Long accountId = cmd.getAccountId();
+        final Long templateId = cmd.getTemplateId();
+
+        try {
+
+            final File downloadDirectory = _storage
+                    .getFile(determineStorageTemplatePath(storagePath,
+                            accountId, templateId));
+            downloadDirectory.mkdirs();
+
+            if (!downloadDirectory.exists()) {
+                final String errMsg = format(
+                        "Unable to create directory "
+                                + "download directory %1$s for download of template id "
+                                + "%2$s from S3.", downloadDirectory.getName(),
+                        templateId);
+                s_logger.error(errMsg);
+                return new Answer(cmd, false, errMsg);
+            }
+
+            getDirectory(s3, s3.getBucketName(),
+                    determineS3TemplateDirectory(accountId, templateId),
+                    downloadDirectory, new FileNamingStrategy() {
+                        @Override
+                        public String determineFileName(final String key) {
+                            return substringAfterLast(key, S3Utils.SEPARATOR);
+                        }
+                    });
+
+            return new Answer(cmd, true, format("Successfully downloaded "
+                    + "template id %1$s from S3 to directory %2$s", templateId,
+                    downloadDirectory.getName()));
+
+        } catch (Exception e) {
+
+            final String errMsg = format("Failed to upload template id %1$s "
+                    + "due to $2%s", templateId, e.getMessage());
+            s_logger.error(errMsg, e);
+            return new Answer(cmd, false, errMsg);
+
+        }
+
+    }
 
     private Answer execute(downloadTemplateFromSwiftToSecondaryStorageCommand cmd) {
         SwiftTO swift = cmd.getSwift();
@@ -256,6 +354,83 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         }
     }
 
+    private Answer execute(UploadTemplateToS3FromSecondaryStorageCommand cmd) {
+
+        final S3TO s3 = cmd.getS3();
+        final Long accountId = cmd.getAccountId();
+        final Long templateId = cmd.getTemplateId();
+
+        try {
+
+            final String templatePath = determineStorageTemplatePath(
+                    cmd.getStoragePath(), accountId, templateId);
+
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("Found template id " + templateId
+                        + " account id " + accountId + " from directory "
+                        + templatePath + " to upload to S3.");
+            }
+
+            if (!_storage.isDirectory(templatePath)) {
+                final String errMsg = format("S3 Sync Failure: Directory %1$s"
+                        + "for template id %2$s does not exist.", templatePath,
+                        templateId);
+                s_logger.error(errMsg);
+                return new Answer(cmd, false, errMsg);
+            }
+
+            if (!_storage.isFile(templatePath + "/template.properties")) {
+                final String errMsg = format("S3 Sync Failure: Template id "
+                        + "%1$s does not exist on the file system.",
+                        templatePath);
+                s_logger.error(errMsg);
+                return new Answer(cmd, false, errMsg);
+            }
+
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug(format(
+                        "Pushing template id %1$s from %2$s to S3...",
+                        templateId, templatePath));
+            }
+
+            final String bucket = s3.getBucketName();
+            putDirectory(s3, bucket, _storage.getFile(templatePath),
+                    new FilenameFilter() {
+                        @Override
+                        public boolean accept(final File directory,
+                                final String fileName) {
+                            return !fileName.startsWith(".");
+                        }
+                    }, new ObjectNamingStrategy() {
+                        @Override
+                        public String determineKey(final File file) {
+                            s_logger.debug(String
+                                    .format("Determining key using account id %1$s and template id %2$s",
+                                            accountId, templateId));
+                            return join(
+                                    asList(determineS3TemplateDirectory(
+                                            accountId, templateId), file
+                                            .getName()), S3Utils.SEPARATOR);
+                        }
+                    });
+
+            return new Answer(
+                    cmd,
+                    true,
+                    format("Uploaded the contents of directory %1$s for template id %2$s to S3 bucket %3$s",
+                            templatePath, templateId, bucket));
+
+        } catch (Exception e) {
+
+            final String errMsg = format("Failed to upload template id %1$s",
+                    templateId);
+            s_logger.error(errMsg, e);
+            return new Answer(cmd, false, errMsg);
+
+        }
+
+    }
+
     private Answer execute(DeleteObjectFromSwiftCommand cmd) {
         SwiftTO swift = cmd.getSwift();
         String container = cmd.getContainer();
@@ -279,6 +454,47 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
 
     }
 
+    private Answer execute(final DeleteTemplateFromS3Command cmd) {
+
+        final S3TO s3 = cmd.getS3();
+        final Long accountId = cmd.getAccountId();
+        final Long templateId = cmd.getTemplateId();
+
+        if (accountId == null || (accountId != null && accountId <= 0)) {
+            final String errorMessage = "No account id specified for S3 template deletion.";
+            s_logger.error(errorMessage);
+            return new Answer(cmd, false, errorMessage);
+        }
+
+        if (templateId == null || (templateId != null && templateId <= 0)) {
+            final String errorMessage = "No template id specified for S3 template deletion.";
+            s_logger.error(errorMessage);
+            return new Answer(cmd, false, errorMessage);
+        }
+
+        if (s3 == null) {
+            final String errorMessge = "No S3 client options provided";
+            s_logger.error(errorMessge);
+            return new Answer(cmd, false, errorMessge);
+        }
+
+        final String bucket = s3.getBucketName();
+        try {
+            deleteDirectory(s3, bucket,
+                    determineS3TemplateDirectory(templateId, accountId));
+            return new Answer(cmd, true, String.format(
+                    "Deleted template %1%s from bucket %2$s.", templateId,
+                    bucket));
+        } catch (Exception e) {
+            final String errorMessage = String
+                    .format("Failed to delete templaet id %1$s from bucket %2$s due to the following error: %3$s",
+                            templateId, bucket, e.getMessage());
+            s_logger.error(errorMessage, e);
+            return new Answer(cmd, false, errorMessage);
+        }
+
+    }
+
     String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) {
         Script command = new Script("/bin/bash", s_logger);
         command.add("-c");
@@ -451,6 +667,110 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         }
     }
 
+    public Answer execute(final DownloadSnapshotFromS3Command cmd) {
+
+        final S3TO s3 = cmd.getS3();
+        final String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
+        final Long accountId = cmd.getAccountId();
+        final Long volumeId = cmd.getVolumeId();
+
+        try {
+
+            executeWithNoWaitLock(determineSnapshotLockId(accountId, volumeId),
+                    new Callable<Void>() {
+
+                        @Override
+                        public Void call() throws Exception {
+
+                            final String directoryName = determineSnapshotLocalDirectory(
+                                    secondaryStorageUrl, accountId, volumeId);
+
+                            String result = createLocalDir(directoryName);
+                            if (result != null) {
+                                throw new InternalErrorException(
+                                        format("Failed to create directory %1$s during S3 snapshot download.",
+                                                directoryName));
+                            }
+
+                            final String snapshotFileName = determineSnapshotBackupFilename(cmd
+                                    .getSnapshotUuid());
+                            final String key = determineSnapshotS3Key(
+                                    accountId, volumeId, snapshotFileName);
+                            final File targetFile = S3Utils.getFile(s3,
+                                    s3.getBucketName(), key,
+                                    _storage.getFile(directoryName),
+                                    new FileNamingStrategy() {
+
+                                        @Override
+                                        public String determineFileName(
+                                                String key) {
+                                            return snapshotFileName;
+                                        }
+
+                                    });
+
+                            if (cmd.getParent() != null) {
+
+                                final String parentPath = join(
+                                        File.pathSeparator, directoryName,
+                                        determineSnapshotBackupFilename(cmd
+                                                .getParent()));
+                                result = setVhdParent(
+                                        targetFile.getAbsolutePath(),
+                                        parentPath);
+                                if (result != null) {
+                                    throw new InternalErrorException(
+                                            format("Failed to set the parent for backup %1$s to %2$s due to %3$s.",
+                                                    targetFile
+                                                            .getAbsolutePath(),
+                                                    parentPath, result));
+                                }
+
+                            }
+
+                            return null;
+
+                        }
+
+                    });
+
+            return new Answer(
+                    cmd,
+                    true,
+                    format("Succesfully retrieved volume id %1$s for account id %2$s to %3$s from S3.",
+                            volumeId, accountId, secondaryStorageUrl));
+
+        } catch (Exception e) {
+            final String errMsg = format(
+                    "Failed to retrieve volume id %1$s for account id %2$s to %3$s from S3 due to exception %4$s",
+                    volumeId, accountId, secondaryStorageUrl, e.getMessage());
+            s_logger.error(errMsg);
+            return new Answer(cmd, false, errMsg);
+        }
+
+    }
+
+    private String determineSnapshotS3Directory(final Long accountId,
+            final Long volumeId) {
+        return join(S3Utils.SEPARATOR, SNAPSHOT_ROOT_DIR, accountId, volumeId);
+    }
+
+    private String determineSnapshotS3Key(final Long accountId,
+            final Long volumeId, final String snapshotFileName) {
+
+        final String directoryName = determineSnapshotS3Directory(accountId,
+                volumeId);
+        return join(S3Utils.SEPARATOR, directoryName, snapshotFileName);
+
+    }
+
+    private String determineSnapshotLocalDirectory(
+            final String secondaryStorageUrl, final Long accountId,
+            final Long volumeId) {
+        return join(File.pathSeparator, getRootDir(secondaryStorageUrl),
+                SNAPSHOT_ROOT_DIR, accountId, volumeId);
+    }
+
     public Answer execute(downloadSnapshotFromSwiftCommand cmd){
         SwiftTO swift = cmd.getSwift();
         String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
@@ -622,6 +942,92 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         }
     }
     
+    private String deleteSnapshotBackupFromLocalFileSystem(
+            final String secondaryStorageUrl, final Long accountId,
+            final Long volumeId, final String name, final Boolean deleteAllFlag) {
+
+        final String lPath = determineSnapshotLocalDirectory(
+                secondaryStorageUrl, accountId, volumeId)
+                + File.pathSeparator
+                + (deleteAllFlag ? "*" : "*" + name + "*");
+
+        final String result = deleteLocalFile(lPath);
+
+        if (result != null) {
+            return "failed to delete snapshot " + lPath + " , err=" + result;
+        }
+
+        return null;
+
+    }
+
+    private String deleteSnapshotBackupfromS3(final S3TO s3,
+            final String secondaryStorageUrl, final Long accountId,
+            final Long volumeId, final String name, final Boolean deleteAllFlag) {
+
+        try {
+
+            final String bucket = s3.getBucketName();
+
+            final String result = executeWithNoWaitLock(
+                    determineSnapshotLockId(accountId, volumeId),
+                    new Callable<String>() {
+
+                        @Override
+                        public String call() throws Exception {
+
+                            final String innerResult = deleteSnapshotBackupFromLocalFileSystem(
+                                    secondaryStorageUrl, accountId, volumeId,
+                                    name, deleteAllFlag);
+                            if (innerResult != null) {
+                                return innerResult;
+                            }
+
+                            if (deleteAllFlag) {
+                                S3Utils.deleteDirectory(
+                                        s3,
+                                        bucket,
+                                        determineSnapshotS3Directory(accountId,
+                                                volumeId));
+                            } else {
+                                S3Utils.deleteObject(
+                                        s3,
+                                        bucket,
+                                        determineSnapshotS3Key(
+                                                accountId,
+                                                volumeId,
+                                                determineSnapshotBackupFilename(name)));
+                            }
+
+                            return null;
+
+                        }
+
+                    });
+
+            return result;
+
+        } catch (Exception e) {
+
+            s_logger.error(
+                    String.format(
+                            "Failed to delete snapshot backup for account id %1$s volume id %2$sfrom S3.",
+                            accountId, volumeId), e);
+            return e.getMessage();
+
+        }
+
+    }
+
+    private String determineSnapshotBackupFilename(final String snapshotUuid) {
+        return snapshotUuid + ".vhd";
+    }
+
+    private String determineSnapshotLockId(final Long accountId,
+            final Long volumeId) {
+        return join("_", "SNAPSHOT", accountId, volumeId);
+    }
+
     protected Answer execute(final DeleteSnapshotBackupCommand cmd) {
         String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
         Long accountId = cmd.getAccountId();
@@ -629,21 +1035,22 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         String name = cmd.getSnapshotUuid();
         try {
             SwiftTO swift = cmd.getSwift();
+            S3TO s3 = cmd.getS3();
             if (swift == null) {
-                String parent = getRootDir(secondaryStorageUrl);
-                String filename;
-                if (cmd.isAll()) {
-                    filename = "*";
-
-                } else {
-                    filename = "*" + name + "*";
-                }
-                String lPath = parent + "/snapshots/" + String.valueOf(accountId) + "/" + String.valueOf(volumeId) + "/" + filename;
-                String result = deleteLocalFile(lPath);
+                final String result = deleteSnapshotBackupFromLocalFileSystem(
+                        secondaryStorageUrl, accountId, volumeId, name,
+                        cmd.isAll());
                 if (result != null) {
-                    String errMsg = "failed to delete snapshot " + lPath + " , err=" + result;
-                    s_logger.warn(errMsg);
-                    return new Answer(cmd, false, errMsg);
+                    s_logger.warn(result);
+                    return new Answer(cmd, false, result);
+                }
+            } else if (s3 != null) {
+                final String result = deleteSnapshotBackupfromS3(s3,
+                        secondaryStorageUrl, accountId, volumeId, name,
+                        cmd.isAll());
+                if (result != null) {
+                    s_logger.warn(result);
+                    return new Answer(cmd, false, result);
                 }
             } else {
                 String filename;

docs/README.txt

@@ -201,7 +201,7 @@ TAGS FOR A SECTION
 <section id="INSERT A UNIQUE SECTION ID HERE, PROBABLY MATCHING THE TITLE BELOW. KEEP THE QUOTE MARKS.">
     <title>Text of the section title</title>
     <para>Here's the text of a paragraph in this section.</para>
-    <para>Always use &PRODUCT; rather than typing CloudStack or CloudPlatform.</para>
+    <para>Always use &PRODUCT; rather than typing CloudStack.</para>
     <para>Indent with 4 spaces, not with tab characters.</para>
     <para>To hyperlink to a URL outside this document: <ulink url="http://external URL here">Display text of the link here</ulink></para>
     <para>To hyperlink to another section in this document: <xref linkend="SECTION ID OF THE OTHER SECTION GOES HERE." />

docs/en-US/Developers_Guide.xml

@@ -24,14 +24,14 @@
 
 <book>
     <bookinfo id="cloudstack_developers">
-        <title>&PRODUCT; API Developer's Guide</title>
+        <title>&PRODUCT; Developer's Guide</title>
         <productname>Apache CloudStack</productname>
         <productnumber>4.0.0-incubating</productnumber>
         <edition></edition>
         <pubsnumber></pubsnumber>
         <abstract>
             <para>
-                How to integrate with &PRODUCT; using the &PRODUCT; API.
+                This guide shows how to develop &PRODUCT;, use the API for operation and integration, access the usage data and use &PRODUCT; specific tools to ease development, testing and integration.
             </para>
         </abstract>
         <corpauthor>
@@ -45,10 +45,12 @@
         <xi:include href="Author_Group.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
     </bookinfo>
     <xi:include href="concepts.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+    <xi:include href="building-with-maven.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
     <xi:include href="developer-introduction.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
     <xi:include href="whats-new.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
     <xi:include href="api-calls.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
     <xi:include href="working-with-usage-data.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+    <xi:include href="tools.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
     <xi:include href="event-types.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
     <xi:include href="alerts.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
     <xi:include href="time-zones.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />

docs/en-US/building-devcloud.xml

@@ -0,0 +1,32 @@
+<?xml version='1.0' encoding='utf-8' ?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % BOOK_ENTITIES SYSTEM "cloudstack.ent">
+%BOOK_ENTITIES;
+]>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<section id="building-devcloud">
+    <title>Building DevCloud</title>
+    <para>The DevCloud appliance can be downloaded from the wiki at <ulink url="https://cwiki.apache.org/confluence/display/CLOUDSTACK/DevCloud"></ulink>. It can also be built from scratch. Code is being developed to provide this alternative build. It is based on <emphasis>veewee</emphasis>, <emphasis>Vagrant</emphasis> and <emphasis>Puppet</emphasis>.</para>
+    <para>The goal is to automate the DevCloud build and make this automation capability available to all within the source release of &PRODUCT;</para>
+    <warning><para>This is under heavy development. The code is located in the source tree under <emphasis>tools/devcloud</emphasis> </para>
+        <para>A preliminary wiki page describes the build at <ulink url="https://cwiki.apache.org/CLOUDSTACK/building-devcloud.html">https://cwiki.pache.org/CLOUDSTACK/building-devcloud.html</ulink></para>
+    </warning>
+</section>

docs/en-US/building-prerequisites.xml

@@ -0,0 +1,66 @@
+<?xml version='1.0' encoding='utf-8' ?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % BOOK_ENTITIES SYSTEM "cloudstack.ent">
+%BOOK_ENTITIES;
+]>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+
+<section id="build-prerequisites">
+    <title>Build Procedure Prerequisites</title>
+    <para>In this section we will assume that you are using the Ubuntu Linux distribution with the Advanced Packaging Tool (APT). If you are using a different distribution or OS and a different packaging tool, adapt the following instructions to your environment. To build &PRODUCT; you will need:</para>
+    <itemizedlist>
+        <listitem>
+            <para>git, <ulink url="http://git-scm.com">http://git-scm.com</ulink></para>
+            <para><programlisting>sudo apt-get install git-core</programlisting></para>
+        </listitem>
+        <listitem>
+            <para>maven, <ulink url="http://maven.apache.org">http://maven.apache.org</ulink></para>
+            <para><programlisting>sudo apt-get install maven</programlisting></para>
+            <para>Make sure that you installed maven 3</para>
+            <para><programlisting>$ mvn --version
+Apache Maven 3.0.4
+Maven home: /usr/share/maven
+Java version: 1.6.0_24, vendor: Sun Microsystems Inc.
+Java home: /usr/lib/jvm/java-6-openjdk-amd64/jre
+Default locale: en_US, platform encoding: UTF-8
+OS name: "linux", version: "3.2.0-33-generic", arch: "amd64", family: "unix"</programlisting></para>
+        </listitem>
+        <listitem>
+            <para>java</para>
+            <para>set the JAVA_HOME environment variable</para>
+            <para><programlisting>$ export JAVA_HOME=/usr/lib/jvm/java-6-openjdk</programlisting></para>
+        </listitem>
+    </itemizedlist>
+    
+    <para>In addition, to deploy and run &PRODUCT; in a development environment you will need:</para>
+    <itemizedlist>
+        <listitem>
+            <para>Mysql</para>
+            <para><programlisting>sudo apt-get install mysql-server-5.5</programlisting></para>
+            <para>Start the mysqld service and create a cloud user with cloud as a password</para>
+        </listitem>
+        <listitem>
+            <para>Tomcat 6</para>
+            <para><programlisting>sudo apt-get install tomcat6</programlisting></para>
+        </listitem>
+    </itemizedlist>
+
+</section>

docs/en-US/building-with-maven-deploy.xml

@@ -0,0 +1,39 @@
+<?xml version='1.0' encoding='utf-8' ?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % BOOK_ENTITIES SYSTEM "cloudstack.ent">
+%BOOK_ENTITIES;
+]>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<section id="building-with-maven-deploy">
+    <title>Deployment and Testing Steps</title>
+    <para>Deploying the &PRODUCT; code that you compiled is a two step process:</para>
+    <orderedlist>
+        <listitem><para>If you have not configured the database or modified its properties do:</para>
+            <para><programlisting>mvn -P developer -pl developer -Ddeploydb</programlisting></para>
+        </listitem>
+        <listitem><para>Then you need to run the &PRODUCT; management server. To attach a debugger to it, do:</para>
+            <para><programlisting>export MAVEN_OPTS="-Xmx1024 -Xdebug -Xrunjdwp:transport=dt_socket,address=8787,server=y,suspend=n"</programlisting></para>
+            <para><programlisting>mvn -pl :cloud-client-ui jetty:run</programlisting></para>
+        </listitem>
+    </orderedlist>
+    <warning><para>When dealing with the database, remember that you may wipe it entirely and lose any data center configuration that you may have set previously.</para></warning>
+</section>
+

docs/en-US/building-with-maven-steps.xml

@@ -0,0 +1,33 @@
+<?xml version='1.0' encoding='utf-8' ?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % BOOK_ENTITIES SYSTEM "cloudstack.ent">
+%BOOK_ENTITIES;
+]>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<section id="building-with-maven-steps">
+    <title>Building Steps</title>
+    <para>&PRODUCT; uses git for source version control, first make sure you have the source code by pulling it:</para>
+    <programlisting>git clone https://git-wip-us.apache.org/repos/asf/incubator-cloudstack.git</programlisting>
+    <para>Several Project Object Models (POM) are defined to deal with the various build targets of &PRODUCT;. Certain features require some packages that are not compatible with the Apache license and therefore need to be downloaded on your own. Check the wiki for additional information <ulink url="https://cwiki.apache.org/CLOUDSTACK/building-with-maven.html">https://cwiki.apache.org/CLOUDSTACK/building-with-maven.html</ulink>. In order to build all the open source targets of &PRODUCT; do:</para>
+    <para><programlisting>mvn clean install</programlisting></para>
+    <para>The resulting jar files will be in the target directory of the subdirectory of the compiled module.</para>
+</section>
+

docs/en-US/building-with-maven.xml

@@ -0,0 +1,32 @@
+<?xml version='1.0' encoding='utf-8' ?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % BOOK_ENTITIES SYSTEM "cloudstack.ent">
+%BOOK_ENTITIES;
+]>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<chapter id="building-with-maven">
+  <title>Using Maven to Build &PRODUCT;</title>
+  <xi:include href="source-build.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+  <xi:include href="building-prerequisites.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+  <xi:include href="building-with-maven-steps.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+  <xi:include href="building-with-maven-deploy.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+</chapter>
+

docs/en-US/devcloud-usage-mode.xml

@@ -0,0 +1,60 @@
+<?xml version='1.0' encoding='utf-8' ?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % BOOK_ENTITIES SYSTEM "cloudstack.ent">
+%BOOK_ENTITIES;
+]>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<section id="devcloud-usage-mode">
+    <title>DevCloud Usage Mode</title>
+    <para>DevCloud can be used in several different ways:</para>
+    <orderedlist>
+        <listitem>
+            <para>Full sandbox. Where &PRODUCT; is run within the DevCloud instance started in Virtual Box.</para>
+            <para>In this mode, the &PRODUCT; management server runs within the instance and nested virtualization allows instantiation of tiny VMs within DevCloud itself. &PRODUCT; code modifications are done within DevCloud.</para>
+            <para>The following diagram shows the architecture of the SandBox mode.</para>
+            <mediaobject>
+                <imageobject>
+                    <imagedata fileref="./images/DevCloud.png" />
+                </imageobject>
+                <textobject>
+                    <phrase>DevCloud.png: Schematic of the DevCloud SandBox architecture</phrase>
+                </textobject>
+            </mediaobject>
+        </listitem>
+        <listitem>
+            <para>A deployment environment. Where &PRODUCT; code is developed in the localhost of the developer and the resulting build is deployed within DevCloud</para>
+           <para>This mode was used in the testing procedure of &PRODUCT; 4.0.0 incubating release. See the following screencast to see how: <ulink url="http://vimeo.com/54621457">http://vimeo.com/54621457</ulink></para>
+        </listitem>
+        <listitem>
+            <para>A host-only mode. Where DevCloud is used only as a host. &PRODUCT; management server is run in the localhost of the developer</para>
+            <para>This mode makes use of a host-only interface defined in the Virtual Box preferences. Check the following screencast to see how: <ulink url="http://vimeo.com/54610161">http://vimeo.com/54610161</ulink></para>
+            <para>The following schematic shows the architecture of the Host-Only mode.</para>
+            <mediaobject>
+                <imageobject>
+                    <imagedata fileref="./images/DevCloud-hostonly.png" />
+                </imageobject>
+                <textobject>
+                    <phrase>DevCloud-hostonly.png: Schematic of the DevCloud host-only architecture </phrase>
+                </textobject>
+            </mediaobject>
+        </listitem>
+    </orderedlist>
+</section>

docs/en-US/devcloud.xml

@@ -0,0 +1,38 @@
+<?xml version='1.0' encoding='utf-8' ?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % BOOK_ENTITIES SYSTEM "cloudstack.ent">
+%BOOK_ENTITIES;
+]>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<section id="devcloud">
+    <title>DevCloud</title>
+    <para>DevCloud is the &PRODUCT; sandbox. It is provided as a Virtual Box appliance. It is meant to be used as a development environment to easily test new &PRODUCT; development. It has also been used for training and &PRODUCT; demos since it provides a <emphasis>Cloud in a box</emphasis>.</para>
+    <note>
+        <para>DevCloud is provided as a convenience by community members. It is not an official &PRODUCT; release artifact.</para>
+        <para>The &PRODUCT; source code however, contains tools to build your own DevCloud.</para>
+    </note>
+    <warning>
+        <para>DevCloud is under development and should be considered a Work In Progress (WIP), the wiki is the most up to date documentation:</para>
+        <para><ulink url="https://cwiki.apache.org/confluence/display/CLOUDSTACK/DevCloud"></ulink></para>
+    </warning>
+    <xi:include href="devcloud-usage-mode.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+    <xi:include href="building-devcloud.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+</section>

docs/en-US/developer-introduction.xml

@@ -23,7 +23,7 @@
 -->
 
 <chapter id="developer-introduction">
-    <title>Introduction for Developers</title>
+    <title>Introduction to the &PRODUCT; API</title>
     <xi:include href="roles.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
     <xi:include href="api-reference.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
     <xi:include href="developer-getting-started.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />    

docs/en-US/marvin.xml

@@ -0,0 +1,32 @@
+<?xml version='1.0' encoding='utf-8' ?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % BOOK_ENTITIES SYSTEM "cloudstack.ent">
+%BOOK_ENTITIES;
+]>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<section id="marvin">
+    <title>Marvin</title>
+    <para>Marvin is the &PRODUCT; automation framework. It originated as a tool for integration testing but is now also used to build DevCloud as well as to provide a Python &PRODUCT; API binding.</para>
+    <note>
+        <para>Marvin's complete documenation is on the wiki at <ulink url="https://cwiki.apache.org/CLOUDSTACK/testing-with-python.html">https://cwiki.apache.org/CLOUDSTACK/testing-with-python.html</ulink></para>
+        <para>The source code is located at <emphasis>tools/marvin</emphasis></para>
+    </note>
+</section>

docs/en-US/source-build.xml

@@ -0,0 +1,49 @@
+<?xml version='1.0' encoding='utf-8' ?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % BOOK_ENTITIES SYSTEM "cloudstack.ent">
+%BOOK_ENTITIES;
+]>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<section id="source-build">
+    <title>Building &PRODUCT; from Source</title>
+    <note>
+        <para>Prior to the 4.0.0 incubating release, Ant was used to build &PRODUCT;. Starting with 4.0.0 a migration to Maven is underway.</para>
+        <para>The website and the wiki contain up to date information on the build procedure at:</para>
+            <itemizedlist>
+                <listitem><para><ulink url="https://cwiki.apache.org/CLOUDSTACK/building-with-maven.html">https://cwiki.apache.org/CLOUDSTACK/building-with-maven.html</ulink></para></listitem>
+                <listitem><para><ulink url="http://incubator.apache.org/cloudstack/develop/environment.html">http://incubator.apache.org/cloudstack/develop/environment.html</ulink></para></listitem>
+            </itemizedlist>
+    </note>
+    <para>The overarching steps to build &PRODUCT; are:.</para>
+    <itemizedlist>
+        <listitem><para>Install the prerequisites and setup your environment</para></listitem>
+        <listitem><para>Understand that various Maven profiles and build targets</para></listitem>
+        <listitem><para>Deploy and test your build </para></listitem>
+        <listitem><para>If needed, learn how to build binaries</para></listitem>
+    </itemizedlist>
+
+    <note>
+        <para>Learning Maven is outside the scope of this documentation.</para>
+        <para>Go to the Maven website at <ulink url="http://maven.apache.org/guides/getting-started/index.html">http://maven.apache.org/guides/getting-started/index.html</ulink></para>
+    </note>
+
+</section>
+

docs/en-US/tools.xml

@@ -0,0 +1,29 @@
+<?xml version='1.0' encoding='utf-8' ?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % BOOK_ENTITIES SYSTEM "cloudstack.ent">
+%BOOK_ENTITIES;
+]>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<chapter id="tools">
+  <title>Tools</title>
+  <xi:include href="devcloud.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+  <xi:include href="marvin.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+</chapter>

docs/publican-devguide.cfg

@@ -20,7 +20,7 @@
 
 xml_lang: en-US
 type: Book
-docname: API_Developers_Guide
+docname: Developers_Guide
 brand: cloudstack
 chunk_first: 1
 chunk_section_depth: 1

patches/systemvm/debian/config/etc/sysctl.conf

@@ -8,7 +8,7 @@
 net.ipv4.ip_forward = 1
 
 # Controls source route verification
-net.ipv4.conf.default.rp_filter = 1
+net.ipv4.conf.default.rp_filter = 0
 
 # Do not accept source routing
 net.ipv4.conf.default.accept_source_route = 0

patches/systemvm/debian/config/root/edithosts.sh

@@ -88,7 +88,7 @@ echo "0 $mac $ip $host *" >> $DHCP_LEASES
 
 #edit hosts file as well
 sed -i  /"$ip "/d $HOSTS
-sed -i  /"$host "/d $HOSTS
+sed -i  /" $host$"/d $HOSTS
 echo "$ip $host" >> $HOSTS
 
 if [ "$dflt" != "" ]

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -1104,8 +1104,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements
         String secondaryStorageUrl = cmd.getSecondaryStorageURL();
         KVMStoragePool secondaryStoragePool = null;
         try {
-            KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(pool
-                    .getUuid());
+            KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(
+                                             pool.getType(),
+                                             pool.getUuid());
             String volumeName = UUID.randomUUID().toString();
 
             if (copyToSecondary) {
@@ -1114,20 +1115,21 @@ public class LibvirtComputingResource extends ServerResourceBase implements
                         .getVolumePath());
                 String volumeDestPath = "/volumes/" + cmd.getVolumeId()
                         + File.separator;
-                secondaryStoragePool = _storagePoolMgr
-                        .getStoragePoolByURI(secondaryStorageUrl);
+                secondaryStoragePool = _storagePoolMgr.getStoragePoolByURI(
+                                                           secondaryStorageUrl);
                 secondaryStoragePool.createFolder(volumeDestPath);
                 secondaryStoragePool.delete();
-                secondaryStoragePool = _storagePoolMgr
-                        .getStoragePoolByURI(secondaryStorageUrl
+                secondaryStoragePool = _storagePoolMgr.getStoragePoolByURI(
+                                                           secondaryStorageUrl
                                                            + volumeDestPath);
-                _storagePoolMgr.copyPhysicalDisk(volume, destVolumeName,
-                        secondaryStoragePool);
+                _storagePoolMgr.copyPhysicalDisk(volume,
+                        destVolumeName,secondaryStoragePool);
                 return new CopyVolumeAnswer(cmd, true, null, null, volumeName);
             } else {
                 volumePath = "/volumes/" + cmd.getVolumeId() + File.separator;
-                secondaryStoragePool = _storagePoolMgr
-                        .getStoragePoolByURI(secondaryStorageUrl + volumePath);
+                secondaryStoragePool = _storagePoolMgr.getStoragePoolByURI(
+                                                           secondaryStorageUrl
+                                                           + volumePath);
                 KVMPhysicalDisk volume = secondaryStoragePool
                         .getPhysicalDisk(cmd.getVolumePath() + ".qcow2");
                 _storagePoolMgr.copyPhysicalDisk(volume, volumeName,
@@ -1145,7 +1147,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements
 
     protected Answer execute(DeleteStoragePoolCommand cmd) {
         try {
-            _storagePoolMgr.deleteStoragePool(cmd.getPool().getUuid());
+            _storagePoolMgr.deleteStoragePool(cmd.getPool().getType(),
+                                              cmd.getPool().getUuid());
             return new Answer(cmd);
         } catch (CloudRuntimeException e) {
             return new Answer(cmd, false, e.toString());
@@ -1186,7 +1189,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements
         KVMPhysicalDisk vol = null;
         long disksize;
         try {
-            primaryPool = _storagePoolMgr.getStoragePool(pool.getUuid());
+            primaryPool = _storagePoolMgr.getStoragePool(pool.getType(),
+                                                         pool.getUuid());
             disksize = dskch.getSize();
 
             if (cmd.getTemplateUrl() != null) {
@@ -1268,8 +1272,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements
         VolumeTO vol = cmd.getVolume();
 
         try {
-            KVMStoragePool pool = _storagePoolMgr.getStoragePool(vol
-                    .getPoolUuid());
+            KVMStoragePool pool = _storagePoolMgr.getStoragePool(
+                                      vol.getPoolType(),
+                                      vol.getPoolUuid());
             pool.deletePhysicalDisk(vol.getPath());
             String vmName = cmd.getVmName();
             String poolPath = pool.getLocalPath();
@@ -1281,7 +1286,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements
                 File patchVbd = new File(poolPath + File.separator + vmName + "-patchdisk");
                 if(patchVbd.exists()){
                     try {
-                        _storagePoolMgr.deleteVbdByPath(patchVbd.getAbsolutePath());
+                        _storagePoolMgr.deleteVbdByPath(vol.getPoolType(),patchVbd.getAbsolutePath());
                     } catch(CloudRuntimeException e) {
                         s_logger.warn("unable to destroy patch disk '" + patchVbd.getAbsolutePath() +
                                       "' while removing root disk for " + vmName + " : " + e);
@@ -1643,8 +1648,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements
                 }
             }
 
-            KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(cmd
-                    .getPool().getUuid());
+            KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(
+                                             cmd.getPool().getType(),
+                                             cmd.getPool().getUuid());
 
             if (primaryPool.getType() == StoragePoolType.RBD) {
                 s_logger.debug("Snapshots are not supported on RBD volumes");
@@ -1721,8 +1727,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements
         try {
             Connect conn = LibvirtConnection.getConnection();
 
-            secondaryStoragePool = _storagePoolMgr
-                    .getStoragePoolByURI(secondaryStoragePoolUrl);
+            secondaryStoragePool = _storagePoolMgr.getStoragePoolByURI(
+                                                   secondaryStoragePoolUrl);
 
             String ssPmountPath = secondaryStoragePool.getLocalPath();
             snapshotRelPath = File.separator + "snapshots" + File.separator
@@ -1732,8 +1738,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements
             snapshotDestPath = ssPmountPath + File.separator + "snapshots"
                     + File.separator + dcId + File.separator + accountId
                     + File.separator + volumeId;
-            KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(cmd
-                    .getPrimaryStoragePoolNameLabel());
+            KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(
+                                             cmd.getPool().getType(),
+                                             cmd.getPrimaryStoragePoolNameLabel());
             KVMPhysicalDisk snapshotDisk = primaryPool.getPhysicalDisk(cmd
                     .getVolumePath());
             Script command = new Script(_manageSnapshotPath, _cmdsTimeout,
@@ -1760,8 +1767,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements
                 }
             }
 
-            KVMStoragePool primaryStorage = _storagePoolMgr.getStoragePool(cmd
-                    .getPool().getUuid());
+            KVMStoragePool primaryStorage = _storagePoolMgr.getStoragePool(
+                                                cmd.getPool().getType(),
+                                                cmd.getPool().getUuid());
             if (state == DomainInfo.DomainState.VIR_DOMAIN_RUNNING
                     && !primaryStorage.isExternalSnapshot()) {
                 String vmUuid = vm.getUUIDString();
@@ -1875,15 +1883,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements
             String snapshotPath = cmd.getSnapshotUuid();
             int index = snapshotPath.lastIndexOf("/");
             snapshotPath = snapshotPath.substring(0, index);
-            KVMStoragePool secondaryPool = _storagePoolMgr
-                    .getStoragePoolByURI(cmd.getSecondaryStorageUrl()
+            KVMStoragePool secondaryPool = _storagePoolMgr.getStoragePoolByURI(
+                                               cmd.getSecondaryStorageUrl()
                                                + snapshotPath);
             KVMPhysicalDisk snapshot = secondaryPool.getPhysicalDisk(cmd
                     .getSnapshotName());
 
             String primaryUuid = cmd.getPrimaryStoragePoolNameLabel();
             KVMStoragePool primaryPool = _storagePoolMgr
-                    .getStoragePool(primaryUuid);
+                    .getStoragePool(cmd.getPool().getType(),
+                    primaryUuid);
             String volUuid = UUID.randomUUID().toString();
             KVMPhysicalDisk disk = _storagePoolMgr.copyPhysicalDisk(snapshot,
                     volUuid, primaryPool);
@@ -1918,8 +1927,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements
             KVMPhysicalDisk snapshot = snapshotPool.getPhysicalDisk(cmd
                     .getSnapshotName());
 
-            secondaryPool = _storagePoolMgr.getStoragePoolByURI(cmd
-                    .getSecondaryStorageUrl());
+            secondaryPool = _storagePoolMgr.getStoragePoolByURI(
+                                            cmd.getSecondaryStorageUrl());
 
             String templatePath = secondaryPool.getLocalPath() + File.separator
                     + templateInstallFolder;
@@ -1968,8 +1977,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements
 
     protected GetStorageStatsAnswer execute(final GetStorageStatsCommand cmd) {
         try {
-            KVMStoragePool sp = _storagePoolMgr.getStoragePool(cmd
-                    .getStorageId());
+            KVMStoragePool sp = _storagePoolMgr.getStoragePool(
+                                    cmd.getPooltype(),
+                                    cmd.getStorageId());
             return new GetStorageStatsAnswer(cmd, sp.getCapacity(),
                     sp.getUsed());
         } catch (CloudRuntimeException e) {
@@ -1988,11 +1998,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements
                     + cmd.getTemplateId() + File.separator;
             String templateInstallFolder = "/template/tmpl/" + templateFolder;
 
-            secondaryStorage = _storagePoolMgr
-                    .getStoragePoolByURI(secondaryStorageURL);
+            secondaryStorage = _storagePoolMgr.getStoragePoolByURI(
+                                               secondaryStorageURL);
 
-            KVMStoragePool primary = _storagePoolMgr.getStoragePool(cmd
-                    .getPrimaryStoragePoolNameLabel());
+            KVMStoragePool primary = _storagePoolMgr.getStoragePool(
+                                         cmd.getPool().getType(),
+                                         cmd.getPrimaryStoragePoolNameLabel());
             KVMPhysicalDisk disk = primary.getPhysicalDisk(cmd.getVolumePath());
             String tmpltPath = secondaryStorage.getLocalPath() + File.separator
                     + templateInstallFolder;
@@ -2114,8 +2125,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements
             }
 
             /* Copy volume to primary storage */
-            KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(cmd
-                    .getPoolUuid());
+            KVMStoragePool primaryPool = _storagePoolMgr.getStoragePool(
+                                             cmd.getPool().getType(),
+                                             cmd.getPoolUuid());
 
             KVMPhysicalDisk primaryVol = _storagePoolMgr.copyPhysicalDisk(
                     tmplVol, UUID.randomUUID().toString(), primaryPool);
@@ -2136,9 +2148,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements
     }
 
     protected Answer execute(ModifyStoragePoolCommand cmd) {
+        String poolType = cmd.getPool().getType().toString();
         KVMStoragePool storagepool = _storagePoolMgr.createStoragePool(cmd
-                .getPool().getUuid(), cmd.getPool().getHost(), cmd.getPool().getPort(),
-                cmd.getPool().getPath(), cmd.getPool().getUserInfo(), cmd.getPool().getType());
+                .getPool().getUuid(), cmd.getPool().getHost(),
+                cmd.getPool().getPort(), cmd.getPool().getPath(),
+                cmd.getPool().getUserInfo(), cmd.getPool().getType());
         if (storagepool == null) {
             return new Answer(cmd, false, " Failed to create storage pool");
         }
@@ -2276,8 +2290,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements
     private AttachVolumeAnswer execute(AttachVolumeCommand cmd) {
         try {
             Connect conn = LibvirtConnection.getConnection();
-            KVMStoragePool primary = _storagePoolMgr.getStoragePool(cmd
-                    .getPoolUuid());
+            KVMStoragePool primary = _storagePoolMgr.getStoragePool(
+                                         cmd.getPooltype(),
+                                         cmd.getPoolUuid());
             KVMPhysicalDisk disk = primary.getPhysicalDisk(cmd.getVolumePath());
             attachOrDetachDisk(conn, cmd.getAttach(), cmd.getVmName(), disk,
                     cmd.getDeviceId().intValue());
@@ -2924,8 +2939,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements
             int index = isoPath.lastIndexOf("/");
             String path = isoPath.substring(0, index);
             String name = isoPath.substring(index + 1);
-            KVMStoragePool secondaryPool = _storagePoolMgr
-                    .getStoragePoolByURI(path);
+            KVMStoragePool secondaryPool = _storagePoolMgr.getStoragePoolByURI(
+                                                                          path);
             KVMPhysicalDisk isoVol = secondaryPool.getPhysicalDisk(name);
             return isoVol.getPath();
         } else {
@@ -2952,11 +2967,13 @@ public class LibvirtComputingResource extends ServerResourceBase implements
                 int index = volPath.lastIndexOf("/");
                 String volDir = volPath.substring(0, index);
                 String volName = volPath.substring(index + 1);
-                KVMStoragePool secondaryStorage = _storagePoolMgr
-                        .getStoragePoolByURI(volDir);
+                KVMStoragePool secondaryStorage = _storagePoolMgr.
+                                                  getStoragePoolByURI(volDir);
                 physicalDisk = secondaryStorage.getPhysicalDisk(volName);
             } else if (volume.getType() != Volume.Type.ISO) {
-                pool = _storagePoolMgr.getStoragePool(volume.getPoolUuid());
+                pool = _storagePoolMgr.getStoragePool(
+                           volume.getPoolType(),
+                           volume.getPoolUuid());
                 physicalDisk = pool.getPhysicalDisk(volume.getPath());
             }
 
@@ -3034,7 +3051,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements
         DiskDef rootDisk = disks.get(0);
         VolumeTO rootVol = getVolume(vmSpec, Volume.Type.ROOT);
         String patchName = vmName + "-patchdisk";
-        KVMStoragePool pool = _storagePoolMgr.getStoragePool(rootVol.getPoolUuid());
+        KVMStoragePool pool = _storagePoolMgr.getStoragePool(
+                                  rootVol.getPoolType(),
+                                  rootVol.getPoolUuid());
         String patchDiskPath = pool.getLocalPath() + "/" + patchName;
 
         List<KVMPhysicalDisk> phyDisks = pool.listPhysicalDisks();
@@ -3130,7 +3149,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements
         }
 
         try {
-            KVMStoragePool pool = _storagePoolMgr.getStoragePool(poolUuid);
+            //we use libvirt since we passed a libvirt connection to cleanupDisk
+            KVMStoragePool pool = _storagePoolMgr.getStoragePool(null, poolUuid);
             if (pool != null) {
                 pool.delete();
             }
@@ -3148,8 +3168,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements
             int index = isoPath.lastIndexOf("/");
             String path = isoPath.substring(0, index);
             String name = isoPath.substring(index + 1);
-            KVMStoragePool secondaryPool = _storagePoolMgr
-                    .getStoragePoolByURI(path);
+            KVMStoragePool secondaryPool = _storagePoolMgr.getStoragePoolByURI(
+                                                           path);
             KVMPhysicalDisk isoVol = secondaryPool.getPhysicalDisk(name);
             isoPath = isoVol.getPath();
 
@@ -4416,4 +4436,5 @@ public class LibvirtComputingResource extends ServerResourceBase implements
 
         return new Answer(cmd, success, "");
     }
+
 }

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java

@@ -16,8 +16,12 @@
 // under the License.
 package com.cloud.hypervisor.kvm.storage;
 
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.HashMap;
+import java.util.UUID;
 
 import com.cloud.hypervisor.kvm.resource.KVMHABase;
 import com.cloud.hypervisor.kvm.resource.KVMHABase.PoolType;
@@ -25,11 +29,22 @@ import com.cloud.hypervisor.kvm.resource.KVMHAMonitor;
 import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk.PhysicalDiskFormat;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StorageLayer;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 public class KVMStoragePoolManager {
     private StorageAdaptor _storageAdaptor;
     private KVMHAMonitor _haMonitor;
     private final Map<String, Object> _storagePools = new ConcurrentHashMap<String, Object>();
+    private final Map<String, StorageAdaptor> _storageMapper = new HashMap<String, StorageAdaptor>();
+
+    private StorageAdaptor getStorageAdaptor(StoragePoolType type) {
+        StorageAdaptor adaptor = _storageMapper.get(type.toString());
+        if (adaptor == null) {
+            // LibvirtStorageAdaptor is selected by default
+            adaptor = _storageMapper.get("libvirt");
+        }
+        return adaptor;
+    }
 
     private void addStoragePool(String uuid) {
         synchronized (_storagePools) {
@@ -42,20 +57,49 @@ public class KVMStoragePoolManager {
     public KVMStoragePoolManager(StorageLayer storagelayer, KVMHAMonitor monitor) {
         this._storageAdaptor = new LibvirtStorageAdaptor(storagelayer);
         this._haMonitor = monitor;
+        this._storageMapper.put("libvirt", new LibvirtStorageAdaptor(storagelayer));
+        // add other storage adaptors here
+	// this._storageMapper.put("newadaptor", new NewStorageAdaptor(storagelayer));
     }
 
-    public KVMStoragePool getStoragePool(String uuid) {
-        return this._storageAdaptor.getStoragePool(uuid);
+    public KVMStoragePool getStoragePool(StoragePoolType type, String uuid) {
+        StorageAdaptor adaptor = getStorageAdaptor(type);
+        return adaptor.getStoragePool(uuid);
     }
 
     public KVMStoragePool getStoragePoolByURI(String uri) {
-        return this._storageAdaptor.getStoragePoolByURI(uri);
+        URI storageUri = null;
+
+        try {
+            storageUri = new URI(uri);
+        } catch (URISyntaxException e) {
+            throw new CloudRuntimeException(e.toString());
         }
 
-    public KVMStoragePool createStoragePool(String name, String host, int port, String path,
-                                            String userInfo, StoragePoolType type) {
-        KVMStoragePool pool = this._storageAdaptor.createStoragePool(name,
+        String sourcePath = null;
+        String uuid = null;
+        String sourceHost = "";
+        StoragePoolType protocol = null;
+        if (storageUri.getScheme().equalsIgnoreCase("nfs")) {
+            sourcePath = storageUri.getPath();
+            sourcePath = sourcePath.replace("//", "/");
+            sourceHost = storageUri.getHost();
+            uuid = UUID.nameUUIDFromBytes(
+                new String(sourceHost + sourcePath).getBytes()).toString();
+            protocol = StoragePoolType.NetworkFilesystem;
+        }
+
+        return createStoragePool(uuid, sourceHost, 0, sourcePath, "", protocol);
+    }
+
+    public KVMStoragePool createStoragePool( String name, String host, int port,
+                                             String path, String userInfo,
+                                             StoragePoolType type) {
+        StorageAdaptor adaptor = getStorageAdaptor(type);
+        KVMStoragePool pool = adaptor.createStoragePool(name,
                                 host, port, path, userInfo, type);
+
+        // LibvirtStorageAdaptor-specific statement
         if (type == StoragePoolType.NetworkFilesystem) {
             KVMHABase.NfsStoragePool nfspool = new KVMHABase.NfsStoragePool(
                     pool.getUuid(), host, path, pool.getLocalPath(),
@@ -66,28 +110,33 @@ public class KVMStoragePoolManager {
         return pool;
     }
 
-    public boolean deleteStoragePool(String uuid) {
+    public boolean deleteStoragePool(StoragePoolType type, String uuid) {
+        StorageAdaptor adaptor = getStorageAdaptor(type);
         _haMonitor.removeStoragePool(uuid);
-        this._storageAdaptor.deleteStoragePool(uuid);
+        adaptor.deleteStoragePool(uuid);
         _storagePools.remove(uuid);
         return true;
     }
 
-    public boolean deleteVbdByPath(String diskPath) {
-        return this._storageAdaptor.deleteVbdByPath(diskPath);
+    public boolean deleteVbdByPath(StoragePoolType type, String diskPath) {
+        StorageAdaptor adaptor = getStorageAdaptor(type);
+        return adaptor.deleteVbdByPath(diskPath);
     }
 
     public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, String name,
                                                     KVMStoragePool destPool) {
+        StorageAdaptor adaptor = getStorageAdaptor(destPool.getType());
+
+        // LibvirtStorageAdaptor-specific statement
         if (destPool.getType() == StoragePoolType.RBD) {
-            return this._storageAdaptor.createDiskFromTemplate(template, name,
+            return adaptor.createDiskFromTemplate(template, name,
                     KVMPhysicalDisk.PhysicalDiskFormat.RAW, template.getSize(), destPool);
         } else if (destPool.getType() == StoragePoolType.CLVM) {
-            return this._storageAdaptor.createDiskFromTemplate(template, name,
+            return adaptor.createDiskFromTemplate(template, name,
                                        KVMPhysicalDisk.PhysicalDiskFormat.RAW, template.getSize(),
                                        destPool);
         } else {
-            return this._storageAdaptor.createDiskFromTemplate(template, name,
+            return adaptor.createDiskFromTemplate(template, name,
                     KVMPhysicalDisk.PhysicalDiskFormat.QCOW2,
             template.getSize(), destPool);
         }
@@ -96,22 +145,22 @@ public class KVMStoragePoolManager {
     public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk,
             String name, PhysicalDiskFormat format, long size,
             KVMStoragePool destPool) {
-        return this._storageAdaptor.createTemplateFromDisk(disk, name, format,
+        StorageAdaptor adaptor = getStorageAdaptor(destPool.getType());
+        return adaptor.createTemplateFromDisk(disk, name, format,
                 size, destPool);
     }
 
     public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name,
             KVMStoragePool destPool) {
-        return this._storageAdaptor.copyPhysicalDisk(disk, name, destPool);
+        StorageAdaptor adaptor = getStorageAdaptor(destPool.getType());
+        return adaptor.copyPhysicalDisk(disk, name, destPool);
     }
 
     public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot,
             String snapshotName, String name, KVMStoragePool destPool) {
-        return this._storageAdaptor.createDiskFromSnapshot(snapshot,
+        StorageAdaptor adaptor = getStorageAdaptor(destPool.getType());
+        return adaptor.createDiskFromSnapshot(snapshot,
                 snapshotName, name, destPool);
     }
 
-    public KVMPhysicalDisk getPhysicalDiskFromUrl(String url) {
-        return this._storageAdaptor.getPhysicalDiskFromURI(url);
-    }
 }

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java

@@ -725,38 +725,6 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
         return newDisk;
     }
 
-    @Override
-    public KVMStoragePool getStoragePoolByURI(String uri) {
-        URI storageUri = null;
-
-        try {
-            storageUri = new URI(uri);
-        } catch (URISyntaxException e) {
-            throw new CloudRuntimeException(e.toString());
-        }
-
-        String sourcePath = null;
-        String uuid = null;
-        String sourceHost = "";
-        StoragePoolType protocal = null;
-        if (storageUri.getScheme().equalsIgnoreCase("nfs")) {
-            sourcePath = storageUri.getPath();
-            sourcePath = sourcePath.replace("//", "/");
-            sourceHost = storageUri.getHost();
-            uuid = UUID.nameUUIDFromBytes(
-                    new String(sourceHost + sourcePath).getBytes()).toString();
-            protocal = StoragePoolType.NetworkFilesystem;
-        }
-
-        return createStoragePool(uuid, sourceHost, 0, sourcePath, "", protocal);
-    }
-
-    @Override
-    public KVMPhysicalDisk getPhysicalDiskFromURI(String uri) {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
     @Override
     public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot,
             String snapshotName, String name, KVMStoragePool destPool) {

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java

@@ -55,10 +55,6 @@ public interface StorageAdaptor {
     public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot,
             String snapshotName, String name, KVMStoragePool destPool);
 
-    public KVMStoragePool getStoragePoolByURI(String uri);
-
-    public KVMPhysicalDisk getPhysicalDiskFromURI(String uri);
-
     public boolean refresh(KVMStoragePool pool);
 
     public boolean deleteStoragePool(KVMStoragePool pool);

plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java

@@ -17,6 +17,10 @@
 package com.cloud.hypervisor.xen.resource;
 
 
+import java.beans.BeanInfo;
+import java.beans.IntrospectionException;
+import java.beans.Introspector;
+import java.beans.PropertyDescriptor;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
@@ -24,11 +28,13 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.StringReader;
+import java.lang.reflect.InvocationTargetException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
@@ -179,6 +185,7 @@ import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand;
 import com.cloud.agent.api.to.IpAddressTO;
 import com.cloud.agent.api.to.NicTO;
 import com.cloud.agent.api.to.PortForwardingRuleTO;
+import com.cloud.agent.api.to.S3TO;
 import com.cloud.agent.api.to.StaticNatRuleTO;
 import com.cloud.agent.api.to.StorageFilerTO;
 import com.cloud.agent.api.to.SwiftTO;
@@ -217,6 +224,8 @@ import com.cloud.storage.template.TemplateInfo;
 import com.cloud.template.VirtualMachineTemplate.BootloaderType;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.Pair;
+import com.cloud.utils.S3Utils;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.Ternary;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.net.NetUtils;
@@ -6507,6 +6516,13 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
                         } finally {
                             deleteSnapshotBackup(conn, dcId, accountId, volumeId, secondaryStorageMountPath, snapshotBackupUuid);
                         }
+                    } else if (cmd.getS3() != null) {
+                        try {
+                            backupSnapshotToS3(conn, cmd.getS3(), snapshotSr.getUuid(conn), snapshotBackupUuid, isISCSI, wait);
+                            snapshotBackupUuid = snapshotBackupUuid + ".vhd";
+                        } finally {
+                            deleteSnapshotBackup(conn, dcId, accountId, volumeId, secondaryStorageMountPath, snapshotBackupUuid);
+                        }
                     }
                     success = true;
                 } finally {
@@ -6524,6 +6540,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
                         snapshotBackupUuid = snapshotPaUuid + ".vhd";
                     }
                     success = true;
+                } else if (cmd.getS3() != null) {
+                    backupSnapshotToS3(conn, cmd.getS3(), primaryStorageSRUuid, snapshotPaUuid, isISCSI, wait);
                 } else {
                     snapshotBackupUuid = backupSnapshot(conn, primaryStorageSRUuid, dcId, accountId, volumeId, secondaryStorageMountPath, snapshotUuid, prevBackupUuid, isISCSI, wait);
                     success = (snapshotBackupUuid != null);
@@ -6546,6 +6564,88 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
         return new BackupSnapshotAnswer(cmd, success, details, snapshotBackupUuid, fullbackup);
     }
 
+    private static List<String> serializeProperties(final Object object,
+            final Class<?> propertySet) {
+
+        assert object != null;
+        assert propertySet != null;
+        assert propertySet.isAssignableFrom(object.getClass());
+
+        try {
+
+            final BeanInfo beanInfo = Introspector.getBeanInfo(propertySet);
+            final PropertyDescriptor[] descriptors = beanInfo
+                    .getPropertyDescriptors();
+
+            final List<String> serializedProperties = new ArrayList<String>();
+            for (final PropertyDescriptor descriptor : descriptors) {
+
+                serializedProperties.add(descriptor.getName());
+                final Object value = descriptor.getReadMethod().invoke(object);
+                serializedProperties.add(value != null ? value.toString()
+                        : "null");
+
+            }
+
+            return Collections.unmodifiableList(serializedProperties);
+
+        } catch (IntrospectionException e) {
+            s_logger.warn(
+                    "Ignored IntrospectionException when serializing class "
+                            + object.getClass().getCanonicalName(), e);
+        } catch (IllegalArgumentException e) {
+            s_logger.warn(
+                    "Ignored IllegalArgumentException when serializing class "
+                            + object.getClass().getCanonicalName(), e);
+        } catch (IllegalAccessException e) {
+            s_logger.warn(
+                    "Ignored IllegalAccessException when serializing class "
+                            + object.getClass().getCanonicalName(), e);
+        } catch (InvocationTargetException e) {
+            s_logger.warn(
+                    "Ignored InvocationTargetException when serializing class "
+                            + object.getClass().getCanonicalName(), e);
+        }
+
+        return Collections.emptyList();
+
+    }
+
+    private boolean backupSnapshotToS3(final Connection connection,
+            final S3TO s3, final String srUuid, final String snapshotUuid,
+            final Boolean iSCSIFlag, final int wait) {
+
+        final String filename = iSCSIFlag ? "VHD-" + snapshotUuid
+                : snapshotUuid + ".vhd";
+        final String dir = (iSCSIFlag ? "/dev/VG_XenStorage-"
+                : "/var/run/sr-mount/") + srUuid;
+        final String key = StringUtils.join("/", "snapshots", snapshotUuid);
+
+        try {
+
+            final List<String> parameters = new ArrayList<String>(
+                    serializeProperties(s3, S3Utils.ClientOptions.class));
+            parameters.addAll(Arrays.asList("operation", "put", "directory",
+                    dir, "filename", filename, "iSCSIFlag",
+                    iSCSIFlag.toString(), "key", key));
+            final String result = callHostPluginAsync(connection, "s3xen",
+                    "s3", wait,
+                    parameters.toArray(new String[parameters.size()]));
+
+            if (result != null && result.equals("true")) {
+                return true;
+            }
+
+        } catch (Exception e) {
+            s_logger.error(String.format(
+                    "S3 upload failed of snapshot %1$s due to %2$s.",
+                    snapshotUuid, e.toString()), e);
+        }
+
+        return false;
+
+    }
+
     protected CreateVolumeFromSnapshotAnswer execute(final CreateVolumeFromSnapshotCommand cmd) {
         Connection conn = getConnection();
         String primaryStorageNameLabel = cmd.getPrimaryStoragePoolNameLabel();

plugins/network-elements/dns-notifier/pom.xml

@@ -0,0 +1,50 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.cloudstack</groupId>
+    <artifactId>cloudstack-plugins</artifactId>
+    <version>4.1.0-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+  <groupId>org.apache.cloudstack</groupId>
+  <artifactId>cloud-plugin-example-dns-notifier</artifactId>
+  <version>4.1.0-SNAPSHOT</version>
+  <name>CloudStack Dns Notifier Example</name>
+  <description>This is sample source code on how to write a plugin for CloudStack</description>
+  <build>
+    <defaultGoal>install</defaultGoal>
+    <sourceDirectory>src</sourceDirectory>
+    <testSourceDirectory>test</testSourceDirectory>
+  </build>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-api</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.cloudstack</groupId>
+      <artifactId>cloud-utils</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+  </dependencies>
+</project>

plugins/network-elements/dns-notifier/resources/components-example.xml

@@ -0,0 +1,220 @@
+<?xml version="1.0"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!--
+  This is an example components.xml that refers to the main components.xml.
+-->
+<components.xml>
+    <system-integrity-checker class="com.cloud.upgrade.DatabaseUpgradeChecker">
+        <checker name="ManagementServerNode" class="com.cloud.cluster.ManagementServerNode"/>
+        <checker name="EncryptionSecretKeyChecker" class="com.cloud.utils.crypt.EncryptionSecretKeyChecker"/>
+        <checker name="DatabaseIntegrityChecker" class="com.cloud.upgrade.DatabaseIntegrityChecker"/>
+        <checker name="DatabaseUpgradeChecker" class="com.cloud.upgrade.PremiumDatabaseUpgradeChecker"/>
+    </system-integrity-checker>
+
+    <interceptor library="com.cloud.configuration.DefaultInterceptorLibrary"/>
+    <management-server class="com.cloud.server.ManagementServerExtImpl" library="com.cloud.configuration.PremiumComponentLibrary">
+        <dao name="Configuration configuration server" class="com.cloud.configuration.dao.ConfigurationDaoImpl">
+             <param name="premium">true</param>
+        </dao>
+        <adapters key="com.cloud.agent.manager.allocator.HostAllocator">
+            <adapter name="FirstFitRouting" class="com.cloud.agent.manager.allocator.impl.FirstFitRoutingAllocator"/>
+            <!--adapter name="FirstFitRouting" class="com.cloud.agent.manager.allocator.impl.RecreateHostAllocator"/-->
+            <!--adapter name="FirstFit" class="com.cloud.agent.manager.allocator.impl.FirstFitAllocator"/-->
+        </adapters>
+        <adapters key="com.cloud.storage.allocator.StoragePoolAllocator">
+            <adapter name="LocalStorage" class="com.cloud.storage.allocator.LocalStoragePoolAllocator"/>
+            <adapter name="Storage" class="com.cloud.storage.allocator.FirstFitStoragePoolAllocator"/>
+        </adapters>
+        <adapters key="com.cloud.agent.manager.allocator.PodAllocator">
+            <adapter name="User First" class="com.cloud.agent.manager.allocator.impl.UserConcentratedAllocator"/>
+        </adapters>
+        <adapters key="com.cloud.consoleproxy.ConsoleProxyAllocator">
+            <adapter name="Balance" class="com.cloud.consoleproxy.ConsoleProxyBalanceAllocator"/>
+        </adapters>
+        <adapters key="com.cloud.network.guru.NetworkGuru">
+            <!--
+                 NOTE: The order of those gurus implicates priority of network traffic types the guru implements.
+                 The upper the higher priority. It effects listTafficTypeImplementors API which returns impelmentor
+                 of a specific network traffic.
+                 A fair question is, if two gurus implement the same two network traffic types, but these traffic types
+                 have cross priority, how to rank them? For example:
+                    GuruA (TrafficTypeA, TrafficTypeB)
+                    GuruB (TrafficTypeA, TrafficTypeB)
+                 we want GuruB.TrafficTypeB > GuruA.TrafficTypeB and GuruB.TrafficTypeA < GuruA.TrafficTypeA. As the priority
+                 implicated by order can not map to multiple traffic type, you have to do implement GuruC which inherits GuruB
+                 for TrafficTypeB. Then ranking them in order of:
+                    GuruC (TrafficTypeB)
+                    GuruA (TrafficTypeA, TrafficTypeB)
+                    GuruB (TrafficTypeA, TrafficTypeB)
+                 now GuruC represents TrafficTypeB with highest priority while GuruA represents TrafficTypeA with highest pirority.
+
+                 However, above case barely happens.
+            -->
+
+            <adapter name="StorageNetworkGuru" class="com.cloud.network.guru.StorageNetworkGuru"/>
+            <adapter name="ExternalGuestNetworkGuru" class="com.cloud.network.guru.ExternalGuestNetworkGuru"/>
+            <adapter name="PublicNetworkGuru" class="com.cloud.network.guru.PublicNetworkGuru"/>
+            <adapter name="PodBasedNetworkGuru" class="com.cloud.network.guru.PodBasedNetworkGuru"/>
+            <adapter name="ControlNetworkGuru" class="com.cloud.network.guru.ControlNetworkGuru"/>
+            <adapter name="DirectNetworkGuru" class="com.cloud.network.guru.DirectNetworkGuru"/>
+            <adapter name="DirectPodBasedNetworkGuru" class="com.cloud.network.guru.DirectPodBasedNetworkGuru"/>
+            <adapter name="OvsGuestNetworkGuru" class="com.cloud.network.guru.OvsGuestNetworkGuru"/>
+            <adapter name="PrivateNetworkGuru" class="com.cloud.network.guru.PrivateNetworkGuru"/>
+            <adapter name="NiciraNvpGuestNetworkGuru" class="com.cloud.network.guru.NiciraNvpGuestNetworkGuru"/>
+        </adapters>
+        <adapters key="com.cloud.cluster.ClusterServiceAdapter">
+            <adapter name="ClusterService" class="com.cloud.cluster.ClusterServiceServletAdapter"/>
+        </adapters>
+        <adapters key="com.cloud.storage.secondary.SecondaryStorageVmAllocator">
+            <adapter name="Balance" class="com.cloud.storage.secondary.SecondaryStorageVmDefaultAllocator"/>
+        </adapters>
+        <adapters key="com.cloud.network.IpAddrAllocator">
+            <adapter name="Basic" class="com.cloud.network.ExteralIpAddressAllocator"/>
+        </adapters>
+        <adapters key="com.cloud.server.auth.UserAuthenticator">
+            <!-- <adapter name="SHA256SALT" class="com.cloud.server.auth.SHA256SaltedUserAuthenticator"/> -->
+            <adapter name="MD5" class="com.cloud.server.auth.MD5UserAuthenticator"/>
+            <adapter name="LDAP" class="com.cloud.server.auth.LDAPUserAuthenticator"/>
+        </adapters>
+        <adapters key="com.cloud.ha.Investigator">
+            <adapter name="SimpleInvestigator" class="com.cloud.ha.CheckOnAgentInvestigator"/>
+            <adapter name="XenServerInvestigator" class="com.cloud.ha.XenServerInvestigator"/>
+            <adapter name="PingInvestigator" class="com.cloud.ha.UserVmDomRInvestigator"/>
+			<adapter name="ManagementIPSysVMInvestigator" class="com.cloud.ha.ManagementIPSystemVMInvestigator"/>
+        </adapters>
+        <adapters key="com.cloud.ha.FenceBuilder">
+            <adapter name="XenServerFenceBuilder" class="com.cloud.ha.XenServerFencer"/>
+            <adapter name="KVMFenceBuilder" class="com.cloud.ha.KVMFencer"/>
+            <adapter name="OvmFenceBuilder" class="com.cloud.ovm.hypervisor.OvmFencer"/>
+        </adapters>
+        <adapters key="com.cloud.hypervisor.HypervisorGuru">
+            <adapter name="XenServerGuru" class="com.cloud.hypervisor.XenServerGuru"/>
+            <adapter name="KVMGuru" class="com.cloud.hypervisor.KVMGuru"/>
+        </adapters>
+        <adapters key="com.cloud.resource.Discoverer">
+            <adapter name="XCP Agent" class="com.cloud.hypervisor.xen.discoverer.XcpServerDiscoverer"/>
+            <adapter name="SecondaryStorage" class="com.cloud.storage.secondary.SecondaryStorageDiscoverer"/>
+            <adapter name="KVM Agent" class="com.cloud.hypervisor.kvm.discoverer.KvmServerDiscoverer"/>
+            <adapter name="Bare Metal Agent" class="com.cloud.baremetal.BareMetalDiscoverer"/>
+            <adapter name="SCVMMServer" class="com.cloud.hypervisor.hyperv.HypervServerDiscoverer"/>            
+			<adapter name="Ovm Discover" class="com.cloud.ovm.hypervisor.OvmDiscoverer" />
+        </adapters>
+        <adapters key="com.cloud.deploy.DeploymentPlanner">
+            <adapter name="First Fit" class="com.cloud.deploy.FirstFitPlanner"/>
+            <adapter name="UserDispersing" class="com.cloud.deploy.UserDispersingPlanner"/>
+            <adapter name="UserConcentratedPod" class="com.cloud.deploy.UserConcentratedPodPlanner"/>
+            <adapter name="BareMetal Fit" class="com.cloud.deploy.BareMetalPlanner"/>
+        </adapters>
+        <adapters key="com.cloud.alert.AlertAdapter">
+            <adapter name="ClusterAlert" class="com.cloud.alert.ClusterAlertAdapter"/>
+            <adapter name="ConsoleProxyAlert" class="com.cloud.alert.ConsoleProxyAlertAdapter"/>
+            <adapter name="SecondaryStorageVmAlert" class="com.cloud.alert.SecondaryStorageVmAlertAdapter"/>
+        </adapters>
+        <adapters key="com.cloud.acl.SecurityChecker">
+            <adapter name="DomainChecker" class="com.cloud.acl.DomainChecker"/>
+        </adapters>
+        <adapters key="com.cloud.network.element.NetworkElement">
+            <adapter name="VirtualRouter" class="com.cloud.network.element.VirtualRouterElement"/>
+            <adapter name="Ovs" class="com.cloud.network.element.OvsElement"/>
+            <adapter name="ExternalDhcpServer" class="com.cloud.network.element.ExternalDhcpElement"/>
+            <adapter name="BareMetal" class="com.cloud.network.element.BareMetalElement"/>
+            <adapter name="SecurityGroupProvider" class="com.cloud.network.element.SecurityGroupElement"/>
+            <adapter name="VpcVirtualRouter" class="com.cloud.network.element.VpcVirtualRouterElement"/>
+            <adapter name="NiciraNvp" class="com.cloud.network.element.NiciraNvpElement"/>
+            <adapter name="DnsNotifier" class="org.apache.cloudstack.network.element.DnsNotifier"/>
+        </adapters>
+        <adapters key="com.cloud.cluster.agentlb.AgentLoadBalancerPlanner">
+            <adapter name="ClusterBasedAgentLbPlanner" class="com.cloud.cluster.agentlb.ClusterBasedAgentLoadBalancerPlanner"/>
+        </adapters>
+        <adapters key="com.cloud.hypervisor.HypervisorGuru">
+            <adapter name="XenServerGuru" class="com.cloud.hypervisor.XenServerGuru"/>
+            <adapter name="KVMGuru" class="com.cloud.hypervisor.KVMGuru"/>
+            <adapter name="BareMetalGuru" class="com.cloud.baremetal.BareMetalGuru"/>
+            <adapter name="HypervGuru" class="com.cloud.hypervisor.guru.HypervGuru"/>            
+            <adapter name="OvmGuru" class="com.cloud.ovm.hypervisor.OvmGuru" />
+        </adapters>
+        <adapters key="com.cloud.agent.StartupCommandProcessor">
+            <adapter name="BasicAgentAuthorizer" class="com.cloud.agent.manager.authn.impl.BasicAgentAuthManager"/>
+        </adapters>
+        <manager name="OvsTunnelManager" key="com.cloud.network.ovs.OvsTunnelManager" class="com.cloud.network.ovs.OvsTunnelManagerImpl"/>
+        <manager name="ElasticLoadBalancerManager" key="com.cloud.network.lb.ElasticLoadBalancerManager" class="com.cloud.network.lb.ElasticLoadBalancerManagerImpl"/>
+        <pluggableservice name="VirtualRouterElementService" key="com.cloud.network.element.VirtualRouterElementService" class="com.cloud.network.element.VirtualRouterElement"/>
+        <pluggableservice name="NiciraNvpElementService" key="com.cloud.network.element.NiciraNvpElementService" class="com.cloud.network.element.NiciraNvpElement"/>
+        <dao name="OvsTunnelInterfaceDao" class="com.cloud.network.ovs.dao.OvsTunnelInterfaceDaoImpl" singleton="false"/>
+        <dao name="OvsTunnelAccountDao" class="com.cloud.network.ovs.dao.OvsTunnelNetworkDaoImpl" singleton="false"/>
+        <dao name="NiciraNvpDao" class="com.cloud.network.dao.NiciraNvpDaoImpl" singleton="false"/>
+        <dao name="NiciraNvpNicMappingDao" class="com.cloud.network.dao.NiciraNvpNicMappingDaoImpl" singleton="false"/>
+        <dao name="NiciraNvpRouterMappingDao" class="com.cloud.network.dao.NiciraNvpRouterMappingDaoImpl" singleton="false"/>
+        <dao name="ElasticLbVmMapDao" class="com.cloud.network.lb.dao.ElasticLbVmMapDaoImpl" singleton="false"/>
+    </management-server>
+
+    <configuration-server class="com.cloud.server.ConfigurationServerImpl">
+        <dao name="Configuration configuration server" class="com.cloud.configuration.dao.ConfigurationDaoImpl" singleton="false">
+             <param name="premium">true</param>
+        </dao>
+        <dao name="Snapshot policy defaults" class="com.cloud.storage.dao.SnapshotPolicyDaoImpl" singleton="false"/>
+        <dao name="DiskOffering configuration server" class="com.cloud.storage.dao.DiskOfferingDaoImpl" singleton="false"/>         
+        <dao name="ServiceOffering configuration server" class="com.cloud.service.dao.ServiceOfferingDaoImpl" singleton="false"/>
+        <dao name="host zone configuration server" class="com.cloud.dc.dao.DataCenterDaoImpl" singleton="false"/>
+        <dao name="host pod configuration server" class="com.cloud.dc.dao.HostPodDaoImpl" singleton="false"/>
+        <dao name="DomainDao" class="com.cloud.domain.dao.DomainDaoImpl" singleton="false"/>
+        <dao name="NetworkOfferingDao" class="com.cloud.offerings.dao.NetworkOfferingDaoImpl" singleton="false"/>
+        <dao name="DataCenterDao" class="com.cloud.dc.dao.DataCenterDaoImpl" singleton="false"/>
+        <dao name="NetworkDao" class="com.cloud.network.dao.NetworkDaoImpl" singleton="false"/>
+        <dao name="IpAddressDao" class="com.cloud.network.dao.IPAddressDaoImpl" singleton="false"/>     
+        <dao name="VlanDao" class="com.cloud.dc.dao.VlanDaoImpl" singleton="false"/>
+        <dao name="ResouceCountDao" class="com.cloud.configuration.dao.ResourceCountDaoImpl" singleton="false"/>
+        <dao name="AccountDao" class="com.cloud.user.dao.AccountDaoImpl" singleton="false"/>
+        <dao name="UserDao" class="com.cloud.user.dao.UserDaoImpl" singleton="false"/>
+        <dao name="NetworkOfferingServiceDao" class="com.cloud.offerings.dao.NetworkOfferingServiceMapDaoImpl" singleton="false"/>
+        <dao name="VirtualRouterProviderDao" class="com.cloud.network.dao.VirtualRouterProviderDaoImpl" singleton="false"/>
+        <dao name="IdentityDao" class="com.cloud.uuididentity.dao.IdentityDaoImpl" singleton="false"/>
+        <dao name="Site2SiteCustomerGatewayDao" class="com.cloud.network.dao.Site2SiteCustomerGatewayDaoImpl" singleton="false"/>
+        <dao name="Site2SiteVpnGatewayDao" class="com.cloud.network.dao.Site2SiteVpnGatewayDaoImpl" singleton="false"/>
+        <dao name="Site2SiteVpnConnectionDao" class="com.cloud.network.dao.Site2SiteVpnConnectionDaoImpl" singleton="false"/>
+    </configuration-server>
+    
+    <awsapi-ec2server class="com.cloud.bridge.service.EC2MainServlet">
+        <dao name="CloudStackConfigurationDao" class="com.cloud.bridge.persist.dao.CloudStackConfigurationDaoImpl" singleton="false"/>
+        <dao name="UserCredentialsDao" class="com.cloud.bridge.persist.dao.UserCredentialsDaoImpl" singleton="false"/>
+        <dao name="CloudStackSvcOfferingDao" class="com.cloud.bridge.persist.dao.CloudStackSvcOfferingDaoImpl" singleton="false"/>
+        <dao name="OfferingDao" class="com.cloud.bridge.persist.dao.OfferingDaoImpl" singleton="false"/>
+        <dao name="CloudStackAccountDao" class="com.cloud.bridge.persist.dao.CloudStackAccountDaoImpl" singleton="false"/>
+    </awsapi-ec2server>
+    
+    <awsapi-s3server class="com.cloud.bridge.service.S3RestServlet">
+        <dao name="CloudStackConfigurationDao" class="com.cloud.bridge.persist.dao.CloudStackConfigurationDaoImpl" singleton="false"/>
+        <dao name="MHostDao" class="com.cloud.bridge.persist.dao.MHostDaoImpl" singleton="false"/>
+        <dao name="SHostDao" class="com.cloud.bridge.persist.dao.SHostDaoImpl" singleton="false"/>
+        <dao name="UserCredentialsDao" class="com.cloud.bridge.persist.dao.UserCredentialsDaoImpl" singleton="false"/>
+        <dao name="BucketPolicyDao" class="com.cloud.bridge.persist.dao.BucketPolicyDaoImpl" singleton="false"/>
+        <dao name="MHostMountDao" class="com.cloud.bridge.persist.dao.MHostMountDaoImpl" singleton="false"/>
+        <dao name="SAclDao" class="com.cloud.bridge.persist.dao.SAclDaoImpl" singleton="false"/>
+        <dao name="SBucketDao" class="com.cloud.bridge.persist.dao.SBucketDaoImpl" singleton="false"/>
+        <dao name="SMetaDao" class="com.cloud.bridge.persist.dao.SMetaDaoImpl" singleton="false"/>
+        <dao name="SObjectDao" class="com.cloud.bridge.persist.dao.SObjectDaoImpl" singleton="false"/>
+        <dao name="SObjectItemDao" class="com.cloud.bridge.persist.dao.SObjectItemDaoImpl" singleton="false"/>
+        <dao name="MultiPartPartsDao" class="com.cloud.bridge.persist.dao.MultiPartPartsDaoImpl" singleton="false"/>
+        <dao name="MultiPartUploadsDao" class="com.cloud.bridge.persist.dao.MultiPartUploadsDaoImpl" singleton="false"/>
+        <dao name="MultipartMetaDao" class="com.cloud.bridge.persist.dao.MultipartMetaDaoImpl" singleton="false"/>
+        <dao name="UserCredentialsDao" class="com.cloud.bridge.persist.dao.UserCredentialsDaoImpl" singleton="false"/>
+    </awsapi-s3server>
+    
+</components.xml>

plugins/network-elements/dns-notifier/src/org/apache/cloudstack/network/element/DnsNotifier.java

@@ -0,0 +1,143 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.network.element;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import javax.ejb.Local;
+import javax.naming.ConfigurationException;
+
+import com.cloud.deploy.DeployDestination;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.network.Network;
+import com.cloud.network.Network.Capability;
+import com.cloud.network.Network.Provider;
+import com.cloud.network.Network.Service;
+import com.cloud.network.PhysicalNetworkServiceProvider;
+import com.cloud.network.element.NetworkElement;
+import com.cloud.offering.NetworkOffering;
+import com.cloud.vm.NicProfile;
+import com.cloud.vm.ReservationContext;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VirtualMachineProfile;
+
+/**
+ * @author ahuang
+ *
+ */
+@Local(NetworkElement.class)
+public class DnsNotifier implements NetworkElement {
+    String _name = null;
+
+    public DnsNotifier() {
+
+    }
+
+    @Override
+    public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
+        _name = name;
+        return true;
+    }
+
+    @Override
+    public String getName() {
+        return _name;
+    }
+
+    @Override
+    public boolean start() {
+        return true;
+    }
+
+    @Override
+    public boolean stop() {
+        return true;
+    }
+
+    @Override
+    public Map<Service, Map<Capability, String>> getCapabilities() {
+        Map<Service, Map<Capability, String>> caps = new HashMap<Service, Map<Capability, String>>(); 
+        caps.put(Service.Dns, new HashMap<Capability, String>());
+        return caps;
+    }
+
+    @Override
+    public Provider getProvider() {
+        return null;
+    }
+
+    @Override
+    public boolean implement(Network network, NetworkOffering offering, DeployDestination dest, ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException,
+    InsufficientCapacityException {
+        return true;
+    }
+
+    @Override
+    public boolean prepare(Network network, NicProfile nic, VirtualMachineProfile<? extends VirtualMachine> vm, DeployDestination dest, ReservationContext context) throws ConcurrentOperationException,
+    ResourceUnavailableException, InsufficientCapacityException {
+        // signal to the dns server that this vm is up and running and set the ip address to hostname mapping.
+        vm.getHostName();
+        nic.getIp4Address();
+        nic.getIp6Address();
+        return true;
+    }
+
+    @Override
+    public boolean release(Network network, NicProfile nic, VirtualMachineProfile<? extends VirtualMachine> vm, ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException {
+        vm.getHostName();
+        nic.getIp4Address();
+        nic.getIp6Address();
+        // signal to the dns server that the vm is being shutdown and remove the mapping.
+        return true;
+    }
+
+    @Override
+    public boolean shutdown(Network network, ReservationContext context, boolean cleanup) throws ConcurrentOperationException, ResourceUnavailableException {
+        return true;
+    }
+
+    @Override
+    public boolean destroy(Network network, ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException {
+        return true;
+    }
+
+    @Override
+    public boolean isReady(PhysicalNetworkServiceProvider provider) {
+        return true;
+    }
+
+    @Override
+    public boolean shutdownProviderInstances(PhysicalNetworkServiceProvider provider, ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException {
+        return true;
+    }
+
+    @Override
+    public boolean canEnableIndividualServices() {
+        return true;
+    }
+
+    @Override
+    public boolean verifyServicesCombination(Set<Service> services) {
+        return true;
+    }
+
+}

plugins/network-elements/nicira-nvp/src/com/cloud/network/guru/NiciraNvpGuestNetworkGuru.java

@@ -39,6 +39,7 @@ import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
 import com.cloud.host.dao.HostDetailsDao;
 import com.cloud.network.Network;
+import com.cloud.network.Network.Service;
 import com.cloud.network.NetworkManager;
 import com.cloud.network.NetworkProfile;
 import com.cloud.network.NetworkVO;
@@ -50,9 +51,11 @@ import com.cloud.network.PhysicalNetwork;
 import com.cloud.network.PhysicalNetwork.IsolationMethod;
 import com.cloud.network.PhysicalNetworkVO;
 import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkServiceMapDao;
 import com.cloud.network.dao.NiciraNvpDao;
 import com.cloud.network.dao.PhysicalNetworkDao;
 import com.cloud.offering.NetworkOffering;
+import com.cloud.offerings.dao.NetworkOfferingServiceMapDao;
 import com.cloud.resource.ResourceManager;
 import com.cloud.user.Account;
 import com.cloud.user.dao.AccountDao;
@@ -88,6 +91,8 @@ public class NiciraNvpGuestNetworkGuru extends GuestNetworkGuru {
     AgentManager _agentMgr;
     @Inject
     HostDetailsDao _hostDetailsDao;
+    @Inject
+    NetworkOfferingServiceMapDao _ntwkOfferingSrvcDao;
     
     public NiciraNvpGuestNetworkGuru() {
         super();
@@ -100,7 +105,8 @@ public class NiciraNvpGuestNetworkGuru extends GuestNetworkGuru {
         if (networkType == NetworkType.Advanced 
                 && isMyTrafficType(offering.getTrafficType()) 
                 && offering.getGuestType() == Network.GuestType.Isolated
-                && isMyIsolationMethod(physicalNetwork)) {
+                && isMyIsolationMethod(physicalNetwork)
+                && _ntwkOfferingSrvcDao.areServicesSupportedByNetworkOffering(offering.getId(), Service.Connectivity)) {
             return true;
         } else {
             s_logger.trace("We only take care of Guest networks of type   " + GuestType.Isolated + " in zone of type " + NetworkType.Advanced);

plugins/network-elements/nicira-nvp/test/com/cloud/network/guru/NiciraNvpGuestNetworkGuruTest.java

@@ -0,0 +1,343 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.guru;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Collections;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Command;
+import com.cloud.agent.api.CreateLogicalSwitchAnswer;
+import com.cloud.agent.api.DeleteLogicalSwitchAnswer;
+import com.cloud.dc.DataCenter;
+import com.cloud.dc.DataCenterVO;
+import com.cloud.dc.DataCenter.NetworkType;
+import com.cloud.dc.dao.DataCenterDao;
+import com.cloud.deploy.DeployDestination;
+import com.cloud.deploy.DeploymentPlan;
+import com.cloud.domain.Domain;
+import com.cloud.exception.InsufficientVirtualNetworkCapcityException;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
+import com.cloud.network.Network;
+import com.cloud.network.Network.GuestType;
+import com.cloud.network.Network.Service;
+import com.cloud.network.Network.State;
+import com.cloud.network.NetworkManager;
+import com.cloud.network.NetworkProfile;
+import com.cloud.network.NetworkVO;
+import com.cloud.network.Networks.BroadcastDomainType;
+import com.cloud.network.Networks.TrafficType;
+import com.cloud.network.NiciraNvpDeviceVO;
+import com.cloud.network.PhysicalNetworkVO;
+import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NiciraNvpDao;
+import com.cloud.network.dao.PhysicalNetworkDao;
+import com.cloud.offering.NetworkOffering;
+import com.cloud.offerings.dao.NetworkOfferingServiceMapDao;
+import com.cloud.user.Account;
+import com.cloud.vm.ReservationContext;
+
+import edu.emory.mathcs.backport.java.util.Arrays;
+
+public class NiciraNvpGuestNetworkGuruTest {
+	PhysicalNetworkDao physnetdao = mock (PhysicalNetworkDao.class);
+	NiciraNvpDao nvpdao = mock(NiciraNvpDao.class);
+	DataCenterDao dcdao = mock(DataCenterDao.class);
+	NetworkOfferingServiceMapDao nosd = mock(NetworkOfferingServiceMapDao.class);
+	AgentManager agentmgr = mock (AgentManager.class);
+	NetworkManager netmgr = mock (NetworkManager.class);
+	HostDao hostdao = mock (HostDao.class);
+	NetworkDao netdao = mock(NetworkDao.class);
+	NiciraNvpGuestNetworkGuru guru;
+	
+	
+	@Before
+	public void setUp() {
+		guru = new NiciraNvpGuestNetworkGuru();
+		((GuestNetworkGuru) guru)._physicalNetworkDao = physnetdao;
+		guru._physicalNetworkDao = physnetdao;
+		guru._niciraNvpDao = nvpdao;
+		guru._dcDao = dcdao;
+		guru._ntwkOfferingSrvcDao = nosd;
+		guru._networkMgr = netmgr;
+		guru._hostDao = hostdao;
+		guru._agentMgr = agentmgr;
+		guru._networkDao = netdao;
+		
+		DataCenterVO dc = mock(DataCenterVO.class);
+		when(dc.getNetworkType()).thenReturn(NetworkType.Advanced);
+		when(dc.getGuestNetworkCidr()).thenReturn("10.1.1.1/24");
+		
+		when(dcdao.findById((Long) any())).thenReturn((DataCenterVO) dc);
+	}
+	
+	@Test
+	public void testCanHandle() {
+		NetworkOffering offering = mock(NetworkOffering.class);
+		when(offering.getId()).thenReturn(42L);
+		when(offering.getTrafficType()).thenReturn(TrafficType.Guest);
+		when(offering.getGuestType()).thenReturn(GuestType.Isolated);
+	
+		PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class);
+		when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "STT" }));
+		when(physnet.getId()).thenReturn(42L);
+		
+		when(nosd.areServicesSupportedByNetworkOffering(42L, Service.Connectivity)).thenReturn(true);
+		
+		assertTrue(guru.canHandle(offering, NetworkType.Advanced, physnet) == true);
+		
+		// Not supported TrafficType != Guest
+		when(offering.getTrafficType()).thenReturn(TrafficType.Management);
+		assertFalse(guru.canHandle(offering, NetworkType.Advanced, physnet) == true);
+		
+		// Not supported: GuestType Shared
+		when(offering.getTrafficType()).thenReturn(TrafficType.Guest);
+		when(offering.getGuestType()).thenReturn(GuestType.Shared);
+		assertFalse(guru.canHandle(offering, NetworkType.Advanced, physnet) == true);
+		
+		// Not supported: Basic networking
+		when(offering.getGuestType()).thenReturn(GuestType.Isolated);
+		assertFalse(guru.canHandle(offering, NetworkType.Basic, physnet) == true);
+		
+		// Not supported: IsolationMethod != STT
+		when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "VLAN" }));
+		assertFalse(guru.canHandle(offering, NetworkType.Advanced, physnet) == true);
+		
+	}
+	
+	
+	@Test
+	public void testDesign() {
+		PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class);
+		when(physnetdao.findById((Long) any())).thenReturn(physnet);
+		when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "STT" }));
+		when(physnet.getId()).thenReturn(42L);
+		
+		NiciraNvpDeviceVO device = mock(NiciraNvpDeviceVO.class);
+		when(nvpdao.listByPhysicalNetwork(42L)).thenReturn(Arrays.asList(new NiciraNvpDeviceVO[] { device }));
+		when(device.getId()).thenReturn(1L);
+		
+		NetworkOffering offering = mock(NetworkOffering.class);
+		when(offering.getId()).thenReturn(42L);
+		when(offering.getTrafficType()).thenReturn(TrafficType.Guest);
+		when(offering.getGuestType()).thenReturn(GuestType.Isolated);
+		
+		when(nosd.areServicesSupportedByNetworkOffering(42L, Service.Connectivity)).thenReturn(true);
+		
+		DeploymentPlan plan = mock(DeploymentPlan.class);
+		Network network = mock(Network.class);
+		Account account = mock(Account.class);
+		
+		Network designednetwork = guru.design(offering, plan, network, account);
+		assertTrue(designednetwork != null);
+		assertTrue(designednetwork.getBroadcastDomainType() == BroadcastDomainType.Lswitch);		
+	}
+	
+	@Test
+	public void testDesignNoElementOnPhysicalNetwork() {
+		PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class);
+		when(physnetdao.findById((Long) any())).thenReturn(physnet);
+		when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "STT" }));
+		when(physnet.getId()).thenReturn(42L);
+		
+		NiciraNvpDeviceVO device = mock(NiciraNvpDeviceVO.class);
+		when(nvpdao.listByPhysicalNetwork(42L)).thenReturn(Collections.<NiciraNvpDeviceVO> emptyList());
+		
+		NetworkOffering offering = mock(NetworkOffering.class);
+		when(offering.getId()).thenReturn(42L);
+		when(offering.getTrafficType()).thenReturn(TrafficType.Guest);
+		when(offering.getGuestType()).thenReturn(GuestType.Isolated);
+		
+		DeploymentPlan plan = mock(DeploymentPlan.class);
+		Network network = mock(Network.class);
+		Account account = mock(Account.class);
+		
+		Network designednetwork = guru.design(offering, plan, network, account);
+		assertTrue(designednetwork == null);	
+	}
+	
+	@Test
+	public void testDesignNoIsolationMethodSTT() {
+		PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class);
+		when(physnetdao.findById((Long) any())).thenReturn(physnet);
+		when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "VLAN" }));
+		when(physnet.getId()).thenReturn(42L);
+		
+		NiciraNvpDeviceVO device = mock(NiciraNvpDeviceVO.class);
+		when(nvpdao.listByPhysicalNetwork(42L)).thenReturn(Collections.<NiciraNvpDeviceVO> emptyList());
+		
+		NetworkOffering offering = mock(NetworkOffering.class);
+		when(offering.getId()).thenReturn(42L);
+		when(offering.getTrafficType()).thenReturn(TrafficType.Guest);
+		when(offering.getGuestType()).thenReturn(GuestType.Isolated);
+		
+		DeploymentPlan plan = mock(DeploymentPlan.class);
+		Network network = mock(Network.class);
+		Account account = mock(Account.class);
+		
+		Network designednetwork = guru.design(offering, plan, network, account);
+		assertTrue(designednetwork == null);			
+	}
+	
+	@Test
+	public void testDesignNoConnectivityInOffering() {
+		PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class);
+		when(physnetdao.findById((Long) any())).thenReturn(physnet);
+		when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "STT" }));
+		when(physnet.getId()).thenReturn(42L);
+		
+		NiciraNvpDeviceVO device = mock(NiciraNvpDeviceVO.class);
+		when(nvpdao.listByPhysicalNetwork(42L)).thenReturn(Arrays.asList(new NiciraNvpDeviceVO[] { device }));
+		when(device.getId()).thenReturn(1L);
+		
+		NetworkOffering offering = mock(NetworkOffering.class);
+		when(offering.getId()).thenReturn(42L);
+		when(offering.getTrafficType()).thenReturn(TrafficType.Guest);
+		when(offering.getGuestType()).thenReturn(GuestType.Isolated);
+		
+		when(nosd.areServicesSupportedByNetworkOffering(42L, Service.Connectivity)).thenReturn(false);
+		
+		DeploymentPlan plan = mock(DeploymentPlan.class);
+		Network network = mock(Network.class);
+		Account account = mock(Account.class);
+		
+		Network designednetwork = guru.design(offering, plan, network, account);
+		assertTrue(designednetwork == null);		
+	}
+	
+	@Test
+	public void testImplement() throws InsufficientVirtualNetworkCapcityException {
+		PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class);
+		when(physnetdao.findById((Long) any())).thenReturn(physnet);
+		when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "STT" }));
+		when(physnet.getId()).thenReturn(42L);
+		
+		NiciraNvpDeviceVO device = mock(NiciraNvpDeviceVO.class);
+		when(nvpdao.listByPhysicalNetwork(42L)).thenReturn(Arrays.asList(new NiciraNvpDeviceVO[] { device }));
+		when(device.getId()).thenReturn(1L);
+		
+		NetworkOffering offering = mock(NetworkOffering.class);
+		when(offering.getId()).thenReturn(42L);
+		when(offering.getTrafficType()).thenReturn(TrafficType.Guest);
+		when(offering.getGuestType()).thenReturn(GuestType.Isolated);
+		
+		when(nosd.areServicesSupportedByNetworkOffering(42L, Service.Connectivity)).thenReturn(false);
+		
+		DeploymentPlan plan = mock(DeploymentPlan.class);
+		
+		NetworkVO network = mock(NetworkVO.class);
+		when(network.getName()).thenReturn("testnetwork");
+		when(network.getState()).thenReturn(State.Implementing);
+		
+		DeployDestination dest = mock(DeployDestination.class);
+		
+		DataCenter dc = mock(DataCenter.class);
+		when(dest.getDataCenter()).thenReturn(dc);
+		
+		HostVO niciraHost = mock(HostVO.class);
+		when(hostdao.findById(anyLong())).thenReturn(niciraHost);
+		when(niciraHost.getDetail("transportzoneuuid")).thenReturn("aaaa");
+		when(niciraHost.getDetail("transportzoneisotype")).thenReturn("stt");
+		when(niciraHost.getId()).thenReturn(42L);
+		
+		when(netmgr.findPhysicalNetworkId(anyLong(), (String) any(), (TrafficType) any())).thenReturn(42L);
+		Domain dom = mock(Domain.class);
+		when(dom.getName()).thenReturn("domain");
+		Account acc = mock(Account.class);
+		when(acc.getAccountName()).thenReturn("accountname");
+		ReservationContext res = mock(ReservationContext.class);
+		when(res.getDomain()).thenReturn(dom);
+		when(res.getAccount()).thenReturn(acc);
+		
+		CreateLogicalSwitchAnswer answer = mock(CreateLogicalSwitchAnswer.class);
+		when(answer.getResult()).thenReturn(true);
+ 		when(agentmgr.easySend(eq(42L), (Command)any())).thenReturn(answer);		
+
+ 		Network implementednetwork = guru.implement(network, offering, dest, res);
+		assertTrue(implementednetwork != null);		
+		verify(agentmgr, times(1)).easySend(eq(42L), (Command)any());
+	}
+	
+	@Test
+	public void testShutdown() throws InsufficientVirtualNetworkCapcityException, URISyntaxException {
+		PhysicalNetworkVO physnet = mock(PhysicalNetworkVO.class);
+		when(physnetdao.findById((Long) any())).thenReturn(physnet);
+		when(physnet.getIsolationMethods()).thenReturn(Arrays.asList(new String[] { "STT" }));
+		when(physnet.getId()).thenReturn(42L);
+		
+		NiciraNvpDeviceVO device = mock(NiciraNvpDeviceVO.class);
+		when(nvpdao.listByPhysicalNetwork(42L)).thenReturn(Arrays.asList(new NiciraNvpDeviceVO[] { device }));
+		when(device.getId()).thenReturn(1L);
+		
+		NetworkOffering offering = mock(NetworkOffering.class);
+		when(offering.getId()).thenReturn(42L);
+		when(offering.getTrafficType()).thenReturn(TrafficType.Guest);
+		when(offering.getGuestType()).thenReturn(GuestType.Isolated);
+		
+		when(nosd.areServicesSupportedByNetworkOffering(42L, Service.Connectivity)).thenReturn(false);
+		
+		DeploymentPlan plan = mock(DeploymentPlan.class);
+		
+		NetworkVO network = mock(NetworkVO.class);
+		when(network.getName()).thenReturn("testnetwork");
+		when(network.getState()).thenReturn(State.Implementing);
+		when(network.getBroadcastDomainType()).thenReturn(BroadcastDomainType.Lswitch);
+		when(network.getBroadcastUri()).thenReturn(new URI("lswitch:aaaaa"));
+		when(network.getPhysicalNetworkId()).thenReturn(42L);
+		when(netdao.findById(42L)).thenReturn(network);
+		
+		DeployDestination dest = mock(DeployDestination.class);
+		
+		DataCenter dc = mock(DataCenter.class);
+		when(dest.getDataCenter()).thenReturn(dc);
+		
+		HostVO niciraHost = mock(HostVO.class);
+		when(hostdao.findById(anyLong())).thenReturn(niciraHost);
+		when(niciraHost.getDetail("transportzoneuuid")).thenReturn("aaaa");
+		when(niciraHost.getDetail("transportzoneisotype")).thenReturn("stt");
+		when(niciraHost.getId()).thenReturn(42L);
+		
+		when(netmgr.findPhysicalNetworkId(anyLong(), (String) any(), (TrafficType) any())).thenReturn(42L);
+		Domain dom = mock(Domain.class);
+		when(dom.getName()).thenReturn("domain");
+		Account acc = mock(Account.class);
+		when(acc.getAccountName()).thenReturn("accountname");
+		ReservationContext res = mock(ReservationContext.class);
+		when(res.getDomain()).thenReturn(dom);
+		when(res.getAccount()).thenReturn(acc);
+		
+		DeleteLogicalSwitchAnswer answer = mock(DeleteLogicalSwitchAnswer.class);
+		when(answer.getResult()).thenReturn(true);
+ 		when(agentmgr.easySend(eq(42L), (Command)any())).thenReturn(answer);		
+
+ 		NetworkProfile implementednetwork = mock(NetworkProfile.class);
+ 		when(implementednetwork.getId()).thenReturn(42L);
+ 		when(implementednetwork.getBroadcastUri()).thenReturn(new URI("lswitch:aaaa"));
+ 		when(offering.getSpecifyVlan()).thenReturn(false);
+ 		
+ 		guru.shutdown(implementednetwork, offering);
+		verify(agentmgr, times(1)).easySend(eq(42L), (Command)any());
+		verify(implementednetwork, times(1)).setBroadcastUri(null);
+	}
+}

plugins/network-elements/ovs/src/com/cloud/network/element/OvsElement.java

@@ -16,7 +16,6 @@
 // under the License.
 package com.cloud.network.element;
 
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 

plugins/pom.xml

@@ -46,6 +46,7 @@
     <module>user-authenticators/md5</module>
     <module>user-authenticators/plain-text</module>
     <module>user-authenticators/sha256salted</module>
+    <module>network-elements/dns-notifier</module>
   </modules>
 
   <dependencies>

pom.xml

@@ -67,7 +67,7 @@
     <cs.gson.version>1.7.1</cs.gson.version>
     <cs.xapi.version>5.6.100-1-SNAPSHOT</cs.xapi.version>
     <cs.httpclient.version>3.1</cs.httpclient.version>
-    <cs.httpcore.version>4.0</cs.httpcore.version>
+    <cs.httpcore.version>4.1</cs.httpcore.version>
     <cs.mysql.version>5.1.21</cs.mysql.version>
     <cs.xstream.version>1.3.1</cs.xstream.version>
     <cs.xmlrpc.version>3.1.3</cs.xmlrpc.version>
@@ -82,7 +82,9 @@
     <cs.selenium.server.version>1.0-20081010.060147</cs.selenium.server.version>
     <cs.vmware.api.version>4.1</cs.vmware.api.version>
     <cs.mockito.version>1.9.5</cs.mockito.version>
-
+    <cs.aws.sdk.version>1.3.21.1</cs.aws.sdk.version>
+    <cs.lang.version>2.6</cs.lang.version>
+    <cs.commons-io.version>1.4</cs.commons-io.version>
   </properties>
 
   <distributionManagement>
@@ -206,8 +208,12 @@
             <excludes>
               <exclude>CHANGES</exclude>
               <exclude>INSTALL.md</exclude>
+              <exclude>.idea/</exclude>
+              <exclude>*.log</exclude>
+              <exclude>**/*.patch</exclude>
               <exclude>**/.classpath</exclude>
               <exclude>**/.project</exclude>
+              <exclude>**/*.iml</exclude>
               <exclude>**/.settings/**</exclude>
               <exclude>.metadata/**</exclude>
               <exclude>.git/**</exclude>

scripts/vm/hypervisor/xenserver/s3xen

@@ -0,0 +1,297 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Version @VERSION@
+#
+# A plugin for executing script needed by cloud  stack
+from __future__ import with_statement
+
+from copy import copy
+from datetime import datetime
+from httplib import *
+from string import join
+
+import os
+import sys
+import time
+import hashlib
+import base64
+import hmac
+import traceback
+import urllib2
+
+import XenAPIPlugin
+sys.path.extend(["/opt/xensource/sm/"])
+import util
+
+NULL = 'null'
+
+# Value conversion utility functions ...
+
+
+def to_none(value):
+    return value if value is not None and value.strip() != NULL else None
+
+
+def to_bool(value):
+    return True if to_none(value) in ['true', 'True', None] else False
+
+
+def to_integer(value, default):
+    return int(value) if to_none(value) is not None else default
+
+
+def optional_str_value(value, default):
+    return value if is_not_blank(value) else default
+
+
+def is_not_blank(value):
+    return True if to_none(value) is not None and value.strip != '' else False
+
+
+def get_optional_key(map, key, default=''):
+    return map[key] if key in map else default
+
+
+def log(message):
+    util.SMlog('#### VMOPS %s ####' % message)
+
+
+def echo(fn):
+    def wrapped(*v, **k):
+        name = fn.__name__
+        log("enter %s ####" % name)
+        res = fn(*v, **k)
+        log("exit %s with result %s" % name, res)
+        return res
+    return wrapped
+
+
+def require_str_value(value, error_message):
+
+    if is_not_blank(value):
+        return value
+
+    raise ValueError(error_message)
+
+
+def retry(max_attempts, fn):
+
+    attempts = 1
+    while attempts <= max_attempts:
+        log("Attempting execution {0}/{1} of {2}".
+            format(attempts, max_attempts, fn.__name__))
+        try:
+            return fn()
+        except:
+            if (attempts >= max_attempts):
+                raise
+            attempts = attempts + 1
+
+
+def compute_md5(filename, buffer_size=8192):
+
+    hasher = hashlib.md5()
+
+    with open(filename, 'rb') as file:
+        data = file.read(buffer_size)
+        while data != "":
+            hasher.update(data)
+            data = file.read(buffer_size)
+
+    return base64.encodestring(hasher.digest())[:-1]
+
+
+class S3Client(object):
+
+    DEFAULT_END_POINT = 's3.amazonaws.com'
+    DEFAULT_CONNECTION_TIMEOUT = 50000
+    DEFAULT_SOCKET_TIMEOUT = 50000
+    DEFAULT_MAX_ERROR_RETRY = 3
+
+    HEADER_CONTENT_MD5 = 'Content-MD5'
+    HEADER_CONTENT_TYPE = 'Content-Type'
+    HEADER_CONTENT_LENGTH = 'Content-Length'
+
+    def __init__(self, access_key, secret_key, end_point=None,
+                 https_flag=None, connection_timeout=None, socket_timeout=None,
+                 max_error_retry=None):
+
+        self.access_key = require_str_value(
+            access_key, 'An access key must be specified.')
+        self.secret_key = require_str_value(
+            secret_key, 'A secret key must be specified.')
+        self.end_point = optional_str_value(end_point, self.DEFAULT_END_POINT)
+        self.https_flag = to_bool(https_flag)
+        self.connection_timeout = to_integer(
+            connection_timeout, self.DEFAULT_CONNECTION_TIMEOUT)
+        self.socket_timeout = to_integer(
+            socket_timeout, self.DEFAULT_SOCKET_TIMEOUT)
+        self.max_error_retry = to_integer(
+            max_error_retry, self.DEFAULT_MAX_ERROR_RETRY)
+
+    def build_canocialized_resource(self, bucket, key):
+
+        return '/{bucket}/{key}'.format(bucket=bucket, key=key)
+
+    def noop_send_body():
+        pass
+
+    def noop_read(response):
+        return response.read()
+
+    def do_operation(
+        self, method, bucket, key, input_headers={},
+            fn_send_body=noop_send_body, fn_read=noop_read):
+
+        headers = copy(input_headers)
+        headers['Expect'] = '100-continue'
+
+        uri = self.build_canocialized_resource(bucket, key)
+        signature, request_date = self.sign_request(method, uri, headers)
+        headers['Authorization'] = "AWS {0}:{1}".format(
+            self.access_key, signature)
+        headers['Date'] = request_date
+
+        connection = HTTPSConnection(self.end_point) \
+            if self.https_flag else HTTPConnection(self.end_point)
+        connection.timeout = self.socket_timeout
+
+        def perform_request():
+
+            connection.request(method, uri, fn_send_body(), headers)
+            response = connection.getresponse()
+            log("Sent {0} request to {1} {2} with headers {3}.  \
+                Got response status {4}: {5}".
+                format(method, self.end_point, uri, headers,
+                response.status, response.reason))
+            return fn_read(response)
+
+        try:
+            return retry(self.max_error_retry, perform_request)
+        finally:
+            connection.close()
+
+    '''
+    See http://bit.ly/MMC5de for more information regarding the creation of
+    AWS authorization tokens and header signing
+    '''
+    def sign_request(self, operation, canocialized_resource, headers):
+
+        request_date = datetime.utcnow(
+        ).strftime('%a, %d %b %Y %H:%M:%S +0000')
+
+        content_hash = get_optional_key(headers, self.HEADER_CONTENT_MD5)
+        content_type = get_optional_key(headers, self.HEADER_CONTENT_TYPE)
+
+        string_to_sign = join(
+            [operation, content_hash, content_type, request_date,
+                canocialized_resource], '\n')
+
+        signature = base64.encodestring(
+            hmac.new(self.secret_key, string_to_sign.encode('utf8'),
+                     hashlib.sha1).digest())[:-1]
+
+        return signature, request_date
+
+    def put(self, bucket, key, src_filename):
+
+        headers = {
+            self.HEADER_CONTENT_MD5: compute_md5(src_filename),
+            self.HEADER_CONTENT_TYPE: 'application/octet-stream',
+            self.HEADER_CONTENT_LENGTH: os.stat(src_filename).st_size,
+        }
+
+        def send_body():
+            return open(src_filename, 'rb')
+
+        self.do_operation('PUT', bucket, key, headers, send_body)
+
+    def get(self, bucket, key, target_filename):
+
+        def read(response):
+
+            with open(target_filename, 'wb') as file:
+                while True:
+                    block = response.read(8192)
+                    if not block:
+                        break
+                        file.write(block)
+
+        return self.do_operation('GET', bucket, key, fn_read=read)
+
+    def delete(self, bucket, key):
+
+        return self.do_operation('DELETE', bucket, key)
+
+
+def parseArguments(args):
+
+    # The keys in the args map will correspond to the properties defined on
+    # the com.cloud.utils.S3Utils#ClientOptions interface
+    client = S3Client(
+        args['accessKey'], args['secretKey'], args['endPoint'],
+        args['isHttps'], args['connectionTimeout'], args['socketTimeout'])
+
+    operation = args['operation']
+    bucket = args['bucket']
+    key = args['key']
+    filename = args['filename']
+
+    if is_blank(operation):
+        raise ValueError('An operation must be specified.')
+
+    if is_blank(bucket):
+        raise ValueError('A bucket must be specified.')
+
+    if is_blank(key):
+        raise ValueError('A value must be specified.')
+
+    if is_blank(filename):
+        raise ValueError('A filename must be specified.')
+
+    return client, operation, bucket, key, filename
+
+
+@echo
+def s3(session, args):
+
+    client, operation, bucket, key, filename = parseArguments(args)
+
+    try:
+
+        if operation == 'put':
+            client.put(bucket, key, filename)
+        elif operation == 'get':
+            client.get(bucket, key, filename)
+        elif operation == 'delete':
+            client.delete(bucket, key, filename)
+        else:
+            raise RuntimeError(
+                "S3 plugin does not support operation {0}.".format(operation))
+
+        return 'true'
+
+    except:
+        log("Operation {0} on file {1} from/in bucket {2} key {3}.".format(
+            operation, filename, bucket, key))
+        log(traceback.format_exc())
+        return 'false'
+
+if __name__ == "__main__":
+    XenAPIPlugin.dispatch({"s3": s3})

scripts/vm/hypervisor/xenserver/xenserver56/patch

@@ -62,3 +62,5 @@ cloud-prepare-upgrade.sh=..,0755,/opt/xensource/bin
 bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin
 swift=..,0755,/opt/xensource/bin
 swiftxen=..,0755,/etc/xapi.d/plugins
+s3xen=..,0755,/etc/xapi.d/plugins
+

scripts/vm/hypervisor/xenserver/xenserver56fp1/patch

@@ -61,3 +61,5 @@ cloud-prepare-upgrade.sh=..,0755,/opt/xensource/bin
 bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin
 swift=..,0755,/opt/xensource/bin
 swiftxen=..,0755,/etc/xapi.d/plugins
+s3xen=..,0755,/etc/xapi.d/plugins
+

scripts/vm/hypervisor/xenserver/xenserver60/patch

@@ -66,3 +66,5 @@ cloud-prepare-upgrade.sh=..,0755,/opt/xensource/bin
 bumpUpPriority.sh=../../../../network/domr/,0755,/opt/xensource/bin
 swift=..,0755,/opt/xensource/bin
 swiftxen=..,0755,/etc/xapi.d/plugins
+s3xen=..,0755,/etc/xapi.d/plugins
+

scripts/vm/network/security_group.py

@@ -24,6 +24,7 @@ import sys
 import os
 import xml.dom.minidom
 from optparse import OptionParser, OptionGroup, OptParseError, BadOptionError, OptionError, OptionConflictError, OptionValueError
+import re
 iptables = Command("iptables")
 bash = Command("/bin/bash")
 virsh = Command("virsh")
@@ -420,12 +421,11 @@ def network_rules_for_rebooted_vm(vmName):
     
     delete_rules_for_vm_in_bridge_firewall_chain(vm_name)
 
-    brName = execute("iptables-save  |grep physdev-is-bridged |grep FORWARD |grep BF |grep '\-o' |awk '{print $9}'").split("\n")
-    if brName is None:
+    brName = execute("iptables-save |grep physdev-is-bridged |grep FORWARD |grep BF |grep '\-o' |awk '{print $9}' | head -1").strip()
+    if brName is None or brName is "":
         brName = "cloudbr0"
     else:
-        brName.pop()
-        brName = brName[0].split("-")[1]
+        brName = re.sub("^BF-", "", brName)
 
     if 1 in [ vm_name.startswith(c) for c in ['r-', 's-', 'v-'] ]:
         

server/pom.xml

@@ -62,6 +62,11 @@
       <artifactId>jstl</artifactId>
       <version>${cs.jstl.version}</version>
     </dependency>
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+      <version>${cs.codec.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.cloudstack</groupId>
       <artifactId>cloud-utils</artifactId>
@@ -69,7 +74,6 @@
       <classifier>tests</classifier>
       <scope>test</scope>
     </dependency>
-    
   </dependencies>
   <build>
     <defaultGoal>install</defaultGoal>

server/src/com/cloud/api/ApiDBUtils.java

@@ -115,6 +115,7 @@ import com.cloud.storage.StoragePoolVO;
 import com.cloud.storage.StorageStats;
 import com.cloud.storage.UploadVO;
 import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.VMTemplateS3VO;
 import com.cloud.storage.VMTemplateSwiftVO;
 import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.Volume.Type;
@@ -129,6 +130,7 @@ import com.cloud.storage.dao.UploadDao;
 import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.storage.dao.VMTemplateDetailsDao;
 import com.cloud.storage.dao.VMTemplateHostDao;
+import com.cloud.storage.dao.VMTemplateS3Dao;
 import com.cloud.storage.dao.VMTemplateSwiftDao;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.storage.dao.VolumeHostDao;
@@ -196,6 +198,7 @@ public class ApiDBUtils {
     private static VMTemplateDetailsDao _templateDetailsDao;
     private static VMTemplateHostDao _templateHostDao;
     private static VMTemplateSwiftDao _templateSwiftDao;
+    private static VMTemplateS3Dao _templateS3Dao;
     private static UploadDao _uploadDao;
     private static UserDao _userDao;
     private static UserStatisticsDao _userStatsDao;
@@ -260,6 +263,7 @@ public class ApiDBUtils {
         _templateDetailsDao = locator.getDao(VMTemplateDetailsDao.class);
         _templateHostDao = locator.getDao(VMTemplateHostDao.class);
         _templateSwiftDao = locator.getDao(VMTemplateSwiftDao.class);
+        _templateS3Dao = locator.getDao(VMTemplateS3Dao.class);
         _uploadDao = locator.getDao(UploadDao.class);
         _userDao = locator.getDao(UserDao.class);
         _userStatsDao = locator.getDao(UserStatisticsDao.class);
@@ -575,6 +579,10 @@ public class ApiDBUtils {
         return _templateSwiftDao.findOneByTemplateId(templateId);
     }
 
+    public static VMTemplateS3VO findTemplateS3Ref(long templateId) {
+        return _templateS3Dao.findOneByTemplateId(templateId);
+    }
+
     public static UploadVO findUploadById(Long id) {
         return _uploadDao.findById(id);
     }

server/src/com/cloud/api/ApiDispatcher.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.api;
 
+import static org.apache.commons.lang.StringUtils.isNotBlank;
+
 import java.lang.reflect.Field;
 import java.text.DateFormat;
 import java.text.ParseException;
@@ -41,7 +43,6 @@ import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.PermissionDeniedException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
-import com.cloud.utils.IdentityProxy;
 import com.cloud.server.ManagementServer;
 import com.cloud.user.Account;
 import com.cloud.user.UserContext;
@@ -417,10 +418,20 @@ public class ApiDispatcher {
                 }
                 break;
             case FLOAT:
+                // Assuming that the parameters have been checked for required before now,
+                // we ignore blank or null values and defer to the command to set a default
+                // value for optional parameters ...
+                if (paramObj != null && isNotBlank(paramObj.toString())) {
                     field.set(cmdObj, Float.valueOf(paramObj.toString()));
+                }
                 break;
             case INTEGER:
+                // Assuming that the parameters have been checked for required before now,
+                // we ignore blank or null values and defer to the command to set a default
+                // value for optional parameters ...
+                if (paramObj != null && isNotBlank(paramObj.toString())) {
                     field.set(cmdObj, Integer.valueOf(paramObj.toString()));
+                }
                 break;
             case LIST:
                 List listParam = new ArrayList();

server/src/com/cloud/api/ApiResponseHelper.java

@@ -16,6 +16,9 @@
 // under the License.
 package com.cloud.api;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+
 import java.text.DecimalFormat;
 import java.util.ArrayList;
 import java.util.Date;
@@ -82,6 +85,7 @@ import com.cloud.api.response.RemoteAccessVpnResponse;
 import com.cloud.api.response.ResourceCountResponse;
 import com.cloud.api.response.ResourceLimitResponse;
 import com.cloud.api.response.ResourceTagResponse;
+import com.cloud.api.response.S3Response;
 import com.cloud.api.response.SecurityGroupResponse;
 import com.cloud.api.response.SecurityGroupResultObject;
 import com.cloud.api.response.SecurityGroupRuleResponse;
@@ -131,7 +135,6 @@ import com.cloud.dc.Vlan.VlanType;
 import com.cloud.dc.VlanVO;
 import com.cloud.domain.Domain;
 import com.cloud.event.Event;
-import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.host.Host;
 import com.cloud.host.HostStats;
 import com.cloud.host.HostVO;
@@ -189,6 +192,7 @@ import com.cloud.server.ResourceTag.TaggedResourceType;
 import com.cloud.storage.DiskOfferingVO;
 import com.cloud.storage.GuestOS;
 import com.cloud.storage.GuestOSCategoryVO;
+import com.cloud.storage.S3;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.ImageFormat;
@@ -200,6 +204,7 @@ import com.cloud.storage.StorageStats;
 import com.cloud.storage.Swift;
 import com.cloud.storage.UploadVO;
 import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.VMTemplateS3VO;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.storage.VMTemplateSwiftVO;
 import com.cloud.storage.VMTemplateVO;
@@ -743,6 +748,25 @@ public class ApiResponseHelper implements ResponseGenerator {
         return swiftResponse;
     }
 
+    @Override
+    public S3Response createS3Response(final S3 result) {
+
+        final S3Response response = new S3Response();
+
+        response.setAccessKey(result.getAccessKey());
+        response.setConnectionTimeout(result.getConnectionTimeout());
+        response.setEndPoint(result.getEndPoint());
+        response.setHttpsFlag(result.getHttpsFlag());
+        response.setMaxErrorRetry(result.getMaxErrorRetry());
+        response.setObjectId(result.getId());
+        response.setSecretKey(result.getSecretKey());
+        response.setSocketTimeout(result.getSocketTimeout());
+        response.setTemplateBucketName(result.getBucketName());
+
+        return response;
+
+    }
+
     @Override
     public VlanIpRangeResponse createVlanIpRangeResponse(Vlan vlan) {
         Long podId = ApiDBUtils.getPodIdForVlan(vlan.getId());
@@ -2153,7 +2177,7 @@ public class ApiResponseHelper implements ResponseGenerator {
     @Override
     public List<TemplateResponse> createIsoResponses(long isoId, Long zoneId, boolean readyOnly) {
 
-        List<TemplateResponse> isoResponses = new ArrayList<TemplateResponse>();
+        final List<TemplateResponse> isoResponses = new ArrayList<TemplateResponse>();
         VirtualMachineTemplate iso = findTemplateById(isoId);
         if (iso.getTemplateType() == TemplateType.PERHOST) {
             TemplateResponse isoResponse = new TemplateResponse();
@@ -2191,11 +2215,17 @@ public class ApiResponseHelper implements ResponseGenerator {
             return isoResponses;
         } else {
             if (zoneId == null || zoneId == -1) {
-                isoResponses = createSwiftIsoResponses(iso);
+                isoResponses.addAll(createSwiftIsoResponses(iso));
                 if (!isoResponses.isEmpty()) {
                     return isoResponses;
                 }
-                List<DataCenterVO> dcs = new ArrayList<DataCenterVO>();
+
+                isoResponses.addAll(createS3IsoResponses(iso));
+                if (!isoResponses.isEmpty()) {
+                    return isoResponses;
+                }
+
+                final List<DataCenterVO> dcs = new ArrayList<DataCenterVO>();
                 dcs.addAll(ApiDBUtils.listZones());
                 for (DataCenterVO dc : dcs) {
                     isoResponses.addAll(createIsoResponses(iso, dc.getId(), readyOnly));
@@ -2207,6 +2237,65 @@ public class ApiResponseHelper implements ResponseGenerator {
         }
     }
 
+    private List<? extends TemplateResponse> createS3IsoResponses(final VirtualMachineTemplate iso) {
+
+        final VMTemplateS3VO s3Iso = ApiDBUtils.findTemplateS3Ref(iso.getId());
+
+        if (s3Iso == null) {
+            return emptyList();
+        }
+
+        final TemplateResponse templateResponse = new TemplateResponse();
+
+        templateResponse.setId(iso.getId());
+        templateResponse.setName(iso.getName());
+        templateResponse.setDisplayText(iso.getDisplayText());
+        templateResponse.setPublic(iso.isPublicTemplate());
+        templateResponse.setExtractable(iso.isExtractable());
+        templateResponse.setCreated(s3Iso.getCreated());
+        templateResponse.setReady(true);
+        templateResponse.setBootable(iso.isBootable());
+        templateResponse.setFeatured(iso.isFeatured());
+        templateResponse.setCrossZones(iso.isCrossZones());
+        templateResponse.setChecksum(iso.getChecksum());
+        templateResponse.setDetails(iso.getDetails());
+
+        final GuestOS os = ApiDBUtils.findGuestOSById(iso.getGuestOSId());
+
+        if (os != null) {
+            templateResponse.setOsTypeId(os.getId());
+            templateResponse.setOsTypeName(os.getDisplayName());
+        } else {
+            templateResponse.setOsTypeId(-1L);
+            templateResponse.setOsTypeName("");
+        }
+
+        final Account account = ApiDBUtils.findAccountByIdIncludingRemoved(iso.getAccountId());
+        populateAccount(templateResponse, account.getId());
+        populateDomain(templateResponse, account.getDomainId());
+
+        boolean isAdmin = false;
+        if ((account == null) || BaseCmd.isAdmin(account.getType())) {
+            isAdmin = true;
+        }
+
+        // If the user is an admin, add the template download status
+        if (isAdmin || account.getId() == iso.getAccountId()) {
+            // add download status
+            templateResponse.setStatus("Successfully Installed");
+        }
+
+        final Long isoSize = s3Iso.getSize();
+        if (isoSize > 0) {
+            templateResponse.setSize(isoSize);
+        }
+
+        templateResponse.setObjectName("iso");
+
+        return singletonList(templateResponse);
+
+    }
+
     private List<TemplateResponse> createSwiftIsoResponses(VirtualMachineTemplate iso) {
         long isoId = iso.getId();
         List<TemplateResponse> isoResponses = new ArrayList<TemplateResponse>();

server/src/com/cloud/api/ApiServer.java

@@ -108,6 +108,7 @@ import com.cloud.user.UserVO;
 import com.cloud.utils.IdentityProxy;
 import com.cloud.utils.Pair;
 import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.component.ComponentLocator;
 import com.cloud.utils.component.PluggableService;
 import com.cloud.utils.concurrency.NamedThreadFactory;
@@ -313,7 +314,7 @@ public class ApiServer implements HttpRequestHandler {
             InetAddress remoteAddr = ((SocketHttpServerConnection) connObj).getRemoteAddress();
             sb.append(remoteAddr.toString() + " -- ");
         }
-        sb.append(request.getRequestLine());
+        sb.append(StringUtils.cleanString(request.getRequestLine().toString()));
 
         try {
             String uri = request.getRequestLine().getUri();
@@ -589,20 +590,8 @@ public class ApiServer implements HttpRequestHandler {
         if (command.equals("createSSHKeyPair")){
             auditTrailSb.append("This result was not logged because it contains sensitive data.");
         } else {
-            auditTrailSb.append(result);
+            auditTrailSb.append(StringUtils.cleanString(result));
         }
-        /*
-         * if (command.equals("queryAsyncJobResult")){ //For this command we need to also log job status and job
-         * resultcode for
-         * (Pair<String,Object> pair : resultValues){ String key = pair.first(); if (key.equals("jobstatus")){
-         * auditTrailSb.append(" "); auditTrailSb.append(key); auditTrailSb.append("=");
-         * auditTrailSb.append(pair.second());
-         * }else if (key.equals("jobresultcode")){ auditTrailSb.append(" "); auditTrailSb.append(key);
-         * auditTrailSb.append("=");
-         * auditTrailSb.append(pair.second()); } } }else { for (Pair<String,Object> pair : resultValues){ if
-         * (pair.first().equals("jobid")){ // Its an async job so report the jobid auditTrailSb.append(" ");
-         * auditTrailSb.append(pair.first()); auditTrailSb.append("="); auditTrailSb.append(pair.second()); } } }
-         */
     }
     
     private static boolean isCommandAvailable(String commandName) {

server/src/com/cloud/api/ApiServlet.java

@@ -36,6 +36,7 @@ import com.cloud.server.ManagementServer;
 import com.cloud.user.Account;
 import com.cloud.user.AccountService;
 import com.cloud.user.UserContext;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.component.ComponentLocator;
 import com.cloud.utils.exception.CloudRuntimeException;
 
@@ -126,7 +127,7 @@ public class ApiServlet extends HttpServlet {
         String reqStr = "";
         if (s_logger.isDebugEnabled()) {
             reqStr = auditTrailSb.toString() + " " + req.getQueryString();
-            s_logger.debug("===START=== " + reqStr);
+            s_logger.debug("===START=== " + StringUtils.cleanString(reqStr));
         }
         
         try {
@@ -343,7 +344,7 @@ public class ApiServlet extends HttpServlet {
         } finally {
             s_accessLogger.info(auditTrailSb.toString());
             if (s_logger.isDebugEnabled()) {
-                s_logger.debug("===END=== " + reqStr);
+                s_logger.debug("===END=== " + StringUtils.cleanString(reqStr));
             }
             // cleanup user context to prevent from being peeked in other request context
             UserContext.unregisterContext();

server/src/com/cloud/api/commands/AddTrafficMonitorCmd.java

@@ -51,10 +51,24 @@ public class AddTrafficMonitorCmd extends BaseCmd {
 	@Parameter(name=ApiConstants.URL, type=CommandType.STRING, required = true, description="URL of the traffic monitor Host")
 	private String url;	 
 
+	@Parameter(name=ApiConstants.INCL_ZONES, type=CommandType.STRING, description="Traffic going into the listed zones will be metered")
+	private String inclZones;	 
+	
+	@Parameter(name=ApiConstants.EXCL_ZONES, type=CommandType.STRING, description="Traffic going into the listed zones will not be metered")
+	private String exclZones;	 
+	
 	///////////////////////////////////////////////////
 	/////////////////// Accessors ///////////////////////
 	/////////////////////////////////////////////////////
 	 
+	public String getInclZones() {
+		return inclZones;
+	}
+	
+	public String getExclZones() {
+		return exclZones;
+	}
+
 	public Long getZoneId() {
 	    return zoneId;
 	}

server/src/com/cloud/api/doc/ApiXmlDocWriter.java

@@ -314,6 +314,11 @@ public class ApiXmlDocWriter {
             impl = clas.getSuperclass().getAnnotation(Implementation.class);
         }
 
+        if (impl == null) {
+            throw new IllegalStateException(String.format("An %1$s annotation is required for class %2$s.", 
+                    Implementation.class.getCanonicalName(), clas.getCanonicalName()));
+        }
+
         if (impl.includeInApiDoc()) {
             String commandDescription = impl.description();
             if (commandDescription != null && !commandDescription.isEmpty()) {

server/src/com/cloud/configuration/Config.java

@@ -140,7 +140,7 @@ public enum Config {
     JobExpireMinutes("Advanced", ManagementServer.class, String.class, "job.expire.minutes", "1440", "Time (in minutes) for async-jobs to be kept in system", null),
     JobCancelThresholdMinutes("Advanced", ManagementServer.class, String.class, "job.cancel.threshold.minutes", "60", "Time (in minutes) for async-jobs to be forcely cancelled if it has been in process for long", null),
     SwiftEnable("Advanced", ManagementServer.class, Boolean.class, "swift.enable", "false", "enable swift ", null),
-	
+    S3Enable("Advanced", ManagementServer.class, Boolean.class, "s3.enable", "false", "enable s3 ", null),
     EventPurgeInterval("Advanced", ManagementServer.class, Integer.class, "event.purge.interval", "86400", "The interval (in seconds) to wait before running the event purge thread", null),
 	AccountCleanupInterval("Advanced", ManagementServer.class, Integer.class, "account.cleanup.interval", "86400", "The interval (in seconds) between cleanup for removed accounts", null),
 	AllowPublicUserTemplates("Advanced", ManagementServer.class, Integer.class, "allow.public.user.templates", "true", "If false, users will not be able to create public templates.", null),
@@ -274,6 +274,8 @@ public enum Config {
     DirectNetworkStatsInterval("Usage", ManagementServer.class, Integer.class, "direct.network.stats.interval", "86400", "Interval (in seconds) to collect stats from Traffic Monitor", null),
     UsageSanityCheckInterval("Usage", ManagementServer.class, Integer.class, "usage.sanity.check.interval", null, "Interval (in days) to check sanity of usage data", null),
     UsageAggregationTimezone("Usage", ManagementServer.class, String.class, "usage.aggregation.timezone", "GMT", "The timezone to use for usage stats aggregation", null),
+    TrafficSentinelIncludeZones("Usage", ManagementServer.class, Integer.class, "traffic.sentinel.include.zones", "EXTERNAL", "Traffic going into specified list of zones is metered. For metering all traffic leave this parameter empty", null),
+    TrafficSentinelExcludeZones("Usage", ManagementServer.class, Integer.class, "traffic.sentinel.exclude.zones", "", "Traffic going into specified list of zones is not metered.", null),
     
 	// Hidden
 	UseSecondaryStorageVm("Hidden", ManagementServer.class, Boolean.class, "secondary.storage.vm", "false", "Deploys a VM per zone to manage secondary storage if true, otherwise secondary storage is mounted on management server", null),

server/src/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -63,7 +63,9 @@ import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.storage.DiskOfferingVO;
 import com.cloud.storage.SwiftVO;
 import com.cloud.storage.dao.DiskOfferingDao;
+import com.cloud.storage.dao.S3Dao;
 import com.cloud.storage.dao.SwiftDao;
+import com.cloud.storage.s3.S3Manager;
 import com.cloud.storage.secondary.SecondaryStorageVmManager;
 import com.cloud.storage.swift.SwiftManager;
 import com.cloud.test.IPRangeConfig;
@@ -118,6 +120,8 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura
     @Inject
     SwiftDao _swiftDao;
     @Inject
+    S3Dao _s3Dao;
+    @Inject
     ServiceOfferingDao _serviceOfferingDao;
     @Inject
     DiskOfferingDao _diskOfferingDao;
@@ -158,6 +162,8 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura
     @Inject
     SwiftManager _swiftMgr;
     @Inject
+    S3Manager _s3Mgr;
+    @Inject
     PhysicalNetworkTrafficTypeDao _trafficTypeDao;
     @Inject
     NicDao _nicDao;
@@ -419,6 +425,14 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura
                 if (swift != null) {
                     return " can not change " + Config.SwiftEnable.key() + " after you have added Swift";
                 }
+                if (this._s3Mgr.isS3Enabled()) {
+                    return String.format("Swift is not supported when S3 is enabled.");
+                }
+            }
+            if (Config.S3Enable.key().equals(name)) {
+                if (this._swiftMgr.isSwiftEnabled()) {
+                    return String.format("S3-backed Secondary Storage is not supported when Swift is enabled.");
+                }
             }
             return null;
         }
@@ -1520,6 +1534,7 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura
             createDefaultSystemNetworks(zone.getId());
 
             _swiftMgr.propagateSwiftTmplteOnZone(zone.getId());
+            _s3Mgr.propagateTemplatesToZone(zone);
             txn.commit();
             return zone;
         } catch (Exception ex) {

server/src/com/cloud/configuration/DefaultComponentLibrary.java

@@ -147,6 +147,7 @@ import com.cloud.storage.dao.DiskOfferingDaoImpl;
 import com.cloud.storage.dao.GuestOSCategoryDaoImpl;
 import com.cloud.storage.dao.GuestOSDaoImpl;
 import com.cloud.storage.dao.LaunchPermissionDaoImpl;
+import com.cloud.storage.dao.S3DaoImpl;
 import com.cloud.storage.dao.SnapshotDaoImpl;
 import com.cloud.storage.dao.SnapshotPolicyDaoImpl;
 import com.cloud.storage.dao.SnapshotScheduleDaoImpl;
@@ -159,11 +160,13 @@ import com.cloud.storage.dao.VMTemplateDaoImpl;
 import com.cloud.storage.dao.VMTemplateDetailsDaoImpl;
 import com.cloud.storage.dao.VMTemplateHostDaoImpl;
 import com.cloud.storage.dao.VMTemplatePoolDaoImpl;
+import com.cloud.storage.dao.VMTemplateS3DaoImpl;
 import com.cloud.storage.dao.VMTemplateSwiftDaoImpl;
 import com.cloud.storage.dao.VMTemplateZoneDaoImpl;
 import com.cloud.storage.dao.VolumeDaoImpl;
 import com.cloud.storage.dao.VolumeHostDaoImpl;
 import com.cloud.storage.download.DownloadMonitorImpl;
+import com.cloud.storage.s3.S3ManagerImpl;
 import com.cloud.storage.secondary.SecondaryStorageManagerImpl;
 import com.cloud.storage.snapshot.SnapshotManagerImpl;
 import com.cloud.storage.snapshot.SnapshotSchedulerImpl;
@@ -270,6 +273,7 @@ public class DefaultComponentLibrary extends ComponentLibraryBase implements Com
         addDao("VMTemplateHostDao", VMTemplateHostDaoImpl.class);
         addDao("VolumeHostDao", VolumeHostDaoImpl.class);
         addDao("VMTemplateSwiftDao", VMTemplateSwiftDaoImpl.class);
+        addDao("VMTemplateS3Dao", VMTemplateS3DaoImpl.class);
         addDao("UploadDao", UploadDaoImpl.class);
         addDao("VMTemplatePoolDao", VMTemplatePoolDaoImpl.class);
         addDao("LaunchPermissionDao", LaunchPermissionDaoImpl.class);
@@ -318,6 +322,7 @@ public class DefaultComponentLibrary extends ComponentLibraryBase implements Com
         addDao("KeystoreDao", KeystoreDaoImpl.class);
         addDao("DcDetailsDao", DcDetailsDaoImpl.class);
         addDao("SwiftDao", SwiftDaoImpl.class);
+        addDao("S3Dao", S3DaoImpl.class);
         addDao("AgentTransferMapDao", HostTransferMapDaoImpl.class);
         addDao("ProjectDao", ProjectDaoImpl.class);
         addDao("InlineLoadBalancerNicMapDao", InlineLoadBalancerNicMapDaoImpl.class);
@@ -403,6 +408,7 @@ public class DefaultComponentLibrary extends ComponentLibraryBase implements Com
         info.addParameter("consoleproxy.sslEnabled", "true");
         addManager("ProjectManager", ProjectManagerImpl.class);
         addManager("SwiftManager", SwiftManagerImpl.class);
+        addManager("S3Manager", S3ManagerImpl.class);
         addManager("StorageNetworkManager", StorageNetworkManagerImpl.class);
         addManager("ExternalLoadBalancerUsageManager", ExternalLoadBalancerUsageManagerImpl.class);
         addManager("HA Manager", HighAvailabilityManagerImpl.class);

server/src/com/cloud/network/NetworkUsageManagerImpl.java

@@ -108,6 +108,8 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta
 	@Inject ResourceManager _resourceMgr;
     ScheduledExecutorService _executor;
     int _networkStatsInterval;
+    String _TSinclZones;
+    String _TSexclZones;
     protected SearchBuilder<IPAddressVO> AllocatedIpSearch;
 
     @Override
@@ -148,8 +150,8 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta
         hostParams.put("zone", String.valueOf(zoneId));
         hostParams.put("ipaddress", ipAddress);
         hostParams.put("url", cmd.getUrl());
-        //hostParams("numRetries", numRetries);
-        //hostParams("timeout", timeout);
+        hostParams.put("inclZones", (cmd.getInclZones() != null) ? cmd.getInclZones() : _TSinclZones);
+        hostParams.put("exclZones", (cmd.getExclZones() != null) ? cmd.getExclZones() : _TSexclZones);
         hostParams.put("guid", guid);
         hostParams.put("name", guid);
 
@@ -162,6 +164,13 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta
         Map<String, String> hostDetails = new HashMap<String, String>();
         hostDetails.put("url", cmd.getUrl());
         hostDetails.put("last_collection", ""+System.currentTimeMillis());
+        if(cmd.getInclZones() != null){
+        	hostDetails.put("inclZones", cmd.getInclZones());
+        }
+        if(cmd.getExclZones() != null){
+        	hostDetails.put("exclZones", cmd.getExclZones());
+        }
+        
         
         Host trafficMonitor = _resourceMgr.addHost(zoneId, resource, Host.Type.TrafficMonitor, hostDetails);
         return trafficMonitor;
@@ -222,6 +231,8 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta
         AllocatedIpSearch.done();
         
         _networkStatsInterval = NumbersUtil.parseInt(_configDao.getValue(Config.DirectNetworkStatsInterval.key()), 86400);
+        _TSinclZones = _configDao.getValue(Config.TrafficSentinelIncludeZones.key());
+        _TSexclZones = _configDao.getValue(Config.TrafficSentinelExcludeZones.key());
         _agentMgr.registerForHostEvents(new DirectNetworkStatsListener( _networkStatsInterval), true, false, false);
         _resourceMgr.registerResourceStateAdapter(this.getClass().getSimpleName(), this);
         return true;
@@ -372,7 +383,7 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta
             
             //Get usage for Ips which were assigned for the entire duration
             if(fullDurationIpUsage.size() > 0){
-                DirectNetworkUsageCommand cmd = new DirectNetworkUsageCommand(IpList, lastCollection, now);
+                DirectNetworkUsageCommand cmd = new DirectNetworkUsageCommand(IpList, lastCollection, now, _TSinclZones, _TSexclZones);
                 DirectNetworkUsageAnswer answer = (DirectNetworkUsageAnswer) _agentMgr.easySend(host.getId(), cmd);
                 if (answer == null || !answer.getResult()) {
                     String details = (answer != null) ? answer.getDetails() : "details unavailable";
@@ -405,7 +416,7 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta
             for(UsageIPAddressVO usageIp : IpPartialUsage){
                 IpList = new ArrayList<String>() ;
                 IpList.add(usageIp.getAddress());
-                DirectNetworkUsageCommand cmd = new DirectNetworkUsageCommand(IpList, usageIp.getAssigned(), usageIp.getReleased());
+                DirectNetworkUsageCommand cmd = new DirectNetworkUsageCommand(IpList, usageIp.getAssigned(), usageIp.getReleased(), _TSinclZones, _TSexclZones);
                 DirectNetworkUsageAnswer answer = (DirectNetworkUsageAnswer) _agentMgr.easySend(host.getId(), cmd);
                 if (answer == null || !answer.getResult()) {
                     String details = (answer != null) ? answer.getDetails() : "details unavailable";
@@ -532,8 +543,11 @@ public class NetworkUsageManagerImpl implements NetworkUsageManager, ResourceSta
 
 	@Override
     public DeleteHostAnswer deleteHost(HostVO host, boolean isForced, boolean isForceDeleteStorage) throws UnableDeleteHostException {
-	    // TODO Auto-generated method stub
+		if(host.getType() != Host.Type.TrafficMonitor){
 			return null;
 		}
 		
+		return new DeleteHostAnswer(true);
+    }
+
 }

server/src/com/cloud/network/guru/GuestNetworkGuru.java

@@ -442,15 +442,17 @@ public abstract class GuestNetworkGuru extends AdapterBase implements NetworkGur
 
     @Override
     public void shutdown(NetworkProfile profile, NetworkOffering offering) {
+        
+        if (profile.getBroadcastDomainType() == BroadcastDomainType.Vlan && 
+        		profile.getBroadcastUri() != null && !offering.getSpecifyVlan()) {
         	s_logger.debug("Releasing vnet for the network id=" + profile.getId());
-        if (profile.getBroadcastUri() != null && !offering.getSpecifyVlan()) {
             _dcDao.releaseVnet(profile.getBroadcastUri().getHost(), profile.getDataCenterId(), 
                     profile.getPhysicalNetworkId(), profile.getAccountId(), profile.getReservationId());
             EventUtils.saveEvent(UserContext.current().getCallerUserId(), profile.getAccountId(), 
                     EventVO.LEVEL_INFO, EventTypes.EVENT_ZONE_VLAN_RELEASE, "Released Zone Vlan: "
                     +profile.getBroadcastUri().getHost()+" for Network: "+profile.getId(), 0);
-            profile.setBroadcastUri(null);
         }
+        profile.setBroadcastUri(null);
     }
 
     @Override

server/src/com/cloud/resource/ResourceManagerImpl.java

@@ -30,6 +30,11 @@ import java.util.Set;
 import javax.ejb.Local;
 import javax.naming.ConfigurationException;
 
+import com.cloud.api.commands.AddS3Cmd;
+import com.cloud.api.commands.ListS3sCmd;
+import com.cloud.storage.S3;
+import com.cloud.storage.S3VO;
+import com.cloud.storage.s3.S3Manager;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.AgentManager;
@@ -181,6 +186,8 @@ public class ResourceManagerImpl implements ResourceManager, ResourceService, Ma
     @Inject
     protected SwiftManager _swiftMgr;
     @Inject
+    protected S3Manager                      _s3Mgr;
+    @Inject
     protected HostDetailsDao                 _hostDetailsDao;
     @Inject
     protected ConfigurationDao _configDao;
@@ -561,6 +568,16 @@ public class ResourceManagerImpl implements ResourceManager, ResourceService, Ma
         return _swiftMgr.listSwifts(cmd);
     }
 
+    @Override
+    public S3 discoverS3(final AddS3Cmd cmd) throws DiscoveryException {
+        return this._s3Mgr.addS3(cmd);
+    }
+
+    @Override
+    public List<S3VO> listS3s(final ListS3sCmd cmd) {
+        return this._s3Mgr.listS3s(cmd);
+    }
+
     private List<HostVO> discoverHostsFull(Long dcId, Long podId, Long clusterId, String clusterName, String url, String username, String password, String hypervisorType, List<String> hostTags,
             Map<String, String> params) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException {
         URI uri = null;

server/src/com/cloud/storage/StorageManagerImpl.java

@@ -151,11 +151,13 @@ import com.cloud.storage.dao.StoragePoolWorkDao;
 import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.storage.dao.VMTemplateHostDao;
 import com.cloud.storage.dao.VMTemplatePoolDao;
+import com.cloud.storage.dao.VMTemplateS3Dao;
 import com.cloud.storage.dao.VMTemplateSwiftDao;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.storage.dao.VolumeHostDao;
 import com.cloud.storage.download.DownloadMonitor;
 import com.cloud.storage.listener.StoragePoolMonitor;
+import com.cloud.storage.s3.S3Manager;
 import com.cloud.storage.secondary.SecondaryStorageVmManager;
 import com.cloud.storage.snapshot.SnapshotManager;
 import com.cloud.storage.snapshot.SnapshotScheduler;
@@ -262,6 +264,10 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
     @Inject
     protected VMTemplateSwiftDao _vmTemplateSwiftDao = null;
     @Inject
+    protected VMTemplateS3Dao _vmTemplateS3Dao;
+    @Inject
+    protected S3Manager _s3Mgr;
+    @Inject
     protected VMTemplateDao _vmTemplateDao = null;
     @Inject
     protected StoragePoolHostDao _poolHostDao = null;
@@ -663,7 +669,6 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
         String vdiUUID = null;
         Long snapshotId = snapshot.getId();
         Long volumeId = snapshot.getVolumeId();
-        String primaryStoragePoolNameLabel = pool.getUuid(); // pool's uuid is actually the namelabel.
         Long dcId = snapshot.getDataCenterId();
         String secondaryStoragePoolUrl = _snapMgr.getSecondaryStorageURL(snapshot);
         long accountId = snapshot.getAccountId();
@@ -707,8 +712,10 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
         try {
             if (snapshot.getSwiftId() != null && snapshot.getSwiftId() != 0) {
                 _snapshotMgr.downloadSnapshotsFromSwift(snapshot);
+            } else if (snapshot.getS3Id() != null && snapshot.getS3Id() != 0) {
+                _snapshotMgr.downloadSnapshotsFromS3(snapshot);
             }
-            CreateVolumeFromSnapshotCommand createVolumeFromSnapshotCommand = new CreateVolumeFromSnapshotCommand(primaryStoragePoolNameLabel, secondaryStoragePoolUrl, dcId, accountId, volumeId,
+            CreateVolumeFromSnapshotCommand createVolumeFromSnapshotCommand = new CreateVolumeFromSnapshotCommand(pool, secondaryStoragePoolUrl, dcId, accountId, volumeId,
                     backedUpSnapshotUuid, snapshot.getName(), _createVolumeFromSnapshotWait);
             CreateVolumeFromSnapshotAnswer answer;
             if (!_snapshotDao.lockInLockTable(snapshotId.toString(), 10)) {
@@ -1281,7 +1288,7 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
         }
         URI uri = null;
         try {
-            uri = new URI(cmd.getUrl());
+            uri = new URI(UriUtils.encodeURIComponent(cmd.getUrl()));
             if (uri.getScheme() == null) {
                 throw new InvalidParameterValueException("scheme is null " + cmd.getUrl() + ", add nfs:// as a prefix");
             } else if (uri.getScheme().equalsIgnoreCase("nfs")) {
@@ -2983,6 +2990,14 @@ public class StorageManagerImpl implements StorageManager, Manager, ClusterManag
         if (tsvs != null && tsvs.size() > 0) {
             size = tsvs.get(0).getSize();
         }
+
+        if (size == null && _s3Mgr.isS3Enabled()) {
+            VMTemplateS3VO vmTemplateS3VO = _vmTemplateS3Dao.findOneByTemplateId(template.getId());
+            if (vmTemplateS3VO != null) {
+                size = vmTemplateS3VO.getSize();
+            }
+        }
+
         if (size == null) {
             List<VMTemplateHostVO> sss = _vmTemplateHostDao.search(sc, null);
             if (sss == null || sss.size() == 0) {

server/src/com/cloud/storage/dao/S3Dao.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage.dao;
+
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.storage.S3VO;
+import com.cloud.utils.db.GenericDao;
+
+public interface S3Dao extends GenericDao<S3VO, Long> {
+
+    S3TO getS3TO(final Long id);
+
+}

server/src/com/cloud/storage/dao/S3DaoImpl.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage.dao;
+
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.storage.S3VO;
+import com.cloud.utils.db.GenericDaoBase;
+
+import javax.ejb.Local;
+
+@Local(S3Dao.class)
+public class S3DaoImpl extends GenericDaoBase<S3VO, Long> implements S3Dao {
+
+    @Override
+    public S3TO getS3TO(final Long id) {
+
+        if (id != null) {
+
+            final S3VO s3VO = findById(id);
+            if (s3VO != null) {
+                return s3VO.toS3TO();
+            }
+
+        }
+
+        // NOTE: Excluded listAll / shuffle operation implemented in SwiftDaoImpl ...
+
+        return null;
+
+    }
+}

server/src/com/cloud/storage/dao/VMTemplateDao.java

@@ -71,4 +71,6 @@ public interface VMTemplateDao extends GenericDao<VMTemplateVO, Long> {
     List<Long> listPrivateTemplatesByHost(Long hostId);
     public Long countTemplatesForAccount(long accountId);
 	
+    List<VMTemplateVO> findTemplatesToSyncToS3();
+
 }

server/src/com/cloud/storage/dao/VMTemplateDaoImpl.java

@@ -92,6 +92,14 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
     
     private final String SELECT_TEMPLATE_SWIFT_REF = "SELECT t.id, t.unique_name, t.name, t.public, t.featured, t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, "
             + "t.checksum, t.display_text, t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type FROM vm_template t";
+
+    private static final String SELECT_S3_CANDIDATE_TEMPLATES = "SELECT t.id, t.unique_name, t.name, t.public, t.featured, " +
+        "t.type, t.hvm, t.bits, t.url, t.format, t.created, t.account_id, t.checksum, t.display_text, " +
+        "t.enable_password, t.guest_os_id, t.bootable, t.prepopulate, t.cross_zones, t.hypervisor_type " +
+        "FROM vm_template t JOIN template_host_ref r ON t.id=r.template_id JOIN host h ON h.id=r.host_id " +
+        "WHERE t.hypervisor_type IN (SELECT hypervisor_type FROM host) AND r.download_state = 'DOWNLOADED' AND " +
+        "r.template_id NOT IN (SELECT template_id FROM template_s3_ref) AND r.destroyed = 0 AND t.type <> 'PERHOST'";
+
     protected SearchBuilder<VMTemplateVO> TemplateNameSearch;
     protected SearchBuilder<VMTemplateVO> UniqueNameSearch;
     protected SearchBuilder<VMTemplateVO> tmpltTypeSearch;
@@ -918,4 +926,9 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
 	            (accountType == Account.ACCOUNT_TYPE_READ_ONLY_ADMIN));
 	}
 
+    @Override
+    public List<VMTemplateVO> findTemplatesToSyncToS3() {
+        return executeList(SELECT_S3_CANDIDATE_TEMPLATES, new Object[] {});
+    }
+
 }

server/src/com/cloud/storage/dao/VMTemplateS3Dao.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage.dao;
+
+import com.cloud.storage.VMTemplateS3VO;
+import com.cloud.utils.db.GenericDao;
+
+import java.util.List;
+
+public interface VMTemplateS3Dao extends GenericDao<VMTemplateS3VO, Long> {
+
+    List<VMTemplateS3VO> listByS3Id(long id);
+
+    VMTemplateS3VO findOneByTemplateId(long id);
+
+    VMTemplateS3VO findOneByS3Template(long s3Id, long templateId);
+
+    void expungeAllByTemplateId(long templateId);
+
+}

server/src/com/cloud/storage/dao/VMTemplateS3DaoImpl.java

@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage.dao;
+
+import static com.cloud.utils.db.SearchCriteria.Op.*;
+import static com.cloud.storage.VMTemplateS3VO.*;
+
+import com.cloud.storage.VMTemplateS3VO;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+
+import javax.ejb.Local;
+import java.util.List;
+
+@Local(VMTemplateS3Dao.class)
+public class VMTemplateS3DaoImpl extends GenericDaoBase<VMTemplateS3VO, Long>
+        implements VMTemplateS3Dao {
+
+    private final SearchBuilder<VMTemplateS3VO> searchBuilder;
+
+    public VMTemplateS3DaoImpl() {
+
+        super();
+
+        this.searchBuilder = createSearchBuilder();
+        this.searchBuilder
+                .and(S3_ID_COLUMN_NAME, this.searchBuilder.entity().getS3Id(),
+                        EQ)
+                .and(TEMPLATE_ID_COLUMN_NAME,
+                        this.searchBuilder.entity().getTemplateId(), EQ).done();
+
+    }
+
+    @Override
+    public List<VMTemplateS3VO> listByS3Id(final long s3id) {
+
+        final SearchCriteria<VMTemplateS3VO> criteria = this.searchBuilder
+                .create();
+
+        criteria.setParameters(S3_ID_COLUMN_NAME, s3id);
+
+        return this.listBy(criteria);
+
+    }
+
+    @Override
+    public VMTemplateS3VO findOneByTemplateId(final long templateId) {
+
+        final SearchCriteria<VMTemplateS3VO> criteria = this.searchBuilder
+                .create();
+
+        criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId);
+
+        return this.findOneBy(criteria);
+
+    }
+
+    @Override
+    public VMTemplateS3VO findOneByS3Template(final long s3Id,
+            final long templateId) {
+
+        final SearchCriteria<VMTemplateS3VO> criteria = this.searchBuilder
+                .create();
+
+        criteria.setParameters(S3_ID_COLUMN_NAME, s3Id);
+        criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId);
+
+        return this.findOneBy(criteria);
+
+    }
+
+    @Override
+    public void expungeAllByTemplateId(long templateId) {
+
+        final SearchCriteria<VMTemplateS3VO> criteria = this.searchBuilder
+                .create();
+
+        criteria.setParameters(TEMPLATE_ID_COLUMN_NAME, templateId);
+
+        this.expunge(criteria);
+
+    }
+
+}

server/src/com/cloud/storage/s3/S3Manager.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.storage.s3;
+
+import java.util.List;
+
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.api.commands.AddS3Cmd;
+import com.cloud.api.commands.ListS3sCmd;
+import com.cloud.dc.DataCenterVO;
+import com.cloud.exception.DiscoveryException;
+import com.cloud.storage.S3;
+import com.cloud.storage.S3VO;
+import com.cloud.storage.VMTemplateS3VO;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.utils.component.Manager;
+
+public interface S3Manager extends Manager {
+
+    S3TO getS3TO();
+
+    S3TO getS3TO(Long s3Id);
+
+    S3 addS3(AddS3Cmd addS3Cmd) throws DiscoveryException;
+
+    Long chooseZoneForTemplateExtract(VMTemplateVO template);
+
+    boolean isS3Enabled();
+
+    boolean isTemplateInstalled(Long templateId);
+
+    void deleteTemplate(final Long accountId, final Long templateId);
+
+    String downloadTemplateFromS3ToSecondaryStorage(final long dcId,
+            final long templateId, final int primaryStorageDownloadWait);
+
+    List<S3VO> listS3s(ListS3sCmd listS3sCmd);
+
+    VMTemplateS3VO findByTemplateId(Long templateId);
+
+    void propagateTemplatesToZone(DataCenterVO zone);
+
+    void propagateTemplateToAllZones(VMTemplateS3VO vmTemplateS3VO);
+
+    void uploadTemplateToS3FromSecondaryStorage(final VMTemplateVO template);
+
+}