Forward merge branch '4.11' to master

ConfigDrive fixes: CLOUDSTACK-10288, CLOUDSTACK-10289 (#2566)
CLOUDSTACK-9677: Adding storage policy support for swift as secondary
storage (#2412)

api/src/main/java/com/cloud/agent/api/to/SwiftTO.java

@@ -26,17 +26,19 @@ public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg {
 
     String userName;
     String key;
+    String storagePolicy;
     private static final String pathSeparator = "/";
 
     public SwiftTO() {
     }
 
-    public SwiftTO(Long id, String url, String account, String userName, String key) {
+    public SwiftTO(Long id, String url, String account, String userName, String key, String storagePolicy) {
         this.id = id;
         this.url = url;
         this.account = account;
         this.userName = userName;
         this.key = key;
+        this.storagePolicy = storagePolicy;
     }
 
     public Long getId() {
@@ -63,6 +65,11 @@ public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg {
         return key;
     }
 
+    @Override
+    public String getStoragePolicy() {
+        return this.storagePolicy;
+    }
+
     @Override
     public DataStoreRole getRole() {
         return DataStoreRole.Image;

api/src/main/java/com/cloud/network/NetworkModel.java

@@ -59,6 +59,9 @@ public interface NetworkModel {
     String SERVICE_OFFERING_FILE = "service-offering";
     String AVAILABILITY_ZONE_FILE = "availability-zone";
     String LOCAL_HOSTNAME_FILE = "local-hostname";
+    String LOCAL_IPV4_FILE = "local-ipv4";
+    String PUBLIC_HOSTNAME_FILE = "public-hostname";
+    String PUBLIC_IPV4_FILE = "public-ipv4";
     String INSTANCE_ID_FILE = "instance-id";
     String VM_ID_FILE = "vm-id";
     String PUBLIC_KEYS_FILE = "public-keys";
@@ -309,8 +312,8 @@ public interface NetworkModel {
 
     boolean getNetworkEgressDefaultPolicy(Long networkId);
 
-    List<String[]> generateVmData(String userData, String serviceOffering, String zoneName,
+    List<String[]> generateVmData(String userData, String serviceOffering, long datacenterId,
-                                  String vmName, long vmId, String publicKey, String password, Boolean isWindows);
+                                  String vmName, long vmId, String vmUuid, String guestIpAddress, String publicKey, String password, Boolean isWindows);
 
     String getValidNetworkCidr(Network guestNetwork);
 

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -289,6 +289,7 @@ public class ApiConstants {
     public static final String STATE = "state";
     public static final String STATUS = "status";
     public static final String STORAGE_TYPE = "storagetype";
+    public static final String STORAGE_POLICY = "storagepolicy";
     public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
     public static final String STORAGE_CAPABILITIES = "storagecapabilities";
     public static final String SYSTEM_VM_TYPE = "systemvmtype";

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -2542,16 +2542,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
             if (defaultNic != null) {
                 UserVmVO userVm = _userVmDao.findById(vm.getId());
                 Map<String, String> details = _vmDetailsDao.listDetailsKeyPairs(vm.getId());
-                vm.setDetails(details);
+                userVm.setDetails(details);
 
                 Network network = _networkModel.getNetwork(defaultNic.getNetworkId());
                 if (_networkModel.isSharedNetworkWithoutServices(network.getId())) {
                     final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText();
-                    final String zoneName = _dcDao.findById(vm.getDataCenterId()).getName();
                     boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
 
-                    vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, zoneName, vm.getInstanceName(), vm.getId(),
+                    vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getId(),
-                            (String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
+                            vm.getUuid(), defaultNic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
                     String vmName = vm.getInstanceName();
                     String configDriveIsoRootFolder = "/tmp";
                     String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso";

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java

@@ -18,23 +18,24 @@
  */
 package com.cloud.hypervisor.xenserver.resource;
 
-import com.cloud.agent.api.Answer;
+import static com.cloud.utils.ReflectUtil.flattenProperties;
-import com.cloud.agent.api.to.DataObjectType;
+import static com.google.common.collect.Lists.newArrayList;
-import com.cloud.agent.api.to.DataStoreTO;
+
-import com.cloud.agent.api.to.DataTO;
+import java.io.File;
-import com.cloud.agent.api.to.DiskTO;
+import java.net.URI;
-import com.cloud.agent.api.to.NfsTO;
+import java.util.ArrayList;
-import com.cloud.agent.api.to.S3TO;
+import java.util.Arrays;
-import com.cloud.agent.api.to.SwiftTO;
+import java.util.HashMap;
-import com.cloud.exception.InternalErrorException;
+import java.util.List;
-import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import java.util.Map;
-import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
+import java.util.Set;
-import com.cloud.storage.DataStoreRole;
+import java.util.UUID;
-import com.cloud.storage.Storage;
+
-import com.cloud.storage.Storage.ImageFormat;
+import org.apache.commons.lang3.BooleanUtils;
-import com.cloud.storage.resource.StorageProcessor;
+import org.apache.log4j.Logger;
-import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.xmlrpc.XmlRpcException;
-import com.cloud.utils.storage.S3.ClientOptions;
+
+import com.google.common.annotations.VisibleForTesting;
 import com.xensource.xenapi.Connection;
 import com.xensource.xenapi.SR;
 import com.xensource.xenapi.Types;
@@ -44,6 +45,7 @@ import com.xensource.xenapi.Types.XenAPIException;
 import com.xensource.xenapi.VBD;
 import com.xensource.xenapi.VDI;
 import com.xensource.xenapi.VM;
+
 import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
 import org.apache.cloudstack.storage.command.AttachAnswer;
 import org.apache.cloudstack.storage.command.AttachCommand;
@@ -65,20 +67,24 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
-import org.apache.log4j.Logger;
-import org.apache.xmlrpc.XmlRpcException;
 
-import java.io.File;
+import com.cloud.agent.api.Answer;
-import java.net.URI;
+import com.cloud.agent.api.to.DataObjectType;
-import java.util.Arrays;
+import com.cloud.agent.api.to.DataStoreTO;
-import java.util.HashMap;
+import com.cloud.agent.api.to.DataTO;
-import java.util.List;
+import com.cloud.agent.api.to.DiskTO;
-import java.util.Map;
+import com.cloud.agent.api.to.NfsTO;
-import java.util.Set;
+import com.cloud.agent.api.to.S3TO;
-import java.util.UUID;
+import com.cloud.agent.api.to.SwiftTO;
-
+import com.cloud.exception.InternalErrorException;
-import static com.cloud.utils.ReflectUtil.flattenProperties;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import static com.google.common.collect.Lists.newArrayList;
+import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Storage;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.resource.StorageProcessor;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.storage.S3.ClientOptions;
 
 public class XenServerStorageProcessor implements StorageProcessor {
     private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class);
@@ -914,20 +920,55 @@ public class XenServerStorageProcessor implements StorageProcessor {
 
     private boolean swiftUpload(final Connection conn, final SwiftTO swift, final String container, final String ldir, final String lfilename, final Boolean isISCSI,
             final int wait) {
-        String result = null;
+
+        List<String> params = getSwiftParams(swift, container, ldir, lfilename, isISCSI);
+
         try {
-            result =
+            String result = hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, params.toArray(new String[params.size()]));
-                    hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, "op", "upload", "url", swift.getUrl(), "account", swift.getAccount(), "username",
+            return BooleanUtils.toBoolean(result);
-                            swift.getUserName(), "key", swift.getKey(), "container", container, "ldir", ldir, "lfilename", lfilename, "isISCSI", isISCSI.toString());
-            if (result != null && result.equals("true")) {
-                return true;
-            }
         } catch (final Exception e) {
             s_logger.warn("swift upload failed due to " + e.toString(), e);
         }
         return false;
     }
 
+    @VisibleForTesting
+    List<String> getSwiftParams(SwiftTO swift, String container, String ldir, String lfilename, Boolean isISCSI) {
+        // ORDER IS IMPORTANT
+        List<String> params = new ArrayList<>();
+
+        //operation
+        params.add("op");
+        params.add("upload");
+
+        //auth
+        params.add("url");
+        params.add(swift.getUrl());
+        params.add("account");
+        params.add(swift.getAccount());
+        params.add("username");
+        params.add(swift.getUserName());
+        params.add("key");
+        params.add(swift.getKey());
+
+        // object info
+        params.add("container");
+        params.add(container);
+        params.add("ldir");
+        params.add(ldir);
+        params.add("lfilename");
+        params.add(lfilename);
+        params.add("isISCSI");
+        params.add(isISCSI.toString());
+
+        if (swift.getStoragePolicy() != null) {
+            params.add("storagepolicy");
+            params.add(swift.getStoragePolicy());
+        }
+
+        return params;
+    }
+
     protected String deleteSnapshotBackup(final Connection conn, final String localMountPoint, final String path, final String secondaryStorageMountPath, final String backupUUID) {
 
         // If anybody modifies the formatting below again, I'll skin them

plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java

@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.hypervisor.xenserver.resource;
+
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.cloud.agent.api.to.SwiftTO;
+
+public class XenServerStorageProcessorTest {
+    @Test
+    public void testOrderOfSwiftUplodScriptParamsWithoutStoragePolicy() {
+        CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
+        XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
+
+        SwiftTO swift = Mockito.mock(SwiftTO.class);
+        when(swift.getStoragePolicy()).thenReturn(null);
+
+        String container = "sample-container-name";
+        String ldir = "sample-ldir";
+        String lfilename = "sample-lfilename";
+        Boolean isISCSI = true;
+
+        List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
+
+        // make sure the params not null and has correct number of items in it
+        Assert.assertNotNull("params is null", params);
+        Assert.assertTrue("Expected param list size is 18 but it was" + params.size(), params.size() == 18);
+
+        // check the order of params
+        Assert.assertEquals("unexpected param.", "op", params.get(0));
+        Assert.assertEquals("unexpected param.", "upload", params.get(1));
+        Assert.assertEquals("unexpected param.", "url", params.get(2));
+        Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
+        Assert.assertEquals("unexpected param.", "account", params.get(4));
+        Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
+        Assert.assertEquals("unexpected param.", "username", params.get(6));
+        Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
+        Assert.assertEquals("unexpected param.", "key", params.get(8));
+        Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
+        Assert.assertEquals("unexpected param.", "container", params.get(10));
+        Assert.assertEquals("unexpected param.", container, params.get(11));
+        Assert.assertEquals("unexpected param.", "ldir", params.get(12));
+        Assert.assertEquals("unexpected param.", ldir, params.get(13));
+        Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
+        Assert.assertEquals("unexpected param.", lfilename, params.get(15));
+        Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
+        Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
+    }
+
+    @Test
+    public void testOrderOfSwiftUplodScriptParamsWithStoragePolicy() {
+        CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
+        XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
+
+        SwiftTO swift = Mockito.mock(SwiftTO.class);
+        when(swift.getStoragePolicy()).thenReturn("sample-storagepolicy");
+
+        String container = "sample-container-name";
+        String ldir = "sample-ldir";
+        String lfilename = "sample-lfilename";
+        Boolean isISCSI = true;
+
+        List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
+
+        // make sure the params not null and has correct number of items in it
+        Assert.assertNotNull("params is null", params);
+        Assert.assertTrue("Expected param list size is 20 but it was" + params.size(), params.size() == 20);
+
+        // check the order of params
+        Assert.assertEquals("unexpected param.", "op", params.get(0));
+        Assert.assertEquals("unexpected param.", "upload", params.get(1));
+        Assert.assertEquals("unexpected param.", "url", params.get(2));
+        Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
+        Assert.assertEquals("unexpected param.", "account", params.get(4));
+        Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
+        Assert.assertEquals("unexpected param.", "username", params.get(6));
+        Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
+        Assert.assertEquals("unexpected param.", "key", params.get(8));
+        Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
+        Assert.assertEquals("unexpected param.", "container", params.get(10));
+        Assert.assertEquals("unexpected param.", container, params.get(11));
+        Assert.assertEquals("unexpected param.", "ldir", params.get(12));
+        Assert.assertEquals("unexpected param.", ldir, params.get(13));
+        Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
+        Assert.assertEquals("unexpected param.", lfilename, params.get(15));
+        Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
+        Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
+        Assert.assertEquals("unexpected param.", "storagepolicy", params.get(18));
+        Assert.assertEquals("unexpected param.", "sample-storagepolicy", params.get(19));
+    }
+}

plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java

@@ -68,7 +68,7 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
     public DataStoreTO getStoreTO(DataStore store) {
         ImageStoreImpl imgStore = (ImageStoreImpl)store;
         Map<String, String> details = _imageStoreDetailsDao.getDetails(imgStore.getId());
-        return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY));
+        return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY), details.get(ApiConstants.STORAGE_POLICY));
     }
 
     @Override

scripts/storage/secondary/swift

@@ -1473,8 +1473,9 @@ post [options] [container] [object]
     Updates meta information for the account, container, or object depending on
     the args given. If the container is not found, it will be created
     automatically; but this is not true for accounts and objects. Containers
-    also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
+    also allow the -r (or --read-acl) and -w (or --write-acl) options.
-    or --meta option is allowed on all and used to define the user meta data
+    The --storage-policy will set a storage policy to the container if the container does not exist.
+    The -m or --meta option is allowed on all and used to define the user meta data
     items to set in the form Name:Value. This option can be repeated. Example:
     post -m Color:Blue -m Size:Large'''.strip('\n')
 
@@ -1493,6 +1494,8 @@ def st_post(options, args, print_queue, error_queue):
     parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
         help='Sets a meta data item with the syntax name:value. This option '
         'may be repeated. Example: -m Color:Blue -m Size:Large')
+    parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+         help='Sets a storage policy to the container if the container does not exist')
     (options, args) = parse_args(parser, args)
     args = args[1:]
     if (options.read_acl or options.write_acl or options.sync_to or
@@ -1529,6 +1532,8 @@ def st_post(options, args, print_queue, error_queue):
             headers['X-Container-Sync-To'] = options.sync_to
         if options.sync_key is not None:
             headers['X-Container-Sync-Key'] = options.sync_key
+        if options.storage_policy is not None:
+             headers['X-Storage-Policy'] = options.storage_policy
         try:
             conn.post_container(args[0], headers=headers)
         except ClientException, err:
@@ -1558,7 +1563,8 @@ upload [options] container file_or_directory [file_or_directory] [...]
     Uploads to the given container the files and directories specified by the
     remaining args. -c or --changed is an option that will only upload files
     that have changed since the last upload. -S <size> or --segment-size <size>
-    and --leave-segments are options as well (see --help for more).
+    and --leave-segments are options as well (see --help for more). --storage-policy
+     Sets a storage policy to the container if the container does not exist.
 '''.strip('\n')
 
 
@@ -1576,6 +1582,8 @@ def st_upload(options, args, print_queue, error_queue):
         dest='leave_segments', default=False, help='Indicates that you want '
         'the older segments of manifest objects left alone (in the case of '
         'overwrites)')
+    parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+         help='Sets a storage policy to the container if the container does not exist')
     (options, args) = parse_args(parser, args)
     args = args[1:]
     if len(args) < 2:
@@ -1749,9 +1757,12 @@ def st_upload(options, args, print_queue, error_queue):
     # permissions, so we'll ignore any error. If there's really a problem,
     # it'll surface on the first object PUT.
     try:
-        conn.put_container(args[0])
+        container_headers = {}
+        if options.storage_policy is not None:
+            container_headers['X-Storage-Policy'] = options.storage_policy
+        conn.put_container(args[0],headers=container_headers)
         if options.segment_size is not None:
-            conn.put_container(args[0] + '_segments')
+            conn.put_container(args[0] + '_segments',headers=container_headers)
     except Exception:
         pass
     try:

scripts/vm/hypervisor/xenserver/swift

@@ -1475,8 +1475,9 @@ post [options] [container] [object]
     Updates meta information for the account, container, or object depending on
     the args given. If the container is not found, it will be created
     automatically; but this is not true for accounts and objects. Containers
-    also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
+    also allow the -r (or --read-acl) and -w (or --write-acl) options.
-    or --meta option is allowed on all and used to define the user meta data
+    The --storage-policy will set a storage policy to the container if the container does not exist.
+    The -m or --meta option is allowed on all and used to define the user meta data
     items to set in the form Name:Value. This option can be repeated. Example:
     post -m Color:Blue -m Size:Large'''.strip('\n')
 
@@ -1495,6 +1496,8 @@ def st_post(options, args, print_queue, error_queue):
     parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
         help='Sets a meta data item with the syntax name:value. This option '
         'may be repeated. Example: -m Color:Blue -m Size:Large')
+    parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+         help='Sets a storage policy to the container if the container does not exist')
     (options, args) = parse_args(parser, args)
     args = args[1:]
     if (options.read_acl or options.write_acl or options.sync_to or
@@ -1531,6 +1534,8 @@ def st_post(options, args, print_queue, error_queue):
             headers['X-Container-Sync-To'] = options.sync_to
         if options.sync_key is not None:
             headers['X-Container-Sync-Key'] = options.sync_key
+        if options.storage_policy is not None:
+            headers['X-Storage-Policy'] = options.storage_policy
         try:
             conn.post_container(args[0], headers=headers)
         except ClientException, err:
@@ -1560,7 +1565,8 @@ upload [options] container file_or_directory [file_or_directory] [...]
     Uploads to the given container the files and directories specified by the
     remaining args. -c or --changed is an option that will only upload files
     that have changed since the last upload. -S <size> or --segment-size <size>
-    and --leave-segments are options as well (see --help for more).
+    and --leave-segments are options as well (see --help for more). --storage-policy
+     Sets a storage policy to the container if the container does not exist.
 '''.strip('\n')
 
 
@@ -1578,6 +1584,8 @@ def st_upload(options, args, print_queue, error_queue):
         dest='leave_segments', default=False, help='Indicates that you want '
         'the older segments of manifest objects left alone (in the case of '
         'overwrites)')
+    parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
+        help='Sets a storage policy to the container if the container does not exist')
     (options, args) = parse_args(parser, args)
     args = args[1:]
     if len(args) < 2:
@@ -1751,9 +1759,12 @@ def st_upload(options, args, print_queue, error_queue):
     # permissions, so we'll ignore any error. If there's really a problem,
     # it'll surface on the first object PUT.
     try:
-        conn.put_container(args[0])
+        container_headers = {}
+        if options.storage_policy is not None:
+            container_headers['X-Storage-Policy'] = options.storage_policy
+        conn.put_container(args[0],headers=container_headers)
         if options.segment_size is not None:
-            conn.put_container(args[0] + '_segments')
+            conn.put_container(args[0] + '_segments',headers=container_headers)
     except Exception:
         pass
     try:

server/src/main/java/com/cloud/network/NetworkModelImpl.java

@@ -22,6 +22,7 @@ import java.security.InvalidParameterException;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -29,24 +30,24 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
-import java.util.Collections;
+
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import org.apache.commons.codec.binary.Base64;
 import org.apache.log4j.Logger;
 
+import org.apache.cloudstack.acl.ControlledEntity.ACLType;
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
-
-import org.apache.cloudstack.acl.ControlledEntity.ACLType;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao;
+
 import com.cloud.api.ApiDBUtils;
 import com.cloud.configuration.Config;
 import com.cloud.configuration.ConfigurationManager;
 import com.cloud.dc.DataCenter;
+import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.PodVlanMapVO;
 import com.cloud.dc.Vlan;
 import com.cloud.dc.Vlan.VlanType;
@@ -2344,18 +2345,47 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi
     }
 
     @Override
-    public List<String[]> generateVmData(String userData, String serviceOffering, String zoneName,
+    public List<String[]> generateVmData(String userData, String serviceOffering, long datacenterId,
-                                         String vmName, long vmId, String publicKey, String password, Boolean isWindows) {
+                                         String vmName, long vmId, String vmUuid,
+                                         String guestIpAddress, String publicKey, String password, Boolean isWindows) {
+
+        DataCenterVO dcVo = _dcDao.findById(datacenterId);
+        final String zoneName = dcVo.getName();
+
+        IPAddressVO publicIp = _ipAddressDao.findByAssociatedVmId(vmId);
+
         final List<String[]> vmData = new ArrayList<String[]>();
 
         if (userData != null) {
-            vmData.add(new String[]{USERDATA_DIR, USERDATA_FILE, new String(Base64.decodeBase64(userData),StringUtils.getPreferredCharset())});
+            vmData.add(new String[]{USERDATA_DIR, USERDATA_FILE, userData});
         }
         vmData.add(new String[]{METATDATA_DIR, SERVICE_OFFERING_FILE, StringUtils.unicodeEscape(serviceOffering)});
         vmData.add(new String[]{METATDATA_DIR, AVAILABILITY_ZONE_FILE, StringUtils.unicodeEscape(zoneName)});
         vmData.add(new String[]{METATDATA_DIR, LOCAL_HOSTNAME_FILE, StringUtils.unicodeEscape(vmName)});
+        vmData.add(new String[]{METATDATA_DIR, LOCAL_IPV4_FILE, guestIpAddress});
+
+        String publicIpAddress = guestIpAddress;
+        String publicHostName = StringUtils.unicodeEscape(vmName);
+
+        if (dcVo.getNetworkType() != DataCenter.NetworkType.Basic) {
+            if (publicIp != null) {
+                publicIpAddress = publicIp.getAddress().addr();
+                publicHostName = publicIp.getAddress().addr();
+            } else {
+                publicHostName = null;
+            }
+        }
+        vmData.add(new String[]{METATDATA_DIR, PUBLIC_IPV4_FILE, publicIpAddress});
+        vmData.add(new String[]{METATDATA_DIR, PUBLIC_HOSTNAME_FILE, publicHostName});
+
+        if (vmUuid == null) {
             vmData.add(new String[]{METATDATA_DIR, INSTANCE_ID_FILE, vmName});
             vmData.add(new String[]{METATDATA_DIR, VM_ID_FILE, String.valueOf(vmId)});
+        } else {
+            vmData.add(new String[]{METATDATA_DIR, INSTANCE_ID_FILE, vmUuid});
+            vmData.add(new String[]{METATDATA_DIR, VM_ID_FILE, vmUuid});
+        }
+
         vmData.add(new String[]{METATDATA_DIR, PUBLIC_KEYS_FILE, publicKey});
 
         String cloudIdentifier = _configDao.getValue("cloud.identifier");

server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java

@@ -206,30 +206,37 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
         return false;
     }
 
+    private String getSshKey(VirtualMachineProfile profile) {
+        UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), "SSH.PublicKey");
+        return (vmDetailSshKey!=null ? vmDetailSshKey.getValue() : null);
+    }
+
     @Override
     public boolean addPasswordAndUserdata(Network network, NicProfile nic, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context)
             throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException {
-        UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), "SSH.PublicKey");
+        String sshPublicKey = getSshKey(profile);
-        return (canHandle(network.getTrafficType()) && updateConfigDrive(profile,
+        return (canHandle(network.getTrafficType())
-                (vmDetailSshKey!=null?vmDetailSshKey.getValue():null)))
+                && updateConfigDrive(profile, sshPublicKey, nic))
                 && updateConfigDriveIso(network, profile, dest.getHost(), false);
     }
 
     @Override
     public boolean savePassword(Network network, NicProfile nic, VirtualMachineProfile profile) throws ResourceUnavailableException {
-        if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, (String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey)))) return false;
+        String sshPublicKey = getSshKey(profile);
+        if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, sshPublicKey, nic))) return false;
         return updateConfigDriveIso(network, profile, true);
     }
 
     @Override
     public boolean saveSSHKey(Network network, NicProfile nic, VirtualMachineProfile vm, String sshPublicKey) throws ResourceUnavailableException {
-        if (!(canHandle(network.getTrafficType()) && updateConfigDrive(vm, sshPublicKey))) return false;
+        if (!(canHandle(network.getTrafficType()) && updateConfigDrive(vm, sshPublicKey, nic))) return false;
         return updateConfigDriveIso(network, vm, true);
     }
 
     @Override
     public boolean saveUserData(Network network, NicProfile nic, VirtualMachineProfile profile) throws ResourceUnavailableException {
-        if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, (String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey)))) return false;
+        String sshPublicKey = getSshKey(profile);
+        if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, sshPublicKey, nic))) return false;
         return updateConfigDriveIso(network, profile, true);
     }
 
@@ -330,7 +337,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
         Answer createIsoAnswer = endpoint.sendMessage(configDriveIsoCommand);
         if (!createIsoAnswer.getResult()) {
             throw new ResourceUnavailableException(String.format("%s ISO failed, details: %s",
-                    (update?"Update":"Create"), createIsoAnswer.getDetails()),ConfigDriveNetworkElement.class,0L);
+                    (update?"Update":"Create"), createIsoAnswer.getDetails()), ConfigDriveNetworkElement.class, 0L);
         }
         configureConfigDriveDisk(profile, secondaryStore);
 
@@ -363,7 +370,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
         }
     }
 
-    private boolean updateConfigDrive(VirtualMachineProfile profile, String publicKey) {
+    private boolean updateConfigDrive(VirtualMachineProfile profile, String publicKey, NicProfile nic) {
         UserVmVO vm = _userVmDao.findById(profile.getId());
         if (vm.getType() != VirtualMachine.Type.User) {
             return false;
@@ -372,11 +379,10 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
         Nic defaultNic = _networkModel.getDefaultNic(vm.getId());
         if (defaultNic != null) {
             final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText();
-            final String zoneName = _dcDao.findById(vm.getDataCenterId()).getName();
             boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
 
-            List<String[]> vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, zoneName, vm.getInstanceName(), vm.getId(),
+            List<String[]> vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getId(),
-                    publicKey, (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
+                    vm.getUuid(), nic.getIPv4Address(), publicKey, (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
             profile.setVmData(vmData);
             profile.setConfigDriveLabel(VirtualMachineManager.VmConfigDriveLabel.value());
         }

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -4085,11 +4085,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             Network network = _networkModel.getNetwork(defaultNic.getNetworkId());
             if (_networkModel.isSharedNetworkWithoutServices(network.getId())) {
                 final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText();
-                final String zoneName = _dcDao.findById(vm.getDataCenterId()).getName();
                 boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
 
-                List<String[]> vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, zoneName, vm.getInstanceName(), vm.getId(),
+                List<String[]> vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getId(),
-                        (String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
+                        vm.getUuid(), defaultNic.getIPv4Address(), vm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
                 String vmName = vm.getInstanceName();
                 String configDriveIsoRootFolder = "/tmp";
                 String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso";

server/src/test/java/com/cloud/network/MockNetworkModelImpl.java

@@ -898,7 +898,7 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel {
     }
 
     @Override
-    public List<String[]> generateVmData(String userData, String serviceOffering, String zoneName, String vmName, long vmId, String publicKey, String password, Boolean isWindows) {
+    public List<String[]> generateVmData(String userData, String serviceOffering, long datacenterId, String vmName, long vmId, String vmUuid, String guestIpAddress, String publicKey, String password, Boolean isWindows) {
         return null;
     }
 

server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java

@@ -36,7 +36,6 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.xerces.impl.dv.util.Base64;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.InjectMocks;
@@ -64,6 +63,8 @@ import com.cloud.host.dao.HostDao;
 import com.cloud.network.Network;
 import com.cloud.network.NetworkModelImpl;
 import com.cloud.network.Networks;
+import com.cloud.network.dao.IPAddressDao;
+import com.cloud.network.dao.IPAddressVO;
 import com.cloud.network.dao.NetworkDao;
 import com.cloud.network.dao.NetworkServiceMapDao;
 import com.cloud.network.dao.NetworkVO;
@@ -77,6 +78,7 @@ import com.cloud.storage.dao.GuestOSDao;
 import com.cloud.utils.fsm.NoTransitionException;
 import com.cloud.utils.fsm.StateListener;
 import com.cloud.utils.fsm.StateMachine2;
+import com.cloud.utils.net.Ip;
 import com.cloud.vm.Nic;
 import com.cloud.vm.NicProfile;
 import com.cloud.vm.UserVmDetailVO;
@@ -99,7 +101,7 @@ public class ConfigDriveNetworkElementTest {
     private final String VMINSTANCENAME = "vm_name";
     private final String VMOFFERING = "custom_instance";
     private final long VMID = 30L;
-    private final String VMUSERDATA = "userdata";
+    private final String VMUSERDATA = "H4sIABCvw1oAAystTi1KSSxJ5AIAUPllwQkAAAA=";
     private final long SOID = 31L;
     private final long HOSTID = NETWORK_ID;
     private final String HOSTNAME = "host1";
@@ -116,6 +118,7 @@ public class ConfigDriveNetworkElementTest {
     @Mock private UserVmDetailsDao _userVmDetailsDao;
     @Mock private NetworkDao _networkDao;
     @Mock private NetworkServiceMapDao _ntwkSrvcDao;
+    @Mock private IPAddressDao _ipAddressDao;
 
     @Mock private DataCenterVO dataCenterVO;
     @Mock private DataStore dataStore;
@@ -130,6 +133,7 @@ public class ConfigDriveNetworkElementTest {
     @Mock private NicProfile nicp;
     @Mock private ServiceOfferingVO serviceOfferingVO;
     @Mock private UserVmVO virtualMachine;
+    @Mock private IPAddressVO publicIp;
 
     @InjectMocks private final ConfigDriveNetworkElement _configDrivesNetworkElement = new ConfigDriveNetworkElement();
     @InjectMocks @Spy private NetworkModelImpl _networkModel = new NetworkModelImpl();
@@ -161,7 +165,7 @@ public class ConfigDriveNetworkElementTest {
         when(virtualMachine.getServiceOfferingId()).thenReturn(SOID);
         when(virtualMachine.getDataCenterId()).thenReturn(DATACENTERID);
         when(virtualMachine.getInstanceName()).thenReturn(VMINSTANCENAME);
-        when(virtualMachine.getUserData()).thenReturn(Base64.encode(VMUSERDATA.getBytes()));
+        when(virtualMachine.getUserData()).thenReturn(VMUSERDATA);
         when(deployDestination.getHost()).thenReturn(hostVO);
         when(hostVO.getId()).thenReturn(HOSTID);
         when(nic.isDefaultNic()).thenReturn(true);
@@ -236,6 +240,71 @@ public class ConfigDriveNetworkElementTest {
 
     @Test
     public void testAddPasswordAndUserdata() throws InsufficientCapacityException, ResourceUnavailableException {
+        List<String[]> actualVmData = getVmData();
+
+        assertThat(actualVmData, containsInAnyOrder(
+                new String[]{"userdata", "user_data", VMUSERDATA},
+                new String[]{"metadata", "service-offering", VMOFFERING},
+                new String[]{"metadata", "availability-zone", ZONENAME},
+                new String[]{"metadata", "local-hostname", VMINSTANCENAME},
+                new String[]{"metadata", "local-ipv4", "192.168.111.111"},
+                new String[]{"metadata", "public-hostname", null},
+                new String[]{"metadata", "public-ipv4", "192.168.111.111"},
+                new String[]{"metadata", "vm-id", String.valueOf(VMID)},
+                new String[]{"metadata", "instance-id", VMINSTANCENAME},
+                new String[]{"metadata", "public-keys", PUBLIC_KEY},
+                new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)},
+                new String[]{PASSWORD, "vm_password", PASSWORD}
+        ));
+    }
+
+    @Test
+    public void testAddPasswordAndUserdataStaticNat() throws InsufficientCapacityException, ResourceUnavailableException {
+        when(_ipAddressDao.findByAssociatedVmId(VMID)).thenReturn(publicIp);
+        when(publicIp.getAddress()).thenReturn(new Ip("7.7.7.7"));
+
+        List<String[]> actualVmData = getVmData();
+
+        assertThat(actualVmData, containsInAnyOrder(
+                new String[]{"userdata", "user_data", VMUSERDATA},
+                new String[]{"metadata", "service-offering", VMOFFERING},
+                new String[]{"metadata", "availability-zone", ZONENAME},
+                new String[]{"metadata", "local-hostname", VMINSTANCENAME},
+                new String[]{"metadata", "local-ipv4", "192.168.111.111"},
+                new String[]{"metadata", "public-hostname", "7.7.7.7"},
+                new String[]{"metadata", "public-ipv4", "7.7.7.7"},
+                new String[]{"metadata", "vm-id", String.valueOf(VMID)},
+                new String[]{"metadata", "instance-id", VMINSTANCENAME},
+                new String[]{"metadata", "public-keys", PUBLIC_KEY},
+                new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)},
+                new String[]{PASSWORD, "vm_password", PASSWORD}
+        ));
+    }
+
+
+    @Test
+    public void testAddPasswordAndUserdataUuid() throws InsufficientCapacityException, ResourceUnavailableException {
+        when(virtualMachine.getUuid()).thenReturn("vm-uuid");
+
+        List<String[]> actualVmData = getVmData();
+
+        assertThat(actualVmData, containsInAnyOrder(
+                new String[]{"userdata", "user_data", VMUSERDATA},
+                new String[]{"metadata", "service-offering", VMOFFERING},
+                new String[]{"metadata", "availability-zone", ZONENAME},
+                new String[]{"metadata", "local-hostname", VMINSTANCENAME},
+                new String[]{"metadata", "local-ipv4", "192.168.111.111"},
+                new String[]{"metadata", "public-hostname", null},
+                new String[]{"metadata", "public-ipv4", "192.168.111.111"},
+                new String[]{"metadata", "vm-id", "vm-uuid"},
+                new String[]{"metadata", "instance-id", "vm-uuid"},
+                new String[]{"metadata", "public-keys", PUBLIC_KEY},
+                new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)},
+                new String[]{PASSWORD, "vm_password", PASSWORD}
+        ));
+    }
+
+    private List<String[]> getVmData() throws InsufficientCapacityException, ResourceUnavailableException {
         final Answer answer = mock(Answer.class);
         final UserVmDetailVO userVmDetailVO = mock(UserVmDetailVO.class);
         when(endpoint.sendMessage(any(HandleConfigDriveIsoCommand.class))).thenReturn(answer);
@@ -243,6 +312,7 @@ public class ConfigDriveNetworkElementTest {
         when(network.getTrafficType()).thenReturn(Networks.TrafficType.Guest);
         when(virtualMachine.getState()).thenReturn(VirtualMachine.State.Stopped);
         when(userVmDetailVO.getValue()).thenReturn(PUBLIC_KEY);
+        when(nicp.getIPv4Address()).thenReturn("192.168.111.111");
         when(_userVmDetailsDao.findDetail(anyLong(), anyString())).thenReturn(userVmDetailVO);
         Map<VirtualMachineProfile.Param, Object> parms = Maps.newHashMap();
         parms.put(VirtualMachineProfile.Param.VmPassword, PASSWORD);
@@ -254,19 +324,6 @@ public class ConfigDriveNetworkElementTest {
         ArgumentCaptor<HandleConfigDriveIsoCommand> commandCaptor = ArgumentCaptor.forClass(HandleConfigDriveIsoCommand.class);
         verify(endpoint, times(1)).sendMessage(commandCaptor.capture());
         HandleConfigDriveIsoCommand result = commandCaptor.getValue();
-        List<String[]> actualVmData = result.getVmData();
+        return result.getVmData();
-
-        assertThat(actualVmData, containsInAnyOrder(
-                new String[]{"userdata", "user_data", VMUSERDATA},
-                new String[]{"metadata", "service-offering", VMOFFERING},
-                new String[]{"metadata", "availability-zone", ZONENAME},
-                new String[]{"metadata", "local-hostname", VMINSTANCENAME},
-                new String[]{"metadata", "vm-id", String.valueOf(VMID)},
-                new String[]{"metadata", "instance-id", String.valueOf(VMINSTANCENAME)},
-                new String[]{"metadata", "public-keys", PUBLIC_KEY},
-                new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)},
-                new String[]{PASSWORD, "vm_password", PASSWORD}
-        ));
-
     }
 }

server/src/test/java/com/cloud/vpc/MockNetworkModelImpl.java

@@ -913,7 +913,7 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel {
     }
 
     @Override
-    public List<String[]> generateVmData(String userData, String serviceOffering, String zoneName, String vmName, long vmId, String publicKey, String password, Boolean isWindows) {
+    public List<String[]> generateVmData(String userData, String serviceOffering, long datacenterId, String vmName, long vmId, String vmUuid, String guestIpAddress, String publicKey, String password, Boolean isWindows) {
         return null;
     }
 

services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java

@@ -40,6 +40,7 @@ import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStreamWriter;
 import java.io.UnsupportedEncodingException;
 import java.net.InetAddress;
 import java.net.URI;
@@ -52,6 +53,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+
 import javax.naming.ConfigurationException;
 
 import io.netty.bootstrap.ServerBootstrap;
@@ -68,6 +70,7 @@ import io.netty.handler.codec.http.HttpResponseEncoder;
 import io.netty.handler.logging.LogLevel;
 import io.netty.handler.logging.LoggingHandler;
 
+import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
@@ -155,11 +158,11 @@ import com.cloud.agent.api.to.DatadiskTO;
 import com.cloud.agent.api.to.NfsTO;
 import com.cloud.agent.api.to.S3TO;
 import com.cloud.agent.api.to.SwiftTO;
+import com.cloud.configuration.Resource;
 import com.cloud.exception.InternalErrorException;
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.host.Host;
 import com.cloud.host.Host.Type;
-import com.cloud.configuration.Resource;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.network.NetworkModel;
 import com.cloud.resource.ServerResourceBase;
@@ -190,9 +193,6 @@ import com.cloud.utils.script.Script;
 import com.cloud.utils.storage.S3.S3Utils;
 import com.cloud.vm.SecondaryStorageVm;
 
-
-import java.io.OutputStreamWriter;
-
 public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
 
     public static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
@@ -492,8 +492,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
                         if (typeFolder.exists() || typeFolder.mkdirs()) {
                             if (StringUtils.isNotEmpty(content)) {
                                 File file = new File(typeFolder, fileName + ".txt");
-                                try (FileWriter fw = new FileWriter(file); BufferedWriter bw = new BufferedWriter(fw)) {
+                                try  {
-                                    bw.write(content);
+                                    if (fileName.equals(USERDATA_FILE)) {
+                                        // User Data is passed as a base64 encoded string
+                                        FileUtils.writeByteArrayToFile(file, Base64.decodeBase64(content));
+                                    } else {
+                                        FileUtils.write(file, content, com.cloud.utils.StringUtils.getPreferredCharset());
+                                    }
                                 } catch (IOException ex) {
                                     s_logger.error("Failed to create file ", ex);
                                     return new Answer(cmd, ex);

test/integration/plugins/nuagevsp/nuageTestCase.py

@@ -733,8 +733,12 @@ class nuageTestCase(cloudstackTestCase):
                                      traffictype=traffic_type
                                      )
 
-    # ssh_into_VM - Gets into the shell of the given VM using its public IP
+    def ssh_into_VM(self, vm, public_ip, reconnect=True, negative_test=False, keypair=None):
-    def ssh_into_VM(self, vm, public_ip, reconnect=True, negative_test=False):
+        """Creates a SSH connection to the VM
+
+        :returns: the SSH connection
+        :rtype: marvin.sshClient.SshClient
+        """
         if self.isSimulator:
             self.debug("Simulator Environment: Skipping ssh into VM")
             return
@@ -748,7 +752,8 @@ class nuageTestCase(cloudstackTestCase):
             ssh_client = vm.get_ssh_client(
                 ipaddress=public_ip.ipaddress.ipaddress,
                 reconnect=reconnect,
-                retries=3 if negative_test else 30
+                retries=3 if negative_test else 30,
+                keyPairFileLocation=keypair.private_key_file if keypair else None
             )
             self.debug("Successful to SSH into VM with ID - %s on "
                        "public IP address - %s" %

test/integration/plugins/nuagevsp/nuage_lib.py

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from marvin.cloudstackAPI import createSSHKeyPair, deleteSSHKeyPair
+
+
+class MySSHKeyPair:
+    """Manage SSH Key pairs"""
+
+    def __init__(self, items):
+        self.__dict__.update(items)
+
+    @classmethod
+    def create(cls, apiclient, name=None, account=None,
+               domainid=None, projectid=None):
+        """Creates SSH keypair"""
+        cmd = createSSHKeyPair.createSSHKeyPairCmd()
+        cmd.name = name
+        if account is not None:
+            cmd.account = account
+        if domainid is not None:
+            cmd.domainid = domainid
+        if projectid is not None:
+            cmd.projectid = projectid
+        return MySSHKeyPair(apiclient.createSSHKeyPair(cmd).__dict__)
+
+    def delete(self, apiclient):
+        """Delete SSH key pair"""
+        cmd = deleteSSHKeyPair.deleteSSHKeyPairCmd()
+        cmd.name = self.name
+        cmd.account = self.account
+        cmd.domainid = self.domainid
+        apiclient.deleteSSHKeyPair(cmd)

test/integration/plugins/nuagevsp/test_nuage_configdrive.py

@@ -19,8 +19,15 @@
     and password reset functionality with
     ConfigDrive and Nuage VSP SDN plugin
 """
-# Import Local Modules
+import base64
-from nuageTestCase import nuageTestCase
+import copy
+import os
+import tempfile
+import threading
+
+import sys
+import time
+from datetime import datetime
 from marvin.cloudstackAPI import updateTemplate, resetSSHKeyForVirtualMachine
 from marvin.lib.base import (Account,
                              createVlanIpRange,
@@ -28,80 +35,24 @@ from marvin.lib.base import (Account,
                              NetworkServiceProvider,
                              PublicIpRange,
                              PublicIPAddress,
-                             createSSHKeyPair,
-                             deleteSSHKeyPair,
                              VirtualMachine)
-
 from marvin.lib.common import list_templates
 from marvin.lib.utils import random_gen
 # Import System Modules
 from nose.plugins.attrib import attr
-from datetime import datetime
+
-import threading
+# Import Local Modules
-import tempfile
+from component.test_configdrive import MySSHKeyPair, ConfigDriveUtils
-import base64
+from nuageTestCase import nuageTestCase, needscleanup
-import sys
+
-import time
+NO_SUCH_FILE = "No such file or directory"
-import os
-import copy
-import json
 
 
-class MySSHKeyPair:
+class TestNuageConfigDrive(nuageTestCase, ConfigDriveUtils):
-    """Manage SSH Key pairs"""
-
-    def __init__(self, items):
-        self.__dict__.update(items)
-
-    @classmethod
-    def create(cls, apiclient, name=None, account=None,
-               domainid=None, projectid=None):
-        """Creates SSH keypair"""
-        cmd = createSSHKeyPair.createSSHKeyPairCmd()
-        cmd.name = name
-        if account is not None:
-            cmd.account = account
-        if domainid is not None:
-            cmd.domainid = domainid
-        if projectid is not None:
-            cmd.projectid = projectid
-        return MySSHKeyPair(apiclient.createSSHKeyPair(cmd).__dict__)
-
-    def delete(self, apiclient):
-        """Delete SSH key pair"""
-        cmd = deleteSSHKeyPair.deleteSSHKeyPairCmd()
-        cmd.name = self.name
-        cmd.account = self.account
-        cmd.domainid = self.domainid
-        apiclient.deleteSSHKeyPair(cmd)
-
-
-class TestNuageConfigDrive(nuageTestCase):
     """Test user data and password reset functionality
     using configDrive with Nuage VSP SDN plugin
     """
 
-    class CreateResult:
-        def __init__(self, success, offering=None, network=None, vpc=None):
-            self.success = success
-            self.network = network
-            self.offering = offering
-            self.vpc = vpc
-
-    class PasswordTest:
-        def __init__(self, password):
-            self.test_presence = False
-            self.presence = None
-            self.password = None
-            if type(password) is bool:
-                self.test_presence = True
-                self.presence = password
-                self.password = None
-            elif type(password) is unicode or type(password) is str:
-                self.test_presence = True
-                self.password = password
-                self.presence = True
-
     class StartVM(threading.Thread):
 
         def __init__(self, nuagetestcase, network, index):
@@ -247,312 +198,11 @@ class TestNuageConfigDrive(nuageTestCase):
         self.updateTemplate(False)
         return
 
-    # updateTemplate - Updates value of the guest VM template's password
+    def validate_firewall_rule(self, fw_rule):
-    # enabled setting
-    def updateTemplate(self, value):
-        self.debug("Updating value of guest VM template's password enabled "
-                   "setting")
-        cmd = updateTemplate.updateTemplateCmd()
-        cmd.id = self.template.id
-        cmd.passwordenabled = value
-        self.api_client.updateTemplate(cmd)
-        list_template_response = list_templates(self.api_client,
-                                                templatefilter="all",
-                                                id=self.template.id
-                                                )
-        self.template = list_template_response[0]
-        self.debug("Updated guest VM template")
-
-    # get_userdata_url - Returns user data URL for the given VM object
-    def get_userdata_url(self, vm):
-        self.debug("Getting user data url")
-        nic = vm.nic[0]
-        gateway = str(nic.gateway)
-        self.debug("Gateway: " + gateway)
-        user_data_url = 'curl "http://' + gateway + ':80/latest/user-data"'
-        return user_data_url
-
-    # create_and_verify_fw - Creates and verifies (Ingress) firewall rule
-    # with a Static NAT rule enabled public IP
-    def create_and_verify_fip_and_fw(self, vm, public_ip, network):
-        self.debug("Creating and verifying firewall rule")
-        self.create_StaticNatRule_For_VM(vm, public_ip, network)
-
-        # VSD verification
-        self.verify_vsd_floating_ip(network, vm, public_ip.ipaddress)
-
-        fw_rule = self.create_FirewallRule(
-                public_ip, self.test_data["ingress_rule"])
-
-        # VSD verification
         self.verify_vsd_firewall_rule(fw_rule)
-        self.debug("Successfully created and verified firewall rule")
 
-    def getConfigDriveContent(self, ssh):
+    def validate_StaticNat_rule_For_VM(self, public_ip, network, vm):
-        """
+        self.verify_vsd_floating_ip(network, vm, public_ip.ipaddress)
-        This method is to verify whether configdrive iso
-        is attached to vm or not
-        Returns mount path if config drive is attached else False
-        """
-        mountdir = "/root/iso"
-        cmd = "blkid -t LABEL='config-2' /dev/sr? /dev/hd? /dev/sd? /dev/xvd? -o device"
-        tmp_cmd = [
-            'bash -c "if [ ! -d /root/iso ] ; then mkdir /root/iso ; fi"',
-            "umount /root/iso"]
-        for tcmd in tmp_cmd:
-            ssh.execute(tcmd)
-        configDrive = ssh.execute(cmd)
-        res = ssh.execute("mount {} {}".format(str(configDrive[0]), mountdir))
-        if str(res).lower().find("mounting read-only") > -1:
-            self.debug("configDrive iso is mounted at location %s" % mountdir)
-            return mountdir
-        else:
-            return None
-
-    def verifyUserData(self, ssh, iso_path, userdata):
-        """
-        verify Userdata
-        """
-        userdata_path = iso_path+"/cloudstack/userdata/user_data.txt"
-        cmd = "cat %s" % userdata_path
-        res = ssh.execute(cmd)
-        vmuserdata = str(res[0])
-        self.debug("Expected userdata is %s" % userdata)
-        self.debug("ConfigDrive userdata acsformat is %s" % vmuserdata)
-        self.assertEqual(vmuserdata, userdata,
-                         'Userdata found: %s is not equal to expected: %s'
-                         % (vmuserdata, userdata))
-
-    def verifyOpenStackUserData(self, ssh, iso_path, userdata):
-        """
-        verify Userdata in Openstack format
-        """
-        userdata_path = iso_path+"/openstack/latest/user_data"
-        cmd = "cat %s" % userdata_path
-        res = ssh.execute(cmd)
-        vmuserdata = str(res[0])
-        self.debug("Expected userdata is %s" % userdata)
-        self.debug("ConfigDrive userdata openstackformat is %s" % vmuserdata)
-        self.assertEqual(vmuserdata, userdata,
-                         'Userdata found: %s is not equal to expected: %s'
-                         % (vmuserdata, userdata))
-
-    def verifyPassword(self, ssh, iso_path, password):
-        self.debug("Expected VM password is %s " % password.password)
-        password_file = iso_path+"/cloudstack/password/vm_password.txt"
-        cmd = "cat %s" % password_file
-        res = ssh.execute(cmd)
-        vmpassword = str(res[0])
-        self.debug("ConfigDrive password is %s " % vmpassword)
-        nosuchfile = "No such file or directory"
-        if nosuchfile in vmpassword:
-            self.debug("Password file is not found")
-            return False, False
-        elif (password.password is not None) \
-                and (password.password in vmpassword):
-                self.debug("Expected Password is found in configDriveIso")
-                return True, True
-        else:
-            self.debug("Expected password is not found in configDriveIso")
-            return True, False
-
-    def verifySshKey(self, ssh, iso_path, sshkey):
-        self.debug("Expected VM sshkey is %s " % sshkey)
-        publicKey_file = iso_path+"/cloudstack/metadata/public-keys.txt"
-        cmd = "cat %s" % publicKey_file
-        res = ssh.execute(cmd)
-        vmsshkey = str(res[0])
-        self.debug("ConfigDrive ssh key is %s " % vmsshkey)
-
-    def verifyMetaData(self, vm, ssh, iso_path):
-
-        metadata_dir = iso_path+"/cloudstack/metadata/"
-        metadata = {}
-        vm_files = ["availability-zone.txt",
-                    "instance-id.txt",
-                    "service-offering.txt",
-                    "vm-id.txt"]
-        for file in vm_files:
-            cmd = "cat %s" % metadata_dir+file
-            res = ssh.execute(cmd)
-            metadata[file] = res
-
-        for mfile in vm_files:
-            if mfile not in metadata:
-                self.fail("{} file is not found in vm metadata".format(mfile))
-        self.assertEqual(
-                str(metadata["availability-zone.txt"][0]),
-                self.zone.name,
-                "Zone name inside metadata does not match with the zone"
-        )
-        self.assertEqual(
-                str(metadata["instance-id.txt"][0]),
-                vm.instancename,
-                "vm name inside metadata does not match with the "
-                "instance name"
-        )
-        self.assertEqual(
-                str(metadata["service-offering.txt"][0]),
-                vm.serviceofferingname,
-                "Service offering inside metadata does not match "
-                "with the instance offering"
-        )
-        return
-
-    def verifyOpenStackData(self, ssh, iso_path):
-
-        openstackdata_dir = iso_path+"/openstack/latest/"
-        openstackdata = {}
-        openstackdata_files = ["user_data",
-                               "meta_data.json",
-                               "vendor_data.json",
-                               "network_data.json"]
-        for file in openstackdata_files:
-            cmd = "cat %s" % openstackdata_dir+file
-            res = ssh.execute(cmd)
-            openstackdata[file] = res
-            if file not in openstackdata:
-                self.fail("{} file not found in vm openstack".format(file))
-        return
-
-    def generate_ssh_keys(self):
-        """
-        This method generates ssh key pair and writes the private key
-        into a temp file and returns the file name
-        """
-        self.keypair = MySSHKeyPair.create(
-                self.api_client,
-                name=random_gen() + ".pem",
-                account=self.account.user[0].account,
-                domainid=self.account.domainid)
-
-        self.cleanup.append(self.keypair)
-        self.debug("Created keypair with name: %s" % self.keypair.name)
-        self.debug("Writing the private key to local file")
-        keyPairFilePath = tempfile.gettempdir() + os.sep + self.keypair.name
-        self.tmp_files.append(keyPairFilePath)
-        self.debug("File path: %s" % keyPairFilePath)
-        with open(keyPairFilePath, "w+") as f:
-            f.write(self.keypair.privatekey)
-        os.system("chmod 400 " + keyPairFilePath)
-        return keyPairFilePath
-
-    def umountConfigDrive(self, ssh, iso_path):
-        """umount config drive iso attached inside guest vm"""
-        ssh.execute("umount -d %s" % iso_path)
-        # Give the VM time to unlock the iso device
-        time.sleep(2)
-        # Verify umount
-        result = ssh.execute("ls %s" % iso_path)
-        self.assertTrue(len(result) == 0,
-                        "After umount directory should be empty "
-                        "but contains: %s" % result)
-
-    def update_provider_state(self, new_state):
-        self.debug("Updating Service Provider ConfigDrive to %s" % new_state)
-        configdriveprovider = NetworkServiceProvider.list(
-            self.api_client,
-            name="ConfigDrive",
-            physicalnetworkid=self.vsp_physical_network.id)[0]
-        orig_state = configdriveprovider.state
-        NetworkServiceProvider.update(self.api_client,
-                                      configdriveprovider.id,
-                                      state=new_state)
-        self.validate_NetworkServiceProvider("ConfigDrive", state=new_state)
-        return orig_state
-
-    def verify_network_creation(self, offering=None,
-                                offering_name=None,
-                                gateway=None,
-                                vpc=None, acl_list=None, testdata=None):
-        if offering is None:
-            self.debug("Creating Nuage VSP network offering...")
-            offering = self.create_NetworkOffering(
-                self.test_data["nuagevsp"][offering_name])
-            self.validate_NetworkOffering(offering, state="Enabled")
-        try:
-            network = self.create_Network(offering,
-                                          gateway=gateway,
-                                          vpc=vpc,
-                                          acl_list=acl_list,
-                                          testdata=testdata)
-            return self.CreateResult(True, offering=offering, network=network)
-        except Exception:
-            self.debug("Exception: %s" % sys.exc_info()[0])
-            return self.CreateResult(False, offering=offering)
-
-    def verify_vpc_creation(self, offering=None, offering_name=None):
-
-        if offering is None:
-            self.debug("Creating Nuage VSP VPC offering...")
-            offering = self.create_VpcOffering(
-                self.test_data["nuagevsp"][offering_name])
-            self.validate_VpcOffering(offering, state="Enabled")
-        try:
-            vpc = self.create_Vpc(offering, cidr='10.1.0.0/16')
-            self.validate_Vpc(vpc, state="Enabled")
-            return self.CreateResult(True, offering=offering, vpc=vpc)
-        except Exception:
-            return self.CreateResult(False, offering=offering)
-
-    def update_password_enable_in_template(self, new_state):
-        self.debug("Updating guest VM template to password %s" % new_state)
-        orig_state = self.template.passwordenabled
-        if self.template.passwordenabled is not new_state:
-            self.updateTemplate(new_state)
-        self.assertEqual(self.template.passwordenabled, new_state,
-                         "Guest VM template is not password enabled")
-        return orig_state
-
-    def verify_config_drive_content(self, vm,
-                                    public_ip,
-                                    password_test,
-                                    userdata=None,
-                                    metadata=False,
-                                    sshkey=None,
-                                    ssh_client=None):
-        if self.isSimulator:
-            self.debug("Simulator Environment: Skipping Config Drive content verification")
-            return
-
-        self.debug("SSHing into the VM %s" % vm.name)
-        if ssh_client is None:
-            ssh = self.ssh_into_VM(vm, public_ip)
-        else:
-            ssh = ssh_client
-        d = {x.name: x for x in ssh.logger.handlers}
-        ssh.logger.handlers = list(d.values())
-        config_drive_path = self.getConfigDriveContent(ssh)
-        self.assertIsNotNone(config_drive_path,
-                             'ConfigdriveIso is not attached to vm')
-        if metadata:
-            self.debug("Verifying metadata for vm: %s" % vm.name)
-            self.verifyMetaData(vm, ssh, config_drive_path)
-            self.debug("Verifying openstackdata for vm: %s" % vm.name)
-            self.verifyOpenStackData(ssh, config_drive_path)
-
-        if userdata is not None:
-            self.debug("Verifying userdata for vm: %s" % vm.name)
-            self.verifyUserData(ssh, config_drive_path, userdata)
-            self.verifyOpenStackUserData(ssh, config_drive_path, userdata)
-        if password_test.test_presence:
-            self.debug("Verifying password for vm: %s" % vm.name)
-            test_result = self.verifyPassword(ssh, config_drive_path,
-                                              password_test)
-            self.assertEqual(test_result[0], password_test.presence,
-                             "Expected is that password is present: %s "
-                             " but found is: %s"
-                             % (test_result[0], password_test.presence))
-        if password_test.password is not None:
-            self.debug("Password for vm is %s" % password_test.password)
-            self.assertEqual(test_result[1], True,
-                             "Password value test failed.")
-        if sshkey is not None:
-            self.debug("Verifying sshkey for vm: %s" % vm.name)
-            self.verifySshKey(ssh, config_drive_path, sshkey)
-
-        self.umountConfigDrive(ssh, config_drive_path)
-        return ssh
 
     def create_guest_vm(self, networks, acl_item=None,
                         vpc=None, keypair=None):
@@ -579,92 +229,13 @@ class TestNuageConfigDrive(nuageTestCase):
             self.verify_vsd_firewall_rule(acl_item)
         return vm
 
-    # nic_operation_VM - Performs NIC operations such as add, remove, and
+    # =========================================================================
-    # update default NIC in the given VM and network
+    # ---                    Gherkin style helper methods                   ---
-    def nic_operation_VM(self, vm, network, operation="add"):
+    # =========================================================================
-        self.debug("Performing %s NIC operation in VM with ID - %s and "
-                   "network with ID - %s" % (operation, vm.id, network.id))
-        if operation is "add":
-            vm.add_nic(self.api_client, network.id)
-            self.debug("Added NIC in VM with ID - %s and network with ID - %s"
-                       % (vm.id, network.id))
-        vm_info = VirtualMachine.list(self.api_client, id=vm.id)[0]
-        for nic in vm_info.nic:
-            if nic.networkid == network.id:
-                nic_id = nic.id
-        if operation is "update":
-            vm.update_default_nic(self.api_client, nic_id)
-            self.debug("Updated default NIC to NIC with ID- %s in VM with ID "
-                       "- %s and network with ID - %s" %
-                       (nic_id, vm.id, network.id))
-        if operation is "remove":
-            vm.remove_nic(self.api_client, nic_id)
-            self.debug("Removed NIC with ID - %s in VM with ID - %s and "
-                       "network with ID - %s" % (nic_id, vm.id, network.id))
 
-    def update_userdata(self, vm, expected_user_data):
+    # =========================================================================
-        updated_user_data = base64.b64encode(expected_user_data)
+    # ---                            TEST CASES                             ---
-        vm.update(self.api_client, userdata=updated_user_data)
+    # =========================================================================
-        return expected_user_data
-
-    def reset_password(self, vm):
-        vm.password = vm.resetPassword(self.api_client)
-        self.debug("Password reset to - %s" % vm.password)
-        self.debug("VM - %s password - %s !" %
-                   (vm.name, vm.password))
-
-    def wait_until_done(self, thread_list, name):
-        for aThread in thread_list:
-            self.debug("[Concurrency]Join %s for vm %s" % (name,
-                                                           aThread.get_vm()))
-            aThread.join()
-
-    def resetsshkey(self, vm, keypair, account=None, domainid=None):
-        """Resets SSH key"""
-        cmd = resetSSHKeyForVirtualMachine.resetSSHKeyForVirtualMachineCmd()
-        cmd.id = vm.id
-        cmd.keypair = keypair
-        cmd.account = account
-        cmd.domainid = domainid
-        return self.api_client.resetSSHKeyForVirtualMachine(cmd)
-
-    def update_sshkeypair(self, vm):
-        vm.stop(self.api_client)
-        self.resetsshkey(vm,
-                         self.keypair.name,
-                         account=self.account.user[0].account,
-                         domainid=self.account.domainid)
-        self.debug("Sshkey reset to - %s" % self.keypair.name)
-        vm.start(self.api_client)
-
-    def add_subnet_verify(self, network, services):
-        """verify required nic is present in the VM"""
-
-        self.debug("Going to add new ip range in shared network %s" %
-                   network.name)
-        cmd = createVlanIpRange.createVlanIpRangeCmd()
-        cmd.networkid = network.id
-        cmd.gateway = services["gateway"]
-        cmd.netmask = services["netmask"]
-        cmd.startip = services["startip"]
-        cmd.endip = services["endip"]
-        cmd.forVirtualNetwork = services["forvirtualnetwork"]
-        addedsubnet = self.api_client.createVlanIpRange(cmd)
-
-        self.debug("verify above iprange is successfully added in shared "
-                   "network %s or not" % network.name)
-
-        cmd1 = listVlanIpRanges.listVlanIpRangesCmd()
-        cmd1.networkid = network.id
-        cmd1.id = addedsubnet.vlan.id
-
-        allsubnets = self.api_client.listVlanIpRanges(cmd1)
-        self.assertEqual(
-            allsubnets[0].id,
-            addedsubnet.vlan.id,
-            "Check New subnet is successfully added to the shared Network"
-        )
-        return addedsubnet
 
     @attr(tags=["advanced", "nuagevsp", "isonw"], required_hardware="true")
     def test_nuage_configdrive_isolated_network(self):
@@ -673,27 +244,23 @@ class TestNuageConfigDrive(nuageTestCase):
            with Nuage VSP SDN plugin
         """
 
-        # 1. When ConfigDrive is disabled as provider in zone
+        # 2. Given ConfigDrive provider is enabled in zone
-        #    Verify Isolated Network creation with a network offering
+        #    And a network offering which has
-        #    which has userdata provided by ConfigDrive fails
+        #    * user data provided by ConfigDrive
-        # 2. When ConfigDrive is enabled as provider in zone
+        #    * No DNS
-        #    Create an Isolated Network with Nuage VSP Isolated Network
+        #    When I create an Isolated Network using that network offering
-        #    offering specifying ConfigDrive as serviceProvider
+        #    Then the network is successfully created,
-        #    for userdata,
+        #    And is in the "Allocated" state.
-        #    make sure no Dns is in the offering so no VR is spawned.
+        #
-        #    check if it is successfully created and
+        # 3. When I deploy a VM in the created Isolated network with user data,
-        #    is in the "Allocated" state.
+        #    Then the Isolated network state is changed to "Implemented"
-        # 3. Deploy a VM in the created Isolated network with user data,
+        #    And the VM is successfully deployed and is in the "Running" state
-        #    check if the Isolated network state is changed to
+        #    And there is no VR is deployed.
-        #    "Implemented", and the VM is successfully deployed and
+        # 4. And the user data in the ConfigDrive device is as expected
-        #    is in the "Running" state.
+        # 5. And the the vm's password in the ConfigDrive device is as expected
-        #    Check that no VR is deployed.
+        #
-        # 4. SSH into the deployed VM and verify its user data in the iso
+        # 6. When I stop, reset the password, and start the VM
-        #    (expected user data == actual user data).
+        # 7. Then I can login into the VM using the new password.
-        # 5. Verify that the guest VM's password in the iso.
-        # 6. Reset VM password, and start the VM.
-        # 7. Verify that the new guest VM template is password enabled by
-        #     checking the VM's password (password != "password").
         # 8. SSH into the VM for verifying its new password
         #     after its password reset.
         # 9. Verify various scenarios and check the data in configdriveIso
@@ -706,6 +273,12 @@ class TestNuageConfigDrive(nuageTestCase):
             # Configure VSD sessions
             self.configureVSDSessions()
 
+            # 1. Given ConfigDrive provider is disabled in zone
+            #    And a network offering which has
+            #      user data provided by ConfigDrive
+            #    Then creating an Isolated Network
+            #    using that network offering fails
+
             self.debug("+++Testing configdrive in an Isolated network fails..."
                        "as provider configdrive is still disabled...")
             self.update_provider_state("Disabled")
@@ -717,6 +290,8 @@ class TestNuageConfigDrive(nuageTestCase):
                              'Network found success = %s, expected success =%s'
                              % (str(create_network.success), 'False'))
 
+
+
             self.debug("+++Test user data & password reset functionality "
                        "using configdrive in an Isolated network without VR")
             self.update_provider_state("Enabled")
@@ -763,7 +338,7 @@ class TestNuageConfigDrive(nuageTestCase):
                 metadata=True,
                 userdata=self.test_data[
                     "virtual_machine_userdata"]["userdata"],
-                sshkey=self.keypair.name)
+                ssh_key=self.keypair)
 
             expected_user_data1 = self.update_userdata(vm1, "helloworld vm1")
             self.verify_config_drive_content(vm1, public_ip_1,
@@ -776,7 +351,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(True),
                                              metadata=True,
                                              userdata=expected_user_data1,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             # After sshkey reset we need to have the vm password again
             vm1.password = vm1.resetPassword(self.api_client)
             self.debug("Password reset to - %s" % vm1.password)
@@ -791,7 +366,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm1.password),
                                              metadata=True,
                                              userdata=expected_user_data1,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             vm1.password = vm1.resetPassword(self.api_client)
             self.debug("Password reset to - %s" % vm1.password)
             self.debug("VM - %s password - %s !" %
@@ -802,7 +377,7 @@ class TestNuageConfigDrive(nuageTestCase):
             self.verify_config_drive_content(vm1, public_ip_1,
                                              self.PasswordTest(vm1.password),
                                              userdata=expected_user_data1,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("updating non-default nic as the default nic "
                        "of the multi-nic VM and enable staticnat...")
@@ -887,7 +462,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm1.password),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("+++ Restarting the created Isolated network without "
                        "VR with cleanup...")
@@ -898,7 +473,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm1.password),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("+++ Upgrade offering of created Isolated network with "
                        "a dns offering which spins a VR")
@@ -996,14 +571,14 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm1.password),
                                              metadata=True,
                                              userdata=expected_user_data1,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("Updating userdata for VM - %s" % vm1.name)
             expected_user_data1 = self.update_userdata(vm1, "hello afterboot")
             self.verify_config_drive_content(vm1, public_ip_1,
                                              self.PasswordTest(vm1.password),
                                              userdata=expected_user_data1,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             self.debug("Resetting password for VM - %s" % vm1.name)
             self.reset_password(vm1)
             self.debug("SSHing into the VM for verifying its new password "
@@ -1018,7 +593,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm1.password),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("Updating userdata after migrating VM - %s" % vm1.name)
             expected_user_data1 = self.update_userdata(vm1,
@@ -1040,7 +615,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(False),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("Updating userdata for VM - %s" % vm1.name)
             expected_user_data1 = self.update_userdata(vm1,
@@ -1064,7 +639,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(False),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             self.update_provider_state("Disabled")
             expected_user_data1 = self.update_userdata(vm1,
                                                        "hello after recover")
@@ -1072,7 +647,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(False),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("+++ When template is not password enabled, "
                        "verify configdrive of VM - %s" % vm1.name)
@@ -1094,7 +669,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(False),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             vm1.delete(self.api_client, expunge=True)
             create_network1.network.delete(self.api_client)
 
@@ -1205,21 +780,21 @@ class TestNuageConfigDrive(nuageTestCase):
             self.verify_config_drive_content(vm, vpc_public_ip_1,
                                              self.PasswordTest(True),
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             expected_user_data = self.update_userdata(vm, "helloworld vm1")
             self.verify_config_drive_content(vm, vpc_public_ip_1,
                                              self.PasswordTest(True),
                                              metadata=True,
                                              userdata=expected_user_data,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("Resetting password for VM - %s" % vm.name)
             self.reset_password(vm)
             self.verify_config_drive_content(vm, vpc_public_ip_1,
                                              self.PasswordTest(vm.password),
                                              userdata=expected_user_data,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.generate_ssh_keys()
             self.update_sshkeypair(vm)
@@ -1227,7 +802,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(True),
                                              metadata=True,
                                              userdata=expected_user_data,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             # After sshkey reset we need to have the vm password again
             vm.password = vm.resetPassword(self.api_client)
             self.debug("Password reset to - %s" % vm.password)
@@ -1242,7 +817,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm.password),
                                              userdata=expected_user_data,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("Adding a non-default nic to the VM "
                        "making it a multi-nic VM...")
@@ -1252,7 +827,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm.password),
                                              metadata=True,
                                              userdata=expected_user_data,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             vm.password = vm.resetPassword(self.api_client)
             self.debug("Password reset to - %s" % vm.password)
             self.debug("VM - %s password - %s !" %
@@ -1262,7 +837,7 @@ class TestNuageConfigDrive(nuageTestCase):
             self.verify_config_drive_content(vm, vpc_public_ip_1,
                                              self.PasswordTest(vm.password),
                                              userdata=expected_user_data1,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("updating non-default nic as the default nic "
                        "of the multi-nic VM and enable staticnat...")
@@ -1323,7 +898,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm.password),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("+++ Restarting the created VPC Tier network without "
                        "cleanup...")
@@ -1334,7 +909,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm.password),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("+++ Restarting the created VPC Tier network with "
                        "cleanup...")
@@ -1345,7 +920,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm.password),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("Testing user data & password reset functionality "
                        " using configdrive in a VPC network with VR...")
@@ -1464,7 +1039,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm.password),
                                              metadata=True,
                                              userdata=expected_user_data1,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("Updating userdata for VM - %s" % vm.name)
             expected_user_data = self.update_userdata(vm,
@@ -1472,7 +1047,7 @@ class TestNuageConfigDrive(nuageTestCase):
             self.verify_config_drive_content(vm, vpc_public_ip_1,
                                              self.PasswordTest(vm.password),
                                              userdata=expected_user_data,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             self.debug("Resetting password for VM - %s" % vm.name)
             self.reset_password(vm)
             self.debug("SSHing into the VM for verifying its new password "
@@ -1487,7 +1062,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm.password),
                                              userdata=expected_user_data,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("Updating userdata after migrating VM - %s" % vm.name)
             expected_user_data = self.update_userdata(vm,
@@ -1495,7 +1070,7 @@ class TestNuageConfigDrive(nuageTestCase):
             self.verify_config_drive_content(vm, vpc_public_ip_1,
                                              self.PasswordTest(vm.password),
                                              userdata=expected_user_data,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             self.debug("Resetting password for VM - %s" % vm.name)
             self.reset_password(vm)
             self.debug("SSHing into the VM for verifying its new password "
@@ -1510,7 +1085,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(False),
                                              userdata=expected_user_data,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("Updating userdata for VM - %s" % vm.name)
             expected_user_data = self.update_userdata(vm,
@@ -1518,7 +1093,7 @@ class TestNuageConfigDrive(nuageTestCase):
             self.verify_config_drive_content(vm, vpc_public_ip_1,
                                              self.PasswordTest(False),
                                              userdata=expected_user_data,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             self.debug("Resetting password for VM - %s" % vm.name)
             self.reset_password(vm)
             self.debug("SSHing into the VM for verifying its new password "
@@ -1535,13 +1110,13 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(False),
                                              userdata=expected_user_data,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             self.update_provider_state("Disabled")
             self.verify_config_drive_content(vm, vpc_public_ip_1,
                                              self.PasswordTest(False),
                                              userdata=expected_user_data,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("+++ When template is not password enabled "
                        "verify configdrive of VM - %s" % vm.name)
@@ -1567,7 +1142,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(False),
                                              userdata=expected_user_data,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             vm.delete(self.api_client, expunge=True)
             create_tiernetwork.network.delete(self.api_client)
 
@@ -1637,7 +1212,6 @@ class TestNuageConfigDrive(nuageTestCase):
 
         try:
             for i in range(2):
-
                 self.debug("\n+++ [Concurrency]Start update on all VM's")
                 #
                 # 5. Concurrently update all VM's
@@ -1684,7 +1258,6 @@ class TestNuageConfigDrive(nuageTestCase):
             # 10. Verify the passwords
             self.debug("\n+++ [Concurrency]Verify passwords on all VM's")
             for aThread in my_reset_threads:
-
                 # create floating ip
                 self.create_and_verify_fip_and_fw(aThread.get_vm(),
                                                   public_ip_1,
@@ -1753,7 +1326,8 @@ class TestNuageConfigDrive(nuageTestCase):
             # Configure VSD sessions
             self.configureVSDSessions()
             if not self.isNuageInfraUnderlay:
-                self.skipTest("Configured Nuage VSP SDN platform infrastructure "
+                self.skipTest(
+                    "Configured Nuage VSP SDN platform infrastructure "
                     "does not support underlay networking: "
                     "skipping test")
 
@@ -1872,7 +1446,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm1.password),
                                              metadata=True,
                                              userdata=expected_user_data,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             vm1.password = vm1.resetPassword(self.api_client)
             self.debug("Password reset to - %s" % vm1.password)
             self.debug("VM - %s password - %s !" %
@@ -1883,7 +1457,7 @@ class TestNuageConfigDrive(nuageTestCase):
             self.verify_config_drive_content(vm1, public_ip,
                                              self.PasswordTest(vm1.password),
                                              userdata=expected_user_data1,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("+++ Updating non-default nic as the default nic "
                        "of the multi-nic VM...")
@@ -1963,14 +1537,14 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm1.password),
                                              metadata=True,
                                              userdata=expected_user_data1,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("Updating userdata for VM - %s" % vm1.name)
             expected_user_data1 = self.update_userdata(vm1, "hello afterboot")
             self.verify_config_drive_content(vm1, public_ip,
                                              self.PasswordTest(vm1.password),
                                              userdata=expected_user_data1,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             self.debug("Resetting password for VM - %s" % vm1.name)
             self.reset_password(vm1)
             self.debug("SSHing into the VM for verifying its new password "
@@ -1985,7 +1559,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(vm1.password),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("Updating userdata after migrating VM - %s" % vm1.name)
             expected_user_data1 = self.update_userdata(vm1,
@@ -2007,7 +1581,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(False),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("Updating userdata for VM - %s" % vm1.name)
             expected_user_data1 = self.update_userdata(vm1,
@@ -2031,7 +1605,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(False),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             self.update_provider_state("Disabled")
             expected_user_data1 = self.update_userdata(vm1,
                                                        "hello after recover")
@@ -2039,7 +1613,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(False),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
 
             self.debug("+++ When template is not password enabled, "
                        "verify configdrive of VM - %s" % vm1.name)
@@ -2059,7 +1633,7 @@ class TestNuageConfigDrive(nuageTestCase):
                                              self.PasswordTest(False),
                                              userdata=expected_user_data1,
                                              metadata=True,
-                                             sshkey=self.keypair.name)
+                                             ssh_key=self.keypair)
             vm1.delete(self.api_client, expunge=True)
             shared_network.network.delete(self.api_client)
 
@@ -2120,7 +1694,7 @@ class TestNuageConfigDrive(nuageTestCase):
                 self.PasswordTest(vm1.password),
                 metadata=True,
                 userdata=expected_user_data,
-                sshkey=self.keypair.name)
+                ssh_key=self.keypair)
 
             for i in range(0, 300):
                 self.verify_config_drive_content(
@@ -2128,8 +1702,11 @@ class TestNuageConfigDrive(nuageTestCase):
                     self.PasswordTest(vm1.password),
                     metadata=True,
                     userdata=expected_user_data,
-                    sshkey=self.keypair.name,
+                    ssh_key=self.keypair,
                     ssh_client=ssh_client)
                 expected_user_data = \
                     self.update_userdata(vm1,
                                          'This is sample data %s' % i)
+
+if __name__ == "__main__" and __package__ is None:
+    __package__ = "integration.plugins.nuage"

tools/marvin/marvin/jsonHelper.py

@@ -43,6 +43,12 @@ class jsonLoader(object):
         else:
             return None
 
+    def __getitem__(self, val):
+        if val in self.__dict__:
+            return self.__dict__[val]
+        else:
+            return None
+
     def __repr__(self):
         return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v)
                                       in self.__dict__.iteritems()))

ui/l10n/en.js

@@ -1627,6 +1627,7 @@ var dictionary = {
 "label.storage.tags":"Storage Tags",
 "label.storage.traffic":"Storage Traffic",
 "label.storage.type":"Storage Type",
+"label.storagepolicy":"Storage policy",
 "label.subdomain.access":"Subdomain Access",
 "label.submit":"Submit",
 "label.submitted.by":"[Submitted by: <span id=\"submitted_by\"></span>]",

ui/scripts/system.js

@@ -19886,6 +19886,7 @@
                                                             $form.find('.form-item[rel=account]').hide();
                                                             $form.find('.form-item[rel=username]').hide();
                                                             $form.find('.form-item[rel=key]').hide();
+                                                            $form.find('.form-item[rel=storagepolicy]').hide();
                                                         } else if ($(this).val() == "SMB") {
                                                             //NFS, SMB
                                                             $form.find('.form-item[rel=zoneid]').css('display', 'inline-block');
@@ -19918,6 +19919,7 @@
                                                             $form.find('.form-item[rel=account]').hide();
                                                             $form.find('.form-item[rel=username]').hide();
                                                             $form.find('.form-item[rel=key]').hide();
+                                                            $form.find('.form-item[rel=storagepolicy]').hide();
                                                         } else if ($(this).val() == "S3") {
                                                             //NFS, SMB
                                                             $form.find('.form-item[rel=zoneid]').hide();
@@ -19952,6 +19954,7 @@
                                                             $form.find('.form-item[rel=account]').hide();
                                                             $form.find('.form-item[rel=username]').hide();
                                                             $form.find('.form-item[rel=key]').hide();
+                                                            $form.find('.form-item[rel=storagepolicy]').hide();
                                                         } else if ($(this).val() == "Swift") {
                                                             //NFS, SMB
                                                             $form.find('.form-item[rel=zoneid]').hide();
@@ -19984,6 +19987,7 @@
                                                             $form.find('.form-item[rel=account]').css('display', 'inline-block');
                                                             $form.find('.form-item[rel=username]').css('display', 'inline-block');
                                                             $form.find('.form-item[rel=key]').css('display', 'inline-block');
+                                                            $form.find('.form-item[rel=storagepolicy]').css('display', 'inline-block');
                                                         }
                                                     });
 
@@ -20174,13 +20178,25 @@
                                                 }
                                             },
                                             account: {
-                                                label: 'label.account'
+                                                label: 'label.account',
+                                                 validation: {
+                                                     required: true
+                                                 }
                                             },
                                             username: {
-                                                label: 'label.username'
+                                                label: 'label.username',
+                                                 validation: {
+                                                     required: true
+                                                 }
                                             },
                                             key: {
-                                                label: 'label.key'
+                                                label: 'label.key',
+                                                 validation: {
+                                                     required: true
+                                                 }
+                                            },
+                                             storagepolicy: {
+                                                 label: 'label.storagepolicy'
                                              }
                                             //Swift (end)
                                         }
@@ -20348,6 +20364,11 @@
                                                 data[ 'details[' + index.toString() + '].value'] = args.data.key;
                                                 index++;
                                             }
+                                            if (args.data.storagepolicy != null && args.data.storagepolicy.length > 0) {
+                                                data[ 'details[' + index.toString() + '].key'] = 'storagepolicy';
+                                                data[ 'details[' + index.toString() + '].value'] = args.data.storagepolicy;
+                                                index++;
+                                            }
                                             $.ajax({
                                                 url: createURL('addImageStore'),
                                                 data: data,

utils/src/main/java/com/cloud/utils/SwiftUtil.java

@@ -24,26 +24,29 @@ import java.net.URL;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.security.SignatureException;
-import java.util.Map;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.Formatter;
+import java.util.Map;
 
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.commons.codec.binary.Hex;
 import org.apache.log4j.Logger;
 
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
 
-import javax.crypto.Mac;
-import javax.crypto.spec.SecretKeySpec;
-
 public class SwiftUtil {
     private static Logger logger = Logger.getLogger(SwiftUtil.class);
-    private static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
+    protected static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
     private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1";
-
+    private static final String CD_SRC = "cd %s;";
-
+    private static final String SWIFT_CMD= "/usr/bin/python %s -A %s -U %s:%s -K %s %s";
+    private static final String WITH_STORAGE_POLICY = " --storage-policy \"%s\"";
+    private static final String WITH_SEGMENTS = " -S "+SWIFT_MAX_SIZE;
+    private static final String[] OPERATIONS_WITH_STORAGE_POLICIES = {"post","upload"};
 
     public interface SwiftClientCfg {
         String getAccount();
@@ -53,6 +56,8 @@ public class SwiftUtil {
         String getKey();
 
         String getEndPoint();
+
+        String getStoragePolicy();
     }
 
     private static String getSwiftCLIPath() {
@@ -65,19 +70,10 @@ public class SwiftUtil {
     }
 
     public static boolean postMeta(SwiftClientCfg cfg, String container, String object, Map<String, String> metas) {
-        String swiftCli = getSwiftCLIPath();
-        StringBuilder cms = new StringBuilder();
-        for (Map.Entry<String, String> entry : metas.entrySet()) {
-            cms.append(" -m ");
-            cms.append(entry.getKey());
-            cms.append(":");
-            cms.append(entry.getValue());
-            cms.append(" ");
-        }
         Script command = new Script("/bin/bash", logger);
         command.add("-c");
-        command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() + " post " +
+        command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(),"post", container, object) + getMeta(metas));
-            container + " " + object + " " + cms.toString());
+
         OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
         String result = command.execute(parser);
         if (result != null) {
@@ -87,21 +83,14 @@ public class SwiftUtil {
     }
 
     public static String putObject(SwiftClientCfg cfg, File srcFile, String container, String fileName) {
-        String swiftCli = getSwiftCLIPath();
         if (fileName == null) {
             fileName = srcFile.getName();
         }
-        String srcDirectory = srcFile.getParent();
+
         Script command = new Script("/bin/bash", logger);
-        long size = srcFile.length();
         command.add("-c");
-        if (size <= SWIFT_MAX_SIZE) {
+        command.add(String.format(CD_SRC, srcFile.getParent())+getUploadObjectCommand(cfg, getSwiftCLIPath(), container,fileName, srcFile.length()));
-            command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() +
+
-                " -K " + cfg.getKey() + " upload " + container + " " + fileName);
-        } else {
-            command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() +
-                " -K " + cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + fileName);
-        }
         OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
         String result = command.execute(parser);
         if (result != null) {
@@ -120,38 +109,19 @@ public class SwiftUtil {
         return container + File.separator + srcFile.getName();
     }
 
-    private static StringBuilder buildSwiftCmd(SwiftClientCfg swift) {
-        String swiftCli = getSwiftCLIPath();
-        StringBuilder sb = new StringBuilder();
-        sb.append(" /usr/bin/python ");
-        sb.append(swiftCli);
-        sb.append(" -A ");
-        sb.append(swift.getEndPoint());
-        sb.append(" -U ");
-        sb.append(swift.getAccount());
-        sb.append(":");
-        sb.append(swift.getUserName());
-        sb.append(" -K ");
-        sb.append(swift.getKey());
-        sb.append(" ");
-        return sb;
-    }
-
     public static String[] list(SwiftClientCfg swift, String container, String rFilename) {
-        getSwiftCLIPath();
+        StringBuilder swiftCmdBuilder = new StringBuilder();
-        Script command = new Script("/bin/bash", logger);
+        swiftCmdBuilder.append(getSwiftContainerCmd(swift, getSwiftCLIPath(), "list", container));
-        command.add("-c");
-
-        StringBuilder swiftCmdBuilder = buildSwiftCmd(swift);
-        swiftCmdBuilder.append(" list ");
-        swiftCmdBuilder.append(container);
 
         if (rFilename != null) {
             swiftCmdBuilder.append(" -p ");
             swiftCmdBuilder.append(rFilename);
         }
 
+        Script command = new Script("/bin/bash", logger);
+        command.add("-c");
         command.add(swiftCmdBuilder.toString());
+
         OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
         String result = command.execute(parser);
         if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase("")) {
@@ -178,11 +148,11 @@ public class SwiftUtil {
         } else {
             destFilePath = destDirectory.getAbsolutePath();
         }
-        String swiftCli = getSwiftCLIPath();
+
         Script command = new Script("/bin/bash", logger);
         command.add("-c");
-        command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() +
+        command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "download", container, srcPath)+" -o " + destFilePath);
-                " download " + container + " " + srcPath + " -o " + destFilePath);
+
         OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
         String result = command.execute(parser);
         if (result != null) {
@@ -203,27 +173,6 @@ public class SwiftUtil {
         return new File(destFilePath);
     }
 
-    public static String getContainerName(String type, Long id) {
-        if (type.startsWith("T")) {
-            return "T-" + id;
-        } else if (type.startsWith("S")) {
-            return "S-" + id;
-        } else if (type.startsWith("V")) {
-            return "V-" + id;
-        }
-        return null;
-    }
-
-    public static String[] splitSwiftPath(String path) {
-        int index = path.indexOf(File.separator);
-        if (index == -1) {
-            return null;
-        }
-        String[] paths = new String[2];
-        paths[0] = path.substring(0, index);
-        paths[1] = path.substring(index + 1);
-        return paths;
-    }
 
     public static boolean deleteObject(SwiftClientCfg cfg, String path) {
         Script command = new Script("/bin/bash", logger);
@@ -236,13 +185,8 @@ public class SwiftUtil {
         String container = paths[0];
         String objectName = paths[1];
 
-        StringBuilder swiftCmdBuilder = buildSwiftCmd(cfg);
+        command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "delete", container, objectName));
-        swiftCmdBuilder.append(" delete ");
-        swiftCmdBuilder.append(container);
-        swiftCmdBuilder.append(" ");
-        swiftCmdBuilder.append(objectName);
 
-        command.add(swiftCmdBuilder.toString());
         OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
         command.execute(parser);
         return true;
@@ -284,7 +228,7 @@ public class SwiftUtil {
 
     }
 
-    public static String calculateRFC2104HMAC(String data, String key)
+    static String calculateRFC2104HMAC(String data, String key)
             throws SignatureException, NoSuchAlgorithmException, InvalidKeyException {
 
         SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM);
@@ -294,12 +238,75 @@ public class SwiftUtil {
 
     }
 
-    public static String toHexString(byte[] bytes) {
+    static String toHexString(byte[] bytes) {
-
+        return Hex.encodeHexString(bytes);
-        Formatter formatter = new Formatter();
-        for (byte b : bytes) {
-            formatter.format("%02x", b);
     }
-        return formatter.toString();
+
+    /////////////// SWIFT CMD STRING HELPERS ///////////////
+    protected static String getSwiftCmd(SwiftClientCfg cfg, String swiftCli, String operation){
+        return String.format(SWIFT_CMD, swiftCli,cfg.getEndPoint(),cfg.getAccount(),cfg.getUserName(),cfg.getKey(),operation);
+    }
+
+    protected static String getSwiftObjectCmd(SwiftClientCfg cfg, String swiftCliPath, String operation,String container, String objectName) {
+        String cmd = getSwiftCmd(cfg,swiftCliPath, operation)  +" "+ container+" "+objectName;
+        if(StringUtils.isNotBlank(cfg.getStoragePolicy()) && supportsStoragePolicies(operation)){
+            return cmd + String.format(WITH_STORAGE_POLICY, cfg.getStoragePolicy());
+        }
+        return cmd;
+    }
+
+    private static boolean supportsStoragePolicies(String operation) {
+        for(String supportedOp: OPERATIONS_WITH_STORAGE_POLICIES){
+            if(supportedOp.equals(operation)){
+                return true;
+            }
+        }
+        return false;
+    }
+
+    protected static String getSwiftContainerCmd(SwiftClientCfg cfg, String swiftCliPath, String operation, String container) {
+        return getSwiftCmd(cfg,swiftCliPath, operation) +" "+ container;
+    }
+
+    protected static String getUploadObjectCommand(SwiftClientCfg cfg, String swiftCliPath, String container, String objectName, long size) {
+        String cmd = getSwiftObjectCmd(cfg, swiftCliPath, "upload", container, objectName);
+        if(size > SWIFT_MAX_SIZE){
+            return cmd + WITH_SEGMENTS;
+        }
+        return cmd;
+    }
+
+    public static String getContainerName(String type, Long id) {
+        if (type.startsWith("T")) {
+            return "T-" + id;
+        } else if (type.startsWith("S")) {
+            return "S-" + id;
+        } else if (type.startsWith("V")) {
+            return "V-" + id;
+        }
+        return null;
+    }
+
+    public static String[] splitSwiftPath(String path) {
+        int index = path.indexOf(File.separator);
+        if (index == -1) {
+            return null;
+        }
+        String[] paths = new String[2];
+        paths[0] = path.substring(0, index);
+        paths[1] = path.substring(index + 1);
+        return paths;
+    }
+
+    private static String getMeta(Map<String, String> metas) {
+        StringBuilder cms = new StringBuilder();
+        for (Map.Entry<String, String> entry : metas.entrySet()) {
+            cms.append(" -m ");
+            cms.append(entry.getKey());
+            cms.append(":");
+            cms.append(entry.getValue());
+            cms.append(" ");
+        }
+        return cms.toString();
     }
 }

utils/src/test/java/com/cloud/utils/SwiftUtilTest.java

@@ -19,9 +19,15 @@
 
 package com.cloud.utils;
 
-
+import static org.mockito.BDDMockito.given;
-import org.junit.Test;
+import static org.mockito.BDDMockito.mock;
-import org.mockito.Mockito;
+import static org.junit.Assert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
 
 import java.io.File;
 import java.net.URL;
@@ -29,10 +35,10 @@ import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.security.SignatureException;
 
-import static org.junit.Assert.assertArrayEquals;
+import org.junit.Test;
-import static org.junit.Assert.assertEquals;
+import org.mockito.Mockito;
-import static org.junit.Assert.assertTrue;
+
-import static org.mockito.Mockito.when;
+import com.cloud.utils.SwiftUtil.SwiftClientCfg;
 
 public class SwiftUtilTest {
 
@@ -90,4 +96,134 @@ public class SwiftUtilTest {
 
         assertEquals(expected, output);
     }
+
+    @Test
+    public void testGetSwiftCmd() {
+        SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+        given(cfg.getEndPoint()).willReturn("swift.endpoint");
+        given(cfg.getAccount()).willReturn("cs");
+        given(cfg.getUserName()).willReturn("sec-storage");
+        given(cfg.getKey()).willReturn("mypassword");
+        given(cfg.getStoragePolicy()).willReturn(null);
+
+        String cmd = SwiftUtil.getSwiftCmd(cfg, "swift", "stat");
+
+        String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword stat";
+        assertThat(cmd, is(equalTo(expected)));
+    }
+
+    @Test
+    public void testGetSwiftObjectCmd() {
+        SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+        given(cfg.getEndPoint()).willReturn("swift.endpoint");
+        given(cfg.getAccount()).willReturn("cs");
+        given(cfg.getUserName()).willReturn("sec-storage");
+        given(cfg.getKey()).willReturn("mypassword");
+        given(cfg.getStoragePolicy()).willReturn(null);
+
+        String objectCmd = SwiftUtil.getSwiftObjectCmd(cfg, "swift", "delete", "T-123", "template.vhd");
+
+        String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword delete T-123 template.vhd";
+        assertThat(objectCmd, is(equalTo(expected)));
+    }
+
+    @Test
+    public void testGetSwiftContainerCmd() {
+        SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+        given(cfg.getEndPoint()).willReturn("swift.endpoint");
+        given(cfg.getAccount()).willReturn("cs");
+        given(cfg.getUserName()).willReturn("sec-storage");
+        given(cfg.getKey()).willReturn("mypassword");
+        given(cfg.getStoragePolicy()).willReturn(null);
+
+        String containerCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-123");
+
+        String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-123";
+        assertThat(containerCmd, is(equalTo(expected)));
+    }
+
+    @Test
+    public void testGetUploadCmd() {
+        SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+        given(cfg.getEndPoint()).willReturn("swift.endpoint");
+        given(cfg.getAccount()).willReturn("cs");
+        given(cfg.getUserName()).willReturn("sec-storage");
+        given(cfg.getKey()).willReturn("mypassword");
+        given(cfg.getStoragePolicy()).willReturn(null);
+
+        String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 1024);
+
+        String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd";
+        assertThat(uploadCmd, is(equalTo(expected)));
+    }
+
+    @Test
+    public void testGetUploadCmdWithSegmentsBecauseOfSize() {
+        SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+        given(cfg.getEndPoint()).willReturn("swift.endpoint");
+        given(cfg.getAccount()).willReturn("cs");
+        given(cfg.getUserName()).willReturn("sec-storage");
+        given(cfg.getKey()).willReturn("mypassword");
+        given(cfg.getStoragePolicy()).willReturn(null);
+
+        String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 5368709121L);
+
+        String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd -S 5368709120";
+        assertThat(uploadCmd, is(equalTo(expected)));
+    }
+
+    @Test
+    public void testGetUploadCmdWithStoragePolicy() {
+        SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+        given(cfg.getEndPoint()).willReturn("swift.endpoint");
+        given(cfg.getAccount()).willReturn("cs");
+        given(cfg.getUserName()).willReturn("sec-storage");
+        given(cfg.getKey()).willReturn("mypassword");
+        given(cfg.getStoragePolicy()).willReturn("policy1");
+
+        String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 1024L);
+        String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy \"policy1\"";
+        assertThat(uploadCmd, is(equalTo(expected)));
+    }
+
+    @Test
+    public void testGetUploadCmdWithSegmentsAndStoragePolicy() {
+        SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+        given(cfg.getEndPoint()).willReturn("swift.endpoint");
+        given(cfg.getAccount()).willReturn("cs");
+        given(cfg.getUserName()).willReturn("sec-storage");
+        given(cfg.getKey()).willReturn("mypassword");
+        given(cfg.getStoragePolicy()).willReturn("policy1");
+        String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 5368709121L);
+        String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy \"policy1\" -S 5368709120";
+        assertThat(uploadCmd, is(equalTo(expected)));
+    }
+
+    @Test
+    public void testListContainerCmdWithStoragePolicyButNotSupportedByOperation() {
+        SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+        given(cfg.getEndPoint()).willReturn("swift.endpoint");
+        given(cfg.getAccount()).willReturn("cs");
+        given(cfg.getUserName()).willReturn("sec-storage");
+        given(cfg.getKey()).willReturn("mypassword");
+        given(cfg.getStoragePolicy()).willReturn("policy1");
+
+        String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-1");
+        String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-1";
+        assertThat(uploadCmd, is(equalTo(expected)));
+    }
+
+    @Test
+    public void testListContainerCmdWithoutStoragePolicy() {
+        SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+        given(cfg.getEndPoint()).willReturn("swift.endpoint");
+        given(cfg.getAccount()).willReturn("cs");
+        given(cfg.getUserName()).willReturn("sec-storage");
+        given(cfg.getKey()).willReturn("mypassword");
+        given(cfg.getStoragePolicy()).willReturn(null);
+
+        String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-1");
+        String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-1";
+        assertThat(uploadCmd, is(equalTo(expected)));
+    }
 }