diff --git a/api/src/main/java/com/cloud/agent/api/to/SwiftTO.java b/api/src/main/java/com/cloud/agent/api/to/SwiftTO.java index c7a98660302..b89dfea40e0 100644 --- a/api/src/main/java/com/cloud/agent/api/to/SwiftTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/SwiftTO.java @@ -26,17 +26,19 @@ public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg { String userName; String key; + String storagePolicy; private static final String pathSeparator = "/"; public SwiftTO() { } - public SwiftTO(Long id, String url, String account, String userName, String key) { + public SwiftTO(Long id, String url, String account, String userName, String key, String storagePolicy) { this.id = id; this.url = url; this.account = account; this.userName = userName; this.key = key; + this.storagePolicy = storagePolicy; } public Long getId() { @@ -63,6 +65,11 @@ public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg { return key; } + @Override + public String getStoragePolicy() { + return this.storagePolicy; + } + @Override public DataStoreRole getRole() { return DataStoreRole.Image; diff --git a/api/src/main/java/com/cloud/network/NetworkModel.java b/api/src/main/java/com/cloud/network/NetworkModel.java index 5e9839449c8..35a6a63a992 100644 --- a/api/src/main/java/com/cloud/network/NetworkModel.java +++ b/api/src/main/java/com/cloud/network/NetworkModel.java @@ -59,6 +59,9 @@ public interface NetworkModel { String SERVICE_OFFERING_FILE = "service-offering"; String AVAILABILITY_ZONE_FILE = "availability-zone"; String LOCAL_HOSTNAME_FILE = "local-hostname"; + String LOCAL_IPV4_FILE = "local-ipv4"; + String PUBLIC_HOSTNAME_FILE = "public-hostname"; + String PUBLIC_IPV4_FILE = "public-ipv4"; String INSTANCE_ID_FILE = "instance-id"; String VM_ID_FILE = "vm-id"; String PUBLIC_KEYS_FILE = "public-keys"; @@ -309,8 +312,8 @@ public interface NetworkModel { boolean getNetworkEgressDefaultPolicy(Long networkId); - List generateVmData(String userData, String serviceOffering, String zoneName, - String vmName, long vmId, String publicKey, String password, Boolean isWindows); + List generateVmData(String userData, String serviceOffering, long datacenterId, + String vmName, long vmId, String vmUuid, String guestIpAddress, String publicKey, String password, Boolean isWindows); String getValidNetworkCidr(Network guestNetwork); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 56d5957f022..3a5c16f4c90 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -289,6 +289,7 @@ public class ApiConstants { public static final String STATE = "state"; public static final String STATUS = "status"; public static final String STORAGE_TYPE = "storagetype"; + public static final String STORAGE_POLICY = "storagepolicy"; public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled"; public static final String STORAGE_CAPABILITIES = "storagecapabilities"; public static final String SYSTEM_VM_TYPE = "systemvmtype"; diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index fb1c1b41933..1d430c6f253 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -2542,16 +2542,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (defaultNic != null) { UserVmVO userVm = _userVmDao.findById(vm.getId()); Map details = _vmDetailsDao.listDetailsKeyPairs(vm.getId()); - vm.setDetails(details); + userVm.setDetails(details); Network network = _networkModel.getNetwork(defaultNic.getNetworkId()); if (_networkModel.isSharedNetworkWithoutServices(network.getId())) { final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText(); - final String zoneName = _dcDao.findById(vm.getDataCenterId()).getName(); boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); - vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, zoneName, vm.getInstanceName(), vm.getId(), - (String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows); + vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getId(), + vm.getUuid(), defaultNic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows); String vmName = vm.getInstanceName(); String configDriveIsoRootFolder = "/tmp"; String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso"; diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java index b575fdb4f02..d2ce43e3229 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java @@ -18,23 +18,24 @@ */ package com.cloud.hypervisor.xenserver.resource; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.to.DataObjectType; -import com.cloud.agent.api.to.DataStoreTO; -import com.cloud.agent.api.to.DataTO; -import com.cloud.agent.api.to.DiskTO; -import com.cloud.agent.api.to.NfsTO; -import com.cloud.agent.api.to.S3TO; -import com.cloud.agent.api.to.SwiftTO; -import com.cloud.exception.InternalErrorException; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.Storage; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.resource.StorageProcessor; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.storage.S3.ClientOptions; +import static com.cloud.utils.ReflectUtil.flattenProperties; +import static com.google.common.collect.Lists.newArrayList; + +import java.io.File; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import org.apache.commons.lang3.BooleanUtils; +import org.apache.log4j.Logger; +import org.apache.xmlrpc.XmlRpcException; + +import com.google.common.annotations.VisibleForTesting; import com.xensource.xenapi.Connection; import com.xensource.xenapi.SR; import com.xensource.xenapi.Types; @@ -44,6 +45,7 @@ import com.xensource.xenapi.Types.XenAPIException; import com.xensource.xenapi.VBD; import com.xensource.xenapi.VDI; import com.xensource.xenapi.VM; + import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; @@ -65,20 +67,24 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.log4j.Logger; -import org.apache.xmlrpc.XmlRpcException; -import java.io.File; -import java.net.URI; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import static com.cloud.utils.ReflectUtil.flattenProperties; -import static com.google.common.collect.Lists.newArrayList; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DiskTO; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.S3TO; +import com.cloud.agent.api.to.SwiftTO; +import com.cloud.exception.InternalErrorException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Storage; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.resource.StorageProcessor; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.storage.S3.ClientOptions; public class XenServerStorageProcessor implements StorageProcessor { private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class); @@ -914,20 +920,55 @@ public class XenServerStorageProcessor implements StorageProcessor { private boolean swiftUpload(final Connection conn, final SwiftTO swift, final String container, final String ldir, final String lfilename, final Boolean isISCSI, final int wait) { - String result = null; + + List params = getSwiftParams(swift, container, ldir, lfilename, isISCSI); + try { - result = - hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, "op", "upload", "url", swift.getUrl(), "account", swift.getAccount(), "username", - swift.getUserName(), "key", swift.getKey(), "container", container, "ldir", ldir, "lfilename", lfilename, "isISCSI", isISCSI.toString()); - if (result != null && result.equals("true")) { - return true; - } + String result = hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, params.toArray(new String[params.size()])); + return BooleanUtils.toBoolean(result); } catch (final Exception e) { s_logger.warn("swift upload failed due to " + e.toString(), e); } return false; } + @VisibleForTesting + List getSwiftParams(SwiftTO swift, String container, String ldir, String lfilename, Boolean isISCSI) { + // ORDER IS IMPORTANT + List params = new ArrayList<>(); + + //operation + params.add("op"); + params.add("upload"); + + //auth + params.add("url"); + params.add(swift.getUrl()); + params.add("account"); + params.add(swift.getAccount()); + params.add("username"); + params.add(swift.getUserName()); + params.add("key"); + params.add(swift.getKey()); + + // object info + params.add("container"); + params.add(container); + params.add("ldir"); + params.add(ldir); + params.add("lfilename"); + params.add(lfilename); + params.add("isISCSI"); + params.add(isISCSI.toString()); + + if (swift.getStoragePolicy() != null) { + params.add("storagepolicy"); + params.add(swift.getStoragePolicy()); + } + + return params; + } + protected String deleteSnapshotBackup(final Connection conn, final String localMountPoint, final String path, final String secondaryStorageMountPath, final String backupUUID) { // If anybody modifies the formatting below again, I'll skin them diff --git a/plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java b/plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java new file mode 100644 index 00000000000..75ac8b8ad4d --- /dev/null +++ b/plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.hypervisor.xenserver.resource; + +import static org.mockito.Mockito.when; + +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import com.cloud.agent.api.to.SwiftTO; + +public class XenServerStorageProcessorTest { + @Test + public void testOrderOfSwiftUplodScriptParamsWithoutStoragePolicy() { + CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class); + XenServerStorageProcessor mock = new XenServerStorageProcessor(resource); + + SwiftTO swift = Mockito.mock(SwiftTO.class); + when(swift.getStoragePolicy()).thenReturn(null); + + String container = "sample-container-name"; + String ldir = "sample-ldir"; + String lfilename = "sample-lfilename"; + Boolean isISCSI = true; + + List params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI); + + // make sure the params not null and has correct number of items in it + Assert.assertNotNull("params is null", params); + Assert.assertTrue("Expected param list size is 18 but it was" + params.size(), params.size() == 18); + + // check the order of params + Assert.assertEquals("unexpected param.", "op", params.get(0)); + Assert.assertEquals("unexpected param.", "upload", params.get(1)); + Assert.assertEquals("unexpected param.", "url", params.get(2)); + Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3)); + Assert.assertEquals("unexpected param.", "account", params.get(4)); + Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5)); + Assert.assertEquals("unexpected param.", "username", params.get(6)); + Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7)); + Assert.assertEquals("unexpected param.", "key", params.get(8)); + Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9)); + Assert.assertEquals("unexpected param.", "container", params.get(10)); + Assert.assertEquals("unexpected param.", container, params.get(11)); + Assert.assertEquals("unexpected param.", "ldir", params.get(12)); + Assert.assertEquals("unexpected param.", ldir, params.get(13)); + Assert.assertEquals("unexpected param.", "lfilename", params.get(14)); + Assert.assertEquals("unexpected param.", lfilename, params.get(15)); + Assert.assertEquals("unexpected param.", "isISCSI", params.get(16)); + Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17)); + } + + @Test + public void testOrderOfSwiftUplodScriptParamsWithStoragePolicy() { + CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class); + XenServerStorageProcessor mock = new XenServerStorageProcessor(resource); + + SwiftTO swift = Mockito.mock(SwiftTO.class); + when(swift.getStoragePolicy()).thenReturn("sample-storagepolicy"); + + String container = "sample-container-name"; + String ldir = "sample-ldir"; + String lfilename = "sample-lfilename"; + Boolean isISCSI = true; + + List params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI); + + // make sure the params not null and has correct number of items in it + Assert.assertNotNull("params is null", params); + Assert.assertTrue("Expected param list size is 20 but it was" + params.size(), params.size() == 20); + + // check the order of params + Assert.assertEquals("unexpected param.", "op", params.get(0)); + Assert.assertEquals("unexpected param.", "upload", params.get(1)); + Assert.assertEquals("unexpected param.", "url", params.get(2)); + Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3)); + Assert.assertEquals("unexpected param.", "account", params.get(4)); + Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5)); + Assert.assertEquals("unexpected param.", "username", params.get(6)); + Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7)); + Assert.assertEquals("unexpected param.", "key", params.get(8)); + Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9)); + Assert.assertEquals("unexpected param.", "container", params.get(10)); + Assert.assertEquals("unexpected param.", container, params.get(11)); + Assert.assertEquals("unexpected param.", "ldir", params.get(12)); + Assert.assertEquals("unexpected param.", ldir, params.get(13)); + Assert.assertEquals("unexpected param.", "lfilename", params.get(14)); + Assert.assertEquals("unexpected param.", lfilename, params.get(15)); + Assert.assertEquals("unexpected param.", "isISCSI", params.get(16)); + Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17)); + Assert.assertEquals("unexpected param.", "storagepolicy", params.get(18)); + Assert.assertEquals("unexpected param.", "sample-storagepolicy", params.get(19)); + } +} diff --git a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java index 2816c6063f6..7e1486214bc 100644 --- a/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java +++ b/plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java @@ -68,7 +68,7 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl { public DataStoreTO getStoreTO(DataStore store) { ImageStoreImpl imgStore = (ImageStoreImpl)store; Map details = _imageStoreDetailsDao.getDetails(imgStore.getId()); - return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY)); + return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY), details.get(ApiConstants.STORAGE_POLICY)); } @Override diff --git a/scripts/storage/secondary/swift b/scripts/storage/secondary/swift index 4138db8b17e..c09c53037f8 100755 --- a/scripts/storage/secondary/swift +++ b/scripts/storage/secondary/swift @@ -1473,8 +1473,9 @@ post [options] [container] [object] Updates meta information for the account, container, or object depending on the args given. If the container is not found, it will be created automatically; but this is not true for accounts and objects. Containers - also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m - or --meta option is allowed on all and used to define the user meta data + also allow the -r (or --read-acl) and -w (or --write-acl) options. + The --storage-policy will set a storage policy to the container if the container does not exist. + The -m or --meta option is allowed on all and used to define the user meta data items to set in the form Name:Value. This option can be repeated. Example: post -m Color:Blue -m Size:Large'''.strip('\n') @@ -1493,6 +1494,8 @@ def st_post(options, args, print_queue, error_queue): parser.add_option('-m', '--meta', action='append', dest='meta', default=[], help='Sets a meta data item with the syntax name:value. This option ' 'may be repeated. Example: -m Color:Blue -m Size:Large') + parser.add_option('', '--storage-policy', action='store', dest='storage_policy', + help='Sets a storage policy to the container if the container does not exist') (options, args) = parse_args(parser, args) args = args[1:] if (options.read_acl or options.write_acl or options.sync_to or @@ -1529,6 +1532,8 @@ def st_post(options, args, print_queue, error_queue): headers['X-Container-Sync-To'] = options.sync_to if options.sync_key is not None: headers['X-Container-Sync-Key'] = options.sync_key + if options.storage_policy is not None: + headers['X-Storage-Policy'] = options.storage_policy try: conn.post_container(args[0], headers=headers) except ClientException, err: @@ -1558,7 +1563,8 @@ upload [options] container file_or_directory [file_or_directory] [...] Uploads to the given container the files and directories specified by the remaining args. -c or --changed is an option that will only upload files that have changed since the last upload. -S or --segment-size - and --leave-segments are options as well (see --help for more). + and --leave-segments are options as well (see --help for more). --storage-policy + Sets a storage policy to the container if the container does not exist. '''.strip('\n') @@ -1576,6 +1582,8 @@ def st_upload(options, args, print_queue, error_queue): dest='leave_segments', default=False, help='Indicates that you want ' 'the older segments of manifest objects left alone (in the case of ' 'overwrites)') + parser.add_option('', '--storage-policy', action='store', dest='storage_policy', + help='Sets a storage policy to the container if the container does not exist') (options, args) = parse_args(parser, args) args = args[1:] if len(args) < 2: @@ -1749,9 +1757,12 @@ def st_upload(options, args, print_queue, error_queue): # permissions, so we'll ignore any error. If there's really a problem, # it'll surface on the first object PUT. try: - conn.put_container(args[0]) + container_headers = {} + if options.storage_policy is not None: + container_headers['X-Storage-Policy'] = options.storage_policy + conn.put_container(args[0],headers=container_headers) if options.segment_size is not None: - conn.put_container(args[0] + '_segments') + conn.put_container(args[0] + '_segments',headers=container_headers) except Exception: pass try: diff --git a/scripts/vm/hypervisor/xenserver/swift b/scripts/vm/hypervisor/xenserver/swift index c9d2cebb04f..603bfdc6bce 100755 --- a/scripts/vm/hypervisor/xenserver/swift +++ b/scripts/vm/hypervisor/xenserver/swift @@ -1475,8 +1475,9 @@ post [options] [container] [object] Updates meta information for the account, container, or object depending on the args given. If the container is not found, it will be created automatically; but this is not true for accounts and objects. Containers - also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m - or --meta option is allowed on all and used to define the user meta data + also allow the -r (or --read-acl) and -w (or --write-acl) options. + The --storage-policy will set a storage policy to the container if the container does not exist. + The -m or --meta option is allowed on all and used to define the user meta data items to set in the form Name:Value. This option can be repeated. Example: post -m Color:Blue -m Size:Large'''.strip('\n') @@ -1495,6 +1496,8 @@ def st_post(options, args, print_queue, error_queue): parser.add_option('-m', '--meta', action='append', dest='meta', default=[], help='Sets a meta data item with the syntax name:value. This option ' 'may be repeated. Example: -m Color:Blue -m Size:Large') + parser.add_option('', '--storage-policy', action='store', dest='storage_policy', + help='Sets a storage policy to the container if the container does not exist') (options, args) = parse_args(parser, args) args = args[1:] if (options.read_acl or options.write_acl or options.sync_to or @@ -1531,6 +1534,8 @@ def st_post(options, args, print_queue, error_queue): headers['X-Container-Sync-To'] = options.sync_to if options.sync_key is not None: headers['X-Container-Sync-Key'] = options.sync_key + if options.storage_policy is not None: + headers['X-Storage-Policy'] = options.storage_policy try: conn.post_container(args[0], headers=headers) except ClientException, err: @@ -1560,7 +1565,8 @@ upload [options] container file_or_directory [file_or_directory] [...] Uploads to the given container the files and directories specified by the remaining args. -c or --changed is an option that will only upload files that have changed since the last upload. -S or --segment-size - and --leave-segments are options as well (see --help for more). + and --leave-segments are options as well (see --help for more). --storage-policy + Sets a storage policy to the container if the container does not exist. '''.strip('\n') @@ -1578,6 +1584,8 @@ def st_upload(options, args, print_queue, error_queue): dest='leave_segments', default=False, help='Indicates that you want ' 'the older segments of manifest objects left alone (in the case of ' 'overwrites)') + parser.add_option('', '--storage-policy', action='store', dest='storage_policy', + help='Sets a storage policy to the container if the container does not exist') (options, args) = parse_args(parser, args) args = args[1:] if len(args) < 2: @@ -1751,9 +1759,12 @@ def st_upload(options, args, print_queue, error_queue): # permissions, so we'll ignore any error. If there's really a problem, # it'll surface on the first object PUT. try: - conn.put_container(args[0]) + container_headers = {} + if options.storage_policy is not None: + container_headers['X-Storage-Policy'] = options.storage_policy + conn.put_container(args[0],headers=container_headers) if options.segment_size is not None: - conn.put_container(args[0] + '_segments') + conn.put_container(args[0] + '_segments',headers=container_headers) except Exception: pass try: diff --git a/server/src/main/java/com/cloud/network/NetworkModelImpl.java b/server/src/main/java/com/cloud/network/NetworkModelImpl.java index 380aabf493a..f67c3697e9d 100644 --- a/server/src/main/java/com/cloud/network/NetworkModelImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkModelImpl.java @@ -22,6 +22,7 @@ import java.security.InvalidParameterException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -29,24 +30,24 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.Collections; + import javax.inject.Inject; import javax.naming.ConfigurationException; -import org.apache.commons.codec.binary.Base64; import org.apache.log4j.Logger; +import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; - -import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao; + import com.cloud.api.ApiDBUtils; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenterVO; import com.cloud.dc.PodVlanMapVO; import com.cloud.dc.Vlan; import com.cloud.dc.Vlan.VlanType; @@ -2344,18 +2345,47 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi } @Override - public List generateVmData(String userData, String serviceOffering, String zoneName, - String vmName, long vmId, String publicKey, String password, Boolean isWindows) { + public List generateVmData(String userData, String serviceOffering, long datacenterId, + String vmName, long vmId, String vmUuid, + String guestIpAddress, String publicKey, String password, Boolean isWindows) { + + DataCenterVO dcVo = _dcDao.findById(datacenterId); + final String zoneName = dcVo.getName(); + + IPAddressVO publicIp = _ipAddressDao.findByAssociatedVmId(vmId); + final List vmData = new ArrayList(); if (userData != null) { - vmData.add(new String[]{USERDATA_DIR, USERDATA_FILE, new String(Base64.decodeBase64(userData),StringUtils.getPreferredCharset())}); + vmData.add(new String[]{USERDATA_DIR, USERDATA_FILE, userData}); } vmData.add(new String[]{METATDATA_DIR, SERVICE_OFFERING_FILE, StringUtils.unicodeEscape(serviceOffering)}); vmData.add(new String[]{METATDATA_DIR, AVAILABILITY_ZONE_FILE, StringUtils.unicodeEscape(zoneName)}); vmData.add(new String[]{METATDATA_DIR, LOCAL_HOSTNAME_FILE, StringUtils.unicodeEscape(vmName)}); - vmData.add(new String[]{METATDATA_DIR, INSTANCE_ID_FILE, vmName}); - vmData.add(new String[]{METATDATA_DIR, VM_ID_FILE, String.valueOf(vmId)}); + vmData.add(new String[]{METATDATA_DIR, LOCAL_IPV4_FILE, guestIpAddress}); + + String publicIpAddress = guestIpAddress; + String publicHostName = StringUtils.unicodeEscape(vmName); + + if (dcVo.getNetworkType() != DataCenter.NetworkType.Basic) { + if (publicIp != null) { + publicIpAddress = publicIp.getAddress().addr(); + publicHostName = publicIp.getAddress().addr(); + } else { + publicHostName = null; + } + } + vmData.add(new String[]{METATDATA_DIR, PUBLIC_IPV4_FILE, publicIpAddress}); + vmData.add(new String[]{METATDATA_DIR, PUBLIC_HOSTNAME_FILE, publicHostName}); + + if (vmUuid == null) { + vmData.add(new String[]{METATDATA_DIR, INSTANCE_ID_FILE, vmName}); + vmData.add(new String[]{METATDATA_DIR, VM_ID_FILE, String.valueOf(vmId)}); + } else { + vmData.add(new String[]{METATDATA_DIR, INSTANCE_ID_FILE, vmUuid}); + vmData.add(new String[]{METATDATA_DIR, VM_ID_FILE, vmUuid}); + } + vmData.add(new String[]{METATDATA_DIR, PUBLIC_KEYS_FILE, publicKey}); String cloudIdentifier = _configDao.getValue("cloud.identifier"); diff --git a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java index 37a30ef6010..cc1df935388 100644 --- a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java +++ b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java @@ -206,30 +206,37 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle return false; } + private String getSshKey(VirtualMachineProfile profile) { + UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), "SSH.PublicKey"); + return (vmDetailSshKey!=null ? vmDetailSshKey.getValue() : null); + } + @Override public boolean addPasswordAndUserdata(Network network, NicProfile nic, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException { - UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), "SSH.PublicKey"); - return (canHandle(network.getTrafficType()) && updateConfigDrive(profile, - (vmDetailSshKey!=null?vmDetailSshKey.getValue():null))) + String sshPublicKey = getSshKey(profile); + return (canHandle(network.getTrafficType()) + && updateConfigDrive(profile, sshPublicKey, nic)) && updateConfigDriveIso(network, profile, dest.getHost(), false); } @Override public boolean savePassword(Network network, NicProfile nic, VirtualMachineProfile profile) throws ResourceUnavailableException { - if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, (String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey)))) return false; + String sshPublicKey = getSshKey(profile); + if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, sshPublicKey, nic))) return false; return updateConfigDriveIso(network, profile, true); } @Override public boolean saveSSHKey(Network network, NicProfile nic, VirtualMachineProfile vm, String sshPublicKey) throws ResourceUnavailableException { - if (!(canHandle(network.getTrafficType()) && updateConfigDrive(vm, sshPublicKey))) return false; + if (!(canHandle(network.getTrafficType()) && updateConfigDrive(vm, sshPublicKey, nic))) return false; return updateConfigDriveIso(network, vm, true); } @Override public boolean saveUserData(Network network, NicProfile nic, VirtualMachineProfile profile) throws ResourceUnavailableException { - if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, (String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey)))) return false; + String sshPublicKey = getSshKey(profile); + if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, sshPublicKey, nic))) return false; return updateConfigDriveIso(network, profile, true); } @@ -330,7 +337,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle Answer createIsoAnswer = endpoint.sendMessage(configDriveIsoCommand); if (!createIsoAnswer.getResult()) { throw new ResourceUnavailableException(String.format("%s ISO failed, details: %s", - (update?"Update":"Create"), createIsoAnswer.getDetails()),ConfigDriveNetworkElement.class,0L); + (update?"Update":"Create"), createIsoAnswer.getDetails()), ConfigDriveNetworkElement.class, 0L); } configureConfigDriveDisk(profile, secondaryStore); @@ -363,7 +370,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle } } - private boolean updateConfigDrive(VirtualMachineProfile profile, String publicKey) { + private boolean updateConfigDrive(VirtualMachineProfile profile, String publicKey, NicProfile nic) { UserVmVO vm = _userVmDao.findById(profile.getId()); if (vm.getType() != VirtualMachine.Type.User) { return false; @@ -372,11 +379,10 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle Nic defaultNic = _networkModel.getDefaultNic(vm.getId()); if (defaultNic != null) { final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText(); - final String zoneName = _dcDao.findById(vm.getDataCenterId()).getName(); boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); - List vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, zoneName, vm.getInstanceName(), vm.getId(), - publicKey, (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows); + List vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getId(), + vm.getUuid(), nic.getIPv4Address(), publicKey, (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows); profile.setVmData(vmData); profile.setConfigDriveLabel(VirtualMachineManager.VmConfigDriveLabel.value()); } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index df81dd3ff2b..7558419661e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4085,11 +4085,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir Network network = _networkModel.getNetwork(defaultNic.getNetworkId()); if (_networkModel.isSharedNetworkWithoutServices(network.getId())) { final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText(); - final String zoneName = _dcDao.findById(vm.getDataCenterId()).getName(); boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); - List vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, zoneName, vm.getInstanceName(), vm.getId(), - (String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows); + List vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getId(), + vm.getUuid(), defaultNic.getIPv4Address(), vm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows); String vmName = vm.getInstanceName(); String configDriveIsoRootFolder = "/tmp"; String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso"; diff --git a/server/src/test/java/com/cloud/network/MockNetworkModelImpl.java b/server/src/test/java/com/cloud/network/MockNetworkModelImpl.java index 3c3fd7097cd..f8580056e22 100644 --- a/server/src/test/java/com/cloud/network/MockNetworkModelImpl.java +++ b/server/src/test/java/com/cloud/network/MockNetworkModelImpl.java @@ -898,7 +898,7 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel { } @Override - public List generateVmData(String userData, String serviceOffering, String zoneName, String vmName, long vmId, String publicKey, String password, Boolean isWindows) { + public List generateVmData(String userData, String serviceOffering, long datacenterId, String vmName, long vmId, String vmUuid, String guestIpAddress, String publicKey, String password, Boolean isWindows) { return null; } diff --git a/server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java b/server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java index 11da24f95df..7d5041536f0 100644 --- a/server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java +++ b/server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java @@ -36,7 +36,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import org.apache.xerces.impl.dv.util.Base64; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; @@ -64,6 +63,8 @@ import com.cloud.host.dao.HostDao; import com.cloud.network.Network; import com.cloud.network.NetworkModelImpl; import com.cloud.network.Networks; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkServiceMapDao; import com.cloud.network.dao.NetworkVO; @@ -77,6 +78,7 @@ import com.cloud.storage.dao.GuestOSDao; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.fsm.StateListener; import com.cloud.utils.fsm.StateMachine2; +import com.cloud.utils.net.Ip; import com.cloud.vm.Nic; import com.cloud.vm.NicProfile; import com.cloud.vm.UserVmDetailVO; @@ -99,7 +101,7 @@ public class ConfigDriveNetworkElementTest { private final String VMINSTANCENAME = "vm_name"; private final String VMOFFERING = "custom_instance"; private final long VMID = 30L; - private final String VMUSERDATA = "userdata"; + private final String VMUSERDATA = "H4sIABCvw1oAAystTi1KSSxJ5AIAUPllwQkAAAA="; private final long SOID = 31L; private final long HOSTID = NETWORK_ID; private final String HOSTNAME = "host1"; @@ -116,6 +118,7 @@ public class ConfigDriveNetworkElementTest { @Mock private UserVmDetailsDao _userVmDetailsDao; @Mock private NetworkDao _networkDao; @Mock private NetworkServiceMapDao _ntwkSrvcDao; + @Mock private IPAddressDao _ipAddressDao; @Mock private DataCenterVO dataCenterVO; @Mock private DataStore dataStore; @@ -130,6 +133,7 @@ public class ConfigDriveNetworkElementTest { @Mock private NicProfile nicp; @Mock private ServiceOfferingVO serviceOfferingVO; @Mock private UserVmVO virtualMachine; + @Mock private IPAddressVO publicIp; @InjectMocks private final ConfigDriveNetworkElement _configDrivesNetworkElement = new ConfigDriveNetworkElement(); @InjectMocks @Spy private NetworkModelImpl _networkModel = new NetworkModelImpl(); @@ -161,7 +165,7 @@ public class ConfigDriveNetworkElementTest { when(virtualMachine.getServiceOfferingId()).thenReturn(SOID); when(virtualMachine.getDataCenterId()).thenReturn(DATACENTERID); when(virtualMachine.getInstanceName()).thenReturn(VMINSTANCENAME); - when(virtualMachine.getUserData()).thenReturn(Base64.encode(VMUSERDATA.getBytes())); + when(virtualMachine.getUserData()).thenReturn(VMUSERDATA); when(deployDestination.getHost()).thenReturn(hostVO); when(hostVO.getId()).thenReturn(HOSTID); when(nic.isDefaultNic()).thenReturn(true); @@ -236,6 +240,71 @@ public class ConfigDriveNetworkElementTest { @Test public void testAddPasswordAndUserdata() throws InsufficientCapacityException, ResourceUnavailableException { + List actualVmData = getVmData(); + + assertThat(actualVmData, containsInAnyOrder( + new String[]{"userdata", "user_data", VMUSERDATA}, + new String[]{"metadata", "service-offering", VMOFFERING}, + new String[]{"metadata", "availability-zone", ZONENAME}, + new String[]{"metadata", "local-hostname", VMINSTANCENAME}, + new String[]{"metadata", "local-ipv4", "192.168.111.111"}, + new String[]{"metadata", "public-hostname", null}, + new String[]{"metadata", "public-ipv4", "192.168.111.111"}, + new String[]{"metadata", "vm-id", String.valueOf(VMID)}, + new String[]{"metadata", "instance-id", VMINSTANCENAME}, + new String[]{"metadata", "public-keys", PUBLIC_KEY}, + new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)}, + new String[]{PASSWORD, "vm_password", PASSWORD} + )); + } + + @Test + public void testAddPasswordAndUserdataStaticNat() throws InsufficientCapacityException, ResourceUnavailableException { + when(_ipAddressDao.findByAssociatedVmId(VMID)).thenReturn(publicIp); + when(publicIp.getAddress()).thenReturn(new Ip("7.7.7.7")); + + List actualVmData = getVmData(); + + assertThat(actualVmData, containsInAnyOrder( + new String[]{"userdata", "user_data", VMUSERDATA}, + new String[]{"metadata", "service-offering", VMOFFERING}, + new String[]{"metadata", "availability-zone", ZONENAME}, + new String[]{"metadata", "local-hostname", VMINSTANCENAME}, + new String[]{"metadata", "local-ipv4", "192.168.111.111"}, + new String[]{"metadata", "public-hostname", "7.7.7.7"}, + new String[]{"metadata", "public-ipv4", "7.7.7.7"}, + new String[]{"metadata", "vm-id", String.valueOf(VMID)}, + new String[]{"metadata", "instance-id", VMINSTANCENAME}, + new String[]{"metadata", "public-keys", PUBLIC_KEY}, + new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)}, + new String[]{PASSWORD, "vm_password", PASSWORD} + )); + } + + + @Test + public void testAddPasswordAndUserdataUuid() throws InsufficientCapacityException, ResourceUnavailableException { + when(virtualMachine.getUuid()).thenReturn("vm-uuid"); + + List actualVmData = getVmData(); + + assertThat(actualVmData, containsInAnyOrder( + new String[]{"userdata", "user_data", VMUSERDATA}, + new String[]{"metadata", "service-offering", VMOFFERING}, + new String[]{"metadata", "availability-zone", ZONENAME}, + new String[]{"metadata", "local-hostname", VMINSTANCENAME}, + new String[]{"metadata", "local-ipv4", "192.168.111.111"}, + new String[]{"metadata", "public-hostname", null}, + new String[]{"metadata", "public-ipv4", "192.168.111.111"}, + new String[]{"metadata", "vm-id", "vm-uuid"}, + new String[]{"metadata", "instance-id", "vm-uuid"}, + new String[]{"metadata", "public-keys", PUBLIC_KEY}, + new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)}, + new String[]{PASSWORD, "vm_password", PASSWORD} + )); + } + + private List getVmData() throws InsufficientCapacityException, ResourceUnavailableException { final Answer answer = mock(Answer.class); final UserVmDetailVO userVmDetailVO = mock(UserVmDetailVO.class); when(endpoint.sendMessage(any(HandleConfigDriveIsoCommand.class))).thenReturn(answer); @@ -243,6 +312,7 @@ public class ConfigDriveNetworkElementTest { when(network.getTrafficType()).thenReturn(Networks.TrafficType.Guest); when(virtualMachine.getState()).thenReturn(VirtualMachine.State.Stopped); when(userVmDetailVO.getValue()).thenReturn(PUBLIC_KEY); + when(nicp.getIPv4Address()).thenReturn("192.168.111.111"); when(_userVmDetailsDao.findDetail(anyLong(), anyString())).thenReturn(userVmDetailVO); Map parms = Maps.newHashMap(); parms.put(VirtualMachineProfile.Param.VmPassword, PASSWORD); @@ -254,19 +324,6 @@ public class ConfigDriveNetworkElementTest { ArgumentCaptor commandCaptor = ArgumentCaptor.forClass(HandleConfigDriveIsoCommand.class); verify(endpoint, times(1)).sendMessage(commandCaptor.capture()); HandleConfigDriveIsoCommand result = commandCaptor.getValue(); - List actualVmData = result.getVmData(); - - assertThat(actualVmData, containsInAnyOrder( - new String[]{"userdata", "user_data", VMUSERDATA}, - new String[]{"metadata", "service-offering", VMOFFERING}, - new String[]{"metadata", "availability-zone", ZONENAME}, - new String[]{"metadata", "local-hostname", VMINSTANCENAME}, - new String[]{"metadata", "vm-id", String.valueOf(VMID)}, - new String[]{"metadata", "instance-id", String.valueOf(VMINSTANCENAME)}, - new String[]{"metadata", "public-keys", PUBLIC_KEY}, - new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)}, - new String[]{PASSWORD, "vm_password", PASSWORD} - )); - + return result.getVmData(); } } diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkModelImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkModelImpl.java index 50d9b0f6425..dc2aab4792c 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkModelImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkModelImpl.java @@ -913,7 +913,7 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel { } @Override - public List generateVmData(String userData, String serviceOffering, String zoneName, String vmName, long vmId, String publicKey, String password, Boolean isWindows) { + public List generateVmData(String userData, String serviceOffering, long datacenterId, String vmName, long vmId, String vmUuid, String guestIpAddress, String publicKey, String password, Boolean isWindows) { return null; } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 1f81f6aa841..da3ab32e203 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -40,6 +40,7 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; import java.net.InetAddress; import java.net.URI; @@ -52,6 +53,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; + import javax.naming.ConfigurationException; import io.netty.bootstrap.ServerBootstrap; @@ -68,6 +70,7 @@ import io.netty.handler.codec.http.HttpResponseEncoder; import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; @@ -155,11 +158,11 @@ import com.cloud.agent.api.to.DatadiskTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; +import com.cloud.configuration.Resource; import com.cloud.exception.InternalErrorException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; import com.cloud.host.Host.Type; -import com.cloud.configuration.Resource; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.network.NetworkModel; import com.cloud.resource.ServerResourceBase; @@ -190,9 +193,6 @@ import com.cloud.utils.script.Script; import com.cloud.utils.storage.S3.S3Utils; import com.cloud.vm.SecondaryStorageVm; - -import java.io.OutputStreamWriter; - public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource { public static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class); @@ -492,8 +492,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S if (typeFolder.exists() || typeFolder.mkdirs()) { if (StringUtils.isNotEmpty(content)) { File file = new File(typeFolder, fileName + ".txt"); - try (FileWriter fw = new FileWriter(file); BufferedWriter bw = new BufferedWriter(fw)) { - bw.write(content); + try { + if (fileName.equals(USERDATA_FILE)) { + // User Data is passed as a base64 encoded string + FileUtils.writeByteArrayToFile(file, Base64.decodeBase64(content)); + } else { + FileUtils.write(file, content, com.cloud.utils.StringUtils.getPreferredCharset()); + } } catch (IOException ex) { s_logger.error("Failed to create file ", ex); return new Answer(cmd, ex); diff --git a/test/integration/component/test_configdrive.py b/test/integration/component/test_configdrive.py index ad9ad8fbcab..364bec22404 100644 --- a/test/integration/component/test_configdrive.py +++ b/test/integration/component/test_configdrive.py @@ -55,13 +55,17 @@ import tempfile import socket import base64 import sys +import time import os +NO_SUCH_FILE = "No such file or directory" + class MySSHKeyPair: """Manage SSH Key pairs""" def __init__(self, items): + self.private_key_file = None self.__dict__.update(items) @classmethod @@ -166,11 +170,7 @@ class Services: } -class TestConfigDrive(cloudstackTestCase): - """Test user data and password reset functionality - using configDrive - """ - +class ConfigDriveUtils: class CreateResult: def __init__(self, success, offering=None, network=None, vpc=None): self.success = success @@ -192,6 +192,626 @@ class TestConfigDrive(cloudstackTestCase): self.password = password self.presence = True + def updateTemplate(self, value): + """Updates value of the guest VM template's password enabled setting + """ + self.debug("Updating value of guest VM template's password enabled " + "setting") + cmd = updateTemplate.updateTemplateCmd() + cmd.id = self.template.id + cmd.passwordenabled = value + self.api_client.updateTemplate(cmd) + list_template_response = list_templates(self.api_client, + templatefilter="all", + id=self.template.id + ) + self.template = list_template_response[0] + self.debug("Updated guest VM template") + + def get_userdata_url(self, vm): + """Returns user data URL for the given VM object""" + self.debug("Getting user data url") + nic = vm.nic[0] + gateway = str(nic.gateway) + self.debug("Gateway: " + gateway) + user_data_url = 'curl "http://' + gateway + ':80/latest/user-data"' + return user_data_url + + def validate_firewall_rule(self, fw_rule): + pass + + def validate_StaticNat_rule_For_VM(self, public_ip, network, vm): + self.validate_PublicIPAddress( + public_ip, network, static_nat=True, vm=vm) + + def create_and_verify_fip_and_fw(self, vm, public_ip, network): + """ + Creates and verifies (Ingress) firewall rule + with a Static NAT rule enabled public IP""" + + self.debug("Creating and verifying firewall rule") + self.create_StaticNatRule_For_VM(vm, public_ip, network) + + # Verification + self.validate_StaticNat_rule_For_VM(public_ip, network, vm) + + fw_rule = self.create_FirewallRule(public_ip, self.test_data["ingress_rule"]) + self.validate_firewall_rule(fw_rule) + self.debug("Successfully created and verified firewall rule") + + def mount_config_drive(self, ssh): + """ + This method is to verify whether configdrive iso + is attached to vm or not + Returns mount path if config drive is attached else False + """ + mountdir = "/root/iso" + cmd = "blkid -t LABEL='config-2' /dev/sr? /dev/hd? /dev/sd? /dev/xvd? -o device" + tmp_cmd = [ + 'bash -c "if [ ! -d /root/iso ] ; then mkdir /root/iso ; fi"', + "umount /root/iso"] + for tcmd in tmp_cmd: + ssh.execute(tcmd) + configDrive = ssh.execute(cmd) + res = ssh.execute("mount {} {}".format(str(configDrive[0]), mountdir)) + if str(res).lower().find("mounting read-only") > -1: + self.debug("configDrive iso is mounted at location %s" % mountdir) + return mountdir + else: + return None + + def _get_config_drive_data(self, ssh, file, name, fail_on_missing=True): + """Fetches the content of a file file on the config drive + + :param ssh: SSH connection to the VM + :param file: path to the file to fetch + :param name: description of the file + :param fail_on_missing: + whether the test should fail if the file is missing + :type ssh: marvin.sshClient.SshClient + :type file: str + :type name: str + :type fail_on_missing: bool + :returns: the content of the file + :rtype: str + """ + cmd = "cat %s" % file + res = ssh.execute(cmd) + content = '\n'.join(res) + + if fail_on_missing and NO_SUCH_FILE in content: + self.debug("{} is not found".format(name)) + self.fail("{} is not found".format(name)) + + return content + + def verify_config_drive_data(self, ssh, file, expected_content, name): + """Verifies that the file contains the expected content + + :param ssh: SSH connection to the VM + :param file: path to the file to verify + :param expected_content: + :param name: + :type ssh: marvin.sshClient.SshClient + :type file: str + :type expected_content: str + :type name: str + """ + actual_content = self._get_config_drive_data(ssh, file, name) + + self.debug("Expected {}: {}".format(name, expected_content)) + self.debug("Actual {}: {}".format(name, actual_content)) + + self.assertEqual(expected_content, actual_content, + 'Userdata found: %s is not equal to expected: %s' + % (actual_content, expected_content)) + + def verifyUserData(self, ssh, iso_path, userdata): + """ + verify Userdata + + :param ssh: SSH connection to the VM + :param iso_path: mount point of the config drive + :param userdata: Expected userdata + :type ssh: marvin.sshClient.SshClient + :type iso_path: str + :type userdata: str + """ + self.verify_config_drive_data( + ssh, + iso_path + "/cloudstack/userdata/user_data.txt", + userdata, + "userdata (ACS)" + ) + + def verifyOpenStackUserData(self, ssh, iso_path, userdata): + """ + verify Userdata in Openstack format + + :param ssh: SSH connection to the VM + :param iso_path: mount point of the config drive + :param userdata: Expected userdata + :type ssh: marvin.sshClient.SshClient + :type iso_path: str + :type userdata: str + """ + self.verify_config_drive_data( + ssh, + iso_path + "/openstack/latest/user_data", + userdata, + "userdata (Openstack)" + ) + + def verifyPassword(self, ssh, iso_path, password): + self.debug("Expected VM password is %s " % password.password) + password_file = iso_path + "/cloudstack/password/vm_password.txt" + vmpassword = self._get_config_drive_data(ssh, password_file, + "ConfigDrive password", + fail_on_missing=False) + + self.debug("ConfigDrive password is %s " % vmpassword) + + if NO_SUCH_FILE in vmpassword: + self.debug("Password file is not found") + return False, False + elif (password.password is not None) \ + and (password.password in vmpassword): + self.debug("Expected Password is found in configDriveIso") + return True, True + else: + self.debug("Expected password is not found in configDriveIso") + return True, False + + def verifySshKey(self, ssh, iso_path, ssh_key): + self.debug("Expected VM sshkey is %s " % ssh_key.name) + publicKey_file = iso_path + "/cloudstack/metadata/public-keys.txt" + cmd = "ssh-keygen -lf %s | cut -f2 -d' '" % publicKey_file + res = ssh.execute(cmd) + vmsshkey = str(res[0]) + + self.debug("ConfigDrive ssh key is %s " % vmsshkey) + + if NO_SUCH_FILE in vmsshkey: + self.fail("SSH keyfile is not found") + + self.assertEqual( + vmsshkey, + ssh_key.fingerprint, + "Fingerprint of authorized key does not match ssh key fingerprint" + ) + + def verifyMetaData(self, vm, ssh, iso_path): + """ + verify metadata files in CloudStack format + + :param vm: the VM + :param ssh: SSH connection to the VM + :param iso_path: mount point of the config drive + :type vm: VirtualMachine + :type ssh: marvin.sshClient.SshClient + :type iso_path: str + """ + + metadata_dir = iso_path + "/cloudstack/metadata/" + vm_files = ["availability-zone.txt", + "service-offering.txt", + "instance-id.txt", + "vm-id.txt", + "local-hostname.txt", + "local-ipv4.txt", + "public-ipv4.txt"] + + get_name = lambda file: \ + "{} metadata".format(file.split('.'[-1].replace('-', ' '))) + + metadata = {vm_file: + self._get_config_drive_data(ssh, + metadata_dir + vm_file, + get_name(vm_file)) + for vm_file in vm_files} + + self.assertEqual( + str(metadata["availability-zone.txt"]), + self.zone.name, + "Zone name inside metadata does not match with the zone" + ) + self.assertEqual( + str(metadata["local-hostname.txt"]), + vm.instancename, + "vm name inside metadata does not match with the " + "instance name" + ) + self.assertEqual( + str(metadata["vm-id.txt"]), + vm.id, + "vm name inside metadata does not match with the " + "instance name" + ) + self.assertEqual( + str(metadata["instance-id.txt"]), + vm.id, + "vm name inside metadata does not match with the " + "instance name" + ) + self.assertEqual( + str(metadata["service-offering.txt"]), + vm.serviceofferingname, + "Service offering inside metadata does not match " + "with the instance offering" + ) + return + + def verifyOpenStackData(self, ssh, iso_path): + """ + verify existence of metadata and user data files in OpenStack format + + :param ssh: SSH connection to the VM + :param iso_path: mount point of the config drive + :type ssh: marvin.sshClient.SshClient + :type iso_path: str + """ + openstackdata_dir = iso_path + "/openstack/latest/" + openstackdata_files = ["meta_data.json", + "vendor_data.json", + "network_data.json"] + for file in openstackdata_files: + res = ssh.execute("cat %s" % openstackdata_dir + file) + if NO_SUCH_FILE in res[0]: + self.fail("{} file not found in vm openstack".format(file)) + + def generate_ssh_keys(self): + """Generates ssh key pair + + Writes the private key into a temp file and returns the file name + + :returns: path of the private key file + + """ + self.keypair = MySSHKeyPair.create( + self.api_client, + name=random_gen() + ".pem", + account=self.account.user[0].account, + domainid=self.account.domainid) + + self.cleanup.append(self.keypair) + self.debug("Created keypair with name: %s" % self.keypair.name) + self.debug("Writing the private key to local file") + pkfile = tempfile.gettempdir() + os.sep + self.keypair.name + self.keypair.private_key_file = pkfile + self.tmp_files.append(pkfile) + self.debug("File path: %s" % pkfile) + with open(pkfile, "w+") as f: + f.write(self.keypair.privatekey) + os.chmod(pkfile, 0o400) + + return self.keypair + + def umount_config_drive(self, ssh, iso_path): + """unmount config drive inside guest vm + + :param ssh: SSH connection to the VM + :param iso_path: mount point of the config drive + :type ssh: marvin.sshClient.SshClient + :type iso_path: str + """ + ssh.execute("umount -d %s" % iso_path) + # Give the VM time to unlock the iso device + time.sleep(2) + # Verify umount + result = ssh.execute("ls %s" % iso_path) + self.assertTrue(len(result) == 0, + "After umount directory should be empty " + "but contains: %s" % result) + + def update_provider_state(self, new_state): + """ + Enables or disables the ConfigDrive Service Provider + + :param new_state: "Enabled" | "Disabled" + :type new_state: str + :return: original state + :rtype: str + """ + self.debug("Updating Service Provider ConfigDrive to %s" % new_state) + configdriveprovider = NetworkServiceProvider.list( + self.api_client, + name="ConfigDrive", + physicalnetworkid=self.vsp_physical_network.id)[0] + orig_state = configdriveprovider.state + NetworkServiceProvider.update(self.api_client, + configdriveprovider.id, + state=new_state) + self.validate_NetworkServiceProvider("ConfigDrive", state=new_state) + return orig_state + + def verify_network_creation(self, offering=None, + offering_name=None, + gateway=None, + vpc=None, acl_list=None, testdata=None): + """ + Creates a network + + :param offering: Network Offering + :type offering: NetworkOffering + :param offering_name: Offering name + :type offering_name: Optional[str] + :param gateway: gateway + :type gateway: str + :param vpc: in case of a VPC tier, the parent VPC + :type vpc: VPC + :param acl_list: in case of a VPC tier, the acl list + :type acl_list: NetworkACLList + :param testdata: Test data + :type testdata: dict + :return: Network Creation Result + :rtype: CreateResult + """ + if offering is None: + self.debug("Creating Nuage VSP network offering...") + offering = self.create_NetworkOffering( + self.test_data["nuagevsp"][offering_name]) + self.validate_NetworkOffering(offering, state="Enabled") + try: + network = self.create_Network(offering, + gateway=gateway, + vpc=vpc, + acl_list=acl_list, + testdata=testdata) + return self.CreateResult(True, offering=offering, network=network) + except Exception: + self.debug("Exception: %s" % sys.exc_info()[0]) + return self.CreateResult(False, offering=offering) + + def verify_vpc_creation(self, offering=None, offering_name=None): + """ + Creates a VPC + + :param offering: VPC Offering + :type offering: VpcOffering + :param offering_name: Offering name + :type offering_name: Optional[str] + :return: VPC Creation Result + :rtype: CreateResult + """ + if offering is None: + self.debug("Creating Nuage VSP VPC offering...") + offering = self.create_VpcOffering( + self.test_data["nuagevsp"][offering_name]) + self.validate_VpcOffering(offering, state="Enabled") + try: + vpc = self.create_Vpc(offering, cidr='10.1.0.0/16') + self.validate_Vpc(vpc, state="Enabled") + return self.CreateResult(True, offering=offering, vpc=vpc) + except Exception: + return self.CreateResult(False, offering=offering) + + def update_password_enable_in_template(self, new_state): + self.debug("Updating guest VM template to password %s" % new_state) + orig_state = self.template.passwordenabled + if self.template.passwordenabled is not new_state: + self.updateTemplate(new_state) + self.assertEqual(self.template.passwordenabled, new_state, + "Guest VM template is not password enabled") + return orig_state + + def verify_config_drive_content(self, vm, + public_ip, + password_test, + userdata=None, + metadata=False, + ssh_key=None, + ssh_client=None): + """Verify Config Drive Content + + :param vm: + :param public_ip: + :param password_test: + :param userdata: + :param metadata: + :param ssh_key: + :param ssh_client: SSH Connection + :type vm: + :type public_ip: + :type password_test: + :type userdata: object + :type metadata: + :type ssh_key: + :type ssh_client: + :return: SSH Connection + """ + + if self.isSimulator: + self.debug( + "Simulator Environment: Skipping Config Drive content verification") + return + + self.debug("SSHing into the VM %s" % vm.name) + if ssh_client is None: + ssh = self.ssh_into_VM(vm, public_ip, keypair=ssh_key) + else: + ssh = ssh_client + d = {x.name: x for x in ssh.logger.handlers} + ssh.logger.handlers = list(d.values()) + config_drive_path = self.mount_config_drive(ssh) + self.assertIsNotNone(config_drive_path, + 'ConfigdriveIso is not attached to vm') + if metadata: + self.debug("Verifying metadata for vm: %s" % vm.name) + self.verifyMetaData(vm, ssh, config_drive_path) + self.debug("Verifying openstackdata for vm: %s" % vm.name) + self.verifyOpenStackData(ssh, config_drive_path) + + if userdata is not None: + self.debug("Verifying userdata for vm: %s" % vm.name) + self.verifyUserData(ssh, config_drive_path, userdata) + self.verifyOpenStackUserData(ssh, config_drive_path, userdata) + + if password_test.test_presence: + self.debug("Verifying password for vm: %s" % vm.name) + test_result = self.verifyPassword(ssh, config_drive_path, + password_test) + self.assertEqual(test_result[0], password_test.presence, + "Expected is that password is present: %s " + " but found is: %s" + % (test_result[0], password_test.presence)) + + if password_test.password is not None: + self.debug("Password for vm is %s" % password_test.password) + self.assertEqual(test_result[1], True, + "Password value test failed.") + if ssh_key is not None: + self.debug("Verifying sshkey for vm: %s" % vm.name) + self.verifySshKey(ssh, config_drive_path, ssh_key) + + self.umount_config_drive(ssh, config_drive_path) + return ssh + + def create_guest_vm(self, networks, acl_item=None, + vpc=None, keypair=None): + vm = self.create_VM( + networks, + testdata=self.test_data["virtual_machine_userdata"], + keypair=keypair) + # Check VM + self.check_VM_state(vm, state="Running") + self.verify_vsd_vm(vm) + # Check networks + network_list = [] + if isinstance(networks, list): + for network in networks: + network_list.append(network) + else: + network_list.append(networks) + + for network in network_list: + self.validate_Network(network, state="Implemented") + self.verify_vsd_network(self.domain.id, network, vpc=vpc) + + if acl_item is not None: + self.verify_vsd_firewall_rule(acl_item) + return vm + + # nic_operation_VM - Performs NIC operations such as add, remove, and + # update default NIC in the given VM and network + def nic_operation_VM(self, vm, network, operation="add"): + self.debug("Performing %s NIC operation in VM with ID - %s and " + "network with ID - %s" % (operation, vm.id, network.id)) + if operation is "add": + vm.add_nic(self.api_client, network.id) + self.debug("Added NIC in VM with ID - %s and network with ID - %s" + % (vm.id, network.id)) + vm_info = VirtualMachine.list(self.api_client, id=vm.id)[0] + for nic in vm_info.nic: + if nic.networkid == network.id: + nic_id = nic.id + if operation is "update": + vm.update_default_nic(self.api_client, nic_id) + self.debug("Updated default NIC to NIC with ID- %s in VM with ID " + "- %s and network with ID - %s" % + (nic_id, vm.id, network.id)) + if operation is "remove": + vm.remove_nic(self.api_client, nic_id) + self.debug("Removed NIC with ID - %s in VM with ID - %s and " + "network with ID - %s" % (nic_id, vm.id, network.id)) + + def update_userdata(self, vm, new_user_data): + """Updates the user data of a VM + + :param vm: the Virtual Machine + :param new_user_data: UserData to set + :type vm: VirtualMachine + :type new_user_data: str + :returns: User data in base64 format + :rtype: str + """ + updated_user_data = base64.b64encode(new_user_data) + vm.update(self.api_client, userdata=updated_user_data) + return new_user_data + + def reset_password(self, vm): + vm.password = vm.resetPassword(self.api_client) + self.debug("Password reset to - %s" % vm.password) + self.debug("VM - %s password - %s !" % + (vm.name, vm.password)) + + def wait_until_done(self, thread_list, name): + for aThread in thread_list: + self.debug("[Concurrency]Join %s for vm %s" % (name, + aThread.get_vm())) + aThread.join() + + def update_sshkeypair(self, vm): + """ + + :type vm: VirtualMachine + """ + vm.stop(self.api_client) + vm_new_ssh = vm.resetSshKey(self.api_client, + keypair=self.keypair.name, + account=self.account.user[0].account, + domainid=self.account.domainid) + + self.debug("Sshkey reset to - %s" % self.keypair.name) + vm.start(self.api_client) + + # reset SSH key also resets the password. + # the new password is available in VM detail, + # named "Encrypted.Password". + # It is encrypted using the SSH Public Key, + # and thus can be decrypted using the SSH Private Key + + try: + from base64 import b64decode + from Crypto.PublicKey import RSA + from Crypto.Cipher import PKCS1_v1_5 + with open(self.keypair.private_key_file, "r") as pkfile: + key = RSA.importKey(pkfile.read()) + cipher = PKCS1_v1_5.new(key) + new_password = cipher.decrypt( + b64decode(vm_new_ssh.details['Encrypted.Password']), None) + if new_password: + vm.password = new_password + else: + self.debug("Failed to decrypt new password") + except: + self.debug("Failed to decrypt new password") + + + def add_subnet_verify(self, network, services): + """verify required nic is present in the VM""" + + self.debug("Going to add new ip range in shared network %s" % + network.name) + cmd = createVlanIpRange.createVlanIpRangeCmd() + cmd.networkid = network.id + cmd.gateway = services["gateway"] + cmd.netmask = services["netmask"] + cmd.startip = services["startip"] + cmd.endip = services["endip"] + cmd.forVirtualNetwork = services["forvirtualnetwork"] + addedsubnet = self.api_client.createVlanIpRange(cmd) + + self.debug("verify above iprange is successfully added in shared " + "network %s or not" % network.name) + + cmd1 = listVlanIpRanges.listVlanIpRangesCmd() + cmd1.networkid = network.id + cmd1.id = addedsubnet.vlan.id + + allsubnets = self.api_client.listVlanIpRanges(cmd1) + self.assertEqual( + allsubnets[0].id, + addedsubnet.vlan.id, + "Check New subnet is successfully added to the shared Network" + ) + return addedsubnet + + +class TestConfigDrive(cloudstackTestCase, ConfigDriveUtils): + """Test user data and password reset functionality + using configDrive + """ + @classmethod def setUpClass(cls): # We want to fail quicker, if it's a failure @@ -266,31 +886,6 @@ class TestConfigDrive(cloudstackTestCase): self.debug("Cleanup complete!") return - # updateTemplate - Updates value of the guest VM template's password - # enabled setting - def updateTemplate(self, value): - self.debug("Updating value of guest VM template's password enabled " - "setting") - cmd = updateTemplate.updateTemplateCmd() - cmd.id = self.template.id - cmd.passwordenabled = value - self.api_client.updateTemplate(cmd) - list_template_response = list_templates(self.api_client, - templatefilter="all", - id=self.template.id - ) - self.template = list_template_response[0] - self.debug("Updated guest VM template") - - # get_userdata_url - Returns user data URL for the given VM object - def get_userdata_url(self, vm): - self.debug("Getting user data url") - nic = vm.nic[0] - gateway = str(nic.gateway) - self.debug("Gateway: " + gateway) - user_data_url = 'curl "http://' + gateway + ':80/latest/user-data"' - return user_data_url - # create_StaticNatRule_For_VM - Creates Static NAT rule on the given # public IP for the given VM in the given network def create_StaticNatRule_For_VM(self, vm, public_ip, network, @@ -310,39 +905,6 @@ class TestConfigDrive(cloudstackTestCase): (public_ip.ipaddress.ipaddress, vm.id, network.id)) return static_nat_rule - # validate_PublicIPAddress - Validates if the given public IP address is in - # the expected state form the list of fetched public IP addresses - def validate_PublicIPAddress(self, public_ip, network, static_nat=False, - vm=None): - """Validates the Public IP Address""" - self.debug("Validating the assignment and state of public IP address " - "- %s" % public_ip.ipaddress.ipaddress) - public_ips = PublicIPAddress.list(self.api_client, - id=public_ip.ipaddress.id, - networkid=network.id, - isstaticnat=static_nat, - listall=True - ) - self.assertEqual(isinstance(public_ips, list), True, - "List public IP for network should return a " - "valid list" - ) - self.assertEqual(public_ips[0].ipaddress, - public_ip.ipaddress.ipaddress, - "List public IP for network should list the assigned " - "public IP address" - ) - self.assertEqual(public_ips[0].state, "Allocated", - "Assigned public IP is not in the allocated state" - ) - if static_nat and vm: - self.assertEqual(public_ips[0].virtualmachineid, vm.id, - "Static NAT rule is not enabled for the VM on " - "the assigned public IP" - ) - self.debug("Successfully validated the assignment and state of public " - "IP address - %s" % public_ip.ipaddress.ipaddress) - # create_FirewallRule - Creates (Ingress) Firewall rule on the given # Static NAT rule enabled public IP for Isolated networks def create_FirewallRule(self, public_ip, rule=None): @@ -358,149 +920,6 @@ class TestConfigDrive(cloudstackTestCase): endport=rule["endport"] ) - # create_and_verify_fw - Creates and verifies (Ingress) firewall rule - # with a Static NAT rule enabled public IP - def create_and_verify_fip_and_fw(self, vm, public_ip, network): - self.debug("Creating and verifying firewall rule") - self.create_StaticNatRule_For_VM(vm, public_ip, network) - - # Verification - self.validate_PublicIPAddress( - public_ip, network, static_nat=True, vm=vm) - - self.create_FirewallRule(public_ip, self.test_data["ingress_rule"]) - self.debug("Successfully created and verified firewall rule") - - def getConfigDriveContent(self, ssh): - """ - This method is to verify whether configdrive iso - is attached to vm or not - Returns mount path if config drive is attached else False - """ - mountdir = "/root/iso" - cmd = "blkid -t LABEL='config-2' /dev/sr? /dev/hd? /dev/sd? /dev/xvd? -o device" - tmp_cmd = [ - 'bash -c "if [ ! -d /root/iso ] ; then mkdir /root/iso ; fi"', - "umount /root/iso"] - for tcmd in tmp_cmd: - ssh.execute(tcmd) - configDrive = ssh.execute(cmd) - res = ssh.execute("mount {} {}".format(str(configDrive[0]), mountdir)) - if str(res).lower().find("mounting read-only") > -1: - self.debug("configDrive iso is mounted at location %s" % mountdir) - return mountdir - else: - return None - - def verifyUserData(self, ssh, iso_path, userdata): - """ - verify Userdata - """ - userdata_path = iso_path+"/cloudstack/userdata/user_data.txt" - cmd = "cat %s" % userdata_path - res = ssh.execute(cmd) - vmuserdata = str(res[0]) - self.debug("Expected userdata is %s" % userdata) - self.debug("ConfigDrive userdata is %s" % vmuserdata) - self.assertEqual(vmuserdata, userdata, - 'Userdata found: %s is not equal to expected: %s' - % (vmuserdata, userdata)) - - def verifyPassword(self, vm, ssh, iso_path, password): - self.debug("Expected VM password is %s " % password.password) - password_file = iso_path+"/cloudstack/password/vm_password.txt" - cmd = "cat %s" % password_file - res = ssh.execute(cmd) - vmpassword = str(res[0]) - self.debug("ConfigDrive password is %s " % vmpassword) - nosuchfile = "No such file or directory" - if nosuchfile in vmpassword: - self.debug("Password file is not found") - return False, False - elif (password.password is not None) \ - and (password.password in vmpassword): - self.debug("Expected Password is found in configDriveIso") - return True, True - else: - self.debug("Expected password is not found in configDriveIso") - return True, False - - def verifySshKey(self, ssh, iso_path, sshkey): - self.debug("Expected VM sshkey is %s " % sshkey) - publicKey_file = iso_path+"/cloudstack/metadata/public-keys.txt" - cmd = "cat %s" % publicKey_file - res = ssh.execute(cmd) - vmsshkey = str(res[0]) - self.debug("ConfigDrive ssh key is %s " % vmsshkey) - - def verifyMetaData(self, vm, ssh, iso_path): - - metadata_dir = iso_path+"/cloudstack/metadata/" - metadata = {} - vm_files = ["availability-zone.txt", - "instance-id.txt", - "service-offering.txt", - "vm-id.txt"] - for file in vm_files: - cmd = "cat %s" % metadata_dir+file - res = ssh.execute(cmd) - metadata[file] = res - - for mfile in vm_files: - if mfile not in metadata: - self.fail("{} file is not found in vm metadata".format(mfile)) - self.assertEqual( - str(metadata["availability-zone.txt"][0]), - self.zone.name, - "Zone name inside metadata does not match with the zone" - ) - self.assertEqual( - str(metadata["instance-id.txt"][0]), - vm.instancename, - "vm name inside metadata does not match with the " - "instance name" - ) - self.assertEqual( - str(metadata["service-offering.txt"][0]), - vm.serviceofferingname, - "Service offering inside metadata does not match " - "with the instance offering" - ) - return - - def generate_ssh_keys(self): - """ - This method generates ssh key pair and writes the private key - into a temp file and returns the file name - """ - self.keypair = MySSHKeyPair.create( - self.api_client, - name=random_gen() + ".pem", - account=self.account.user[0].account, - domainid=self.account.domainid) - - self.cleanup.append(self.keypair) - self.debug("Created keypair with name: %s" % self.keypair.name) - self.debug("Writing the private key to local file") - keyPairFilePath = tempfile.gettempdir() + os.sep + self.keypair.name - self.tmp_files.append(keyPairFilePath) - self.debug("File path: %s" % keyPairFilePath) - with open(keyPairFilePath, "w+") as f: - f.write(self.keypair.privatekey) - os.system("chmod 400 " + keyPairFilePath) - return keyPairFilePath - - def umountConfigDrive(self, ssh, iso_path): - """umount config drive iso attached inside guest vm""" - ssh.execute("umount -d %s" % iso_path) - # Give the VM time to unlock the iso device - # time.sleep(2) - # Verify umount - result = ssh.execute("ls %s" % iso_path) - self.assertTrue(len(result) == 0, - "After umount directory should be empty " - "but contains: %s" % result) - # validate_NetworkServiceProvider - Validates the given Network Service # Provider in the Nuage VSP Physical Network, matches the given provider # name and state against the list of providers fetched @@ -529,18 +948,6 @@ class TestConfigDrive(cloudstackTestCase): self.debug("Successfully validated the creation and state of Network " "Service Provider - %s" % provider_name) - def update_provider_state(self, new_state): - self.debug("Updating Service Provider ConfigDrive to %s" % new_state) - configdriveprovider = NetworkServiceProvider.list( - self.api_client, - name="ConfigDrive")[0] - orig_state = configdriveprovider.state - NetworkServiceProvider.update(self.api_client, - configdriveprovider.id, - state=new_state) - self.validate_NetworkServiceProvider("ConfigDrive", state=new_state) - return orig_state - # create_NetworkOffering - Creates Network offering def create_NetworkOffering(self, net_offering, suffix=None, conserve_mode=False): @@ -606,26 +1013,6 @@ class TestConfigDrive(cloudstackTestCase): self.debug("Created network with ID - %s" % network.id) return network - def verify_network_creation(self, offering=None, - offering_name=None, - gateway=None, - vpc=None, acl_list=None, testdata=None): - if offering is None: - self.debug("Creating network offering...") - offering = self.create_NetworkOffering( - self.test_data[offering_name]) - self.validate_NetworkOffering(offering, state="Enabled") - try: - network = self.create_Network(offering, - gateway=gateway, - vpc=vpc, - acl_list=acl_list, - testdata=testdata) - return self.CreateResult(True, offering=offering, network=network) - except Exception: - self.debug("Exception: %s" % sys.exc_info()[0]) - return self.CreateResult(False, offering=offering) - # create_VpcOffering - Creates VPC offering def create_VpcOffering(self, vpc_offering, suffix=None): self.debug("Creating VPC offering") @@ -707,29 +1094,6 @@ class TestConfigDrive(cloudstackTestCase): self.debug("Successfully validated the creation and state of VPC - %s" % vpc.name) - def verify_vpc_creation(self, offering=None, offering_name=None): - - if offering is None: - self.debug("Creating VPC offering...") - offering = self.create_VpcOffering( - self.test_data[offering_name]) - self.validate_VpcOffering(offering, state="Enabled") - try: - vpc = self.create_Vpc(offering, cidr='10.1.0.0/16') - self.validate_Vpc(vpc, state="Enabled") - return self.CreateResult(True, offering=offering, vpc=vpc) - except Exception: - return self.CreateResult(False, offering=offering) - - def update_password_enable_in_template(self, new_state): - self.debug("Updating guest VM template to password %s" % new_state) - orig_state = self.template.passwordenabled - if self.template.passwordenabled is not new_state: - self.updateTemplate(new_state) - self.assertEqual(self.template.passwordenabled, new_state, - "Guest VM template is not password enabled") - return orig_state - # ssh_into_VM - Gets into the shell of the given VM using its public IP def ssh_into_VM(self, vm, public_ip, reconnect=True, negative_test=False): self.debug("SSH into VM with ID - %s on public IP address - %s" % @@ -750,41 +1114,6 @@ class TestConfigDrive(cloudstackTestCase): return retry_ssh() - def verify_config_drive_content(self, vm, - public_ip, - password_test, - userdata=None, - metadata=False, - sshkey=None): - self.debug("SSHing into the VM %s" % vm.name) - ssh = self.ssh_into_VM(vm, public_ip) - config_drive_path = self.getConfigDriveContent(ssh) - self.assertIsNotNone(config_drive_path, - 'ConfigdriveIso is not attached to vm') - if metadata: - self.debug("Verifying metadata for vm: %s" % vm.name) - self.verifyMetaData(vm, ssh, config_drive_path) - if userdata is not None: - self.debug("Verifying userdata for vm: %s" % vm.name) - self.verifyUserData(ssh, config_drive_path, userdata) - if password_test.test_presence: - self.debug("Verifying password for vm: %s" % vm.name) - test_result = self.verifyPassword(vm, ssh, config_drive_path, - password_test) - self.assertEqual(test_result[0], password_test.presence, - "Expected is that password is present: %s " - " but found is: %s" - % (test_result[0], password_test.presence)) - if password_test.password is not None: - self.debug("Password for vm is %s" % password_test.password) - self.assertEqual(test_result[1], True, - "Password value test failed.") - if sshkey is not None: - self.debug("Verifying sshkey for vm: %s" % vm.name) - self.verifySshKey(ssh, config_drive_path, sshkey) - - self.umountConfigDrive(ssh, config_drive_path) - # create_VM - Creates VM in the given network(s) def create_VM(self, network_list, host_id=None, start_vm=True, testdata=None, account=None, keypair=None): @@ -858,116 +1187,6 @@ class TestConfigDrive(cloudstackTestCase): self.debug("Successfully validated the creation and state of Network " "- %s" % network.name) - def create_guest_vm(self, networks, acl_item=None, - vpc=None, keypair=None): - vm = self.create_VM( - networks, - testdata=self.test_data["virtual_machine_userdata"], - keypair=keypair) - - # Check VM - self.check_VM_state(vm, state="Running") - - # Check networks - network_list = [] - if isinstance(networks, list): - for network in networks: - network_list.append(network) - else: - network_list.append(networks) - - for network in network_list: - self.validate_Network(network, state="Implemented") - - return vm - - # nic_operation_VM - Performs NIC operations such as add, remove, and - # update default NIC in the given VM and network - def nic_operation_VM(self, vm, network, operation="add"): - self.debug("Performing %s NIC operation in VM with ID - %s and " - "network with ID - %s" % (operation, vm.id, network.id)) - if operation is "add": - vm.add_nic(self.api_client, network.id) - self.debug("Added NIC in VM with ID - %s and network with ID - %s" - % (vm.id, network.id)) - vm_info = VirtualMachine.list(self.api_client, id=vm.id)[0] - for nic in vm_info.nic: - if nic.networkid == network.id: - nic_id = nic.id - if operation is "update": - vm.update_default_nic(self.api_client, nic_id) - self.debug("Updated default NIC to NIC with ID- %s in VM with ID " - "- %s and network with ID - %s" % - (nic_id, vm.id, network.id)) - if operation is "remove": - vm.remove_nic(self.api_client, nic_id) - self.debug("Removed NIC with ID - %s in VM with ID - %s and " - "network with ID - %s" % (nic_id, vm.id, network.id)) - - def update_userdata(self, vm, expected_user_data): - updated_user_data = base64.b64encode(expected_user_data) - vm.update(self.api_client, userdata=updated_user_data) - return expected_user_data - - def reset_password(self, vm): - vm.password = vm.resetPassword(self.api_client) - self.debug("Password reset to - %s" % vm.password) - self.debug("VM - %s password - %s !" % - (vm.name, vm.password)) - - def wait_until_done(self, thread_list, name): - for aThread in thread_list: - self.debug("[Concurrency]Join %s for vm %s" % (name, - aThread.get_vm())) - aThread.join() - - def resetsshkey(self, vm, keypair, account=None, domainid=None): - """Resets SSH key""" - cmd = resetSSHKeyForVirtualMachine.resetSSHKeyForVirtualMachineCmd() - cmd.id = vm.id - cmd.keypair = keypair - cmd.account = account - cmd.domainid = domainid - return(self.api_client.resetSSHKeyForVirtualMachine(cmd)) - - def update_sshkeypair(self, vm): - vm.stop(self.api_client) - self.resetsshkey(vm, - self.keypair.name, - account=self.account.user[0].account, - domainid=self.account.domainid) - self.debug("Sshkey reset to - %s" % self.keypair.name) - vm.start(self.api_client) - - def add_subnet_verify(self, network, services): - """verify required nic is present in the VM""" - - self.debug("Going to add new ip range in shared network %s" % - network.name) - cmd = createVlanIpRange.createVlanIpRangeCmd() - cmd.networkid = network.id - cmd.gateway = services["gateway"] - cmd.netmask = services["netmask"] - cmd.startip = services["startip"] - cmd.endip = services["endip"] - cmd.forVirtualNetwork = services["forvirtualnetwork"] - addedsubnet = self.api_client.createVlanIpRange(cmd) - - self.debug("verify above iprange is successfully added in shared " - "network %s or not" % network.name) - - cmd1 = listVlanIpRanges.listVlanIpRangesCmd() - cmd1.networkid = network.id - cmd1.id = addedsubnet.vlan.id - - allsubnets = self.api_client.listVlanIpRanges(cmd1) - self.assertEqual( - allsubnets[0].id, - addedsubnet.vlan.id, - "Check New subnet is successfully added to the shared Network" - ) - return addedsubnet - # get_Router - Returns router for the given network def get_Router(self, network): self.debug("Finding the virtual router for network with ID - %s" % @@ -1129,7 +1348,7 @@ class TestConfigDrive(cloudstackTestCase): metadata=True, userdata=self.test_data[ "virtual_machine_userdata"]["userdata"], - sshkey=self.keypair.name) + ssh_key=self.keypair) expected_user_data1 = self.update_userdata(vm1, "helloworld vm1") self.verify_config_drive_content(vm1, public_ip_1, @@ -1147,7 +1366,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm1.password), metadata=True, userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Adding a non-default nic to the VM " "making it a multi-nic VM...") @@ -1157,7 +1376,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm1.password), metadata=True, userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) vm1.password = vm1.resetPassword(self.api_client) self.debug("Password reset to - %s" % vm1.password) self.debug("VM - %s password - %s !" % @@ -1168,7 +1387,7 @@ class TestConfigDrive(cloudstackTestCase): self.verify_config_drive_content(vm1, public_ip_1, self.PasswordTest(vm1.password), userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("updating non-default nic as the default nic " "of the multi-nic VM and enable staticnat...") @@ -1253,7 +1472,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm1.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ Restarting the created Isolated network with " "cleanup...") @@ -1264,7 +1483,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm1.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++Verifying userdata after rebootVM - %s" % vm1.name) vm1.reboot(self.api_client) @@ -1272,14 +1491,14 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm1.password), metadata=True, userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata for VM - %s" % vm1.name) expected_user_data1 = self.update_userdata(vm1, "hello afterboot") self.verify_config_drive_content(vm1, public_ip_1, self.PasswordTest(vm1.password), userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Resetting password for VM - %s" % vm1.name) self.reset_password(vm1) self.debug("SSHing into the VM for verifying its new password " @@ -1294,7 +1513,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm1.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata after migrating VM - %s" % vm1.name) expected_user_data1 = self.update_userdata(vm1, @@ -1316,7 +1535,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(False), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata for VM - %s" % vm1.name) expected_user_data1 = self.update_userdata(vm1, @@ -1340,7 +1559,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(False), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.update_provider_state("Disabled") expected_user_data1 = self.update_userdata(vm1, "hello after recover") @@ -1348,7 +1567,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(False), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ When template is not password enabled, " "verify configdrive of VM - %s" % vm1.name) @@ -1370,7 +1589,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(False), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) vm1.delete(self.api_client, expunge=True) create_network1.network.delete(self.api_client) @@ -1514,21 +1733,21 @@ class TestConfigDrive(cloudstackTestCase): self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(True), metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) expected_user_data = self.update_userdata(vm, "helloworld vm1") self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(True), metadata=True, userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Resetting password for VM - %s" % vm.name) self.reset_password(vm) self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(vm.password), userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.generate_ssh_keys() self.update_sshkeypair(vm) @@ -1542,7 +1761,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm.password), metadata=True, userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ Restarting the created vpc without " "cleanup...") @@ -1552,7 +1771,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm.password), userdata=expected_user_data, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Adding a non-default nic to the VM " "making it a multi-nic VM...") @@ -1562,7 +1781,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm.password), metadata=True, userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) vpc_public_ip_2 = \ self.acquire_PublicIPAddress(create_tiernetwork2.network, create_vpc.vpc) @@ -1576,12 +1795,12 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm.password), metadata=True, userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) expected_user_data1 = self.update_userdata(vm, "hellomultinicvm1") self.verify_config_drive_content(vm, vpc_public_ip_2, self.PasswordTest(vm.password), userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("updating non-default nic as the default nic " "of the multi-nic VM and enable staticnat...") @@ -1628,7 +1847,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ Restarting the created vpc with " "cleanup...") @@ -1638,7 +1857,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ Restarting the created VPC Tier network without " "cleanup...") @@ -1649,7 +1868,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ Restarting the created VPC Tier network with " "cleanup...") @@ -1660,7 +1879,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ Restarting the created vpc without " "cleanup...") @@ -1686,7 +1905,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm.password), metadata=True, userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata for VM - %s" % vm.name) expected_user_data = self.update_userdata(vm, @@ -1694,7 +1913,7 @@ class TestConfigDrive(cloudstackTestCase): self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(vm.password), userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Resetting password for VM - %s" % vm.name) self.reset_password(vm) self.debug("SSHing into the VM for verifying its new password " @@ -1709,7 +1928,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(vm.password), userdata=expected_user_data, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata after migrating VM - %s" % vm.name) expected_user_data = self.update_userdata(vm, @@ -1717,7 +1936,7 @@ class TestConfigDrive(cloudstackTestCase): self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(vm.password), userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Resetting password for VM - %s" % vm.name) self.reset_password(vm) self.debug("SSHing into the VM for verifying its new password " @@ -1732,7 +1951,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(False), userdata=expected_user_data, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata for VM - %s" % vm.name) expected_user_data = self.update_userdata(vm, @@ -1740,7 +1959,7 @@ class TestConfigDrive(cloudstackTestCase): self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(False), userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Resetting password for VM - %s" % vm.name) self.reset_password(vm) self.debug("SSHing into the VM for verifying its new password " @@ -1757,13 +1976,13 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(False), userdata=expected_user_data, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.update_provider_state("Disabled") self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(False), userdata=expected_user_data, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ When template is not password enabled " "verify configdrive of VM - %s" % vm.name) @@ -1789,7 +2008,7 @@ class TestConfigDrive(cloudstackTestCase): self.PasswordTest(False), userdata=expected_user_data, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) vm.delete(self.api_client, expunge=True) create_tiernetwork.network.delete(self.api_client) diff --git a/test/integration/plugins/nuagevsp/nuageTestCase.py b/test/integration/plugins/nuagevsp/nuageTestCase.py index 7e6468b77dd..9d842aa3a93 100644 --- a/test/integration/plugins/nuagevsp/nuageTestCase.py +++ b/test/integration/plugins/nuagevsp/nuageTestCase.py @@ -733,8 +733,12 @@ class nuageTestCase(cloudstackTestCase): traffictype=traffic_type ) - # ssh_into_VM - Gets into the shell of the given VM using its public IP - def ssh_into_VM(self, vm, public_ip, reconnect=True, negative_test=False): + def ssh_into_VM(self, vm, public_ip, reconnect=True, negative_test=False, keypair=None): + """Creates a SSH connection to the VM + + :returns: the SSH connection + :rtype: marvin.sshClient.SshClient + """ if self.isSimulator: self.debug("Simulator Environment: Skipping ssh into VM") return @@ -748,7 +752,8 @@ class nuageTestCase(cloudstackTestCase): ssh_client = vm.get_ssh_client( ipaddress=public_ip.ipaddress.ipaddress, reconnect=reconnect, - retries=3 if negative_test else 30 + retries=3 if negative_test else 30, + keyPairFileLocation=keypair.private_key_file if keypair else None ) self.debug("Successful to SSH into VM with ID - %s on " "public IP address - %s" % diff --git a/test/integration/plugins/nuagevsp/nuage_lib.py b/test/integration/plugins/nuagevsp/nuage_lib.py new file mode 100644 index 00000000000..fc14d297518 --- /dev/null +++ b/test/integration/plugins/nuagevsp/nuage_lib.py @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from marvin.cloudstackAPI import createSSHKeyPair, deleteSSHKeyPair + + +class MySSHKeyPair: + """Manage SSH Key pairs""" + + def __init__(self, items): + self.__dict__.update(items) + + @classmethod + def create(cls, apiclient, name=None, account=None, + domainid=None, projectid=None): + """Creates SSH keypair""" + cmd = createSSHKeyPair.createSSHKeyPairCmd() + cmd.name = name + if account is not None: + cmd.account = account + if domainid is not None: + cmd.domainid = domainid + if projectid is not None: + cmd.projectid = projectid + return MySSHKeyPair(apiclient.createSSHKeyPair(cmd).__dict__) + + def delete(self, apiclient): + """Delete SSH key pair""" + cmd = deleteSSHKeyPair.deleteSSHKeyPairCmd() + cmd.name = self.name + cmd.account = self.account + cmd.domainid = self.domainid + apiclient.deleteSSHKeyPair(cmd) \ No newline at end of file diff --git a/test/integration/plugins/nuagevsp/test_nuage_configdrive.py b/test/integration/plugins/nuagevsp/test_nuage_configdrive.py index a7b80b14234..2a5495af4c9 100644 --- a/test/integration/plugins/nuagevsp/test_nuage_configdrive.py +++ b/test/integration/plugins/nuagevsp/test_nuage_configdrive.py @@ -19,8 +19,15 @@ and password reset functionality with ConfigDrive and Nuage VSP SDN plugin """ -# Import Local Modules -from nuageTestCase import nuageTestCase +import base64 +import copy +import os +import tempfile +import threading + +import sys +import time +from datetime import datetime from marvin.cloudstackAPI import updateTemplate, resetSSHKeyForVirtualMachine from marvin.lib.base import (Account, createVlanIpRange, @@ -28,80 +35,24 @@ from marvin.lib.base import (Account, NetworkServiceProvider, PublicIpRange, PublicIPAddress, - createSSHKeyPair, - deleteSSHKeyPair, VirtualMachine) - from marvin.lib.common import list_templates from marvin.lib.utils import random_gen # Import System Modules from nose.plugins.attrib import attr -from datetime import datetime -import threading -import tempfile -import base64 -import sys -import time -import os -import copy -import json + +# Import Local Modules +from component.test_configdrive import MySSHKeyPair, ConfigDriveUtils +from nuageTestCase import nuageTestCase, needscleanup + +NO_SUCH_FILE = "No such file or directory" -class MySSHKeyPair: - """Manage SSH Key pairs""" - - def __init__(self, items): - self.__dict__.update(items) - - @classmethod - def create(cls, apiclient, name=None, account=None, - domainid=None, projectid=None): - """Creates SSH keypair""" - cmd = createSSHKeyPair.createSSHKeyPairCmd() - cmd.name = name - if account is not None: - cmd.account = account - if domainid is not None: - cmd.domainid = domainid - if projectid is not None: - cmd.projectid = projectid - return MySSHKeyPair(apiclient.createSSHKeyPair(cmd).__dict__) - - def delete(self, apiclient): - """Delete SSH key pair""" - cmd = deleteSSHKeyPair.deleteSSHKeyPairCmd() - cmd.name = self.name - cmd.account = self.account - cmd.domainid = self.domainid - apiclient.deleteSSHKeyPair(cmd) - - -class TestNuageConfigDrive(nuageTestCase): +class TestNuageConfigDrive(nuageTestCase, ConfigDriveUtils): """Test user data and password reset functionality using configDrive with Nuage VSP SDN plugin """ - class CreateResult: - def __init__(self, success, offering=None, network=None, vpc=None): - self.success = success - self.network = network - self.offering = offering - self.vpc = vpc - - class PasswordTest: - def __init__(self, password): - self.test_presence = False - self.presence = None - self.password = None - if type(password) is bool: - self.test_presence = True - self.presence = password - self.password = None - elif type(password) is unicode or type(password) is str: - self.test_presence = True - self.password = password - self.presence = True - class StartVM(threading.Thread): def __init__(self, nuagetestcase, network, index): @@ -247,312 +198,11 @@ class TestNuageConfigDrive(nuageTestCase): self.updateTemplate(False) return - # updateTemplate - Updates value of the guest VM template's password - # enabled setting - def updateTemplate(self, value): - self.debug("Updating value of guest VM template's password enabled " - "setting") - cmd = updateTemplate.updateTemplateCmd() - cmd.id = self.template.id - cmd.passwordenabled = value - self.api_client.updateTemplate(cmd) - list_template_response = list_templates(self.api_client, - templatefilter="all", - id=self.template.id - ) - self.template = list_template_response[0] - self.debug("Updated guest VM template") - - # get_userdata_url - Returns user data URL for the given VM object - def get_userdata_url(self, vm): - self.debug("Getting user data url") - nic = vm.nic[0] - gateway = str(nic.gateway) - self.debug("Gateway: " + gateway) - user_data_url = 'curl "http://' + gateway + ':80/latest/user-data"' - return user_data_url - - # create_and_verify_fw - Creates and verifies (Ingress) firewall rule - # with a Static NAT rule enabled public IP - def create_and_verify_fip_and_fw(self, vm, public_ip, network): - self.debug("Creating and verifying firewall rule") - self.create_StaticNatRule_For_VM(vm, public_ip, network) - - # VSD verification - self.verify_vsd_floating_ip(network, vm, public_ip.ipaddress) - - fw_rule = self.create_FirewallRule( - public_ip, self.test_data["ingress_rule"]) - - # VSD verification + def validate_firewall_rule(self, fw_rule): self.verify_vsd_firewall_rule(fw_rule) - self.debug("Successfully created and verified firewall rule") - def getConfigDriveContent(self, ssh): - """ - This method is to verify whether configdrive iso - is attached to vm or not - Returns mount path if config drive is attached else False - """ - mountdir = "/root/iso" - cmd = "blkid -t LABEL='config-2' /dev/sr? /dev/hd? /dev/sd? /dev/xvd? -o device" - tmp_cmd = [ - 'bash -c "if [ ! -d /root/iso ] ; then mkdir /root/iso ; fi"', - "umount /root/iso"] - for tcmd in tmp_cmd: - ssh.execute(tcmd) - configDrive = ssh.execute(cmd) - res = ssh.execute("mount {} {}".format(str(configDrive[0]), mountdir)) - if str(res).lower().find("mounting read-only") > -1: - self.debug("configDrive iso is mounted at location %s" % mountdir) - return mountdir - else: - return None - - def verifyUserData(self, ssh, iso_path, userdata): - """ - verify Userdata - """ - userdata_path = iso_path+"/cloudstack/userdata/user_data.txt" - cmd = "cat %s" % userdata_path - res = ssh.execute(cmd) - vmuserdata = str(res[0]) - self.debug("Expected userdata is %s" % userdata) - self.debug("ConfigDrive userdata acsformat is %s" % vmuserdata) - self.assertEqual(vmuserdata, userdata, - 'Userdata found: %s is not equal to expected: %s' - % (vmuserdata, userdata)) - - def verifyOpenStackUserData(self, ssh, iso_path, userdata): - """ - verify Userdata in Openstack format - """ - userdata_path = iso_path+"/openstack/latest/user_data" - cmd = "cat %s" % userdata_path - res = ssh.execute(cmd) - vmuserdata = str(res[0]) - self.debug("Expected userdata is %s" % userdata) - self.debug("ConfigDrive userdata openstackformat is %s" % vmuserdata) - self.assertEqual(vmuserdata, userdata, - 'Userdata found: %s is not equal to expected: %s' - % (vmuserdata, userdata)) - - def verifyPassword(self, ssh, iso_path, password): - self.debug("Expected VM password is %s " % password.password) - password_file = iso_path+"/cloudstack/password/vm_password.txt" - cmd = "cat %s" % password_file - res = ssh.execute(cmd) - vmpassword = str(res[0]) - self.debug("ConfigDrive password is %s " % vmpassword) - nosuchfile = "No such file or directory" - if nosuchfile in vmpassword: - self.debug("Password file is not found") - return False, False - elif (password.password is not None) \ - and (password.password in vmpassword): - self.debug("Expected Password is found in configDriveIso") - return True, True - else: - self.debug("Expected password is not found in configDriveIso") - return True, False - - def verifySshKey(self, ssh, iso_path, sshkey): - self.debug("Expected VM sshkey is %s " % sshkey) - publicKey_file = iso_path+"/cloudstack/metadata/public-keys.txt" - cmd = "cat %s" % publicKey_file - res = ssh.execute(cmd) - vmsshkey = str(res[0]) - self.debug("ConfigDrive ssh key is %s " % vmsshkey) - - def verifyMetaData(self, vm, ssh, iso_path): - - metadata_dir = iso_path+"/cloudstack/metadata/" - metadata = {} - vm_files = ["availability-zone.txt", - "instance-id.txt", - "service-offering.txt", - "vm-id.txt"] - for file in vm_files: - cmd = "cat %s" % metadata_dir+file - res = ssh.execute(cmd) - metadata[file] = res - - for mfile in vm_files: - if mfile not in metadata: - self.fail("{} file is not found in vm metadata".format(mfile)) - self.assertEqual( - str(metadata["availability-zone.txt"][0]), - self.zone.name, - "Zone name inside metadata does not match with the zone" - ) - self.assertEqual( - str(metadata["instance-id.txt"][0]), - vm.instancename, - "vm name inside metadata does not match with the " - "instance name" - ) - self.assertEqual( - str(metadata["service-offering.txt"][0]), - vm.serviceofferingname, - "Service offering inside metadata does not match " - "with the instance offering" - ) - return - - def verifyOpenStackData(self, ssh, iso_path): - - openstackdata_dir = iso_path+"/openstack/latest/" - openstackdata = {} - openstackdata_files = ["user_data", - "meta_data.json", - "vendor_data.json", - "network_data.json"] - for file in openstackdata_files: - cmd = "cat %s" % openstackdata_dir+file - res = ssh.execute(cmd) - openstackdata[file] = res - if file not in openstackdata: - self.fail("{} file not found in vm openstack".format(file)) - return - - def generate_ssh_keys(self): - """ - This method generates ssh key pair and writes the private key - into a temp file and returns the file name - """ - self.keypair = MySSHKeyPair.create( - self.api_client, - name=random_gen() + ".pem", - account=self.account.user[0].account, - domainid=self.account.domainid) - - self.cleanup.append(self.keypair) - self.debug("Created keypair with name: %s" % self.keypair.name) - self.debug("Writing the private key to local file") - keyPairFilePath = tempfile.gettempdir() + os.sep + self.keypair.name - self.tmp_files.append(keyPairFilePath) - self.debug("File path: %s" % keyPairFilePath) - with open(keyPairFilePath, "w+") as f: - f.write(self.keypair.privatekey) - os.system("chmod 400 " + keyPairFilePath) - return keyPairFilePath - - def umountConfigDrive(self, ssh, iso_path): - """umount config drive iso attached inside guest vm""" - ssh.execute("umount -d %s" % iso_path) - # Give the VM time to unlock the iso device - time.sleep(2) - # Verify umount - result = ssh.execute("ls %s" % iso_path) - self.assertTrue(len(result) == 0, - "After umount directory should be empty " - "but contains: %s" % result) - - def update_provider_state(self, new_state): - self.debug("Updating Service Provider ConfigDrive to %s" % new_state) - configdriveprovider = NetworkServiceProvider.list( - self.api_client, - name="ConfigDrive", - physicalnetworkid=self.vsp_physical_network.id)[0] - orig_state = configdriveprovider.state - NetworkServiceProvider.update(self.api_client, - configdriveprovider.id, - state=new_state) - self.validate_NetworkServiceProvider("ConfigDrive", state=new_state) - return orig_state - - def verify_network_creation(self, offering=None, - offering_name=None, - gateway=None, - vpc=None, acl_list=None, testdata=None): - if offering is None: - self.debug("Creating Nuage VSP network offering...") - offering = self.create_NetworkOffering( - self.test_data["nuagevsp"][offering_name]) - self.validate_NetworkOffering(offering, state="Enabled") - try: - network = self.create_Network(offering, - gateway=gateway, - vpc=vpc, - acl_list=acl_list, - testdata=testdata) - return self.CreateResult(True, offering=offering, network=network) - except Exception: - self.debug("Exception: %s" % sys.exc_info()[0]) - return self.CreateResult(False, offering=offering) - - def verify_vpc_creation(self, offering=None, offering_name=None): - - if offering is None: - self.debug("Creating Nuage VSP VPC offering...") - offering = self.create_VpcOffering( - self.test_data["nuagevsp"][offering_name]) - self.validate_VpcOffering(offering, state="Enabled") - try: - vpc = self.create_Vpc(offering, cidr='10.1.0.0/16') - self.validate_Vpc(vpc, state="Enabled") - return self.CreateResult(True, offering=offering, vpc=vpc) - except Exception: - return self.CreateResult(False, offering=offering) - - def update_password_enable_in_template(self, new_state): - self.debug("Updating guest VM template to password %s" % new_state) - orig_state = self.template.passwordenabled - if self.template.passwordenabled is not new_state: - self.updateTemplate(new_state) - self.assertEqual(self.template.passwordenabled, new_state, - "Guest VM template is not password enabled") - return orig_state - - def verify_config_drive_content(self, vm, - public_ip, - password_test, - userdata=None, - metadata=False, - sshkey=None, - ssh_client=None): - if self.isSimulator: - self.debug("Simulator Environment: Skipping Config Drive content verification") - return - - self.debug("SSHing into the VM %s" % vm.name) - if ssh_client is None: - ssh = self.ssh_into_VM(vm, public_ip) - else: - ssh = ssh_client - d = {x.name: x for x in ssh.logger.handlers} - ssh.logger.handlers = list(d.values()) - config_drive_path = self.getConfigDriveContent(ssh) - self.assertIsNotNone(config_drive_path, - 'ConfigdriveIso is not attached to vm') - if metadata: - self.debug("Verifying metadata for vm: %s" % vm.name) - self.verifyMetaData(vm, ssh, config_drive_path) - self.debug("Verifying openstackdata for vm: %s" % vm.name) - self.verifyOpenStackData(ssh, config_drive_path) - - if userdata is not None: - self.debug("Verifying userdata for vm: %s" % vm.name) - self.verifyUserData(ssh, config_drive_path, userdata) - self.verifyOpenStackUserData(ssh, config_drive_path, userdata) - if password_test.test_presence: - self.debug("Verifying password for vm: %s" % vm.name) - test_result = self.verifyPassword(ssh, config_drive_path, - password_test) - self.assertEqual(test_result[0], password_test.presence, - "Expected is that password is present: %s " - " but found is: %s" - % (test_result[0], password_test.presence)) - if password_test.password is not None: - self.debug("Password for vm is %s" % password_test.password) - self.assertEqual(test_result[1], True, - "Password value test failed.") - if sshkey is not None: - self.debug("Verifying sshkey for vm: %s" % vm.name) - self.verifySshKey(ssh, config_drive_path, sshkey) - - self.umountConfigDrive(ssh, config_drive_path) - return ssh + def validate_StaticNat_rule_For_VM(self, public_ip, network, vm): + self.verify_vsd_floating_ip(network, vm, public_ip.ipaddress) def create_guest_vm(self, networks, acl_item=None, vpc=None, keypair=None): @@ -579,92 +229,13 @@ class TestNuageConfigDrive(nuageTestCase): self.verify_vsd_firewall_rule(acl_item) return vm - # nic_operation_VM - Performs NIC operations such as add, remove, and - # update default NIC in the given VM and network - def nic_operation_VM(self, vm, network, operation="add"): - self.debug("Performing %s NIC operation in VM with ID - %s and " - "network with ID - %s" % (operation, vm.id, network.id)) - if operation is "add": - vm.add_nic(self.api_client, network.id) - self.debug("Added NIC in VM with ID - %s and network with ID - %s" - % (vm.id, network.id)) - vm_info = VirtualMachine.list(self.api_client, id=vm.id)[0] - for nic in vm_info.nic: - if nic.networkid == network.id: - nic_id = nic.id - if operation is "update": - vm.update_default_nic(self.api_client, nic_id) - self.debug("Updated default NIC to NIC with ID- %s in VM with ID " - "- %s and network with ID - %s" % - (nic_id, vm.id, network.id)) - if operation is "remove": - vm.remove_nic(self.api_client, nic_id) - self.debug("Removed NIC with ID - %s in VM with ID - %s and " - "network with ID - %s" % (nic_id, vm.id, network.id)) + # ========================================================================= + # --- Gherkin style helper methods --- + # ========================================================================= - def update_userdata(self, vm, expected_user_data): - updated_user_data = base64.b64encode(expected_user_data) - vm.update(self.api_client, userdata=updated_user_data) - return expected_user_data - - def reset_password(self, vm): - vm.password = vm.resetPassword(self.api_client) - self.debug("Password reset to - %s" % vm.password) - self.debug("VM - %s password - %s !" % - (vm.name, vm.password)) - - def wait_until_done(self, thread_list, name): - for aThread in thread_list: - self.debug("[Concurrency]Join %s for vm %s" % (name, - aThread.get_vm())) - aThread.join() - - def resetsshkey(self, vm, keypair, account=None, domainid=None): - """Resets SSH key""" - cmd = resetSSHKeyForVirtualMachine.resetSSHKeyForVirtualMachineCmd() - cmd.id = vm.id - cmd.keypair = keypair - cmd.account = account - cmd.domainid = domainid - return self.api_client.resetSSHKeyForVirtualMachine(cmd) - - def update_sshkeypair(self, vm): - vm.stop(self.api_client) - self.resetsshkey(vm, - self.keypair.name, - account=self.account.user[0].account, - domainid=self.account.domainid) - self.debug("Sshkey reset to - %s" % self.keypair.name) - vm.start(self.api_client) - - def add_subnet_verify(self, network, services): - """verify required nic is present in the VM""" - - self.debug("Going to add new ip range in shared network %s" % - network.name) - cmd = createVlanIpRange.createVlanIpRangeCmd() - cmd.networkid = network.id - cmd.gateway = services["gateway"] - cmd.netmask = services["netmask"] - cmd.startip = services["startip"] - cmd.endip = services["endip"] - cmd.forVirtualNetwork = services["forvirtualnetwork"] - addedsubnet = self.api_client.createVlanIpRange(cmd) - - self.debug("verify above iprange is successfully added in shared " - "network %s or not" % network.name) - - cmd1 = listVlanIpRanges.listVlanIpRangesCmd() - cmd1.networkid = network.id - cmd1.id = addedsubnet.vlan.id - - allsubnets = self.api_client.listVlanIpRanges(cmd1) - self.assertEqual( - allsubnets[0].id, - addedsubnet.vlan.id, - "Check New subnet is successfully added to the shared Network" - ) - return addedsubnet + # ========================================================================= + # --- TEST CASES --- + # ========================================================================= @attr(tags=["advanced", "nuagevsp", "isonw"], required_hardware="true") def test_nuage_configdrive_isolated_network(self): @@ -673,27 +244,23 @@ class TestNuageConfigDrive(nuageTestCase): with Nuage VSP SDN plugin """ - # 1. When ConfigDrive is disabled as provider in zone - # Verify Isolated Network creation with a network offering - # which has userdata provided by ConfigDrive fails - # 2. When ConfigDrive is enabled as provider in zone - # Create an Isolated Network with Nuage VSP Isolated Network - # offering specifying ConfigDrive as serviceProvider - # for userdata, - # make sure no Dns is in the offering so no VR is spawned. - # check if it is successfully created and - # is in the "Allocated" state. - # 3. Deploy a VM in the created Isolated network with user data, - # check if the Isolated network state is changed to - # "Implemented", and the VM is successfully deployed and - # is in the "Running" state. - # Check that no VR is deployed. - # 4. SSH into the deployed VM and verify its user data in the iso - # (expected user data == actual user data). - # 5. Verify that the guest VM's password in the iso. - # 6. Reset VM password, and start the VM. - # 7. Verify that the new guest VM template is password enabled by - # checking the VM's password (password != "password"). + # 2. Given ConfigDrive provider is enabled in zone + # And a network offering which has + # * user data provided by ConfigDrive + # * No DNS + # When I create an Isolated Network using that network offering + # Then the network is successfully created, + # And is in the "Allocated" state. + # + # 3. When I deploy a VM in the created Isolated network with user data, + # Then the Isolated network state is changed to "Implemented" + # And the VM is successfully deployed and is in the "Running" state + # And there is no VR is deployed. + # 4. And the user data in the ConfigDrive device is as expected + # 5. And the the vm's password in the ConfigDrive device is as expected + # + # 6. When I stop, reset the password, and start the VM + # 7. Then I can login into the VM using the new password. # 8. SSH into the VM for verifying its new password # after its password reset. # 9. Verify various scenarios and check the data in configdriveIso @@ -706,6 +273,12 @@ class TestNuageConfigDrive(nuageTestCase): # Configure VSD sessions self.configureVSDSessions() + # 1. Given ConfigDrive provider is disabled in zone + # And a network offering which has + # user data provided by ConfigDrive + # Then creating an Isolated Network + # using that network offering fails + self.debug("+++Testing configdrive in an Isolated network fails..." "as provider configdrive is still disabled...") self.update_provider_state("Disabled") @@ -717,6 +290,8 @@ class TestNuageConfigDrive(nuageTestCase): 'Network found success = %s, expected success =%s' % (str(create_network.success), 'False')) + + self.debug("+++Test user data & password reset functionality " "using configdrive in an Isolated network without VR") self.update_provider_state("Enabled") @@ -763,7 +338,7 @@ class TestNuageConfigDrive(nuageTestCase): metadata=True, userdata=self.test_data[ "virtual_machine_userdata"]["userdata"], - sshkey=self.keypair.name) + ssh_key=self.keypair) expected_user_data1 = self.update_userdata(vm1, "helloworld vm1") self.verify_config_drive_content(vm1, public_ip_1, @@ -776,7 +351,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(True), metadata=True, userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) # After sshkey reset we need to have the vm password again vm1.password = vm1.resetPassword(self.api_client) self.debug("Password reset to - %s" % vm1.password) @@ -791,7 +366,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm1.password), metadata=True, userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) vm1.password = vm1.resetPassword(self.api_client) self.debug("Password reset to - %s" % vm1.password) self.debug("VM - %s password - %s !" % @@ -802,7 +377,7 @@ class TestNuageConfigDrive(nuageTestCase): self.verify_config_drive_content(vm1, public_ip_1, self.PasswordTest(vm1.password), userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("updating non-default nic as the default nic " "of the multi-nic VM and enable staticnat...") @@ -887,7 +462,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm1.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ Restarting the created Isolated network without " "VR with cleanup...") @@ -898,12 +473,12 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm1.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ Upgrade offering of created Isolated network with " "a dns offering which spins a VR") self.upgrade_Network(self.test_data["nuagevsp"][ - "isolated_configdrive_network_offering"], + "isolated_configdrive_network_offering"], create_network1.network) vr = self.get_Router(create_network1.network) self.check_Router_state(vr, state="Running") @@ -996,14 +571,14 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm1.password), metadata=True, userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata for VM - %s" % vm1.name) expected_user_data1 = self.update_userdata(vm1, "hello afterboot") self.verify_config_drive_content(vm1, public_ip_1, self.PasswordTest(vm1.password), userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Resetting password for VM - %s" % vm1.name) self.reset_password(vm1) self.debug("SSHing into the VM for verifying its new password " @@ -1018,7 +593,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm1.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata after migrating VM - %s" % vm1.name) expected_user_data1 = self.update_userdata(vm1, @@ -1040,7 +615,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(False), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata for VM - %s" % vm1.name) expected_user_data1 = self.update_userdata(vm1, @@ -1064,7 +639,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(False), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.update_provider_state("Disabled") expected_user_data1 = self.update_userdata(vm1, "hello after recover") @@ -1072,7 +647,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(False), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ When template is not password enabled, " "verify configdrive of VM - %s" % vm1.name) @@ -1094,7 +669,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(False), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) vm1.delete(self.api_client, expunge=True) create_network1.network.delete(self.api_client) @@ -1205,21 +780,21 @@ class TestNuageConfigDrive(nuageTestCase): self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(True), metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) expected_user_data = self.update_userdata(vm, "helloworld vm1") self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(True), metadata=True, userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Resetting password for VM - %s" % vm.name) self.reset_password(vm) self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(vm.password), userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.generate_ssh_keys() self.update_sshkeypair(vm) @@ -1227,7 +802,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(True), metadata=True, userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) # After sshkey reset we need to have the vm password again vm.password = vm.resetPassword(self.api_client) self.debug("Password reset to - %s" % vm.password) @@ -1242,7 +817,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm.password), userdata=expected_user_data, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Adding a non-default nic to the VM " "making it a multi-nic VM...") @@ -1252,7 +827,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm.password), metadata=True, userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) vm.password = vm.resetPassword(self.api_client) self.debug("Password reset to - %s" % vm.password) self.debug("VM - %s password - %s !" % @@ -1262,7 +837,7 @@ class TestNuageConfigDrive(nuageTestCase): self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(vm.password), userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("updating non-default nic as the default nic " "of the multi-nic VM and enable staticnat...") @@ -1323,7 +898,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ Restarting the created VPC Tier network without " "cleanup...") @@ -1334,7 +909,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ Restarting the created VPC Tier network with " "cleanup...") @@ -1345,7 +920,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Testing user data & password reset functionality " " using configdrive in a VPC network with VR...") @@ -1355,9 +930,9 @@ class TestNuageConfigDrive(nuageTestCase): 'Vpc found success = %s, expected success = %s' % (str(create_vrvpc.success), 'True')) acl_list2 = self.create_NetworkAclList( - name="acl", description="acl", vpc=create_vrvpc.vpc) + name="acl", description="acl", vpc=create_vrvpc.vpc) acl_item2 = self.create_NetworkAclRule( - self.test_data["ingress_rule"], acl_list=acl_list2) + self.test_data["ingress_rule"], acl_list=acl_list2) create_vrnetwork = \ self.verify_network_creation( offering_name="vpc_network_offering_configdrive_withdns", @@ -1445,8 +1020,8 @@ class TestNuageConfigDrive(nuageTestCase): self.debug("+++ Upgrade offering of created VPC network with " "an offering which removes the VR...") self.upgrade_Network(self.test_data["nuagevsp"][ - "vpc_network_offering_configdrive_" - "withoutdns"], + "vpc_network_offering_configdrive_" + "withoutdns"], create_vrnetwork.network) self.verify_config_drive_content(vm2, vpc_public_ip_2, @@ -1464,7 +1039,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm.password), metadata=True, userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata for VM - %s" % vm.name) expected_user_data = self.update_userdata(vm, @@ -1472,7 +1047,7 @@ class TestNuageConfigDrive(nuageTestCase): self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(vm.password), userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Resetting password for VM - %s" % vm.name) self.reset_password(vm) self.debug("SSHing into the VM for verifying its new password " @@ -1487,7 +1062,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm.password), userdata=expected_user_data, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata after migrating VM - %s" % vm.name) expected_user_data = self.update_userdata(vm, @@ -1495,7 +1070,7 @@ class TestNuageConfigDrive(nuageTestCase): self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(vm.password), userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Resetting password for VM - %s" % vm.name) self.reset_password(vm) self.debug("SSHing into the VM for verifying its new password " @@ -1510,7 +1085,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(False), userdata=expected_user_data, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata for VM - %s" % vm.name) expected_user_data = self.update_userdata(vm, @@ -1518,7 +1093,7 @@ class TestNuageConfigDrive(nuageTestCase): self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(False), userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Resetting password for VM - %s" % vm.name) self.reset_password(vm) self.debug("SSHing into the VM for verifying its new password " @@ -1535,13 +1110,13 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(False), userdata=expected_user_data, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.update_provider_state("Disabled") self.verify_config_drive_content(vm, vpc_public_ip_1, self.PasswordTest(False), userdata=expected_user_data, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ When template is not password enabled " "verify configdrive of VM - %s" % vm.name) @@ -1567,7 +1142,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(False), userdata=expected_user_data, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) vm.delete(self.api_client, expunge=True) create_tiernetwork.network.delete(self.api_client) @@ -1637,7 +1212,6 @@ class TestNuageConfigDrive(nuageTestCase): try: for i in range(2): - self.debug("\n+++ [Concurrency]Start update on all VM's") # # 5. Concurrently update all VM's @@ -1684,7 +1258,6 @@ class TestNuageConfigDrive(nuageTestCase): # 10. Verify the passwords self.debug("\n+++ [Concurrency]Verify passwords on all VM's") for aThread in my_reset_threads: - # create floating ip self.create_and_verify_fip_and_fw(aThread.get_vm(), public_ip_1, @@ -1753,9 +1326,10 @@ class TestNuageConfigDrive(nuageTestCase): # Configure VSD sessions self.configureVSDSessions() if not self.isNuageInfraUnderlay: - self.skipTest("Configured Nuage VSP SDN platform infrastructure " - "does not support underlay networking: " - "skipping test") + self.skipTest( + "Configured Nuage VSP SDN platform infrastructure " + "does not support underlay networking: " + "skipping test") self.debug("+++Testing configdrive in an shared network fails..." "as provider configdrive is still disabled...") @@ -1816,9 +1390,9 @@ class TestNuageConfigDrive(nuageTestCase): self.test_data["nuagevsp"]["network_all"]["endip"] with self.assertRaises(Exception): - self.create_VM( - [shared_network.network], - testdata=tmp_test_data) + self.create_VM( + [shared_network.network], + testdata=tmp_test_data) self.debug("+++ In a shared network with multiple ip ranges, " "userdata with config drive must be allowed.") @@ -1872,7 +1446,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm1.password), metadata=True, userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) vm1.password = vm1.resetPassword(self.api_client) self.debug("Password reset to - %s" % vm1.password) self.debug("VM - %s password - %s !" % @@ -1883,7 +1457,7 @@ class TestNuageConfigDrive(nuageTestCase): self.verify_config_drive_content(vm1, public_ip, self.PasswordTest(vm1.password), userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ Updating non-default nic as the default nic " "of the multi-nic VM...") @@ -1939,9 +1513,9 @@ class TestNuageConfigDrive(nuageTestCase): {"ipaddress": VirtualMachine.list( self.api_client, id=multinicvm1.id)[0].nic[0]}) self.verify_config_drive_content( - multinicvm1, public_ip_3, - self.PasswordTest(multinicvm1.password), - metadata=True) + multinicvm1, public_ip_3, + self.PasswordTest(multinicvm1.password), + metadata=True) expected_user_data2 = self.update_userdata(multinicvm1, "hello multinicvm1") self.verify_config_drive_content(multinicvm1, public_ip_3, @@ -1963,14 +1537,14 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm1.password), metadata=True, userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata for VM - %s" % vm1.name) expected_user_data1 = self.update_userdata(vm1, "hello afterboot") self.verify_config_drive_content(vm1, public_ip, self.PasswordTest(vm1.password), userdata=expected_user_data1, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Resetting password for VM - %s" % vm1.name) self.reset_password(vm1) self.debug("SSHing into the VM for verifying its new password " @@ -1985,7 +1559,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm1.password), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata after migrating VM - %s" % vm1.name) expected_user_data1 = self.update_userdata(vm1, @@ -2007,7 +1581,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(False), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("Updating userdata for VM - %s" % vm1.name) expected_user_data1 = self.update_userdata(vm1, @@ -2031,7 +1605,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(False), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.update_provider_state("Disabled") expected_user_data1 = self.update_userdata(vm1, "hello after recover") @@ -2039,7 +1613,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(False), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) self.debug("+++ When template is not password enabled, " "verify configdrive of VM - %s" % vm1.name) @@ -2049,9 +1623,9 @@ class TestNuageConfigDrive(nuageTestCase): self.generate_ssh_keys() self.debug("keypair name %s " % self.keypair.name) vm1 = self.create_VM( - [shared_network.network], - testdata=self.test_data["virtual_machine_userdata"], - keypair=self.keypair.name) + [shared_network.network], + testdata=self.test_data["virtual_machine_userdata"], + keypair=self.keypair.name) expected_user_data1 = self.update_userdata(vm1, "This is sample data") public_ip = PublicIPAddress({"ipaddress": vm1}) @@ -2059,7 +1633,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(False), userdata=expected_user_data1, metadata=True, - sshkey=self.keypair.name) + ssh_key=self.keypair) vm1.delete(self.api_client, expunge=True) shared_network.network.delete(self.api_client) @@ -2120,7 +1694,7 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm1.password), metadata=True, userdata=expected_user_data, - sshkey=self.keypair.name) + ssh_key=self.keypair) for i in range(0, 300): self.verify_config_drive_content( @@ -2128,8 +1702,11 @@ class TestNuageConfigDrive(nuageTestCase): self.PasswordTest(vm1.password), metadata=True, userdata=expected_user_data, - sshkey=self.keypair.name, + ssh_key=self.keypair, ssh_client=ssh_client) expected_user_data = \ self.update_userdata(vm1, 'This is sample data %s' % i) + +if __name__ == "__main__" and __package__ is None: + __package__ = "integration.plugins.nuage" diff --git a/tools/marvin/marvin/jsonHelper.py b/tools/marvin/marvin/jsonHelper.py index 324d339606b..2cd342f615b 100644 --- a/tools/marvin/marvin/jsonHelper.py +++ b/tools/marvin/marvin/jsonHelper.py @@ -43,6 +43,12 @@ class jsonLoader(object): else: return None + def __getitem__(self, val): + if val in self.__dict__: + return self.__dict__[val] + else: + return None + def __repr__(self): return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v) in self.__dict__.iteritems())) diff --git a/ui/l10n/en.js b/ui/l10n/en.js index 2bb127db922..1c4814291d2 100644 --- a/ui/l10n/en.js +++ b/ui/l10n/en.js @@ -1627,6 +1627,7 @@ var dictionary = { "label.storage.tags":"Storage Tags", "label.storage.traffic":"Storage Traffic", "label.storage.type":"Storage Type", +"label.storagepolicy":"Storage policy", "label.subdomain.access":"Subdomain Access", "label.submit":"Submit", "label.submitted.by":"[Submitted by: ]", diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 950f447cacf..050c5ebc4b5 100755 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -19886,6 +19886,7 @@ $form.find('.form-item[rel=account]').hide(); $form.find('.form-item[rel=username]').hide(); $form.find('.form-item[rel=key]').hide(); + $form.find('.form-item[rel=storagepolicy]').hide(); } else if ($(this).val() == "SMB") { //NFS, SMB $form.find('.form-item[rel=zoneid]').css('display', 'inline-block'); @@ -19918,6 +19919,7 @@ $form.find('.form-item[rel=account]').hide(); $form.find('.form-item[rel=username]').hide(); $form.find('.form-item[rel=key]').hide(); + $form.find('.form-item[rel=storagepolicy]').hide(); } else if ($(this).val() == "S3") { //NFS, SMB $form.find('.form-item[rel=zoneid]').hide(); @@ -19952,6 +19954,7 @@ $form.find('.form-item[rel=account]').hide(); $form.find('.form-item[rel=username]').hide(); $form.find('.form-item[rel=key]').hide(); + $form.find('.form-item[rel=storagepolicy]').hide(); } else if ($(this).val() == "Swift") { //NFS, SMB $form.find('.form-item[rel=zoneid]').hide(); @@ -19984,6 +19987,7 @@ $form.find('.form-item[rel=account]').css('display', 'inline-block'); $form.find('.form-item[rel=username]').css('display', 'inline-block'); $form.find('.form-item[rel=key]').css('display', 'inline-block'); + $form.find('.form-item[rel=storagepolicy]').css('display', 'inline-block'); } }); @@ -20174,14 +20178,26 @@ } }, account: { - label: 'label.account' + label: 'label.account', + validation: { + required: true + } }, username: { - label: 'label.username' + label: 'label.username', + validation: { + required: true + } }, key: { - label: 'label.key' - } + label: 'label.key', + validation: { + required: true + } + }, + storagepolicy: { + label: 'label.storagepolicy' + } //Swift (end) } }, @@ -20348,6 +20364,11 @@ data[ 'details[' + index.toString() + '].value'] = args.data.key; index++; } + if (args.data.storagepolicy != null && args.data.storagepolicy.length > 0) { + data[ 'details[' + index.toString() + '].key'] = 'storagepolicy'; + data[ 'details[' + index.toString() + '].value'] = args.data.storagepolicy; + index++; + } $.ajax({ url: createURL('addImageStore'), data: data, diff --git a/utils/src/main/java/com/cloud/utils/SwiftUtil.java b/utils/src/main/java/com/cloud/utils/SwiftUtil.java index ce1bee36b62..34aceb51553 100644 --- a/utils/src/main/java/com/cloud/utils/SwiftUtil.java +++ b/utils/src/main/java/com/cloud/utils/SwiftUtil.java @@ -24,26 +24,29 @@ import java.net.URL; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.SignatureException; -import java.util.Map; import java.util.Arrays; import java.util.HashMap; -import java.util.Formatter; +import java.util.Map; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.commons.codec.binary.Hex; import org.apache.log4j.Logger; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; - public class SwiftUtil { private static Logger logger = Logger.getLogger(SwiftUtil.class); - private static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L; + protected static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L; private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; - - + private static final String CD_SRC = "cd %s;"; + private static final String SWIFT_CMD= "/usr/bin/python %s -A %s -U %s:%s -K %s %s"; + private static final String WITH_STORAGE_POLICY = " --storage-policy \"%s\""; + private static final String WITH_SEGMENTS = " -S "+SWIFT_MAX_SIZE; + private static final String[] OPERATIONS_WITH_STORAGE_POLICIES = {"post","upload"}; public interface SwiftClientCfg { String getAccount(); @@ -53,6 +56,8 @@ public class SwiftUtil { String getKey(); String getEndPoint(); + + String getStoragePolicy(); } private static String getSwiftCLIPath() { @@ -65,19 +70,10 @@ public class SwiftUtil { } public static boolean postMeta(SwiftClientCfg cfg, String container, String object, Map metas) { - String swiftCli = getSwiftCLIPath(); - StringBuilder cms = new StringBuilder(); - for (Map.Entry entry : metas.entrySet()) { - cms.append(" -m "); - cms.append(entry.getKey()); - cms.append(":"); - cms.append(entry.getValue()); - cms.append(" "); - } Script command = new Script("/bin/bash", logger); command.add("-c"); - command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() + " post " + - container + " " + object + " " + cms.toString()); + command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(),"post", container, object) + getMeta(metas)); + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); String result = command.execute(parser); if (result != null) { @@ -87,21 +83,14 @@ public class SwiftUtil { } public static String putObject(SwiftClientCfg cfg, File srcFile, String container, String fileName) { - String swiftCli = getSwiftCLIPath(); if (fileName == null) { fileName = srcFile.getName(); } - String srcDirectory = srcFile.getParent(); + Script command = new Script("/bin/bash", logger); - long size = srcFile.length(); command.add("-c"); - if (size <= SWIFT_MAX_SIZE) { - command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + - " -K " + cfg.getKey() + " upload " + container + " " + fileName); - } else { - command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + - " -K " + cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + fileName); - } + command.add(String.format(CD_SRC, srcFile.getParent())+getUploadObjectCommand(cfg, getSwiftCLIPath(), container,fileName, srcFile.length())); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); String result = command.execute(parser); if (result != null) { @@ -120,38 +109,19 @@ public class SwiftUtil { return container + File.separator + srcFile.getName(); } - private static StringBuilder buildSwiftCmd(SwiftClientCfg swift) { - String swiftCli = getSwiftCLIPath(); - StringBuilder sb = new StringBuilder(); - sb.append(" /usr/bin/python "); - sb.append(swiftCli); - sb.append(" -A "); - sb.append(swift.getEndPoint()); - sb.append(" -U "); - sb.append(swift.getAccount()); - sb.append(":"); - sb.append(swift.getUserName()); - sb.append(" -K "); - sb.append(swift.getKey()); - sb.append(" "); - return sb; - } - public static String[] list(SwiftClientCfg swift, String container, String rFilename) { - getSwiftCLIPath(); - Script command = new Script("/bin/bash", logger); - command.add("-c"); - - StringBuilder swiftCmdBuilder = buildSwiftCmd(swift); - swiftCmdBuilder.append(" list "); - swiftCmdBuilder.append(container); + StringBuilder swiftCmdBuilder = new StringBuilder(); + swiftCmdBuilder.append(getSwiftContainerCmd(swift, getSwiftCLIPath(), "list", container)); if (rFilename != null) { swiftCmdBuilder.append(" -p "); swiftCmdBuilder.append(rFilename); } + Script command = new Script("/bin/bash", logger); + command.add("-c"); command.add(swiftCmdBuilder.toString()); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); String result = command.execute(parser); if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase("")) { @@ -178,11 +148,11 @@ public class SwiftUtil { } else { destFilePath = destDirectory.getAbsolutePath(); } - String swiftCli = getSwiftCLIPath(); + Script command = new Script("/bin/bash", logger); command.add("-c"); - command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() + - " download " + container + " " + srcPath + " -o " + destFilePath); + command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "download", container, srcPath)+" -o " + destFilePath); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); String result = command.execute(parser); if (result != null) { @@ -203,27 +173,6 @@ public class SwiftUtil { return new File(destFilePath); } - public static String getContainerName(String type, Long id) { - if (type.startsWith("T")) { - return "T-" + id; - } else if (type.startsWith("S")) { - return "S-" + id; - } else if (type.startsWith("V")) { - return "V-" + id; - } - return null; - } - - public static String[] splitSwiftPath(String path) { - int index = path.indexOf(File.separator); - if (index == -1) { - return null; - } - String[] paths = new String[2]; - paths[0] = path.substring(0, index); - paths[1] = path.substring(index + 1); - return paths; - } public static boolean deleteObject(SwiftClientCfg cfg, String path) { Script command = new Script("/bin/bash", logger); @@ -236,13 +185,8 @@ public class SwiftUtil { String container = paths[0]; String objectName = paths[1]; - StringBuilder swiftCmdBuilder = buildSwiftCmd(cfg); - swiftCmdBuilder.append(" delete "); - swiftCmdBuilder.append(container); - swiftCmdBuilder.append(" "); - swiftCmdBuilder.append(objectName); + command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "delete", container, objectName)); - command.add(swiftCmdBuilder.toString()); OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); command.execute(parser); return true; @@ -284,7 +228,7 @@ public class SwiftUtil { } - public static String calculateRFC2104HMAC(String data, String key) + static String calculateRFC2104HMAC(String data, String key) throws SignatureException, NoSuchAlgorithmException, InvalidKeyException { SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM); @@ -294,12 +238,75 @@ public class SwiftUtil { } - public static String toHexString(byte[] bytes) { - - Formatter formatter = new Formatter(); - for (byte b : bytes) { - formatter.format("%02x", b); - } - return formatter.toString(); + static String toHexString(byte[] bytes) { + return Hex.encodeHexString(bytes); } -} + + /////////////// SWIFT CMD STRING HELPERS /////////////// + protected static String getSwiftCmd(SwiftClientCfg cfg, String swiftCli, String operation){ + return String.format(SWIFT_CMD, swiftCli,cfg.getEndPoint(),cfg.getAccount(),cfg.getUserName(),cfg.getKey(),operation); + } + + protected static String getSwiftObjectCmd(SwiftClientCfg cfg, String swiftCliPath, String operation,String container, String objectName) { + String cmd = getSwiftCmd(cfg,swiftCliPath, operation) +" "+ container+" "+objectName; + if(StringUtils.isNotBlank(cfg.getStoragePolicy()) && supportsStoragePolicies(operation)){ + return cmd + String.format(WITH_STORAGE_POLICY, cfg.getStoragePolicy()); + } + return cmd; + } + + private static boolean supportsStoragePolicies(String operation) { + for(String supportedOp: OPERATIONS_WITH_STORAGE_POLICIES){ + if(supportedOp.equals(operation)){ + return true; + } + } + return false; + } + + protected static String getSwiftContainerCmd(SwiftClientCfg cfg, String swiftCliPath, String operation, String container) { + return getSwiftCmd(cfg,swiftCliPath, operation) +" "+ container; + } + + protected static String getUploadObjectCommand(SwiftClientCfg cfg, String swiftCliPath, String container, String objectName, long size) { + String cmd = getSwiftObjectCmd(cfg, swiftCliPath, "upload", container, objectName); + if(size > SWIFT_MAX_SIZE){ + return cmd + WITH_SEGMENTS; + } + return cmd; + } + + public static String getContainerName(String type, Long id) { + if (type.startsWith("T")) { + return "T-" + id; + } else if (type.startsWith("S")) { + return "S-" + id; + } else if (type.startsWith("V")) { + return "V-" + id; + } + return null; + } + + public static String[] splitSwiftPath(String path) { + int index = path.indexOf(File.separator); + if (index == -1) { + return null; + } + String[] paths = new String[2]; + paths[0] = path.substring(0, index); + paths[1] = path.substring(index + 1); + return paths; + } + + private static String getMeta(Map metas) { + StringBuilder cms = new StringBuilder(); + for (Map.Entry entry : metas.entrySet()) { + cms.append(" -m "); + cms.append(entry.getKey()); + cms.append(":"); + cms.append(entry.getValue()); + cms.append(" "); + } + return cms.toString(); + } +} \ No newline at end of file diff --git a/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java b/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java index 20c1623d822..6dc2cc78cac 100644 --- a/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java +++ b/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java @@ -19,9 +19,15 @@ package com.cloud.utils; - -import org.junit.Test; -import org.mockito.Mockito; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.mock; +import static org.junit.Assert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; import java.io.File; import java.net.URL; @@ -29,10 +35,10 @@ import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.SignatureException; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.when; +import org.junit.Test; +import org.mockito.Mockito; + +import com.cloud.utils.SwiftUtil.SwiftClientCfg; public class SwiftUtilTest { @@ -90,4 +96,134 @@ public class SwiftUtilTest { assertEquals(expected, output); } + + @Test + public void testGetSwiftCmd() { + SwiftClientCfg cfg = mock(SwiftClientCfg.class); + given(cfg.getEndPoint()).willReturn("swift.endpoint"); + given(cfg.getAccount()).willReturn("cs"); + given(cfg.getUserName()).willReturn("sec-storage"); + given(cfg.getKey()).willReturn("mypassword"); + given(cfg.getStoragePolicy()).willReturn(null); + + String cmd = SwiftUtil.getSwiftCmd(cfg, "swift", "stat"); + + String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword stat"; + assertThat(cmd, is(equalTo(expected))); + } + + @Test + public void testGetSwiftObjectCmd() { + SwiftClientCfg cfg = mock(SwiftClientCfg.class); + given(cfg.getEndPoint()).willReturn("swift.endpoint"); + given(cfg.getAccount()).willReturn("cs"); + given(cfg.getUserName()).willReturn("sec-storage"); + given(cfg.getKey()).willReturn("mypassword"); + given(cfg.getStoragePolicy()).willReturn(null); + + String objectCmd = SwiftUtil.getSwiftObjectCmd(cfg, "swift", "delete", "T-123", "template.vhd"); + + String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword delete T-123 template.vhd"; + assertThat(objectCmd, is(equalTo(expected))); + } + + @Test + public void testGetSwiftContainerCmd() { + SwiftClientCfg cfg = mock(SwiftClientCfg.class); + given(cfg.getEndPoint()).willReturn("swift.endpoint"); + given(cfg.getAccount()).willReturn("cs"); + given(cfg.getUserName()).willReturn("sec-storage"); + given(cfg.getKey()).willReturn("mypassword"); + given(cfg.getStoragePolicy()).willReturn(null); + + String containerCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-123"); + + String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-123"; + assertThat(containerCmd, is(equalTo(expected))); + } + + @Test + public void testGetUploadCmd() { + SwiftClientCfg cfg = mock(SwiftClientCfg.class); + given(cfg.getEndPoint()).willReturn("swift.endpoint"); + given(cfg.getAccount()).willReturn("cs"); + given(cfg.getUserName()).willReturn("sec-storage"); + given(cfg.getKey()).willReturn("mypassword"); + given(cfg.getStoragePolicy()).willReturn(null); + + String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 1024); + + String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd"; + assertThat(uploadCmd, is(equalTo(expected))); + } + + @Test + public void testGetUploadCmdWithSegmentsBecauseOfSize() { + SwiftClientCfg cfg = mock(SwiftClientCfg.class); + given(cfg.getEndPoint()).willReturn("swift.endpoint"); + given(cfg.getAccount()).willReturn("cs"); + given(cfg.getUserName()).willReturn("sec-storage"); + given(cfg.getKey()).willReturn("mypassword"); + given(cfg.getStoragePolicy()).willReturn(null); + + String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 5368709121L); + + String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd -S 5368709120"; + assertThat(uploadCmd, is(equalTo(expected))); + } + + @Test + public void testGetUploadCmdWithStoragePolicy() { + SwiftClientCfg cfg = mock(SwiftClientCfg.class); + given(cfg.getEndPoint()).willReturn("swift.endpoint"); + given(cfg.getAccount()).willReturn("cs"); + given(cfg.getUserName()).willReturn("sec-storage"); + given(cfg.getKey()).willReturn("mypassword"); + given(cfg.getStoragePolicy()).willReturn("policy1"); + + String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 1024L); + String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy \"policy1\""; + assertThat(uploadCmd, is(equalTo(expected))); + } + + @Test + public void testGetUploadCmdWithSegmentsAndStoragePolicy() { + SwiftClientCfg cfg = mock(SwiftClientCfg.class); + given(cfg.getEndPoint()).willReturn("swift.endpoint"); + given(cfg.getAccount()).willReturn("cs"); + given(cfg.getUserName()).willReturn("sec-storage"); + given(cfg.getKey()).willReturn("mypassword"); + given(cfg.getStoragePolicy()).willReturn("policy1"); + String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 5368709121L); + String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy \"policy1\" -S 5368709120"; + assertThat(uploadCmd, is(equalTo(expected))); + } + + @Test + public void testListContainerCmdWithStoragePolicyButNotSupportedByOperation() { + SwiftClientCfg cfg = mock(SwiftClientCfg.class); + given(cfg.getEndPoint()).willReturn("swift.endpoint"); + given(cfg.getAccount()).willReturn("cs"); + given(cfg.getUserName()).willReturn("sec-storage"); + given(cfg.getKey()).willReturn("mypassword"); + given(cfg.getStoragePolicy()).willReturn("policy1"); + + String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-1"); + String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-1"; + assertThat(uploadCmd, is(equalTo(expected))); + } + + @Test + public void testListContainerCmdWithoutStoragePolicy() { + SwiftClientCfg cfg = mock(SwiftClientCfg.class); + given(cfg.getEndPoint()).willReturn("swift.endpoint"); + given(cfg.getAccount()).willReturn("cs"); + given(cfg.getUserName()).willReturn("sec-storage"); + given(cfg.getKey()).willReturn("mypassword"); + given(cfg.getStoragePolicy()).willReturn(null); + + String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-1"); + String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-1"; + assertThat(uploadCmd, is(equalTo(expected))); + } }