apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Forward merge branch '4.11' to master

Browse Source 
ConfigDrive fixes: CLOUDSTACK-10288, CLOUDSTACK-10289 (#2566)
CLOUDSTACK-9677: Adding storage policy support for swift as secondary
storage (#2412)
This commit is contained in:
Rafael Weingärtner 2018-04-26 10:14:49 -03:00
commit b3c22df71d
25 changed files with 1488 additions and 1186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
api/src/main/java/com/cloud/agent/api/to/SwiftTO.java
View File

@ -26,17 +26,19 @@ public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg {
String userName;
String key;
String storagePolicy;
private static final String pathSeparator = "/";
public SwiftTO() {
}
public SwiftTO(Long id, String url, String account, String userName, String key) {
public SwiftTO(Long id, String url, String account, String userName, String key, String storagePolicy) {
this.id = id;
this.url = url;
this.account = account;
this.userName = userName;
this.key = key;
this.storagePolicy = storagePolicy;
}
public Long getId() {
@ -63,6 +65,11 @@ public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg {
return key;
}
@Override
public String getStoragePolicy() {
return this.storagePolicy;
}
@Override
public DataStoreRole getRole() {
return DataStoreRole.Image;

7
api/src/main/java/com/cloud/network/NetworkModel.java
View File

@ -59,6 +59,9 @@ public interface NetworkModel {
String SERVICE_OFFERING_FILE = "service-offering";
String AVAILABILITY_ZONE_FILE = "availability-zone";
String LOCAL_HOSTNAME_FILE = "local-hostname";
String LOCAL_IPV4_FILE = "local-ipv4";
String PUBLIC_HOSTNAME_FILE = "public-hostname";
String PUBLIC_IPV4_FILE = "public-ipv4";
String INSTANCE_ID_FILE = "instance-id";
String VM_ID_FILE = "vm-id";
String PUBLIC_KEYS_FILE = "public-keys";
@ -309,8 +312,8 @@ public interface NetworkModel {
boolean getNetworkEgressDefaultPolicy(Long networkId);
List<String[]> generateVmData(String userData, String serviceOffering, String zoneName,
String vmName, long vmId, String publicKey, String password, Boolean isWindows);
List<String[]> generateVmData(String userData, String serviceOffering, long datacenterId,
String vmName, long vmId, String vmUuid, String guestIpAddress, String publicKey, String password, Boolean isWindows);
String getValidNetworkCidr(Network guestNetwork);

1
api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
View File

@ -289,6 +289,7 @@ public class ApiConstants {
public static final String STATE = "state";
public static final String STATUS = "status";
public static final String STORAGE_TYPE = "storagetype";
public static final String STORAGE_POLICY = "storagepolicy";
public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
public static final String STORAGE_CAPABILITIES = "storagecapabilities";
public static final String SYSTEM_VM_TYPE = "systemvmtype";

7
engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
View File

@ -2542,16 +2542,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
if (defaultNic != null) {
UserVmVO userVm = _userVmDao.findById(vm.getId());
Map<String, String> details = _vmDetailsDao.listDetailsKeyPairs(vm.getId());
vm.setDetails(details);
userVm.setDetails(details);
Network network = _networkModel.getNetwork(defaultNic.getNetworkId());
if (_networkModel.isSharedNetworkWithoutServices(network.getId())) {
final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText();
final String zoneName = _dcDao.findById(vm.getDataCenterId()).getName();
boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, zoneName, vm.getInstanceName(), vm.getId(),
(String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getId(),
vm.getUuid(), defaultNic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
String vmName = vm.getInstanceName();
String configDriveIsoRootFolder = "/tmp";
String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso";

115
plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
View File

@ -18,23 +18,24 @@
*/
package com.cloud.hypervisor.xenserver.resource;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.exception.InternalErrorException;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.resource.StorageProcessor;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.storage.S3.ClientOptions;
import static com.cloud.utils.ReflectUtil.flattenProperties;
import static com.google.common.collect.Lists.newArrayList;
import java.io.File;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.log4j.Logger;
import org.apache.xmlrpc.XmlRpcException;
import com.google.common.annotations.VisibleForTesting;
import com.xensource.xenapi.Connection;
import com.xensource.xenapi.SR;
import com.xensource.xenapi.Types;
@ -44,6 +45,7 @@ import com.xensource.xenapi.Types.XenAPIException;
import com.xensource.xenapi.VBD;
import com.xensource.xenapi.VDI;
import com.xensource.xenapi.VM;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
@ -65,20 +67,24 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.log4j.Logger;
import org.apache.xmlrpc.XmlRpcException;
import java.io.File;
import java.net.URI;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import static com.cloud.utils.ReflectUtil.flattenProperties;
import static com.google.common.collect.Lists.newArrayList;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.exception.InternalErrorException;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.resource.StorageProcessor;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.storage.S3.ClientOptions;
public class XenServerStorageProcessor implements StorageProcessor {
private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class);
@ -914,20 +920,55 @@ public class XenServerStorageProcessor implements StorageProcessor {
private boolean swiftUpload(final Connection conn, final SwiftTO swift, final String container, final String ldir, final String lfilename, final Boolean isISCSI,
final int wait) {
String result = null;
List<String> params = getSwiftParams(swift, container, ldir, lfilename, isISCSI);
try {
result =
hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, "op", "upload", "url", swift.getUrl(), "account", swift.getAccount(), "username",
swift.getUserName(), "key", swift.getKey(), "container", container, "ldir", ldir, "lfilename", lfilename, "isISCSI", isISCSI.toString());
if (result != null && result.equals("true")) {
return true;
}
String result = hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, params.toArray(new String[params.size()]));
return BooleanUtils.toBoolean(result);
} catch (final Exception e) {
s_logger.warn("swift upload failed due to " + e.toString(), e);
}
return false;
}
@VisibleForTesting
List<String> getSwiftParams(SwiftTO swift, String container, String ldir, String lfilename, Boolean isISCSI) {
// ORDER IS IMPORTANT
List<String> params = new ArrayList<>();
//operation
params.add("op");
params.add("upload");
//auth
params.add("url");
params.add(swift.getUrl());
params.add("account");
params.add(swift.getAccount());
params.add("username");
params.add(swift.getUserName());
params.add("key");
params.add(swift.getKey());
// object info
params.add("container");
params.add(container);
params.add("ldir");
params.add(ldir);
params.add("lfilename");
params.add(lfilename);
params.add("isISCSI");
params.add(isISCSI.toString());
if (swift.getStoragePolicy() != null) {
params.add("storagepolicy");
params.add(swift.getStoragePolicy());
}
return params;
}
protected String deleteSnapshotBackup(final Connection conn, final String localMountPoint, final String path, final String secondaryStorageMountPath, final String backupUUID) {
// If anybody modifies the formatting below again, I'll skin them

113
plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java Normal file
View File

@ -0,0 +1,113 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.hypervisor.xenserver.resource;
import static org.mockito.Mockito.when;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import com.cloud.agent.api.to.SwiftTO;
public class XenServerStorageProcessorTest {
@Test
public void testOrderOfSwiftUplodScriptParamsWithoutStoragePolicy() {
CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
SwiftTO swift = Mockito.mock(SwiftTO.class);
when(swift.getStoragePolicy()).thenReturn(null);
String container = "sample-container-name";
String ldir = "sample-ldir";
String lfilename = "sample-lfilename";
Boolean isISCSI = true;
List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
// make sure the params not null and has correct number of items in it
Assert.assertNotNull("params is null", params);
Assert.assertTrue("Expected param list size is 18 but it was" + params.size(), params.size() == 18);
// check the order of params
Assert.assertEquals("unexpected param.", "op", params.get(0));
Assert.assertEquals("unexpected param.", "upload", params.get(1));
Assert.assertEquals("unexpected param.", "url", params.get(2));
Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
Assert.assertEquals("unexpected param.", "account", params.get(4));
Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
Assert.assertEquals("unexpected param.", "username", params.get(6));
Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
Assert.assertEquals("unexpected param.", "key", params.get(8));
Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
Assert.assertEquals("unexpected param.", "container", params.get(10));
Assert.assertEquals("unexpected param.", container, params.get(11));
Assert.assertEquals("unexpected param.", "ldir", params.get(12));
Assert.assertEquals("unexpected param.", ldir, params.get(13));
Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
Assert.assertEquals("unexpected param.", lfilename, params.get(15));
Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
}
@Test
public void testOrderOfSwiftUplodScriptParamsWithStoragePolicy() {
CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
SwiftTO swift = Mockito.mock(SwiftTO.class);
when(swift.getStoragePolicy()).thenReturn("sample-storagepolicy");
String container = "sample-container-name";
String ldir = "sample-ldir";
String lfilename = "sample-lfilename";
Boolean isISCSI = true;
List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
// make sure the params not null and has correct number of items in it
Assert.assertNotNull("params is null", params);
Assert.assertTrue("Expected param list size is 20 but it was" + params.size(), params.size() == 20);
// check the order of params
Assert.assertEquals("unexpected param.", "op", params.get(0));
Assert.assertEquals("unexpected param.", "upload", params.get(1));
Assert.assertEquals("unexpected param.", "url", params.get(2));
Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
Assert.assertEquals("unexpected param.", "account", params.get(4));
Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
Assert.assertEquals("unexpected param.", "username", params.get(6));
Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
Assert.assertEquals("unexpected param.", "key", params.get(8));
Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
Assert.assertEquals("unexpected param.", "container", params.get(10));
Assert.assertEquals("unexpected param.", container, params.get(11));
Assert.assertEquals("unexpected param.", "ldir", params.get(12));
Assert.assertEquals("unexpected param.", ldir, params.get(13));
Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
Assert.assertEquals("unexpected param.", lfilename, params.get(15));
Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
Assert.assertEquals("unexpected param.", "storagepolicy", params.get(18));
Assert.assertEquals("unexpected param.", "sample-storagepolicy", params.get(19));
}
}

2
plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
View File

@ -68,7 +68,7 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
public DataStoreTO getStoreTO(DataStore store) {
ImageStoreImpl imgStore = (ImageStoreImpl)store;
Map<String, String> details = _imageStoreDetailsDao.getDetails(imgStore.getId());
return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY));
return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY), details.get(ApiConstants.STORAGE_POLICY));
}
@Override

21
scripts/storage/secondary/swift
View File

@ -1473,8 +1473,9 @@ post [options] [container] [object]
Updates meta information for the account, container, or object depending on
the args given. If the container is not found, it will be created
automatically; but this is not true for accounts and objects. Containers
also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
or --meta option is allowed on all and used to define the user meta data
also allow the -r (or --read-acl) and -w (or --write-acl) options.
The --storage-policy will set a storage policy to the container if the container does not exist.
The -m or --meta option is allowed on all and used to define the user meta data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
@ -1493,6 +1494,8 @@ def st_post(options, args, print_queue, error_queue):
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item with the syntax name:value. This option '
'may be repeated. Example: -m Color:Blue -m Size:Large')
parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if (options.read_acl or options.write_acl or options.sync_to or
@ -1529,6 +1532,8 @@ def st_post(options, args, print_queue, error_queue):
headers['X-Container-Sync-To'] = options.sync_to
if options.sync_key is not None:
headers['X-Container-Sync-Key'] = options.sync_key
if options.storage_policy is not None:
headers['X-Storage-Policy'] = options.storage_policy
try:
conn.post_container(args[0], headers=headers)
except ClientException, err:
@ -1558,7 +1563,8 @@ upload [options] container file_or_directory [file_or_directory] [...]
Uploads to the given container the files and directories specified by the
remaining args. -c or --changed is an option that will only upload files
that have changed since the last upload. -S <size> or --segment-size <size>
and --leave-segments are options as well (see --help for more).
and --leave-segments are options as well (see --help for more). --storage-policy
Sets a storage policy to the container if the container does not exist.
'''.strip('\n')
@ -1576,6 +1582,8 @@ def st_upload(options, args, print_queue, error_queue):
dest='leave_segments', default=False, help='Indicates that you want '
'the older segments of manifest objects left alone (in the case of '
'overwrites)')
parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if len(args) < 2:
@ -1749,9 +1757,12 @@ def st_upload(options, args, print_queue, error_queue):
# permissions, so we'll ignore any error. If there's really a problem,
# it'll surface on the first object PUT.
try:
conn.put_container(args[0])
container_headers = {}
if options.storage_policy is not None:
container_headers['X-Storage-Policy'] = options.storage_policy
conn.put_container(args[0],headers=container_headers)
if options.segment_size is not None:
conn.put_container(args[0] + '_segments')
conn.put_container(args[0] + '_segments',headers=container_headers)
except Exception:
pass
try:

21
scripts/vm/hypervisor/xenserver/swift
View File

@ -1475,8 +1475,9 @@ post [options] [container] [object]
Updates meta information for the account, container, or object depending on
the args given. If the container is not found, it will be created
automatically; but this is not true for accounts and objects. Containers
also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
or --meta option is allowed on all and used to define the user meta data
also allow the -r (or --read-acl) and -w (or --write-acl) options.
The --storage-policy will set a storage policy to the container if the container does not exist.
The -m or --meta option is allowed on all and used to define the user meta data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
@ -1495,6 +1496,8 @@ def st_post(options, args, print_queue, error_queue):
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item with the syntax name:value. This option '
'may be repeated. Example: -m Color:Blue -m Size:Large')
parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if (options.read_acl or options.write_acl or options.sync_to or
@ -1531,6 +1534,8 @@ def st_post(options, args, print_queue, error_queue):
headers['X-Container-Sync-To'] = options.sync_to
if options.sync_key is not None:
headers['X-Container-Sync-Key'] = options.sync_key
if options.storage_policy is not None:
headers['X-Storage-Policy'] = options.storage_policy
try:
conn.post_container(args[0], headers=headers)
except ClientException, err:
@ -1560,7 +1565,8 @@ upload [options] container file_or_directory [file_or_directory] [...]
Uploads to the given container the files and directories specified by the
remaining args. -c or --changed is an option that will only upload files
that have changed since the last upload. -S <size> or --segment-size <size>
and --leave-segments are options as well (see --help for more).
and --leave-segments are options as well (see --help for more). --storage-policy
Sets a storage policy to the container if the container does not exist.
'''.strip('\n')
@ -1578,6 +1584,8 @@ def st_upload(options, args, print_queue, error_queue):
dest='leave_segments', default=False, help='Indicates that you want '
'the older segments of manifest objects left alone (in the case of '
'overwrites)')
parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if len(args) < 2:
@ -1751,9 +1759,12 @@ def st_upload(options, args, print_queue, error_queue):
# permissions, so we'll ignore any error. If there's really a problem,
# it'll surface on the first object PUT.
try:
conn.put_container(args[0])
container_headers = {}
if options.storage_policy is not None:
container_headers['X-Storage-Policy'] = options.storage_policy
conn.put_container(args[0],headers=container_headers)
if options.segment_size is not None:
conn.put_container(args[0] + '_segments')
conn.put_container(args[0] + '_segments',headers=container_headers)
except Exception:
pass
try:

48
server/src/main/java/com/cloud/network/NetworkModelImpl.java
View File

@ -22,6 +22,7 @@ import java.security.InvalidParameterException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@ -29,24 +30,24 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.Collections;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.Logger;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao;
import com.cloud.api.ApiDBUtils;
import com.cloud.configuration.Config;
import com.cloud.configuration.ConfigurationManager;
import com.cloud.dc.DataCenter;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.PodVlanMapVO;
import com.cloud.dc.Vlan;
import com.cloud.dc.Vlan.VlanType;
@ -2344,18 +2345,47 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi
}
@Override
public List<String[]> generateVmData(String userData, String serviceOffering, String zoneName,
String vmName, long vmId, String publicKey, String password, Boolean isWindows) {
public List<String[]> generateVmData(String userData, String serviceOffering, long datacenterId,
String vmName, long vmId, String vmUuid,
String guestIpAddress, String publicKey, String password, Boolean isWindows) {
DataCenterVO dcVo = _dcDao.findById(datacenterId);
final String zoneName = dcVo.getName();
IPAddressVO publicIp = _ipAddressDao.findByAssociatedVmId(vmId);
final List<String[]> vmData = new ArrayList<String[]>();
if (userData != null) {
vmData.add(new String[]{USERDATA_DIR, USERDATA_FILE, new String(Base64.decodeBase64(userData),StringUtils.getPreferredCharset())});
vmData.add(new String[]{USERDATA_DIR, USERDATA_FILE, userData});
}
vmData.add(new String[]{METATDATA_DIR, SERVICE_OFFERING_FILE, StringUtils.unicodeEscape(serviceOffering)});
vmData.add(new String[]{METATDATA_DIR, AVAILABILITY_ZONE_FILE, StringUtils.unicodeEscape(zoneName)});
vmData.add(new String[]{METATDATA_DIR, LOCAL_HOSTNAME_FILE, StringUtils.unicodeEscape(vmName)});
vmData.add(new String[]{METATDATA_DIR, INSTANCE_ID_FILE, vmName});
vmData.add(new String[]{METATDATA_DIR, VM_ID_FILE, String.valueOf(vmId)});
vmData.add(new String[]{METATDATA_DIR, LOCAL_IPV4_FILE, guestIpAddress});
String publicIpAddress = guestIpAddress;
String publicHostName = StringUtils.unicodeEscape(vmName);
if (dcVo.getNetworkType() != DataCenter.NetworkType.Basic) {
if (publicIp != null) {
publicIpAddress = publicIp.getAddress().addr();
publicHostName = publicIp.getAddress().addr();
} else {
publicHostName = null;
}
}
vmData.add(new String[]{METATDATA_DIR, PUBLIC_IPV4_FILE, publicIpAddress});
vmData.add(new String[]{METATDATA_DIR, PUBLIC_HOSTNAME_FILE, publicHostName});
if (vmUuid == null) {
vmData.add(new String[]{METATDATA_DIR, INSTANCE_ID_FILE, vmName});
vmData.add(new String[]{METATDATA_DIR, VM_ID_FILE, String.valueOf(vmId)});
} else {
vmData.add(new String[]{METATDATA_DIR, INSTANCE_ID_FILE, vmUuid});
vmData.add(new String[]{METATDATA_DIR, VM_ID_FILE, vmUuid});
}
vmData.add(new String[]{METATDATA_DIR, PUBLIC_KEYS_FILE, publicKey});
String cloudIdentifier = _configDao.getValue("cloud.identifier");

28
server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java
View File

@ -206,30 +206,37 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
return false;
}
private String getSshKey(VirtualMachineProfile profile) {
UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), "SSH.PublicKey");
return (vmDetailSshKey!=null ? vmDetailSshKey.getValue() : null);
}
@Override
public boolean addPasswordAndUserdata(Network network, NicProfile nic, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context)
throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException {
UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), "SSH.PublicKey");
return (canHandle(network.getTrafficType()) && updateConfigDrive(profile,
(vmDetailSshKey!=null?vmDetailSshKey.getValue():null)))
String sshPublicKey = getSshKey(profile);
return (canHandle(network.getTrafficType())
&& updateConfigDrive(profile, sshPublicKey, nic))
&& updateConfigDriveIso(network, profile, dest.getHost(), false);
}
@Override
public boolean savePassword(Network network, NicProfile nic, VirtualMachineProfile profile) throws ResourceUnavailableException {
if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, (String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey)))) return false;
String sshPublicKey = getSshKey(profile);
if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, sshPublicKey, nic))) return false;
return updateConfigDriveIso(network, profile, true);
}
@Override
public boolean saveSSHKey(Network network, NicProfile nic, VirtualMachineProfile vm, String sshPublicKey) throws ResourceUnavailableException {
if (!(canHandle(network.getTrafficType()) && updateConfigDrive(vm, sshPublicKey))) return false;
if (!(canHandle(network.getTrafficType()) && updateConfigDrive(vm, sshPublicKey, nic))) return false;
return updateConfigDriveIso(network, vm, true);
}
@Override
public boolean saveUserData(Network network, NicProfile nic, VirtualMachineProfile profile) throws ResourceUnavailableException {
if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, (String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey)))) return false;
String sshPublicKey = getSshKey(profile);
if (!(canHandle(network.getTrafficType()) && updateConfigDrive(profile, sshPublicKey, nic))) return false;
return updateConfigDriveIso(network, profile, true);
}
@ -330,7 +337,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
Answer createIsoAnswer = endpoint.sendMessage(configDriveIsoCommand);
if (!createIsoAnswer.getResult()) {
throw new ResourceUnavailableException(String.format("%s ISO failed, details: %s",
(update?"Update":"Create"), createIsoAnswer.getDetails()),ConfigDriveNetworkElement.class,0L);
(update?"Update":"Create"), createIsoAnswer.getDetails()), ConfigDriveNetworkElement.class, 0L);
}
configureConfigDriveDisk(profile, secondaryStore);
@ -363,7 +370,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
}
}
private boolean updateConfigDrive(VirtualMachineProfile profile, String publicKey) {
private boolean updateConfigDrive(VirtualMachineProfile profile, String publicKey, NicProfile nic) {
UserVmVO vm = _userVmDao.findById(profile.getId());
if (vm.getType() != VirtualMachine.Type.User) {
return false;
@ -372,11 +379,10 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
Nic defaultNic = _networkModel.getDefaultNic(vm.getId());
if (defaultNic != null) {
final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText();
final String zoneName = _dcDao.findById(vm.getDataCenterId()).getName();
boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
List<String[]> vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, zoneName, vm.getInstanceName(), vm.getId(),
publicKey, (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
List<String[]> vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getId(),
vm.getUuid(), nic.getIPv4Address(), publicKey, (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
profile.setVmData(vmData);
profile.setConfigDriveLabel(VirtualMachineManager.VmConfigDriveLabel.value());
}

5
server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
View File

@ -4085,11 +4085,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
Network network = _networkModel.getNetwork(defaultNic.getNetworkId());
if (_networkModel.isSharedNetworkWithoutServices(network.getId())) {
final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText();
final String zoneName = _dcDao.findById(vm.getDataCenterId()).getName();
boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
List<String[]> vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, zoneName, vm.getInstanceName(), vm.getId(),
(String) profile.getParameter(VirtualMachineProfile.Param.VmSshPubKey), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
List<String[]> vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getId(),
vm.getUuid(), defaultNic.getIPv4Address(), vm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
String vmName = vm.getInstanceName();
String configDriveIsoRootFolder = "/tmp";
String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso";

2
server/src/test/java/com/cloud/network/MockNetworkModelImpl.java
View File

@ -898,7 +898,7 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel {
}
@Override
public List<String[]> generateVmData(String userData, String serviceOffering, String zoneName, String vmName, long vmId, String publicKey, String password, Boolean isWindows) {
public List<String[]> generateVmData(String userData, String serviceOffering, long datacenterId, String vmName, long vmId, String vmUuid, String guestIpAddress, String publicKey, String password, Boolean isWindows) {
return null;
}

91
server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java
View File

@ -36,7 +36,6 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.apache.xerces.impl.dv.util.Base64;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
@ -64,6 +63,8 @@ import com.cloud.host.dao.HostDao;
import com.cloud.network.Network;
import com.cloud.network.NetworkModelImpl;
import com.cloud.network.Networks;
import com.cloud.network.dao.IPAddressDao;
import com.cloud.network.dao.IPAddressVO;
import com.cloud.network.dao.NetworkDao;
import com.cloud.network.dao.NetworkServiceMapDao;
import com.cloud.network.dao.NetworkVO;
@ -77,6 +78,7 @@ import com.cloud.storage.dao.GuestOSDao;
import com.cloud.utils.fsm.NoTransitionException;
import com.cloud.utils.fsm.StateListener;
import com.cloud.utils.fsm.StateMachine2;
import com.cloud.utils.net.Ip;
import com.cloud.vm.Nic;
import com.cloud.vm.NicProfile;
import com.cloud.vm.UserVmDetailVO;
@ -99,7 +101,7 @@ public class ConfigDriveNetworkElementTest {
private final String VMINSTANCENAME = "vm_name";
private final String VMOFFERING = "custom_instance";
private final long VMID = 30L;
private final String VMUSERDATA = "userdata";
private final String VMUSERDATA = "H4sIABCvw1oAAystTi1KSSxJ5AIAUPllwQkAAAA=";
private final long SOID = 31L;
private final long HOSTID = NETWORK_ID;
private final String HOSTNAME = "host1";
@ -116,6 +118,7 @@ public class ConfigDriveNetworkElementTest {
@Mock private UserVmDetailsDao _userVmDetailsDao;
@Mock private NetworkDao _networkDao;
@Mock private NetworkServiceMapDao _ntwkSrvcDao;
@Mock private IPAddressDao _ipAddressDao;
@Mock private DataCenterVO dataCenterVO;
@Mock private DataStore dataStore;
@ -130,6 +133,7 @@ public class ConfigDriveNetworkElementTest {
@Mock private NicProfile nicp;
@Mock private ServiceOfferingVO serviceOfferingVO;
@Mock private UserVmVO virtualMachine;
@Mock private IPAddressVO publicIp;
@InjectMocks private final ConfigDriveNetworkElement _configDrivesNetworkElement = new ConfigDriveNetworkElement();
@InjectMocks @Spy private NetworkModelImpl _networkModel = new NetworkModelImpl();
@ -161,7 +165,7 @@ public class ConfigDriveNetworkElementTest {
when(virtualMachine.getServiceOfferingId()).thenReturn(SOID);
when(virtualMachine.getDataCenterId()).thenReturn(DATACENTERID);
when(virtualMachine.getInstanceName()).thenReturn(VMINSTANCENAME);
when(virtualMachine.getUserData()).thenReturn(Base64.encode(VMUSERDATA.getBytes()));
when(virtualMachine.getUserData()).thenReturn(VMUSERDATA);
when(deployDestination.getHost()).thenReturn(hostVO);
when(hostVO.getId()).thenReturn(HOSTID);
when(nic.isDefaultNic()).thenReturn(true);
@ -236,6 +240,71 @@ public class ConfigDriveNetworkElementTest {
@Test
public void testAddPasswordAndUserdata() throws InsufficientCapacityException, ResourceUnavailableException {
List<String[]> actualVmData = getVmData();
assertThat(actualVmData, containsInAnyOrder(
new String[]{"userdata", "user_data", VMUSERDATA},
new String[]{"metadata", "service-offering", VMOFFERING},
new String[]{"metadata", "availability-zone", ZONENAME},
new String[]{"metadata", "local-hostname", VMINSTANCENAME},
new String[]{"metadata", "local-ipv4", "192.168.111.111"},
new String[]{"metadata", "public-hostname", null},
new String[]{"metadata", "public-ipv4", "192.168.111.111"},
new String[]{"metadata", "vm-id", String.valueOf(VMID)},
new String[]{"metadata", "instance-id", VMINSTANCENAME},
new String[]{"metadata", "public-keys", PUBLIC_KEY},
new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)},
new String[]{PASSWORD, "vm_password", PASSWORD}
));
}
@Test
public void testAddPasswordAndUserdataStaticNat() throws InsufficientCapacityException, ResourceUnavailableException {
when(_ipAddressDao.findByAssociatedVmId(VMID)).thenReturn(publicIp);
when(publicIp.getAddress()).thenReturn(new Ip("7.7.7.7"));
List<String[]> actualVmData = getVmData();
assertThat(actualVmData, containsInAnyOrder(
new String[]{"userdata", "user_data", VMUSERDATA},
new String[]{"metadata", "service-offering", VMOFFERING},
new String[]{"metadata", "availability-zone", ZONENAME},
new String[]{"metadata", "local-hostname", VMINSTANCENAME},
new String[]{"metadata", "local-ipv4", "192.168.111.111"},
new String[]{"metadata", "public-hostname", "7.7.7.7"},
new String[]{"metadata", "public-ipv4", "7.7.7.7"},
new String[]{"metadata", "vm-id", String.valueOf(VMID)},
new String[]{"metadata", "instance-id", VMINSTANCENAME},
new String[]{"metadata", "public-keys", PUBLIC_KEY},
new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)},
new String[]{PASSWORD, "vm_password", PASSWORD}
));
}
@Test
public void testAddPasswordAndUserdataUuid() throws InsufficientCapacityException, ResourceUnavailableException {
when(virtualMachine.getUuid()).thenReturn("vm-uuid");
List<String[]> actualVmData = getVmData();
assertThat(actualVmData, containsInAnyOrder(
new String[]{"userdata", "user_data", VMUSERDATA},
new String[]{"metadata", "service-offering", VMOFFERING},
new String[]{"metadata", "availability-zone", ZONENAME},
new String[]{"metadata", "local-hostname", VMINSTANCENAME},
new String[]{"metadata", "local-ipv4", "192.168.111.111"},
new String[]{"metadata", "public-hostname", null},
new String[]{"metadata", "public-ipv4", "192.168.111.111"},
new String[]{"metadata", "vm-id", "vm-uuid"},
new String[]{"metadata", "instance-id", "vm-uuid"},
new String[]{"metadata", "public-keys", PUBLIC_KEY},
new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)},
new String[]{PASSWORD, "vm_password", PASSWORD}
));
}
private List<String[]> getVmData() throws InsufficientCapacityException, ResourceUnavailableException {
final Answer answer = mock(Answer.class);
final UserVmDetailVO userVmDetailVO = mock(UserVmDetailVO.class);
when(endpoint.sendMessage(any(HandleConfigDriveIsoCommand.class))).thenReturn(answer);
@ -243,6 +312,7 @@ public class ConfigDriveNetworkElementTest {
when(network.getTrafficType()).thenReturn(Networks.TrafficType.Guest);
when(virtualMachine.getState()).thenReturn(VirtualMachine.State.Stopped);
when(userVmDetailVO.getValue()).thenReturn(PUBLIC_KEY);
when(nicp.getIPv4Address()).thenReturn("192.168.111.111");
when(_userVmDetailsDao.findDetail(anyLong(), anyString())).thenReturn(userVmDetailVO);
Map<VirtualMachineProfile.Param, Object> parms = Maps.newHashMap();
parms.put(VirtualMachineProfile.Param.VmPassword, PASSWORD);
@ -254,19 +324,6 @@ public class ConfigDriveNetworkElementTest {
ArgumentCaptor<HandleConfigDriveIsoCommand> commandCaptor = ArgumentCaptor.forClass(HandleConfigDriveIsoCommand.class);
verify(endpoint, times(1)).sendMessage(commandCaptor.capture());
HandleConfigDriveIsoCommand result = commandCaptor.getValue();
List<String[]> actualVmData = result.getVmData();
assertThat(actualVmData, containsInAnyOrder(
new String[]{"userdata", "user_data", VMUSERDATA},
new String[]{"metadata", "service-offering", VMOFFERING},
new String[]{"metadata", "availability-zone", ZONENAME},
new String[]{"metadata", "local-hostname", VMINSTANCENAME},
new String[]{"metadata", "vm-id", String.valueOf(VMID)},
new String[]{"metadata", "instance-id", String.valueOf(VMINSTANCENAME)},
new String[]{"metadata", "public-keys", PUBLIC_KEY},
new String[]{"metadata", "cloud-identifier", String.format("CloudStack-{%s}", CLOUD_ID)},
new String[]{PASSWORD, "vm_password", PASSWORD}
));
return result.getVmData();
}
}

2
server/src/test/java/com/cloud/vpc/MockNetworkModelImpl.java
View File

@ -913,7 +913,7 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel {
}
@Override
public List<String[]> generateVmData(String userData, String serviceOffering, String zoneName, String vmName, long vmId, String publicKey, String password, Boolean isWindows) {
public List<String[]> generateVmData(String userData, String serviceOffering, long datacenterId, String vmName, long vmId, String vmUuid, String guestIpAddress, String publicKey, String password, Boolean isWindows) {
return null;
}

17
services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
View File

@ -40,6 +40,7 @@ import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.URI;
@ -52,6 +53,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.naming.ConfigurationException;
import io.netty.bootstrap.ServerBootstrap;
@ -68,6 +70,7 @@ import io.netty.handler.codec.http.HttpResponseEncoder;
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
@ -155,11 +158,11 @@ import com.cloud.agent.api.to.DatadiskTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.configuration.Resource;
import com.cloud.exception.InternalErrorException;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.host.Host;
import com.cloud.host.Host.Type;
import com.cloud.configuration.Resource;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.network.NetworkModel;
import com.cloud.resource.ServerResourceBase;
@ -190,9 +193,6 @@ import com.cloud.utils.script.Script;
import com.cloud.utils.storage.S3.S3Utils;
import com.cloud.vm.SecondaryStorageVm;
import java.io.OutputStreamWriter;
public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
public static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
@ -492,8 +492,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
if (typeFolder.exists() || typeFolder.mkdirs()) {
if (StringUtils.isNotEmpty(content)) {
File file = new File(typeFolder, fileName + ".txt");
try (FileWriter fw = new FileWriter(file); BufferedWriter bw = new BufferedWriter(fw)) {
bw.write(content);
try {
if (fileName.equals(USERDATA_FILE)) {
// User Data is passed as a base64 encoded string
FileUtils.writeByteArrayToFile(file, Base64.decodeBase64(content));
} else {
FileUtils.write(file, content, com.cloud.utils.StringUtils.getPreferredCharset());
}
} catch (IOException ex) {
s_logger.error("Failed to create file ", ex);
return new Answer(cmd, ex);

1099
test/integration/component/test_configdrive.py
View File

File diff suppressed because it is too large Load Diff

11
test/integration/plugins/nuagevsp/nuageTestCase.py
View File

@ -733,8 +733,12 @@ class nuageTestCase(cloudstackTestCase):
traffictype=traffic_type
)
# ssh_into_VM - Gets into the shell of the given VM using its public IP
def ssh_into_VM(self, vm, public_ip, reconnect=True, negative_test=False):
def ssh_into_VM(self, vm, public_ip, reconnect=True, negative_test=False, keypair=None):
"""Creates a SSH connection to the VM
:returns: the SSH connection
:rtype: marvin.sshClient.SshClient
"""
if self.isSimulator:
self.debug("Simulator Environment: Skipping ssh into VM")
return
@ -748,7 +752,8 @@ class nuageTestCase(cloudstackTestCase):
ssh_client = vm.get_ssh_client(
ipaddress=public_ip.ipaddress.ipaddress,
reconnect=reconnect,
retries=3 if negative_test else 30
retries=3 if negative_test else 30,
keyPairFileLocation=keypair.private_key_file if keypair else None
)
self.debug("Successful to SSH into VM with ID - %s on "
"public IP address - %s" %

47
test/integration/plugins/nuagevsp/nuage_lib.py Normal file
View File

@ -0,0 +1,47 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from marvin.cloudstackAPI import createSSHKeyPair, deleteSSHKeyPair
class MySSHKeyPair:
"""Manage SSH Key pairs"""
def __init__(self, items):
self.__dict__.update(items)
@classmethod
def create(cls, apiclient, name=None, account=None,
domainid=None, projectid=None):
"""Creates SSH keypair"""
cmd = createSSHKeyPair.createSSHKeyPairCmd()
cmd.name = name
if account is not None:
cmd.account = account
if domainid is not None:
cmd.domainid = domainid
if projectid is not None:
cmd.projectid = projectid
return MySSHKeyPair(apiclient.createSSHKeyPair(cmd).__dict__)
def delete(self, apiclient):
"""Delete SSH key pair"""
cmd = deleteSSHKeyPair.deleteSSHKeyPairCmd()
cmd.name = self.name
cmd.account = self.account
cmd.domainid = self.domainid
apiclient.deleteSSHKeyPair(cmd)

651
test/integration/plugins/nuagevsp/test_nuage_configdrive.py
View File

File diff suppressed because it is too large Load Diff

6
tools/marvin/marvin/jsonHelper.py
View File

@ -43,6 +43,12 @@ class jsonLoader(object):
else:
return None
def __getitem__(self, val):
if val in self.__dict__:
return self.__dict__[val]
else:
return None
def __repr__(self):
return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v)
in self.__dict__.iteritems()))

1
ui/l10n/en.js
View File

@ -1627,6 +1627,7 @@ var dictionary = {
"label.storage.tags":"Storage Tags",
"label.storage.traffic":"Storage Traffic",
"label.storage.type":"Storage Type",
"label.storagepolicy":"Storage policy",
"label.subdomain.access":"Subdomain Access",
"label.submit":"Submit",
"label.submitted.by":"[Submitted by: <span id=\"submitted_by\"></span>]",

29
ui/scripts/system.js
View File

@ -19886,6 +19886,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
$form.find('.form-item[rel=storagepolicy]').hide();
} else if ($(this).val() == "SMB") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').css('display', 'inline-block');
@ -19918,6 +19919,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
$form.find('.form-item[rel=storagepolicy]').hide();
} else if ($(this).val() == "S3") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').hide();
@ -19952,6 +19954,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
$form.find('.form-item[rel=storagepolicy]').hide();
} else if ($(this).val() == "Swift") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').hide();
@ -19984,6 +19987,7 @@
$form.find('.form-item[rel=account]').css('display', 'inline-block');
$form.find('.form-item[rel=username]').css('display', 'inline-block');
$form.find('.form-item[rel=key]').css('display', 'inline-block');
$form.find('.form-item[rel=storagepolicy]').css('display', 'inline-block');
}
});
@ -20174,14 +20178,26 @@
}
},
account: {
label: 'label.account'
label: 'label.account',
validation: {
required: true
}
},
username: {
label: 'label.username'
label: 'label.username',
validation: {
required: true
}
},
key: {
label: 'label.key'
}
label: 'label.key',
validation: {
required: true
}
},
storagepolicy: {
label: 'label.storagepolicy'
}
//Swift (end)
}
},
@ -20348,6 +20364,11 @@
data[ 'details[' + index.toString() + '].value'] = args.data.key;
index++;
}
if (args.data.storagepolicy != null && args.data.storagepolicy.length > 0) {
data[ 'details[' + index.toString() + '].key'] = 'storagepolicy';
data[ 'details[' + index.toString() + '].value'] = args.data.storagepolicy;
index++;
}
$.ajax({
url: createURL('addImageStore'),
data: data,

191
utils/src/main/java/com/cloud/utils/SwiftUtil.java
View File

@ -24,26 +24,29 @@ import java.net.URL;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
import java.util.Map;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Formatter;
import java.util.Map;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Hex;
import org.apache.log4j.Logger;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
public class SwiftUtil {
private static Logger logger = Logger.getLogger(SwiftUtil.class);
private static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
protected static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1";
private static final String CD_SRC = "cd %s;";
private static final String SWIFT_CMD= "/usr/bin/python %s -A %s -U %s:%s -K %s %s";
private static final String WITH_STORAGE_POLICY = " --storage-policy \"%s\"";
private static final String WITH_SEGMENTS = " -S "+SWIFT_MAX_SIZE;
private static final String[] OPERATIONS_WITH_STORAGE_POLICIES = {"post","upload"};
public interface SwiftClientCfg {
String getAccount();
@ -53,6 +56,8 @@ public class SwiftUtil {
String getKey();
String getEndPoint();
String getStoragePolicy();
}
private static String getSwiftCLIPath() {
@ -65,19 +70,10 @@ public class SwiftUtil {
}
public static boolean postMeta(SwiftClientCfg cfg, String container, String object, Map<String, String> metas) {
String swiftCli = getSwiftCLIPath();
StringBuilder cms = new StringBuilder();
for (Map.Entry<String, String> entry : metas.entrySet()) {
cms.append(" -m ");
cms.append(entry.getKey());
cms.append(":");
cms.append(entry.getValue());
cms.append(" ");
}
Script command = new Script("/bin/bash", logger);
command.add("-c");
command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() + " post " +
container + " " + object + " " + cms.toString());
command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(),"post", container, object) + getMeta(metas));
OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
String result = command.execute(parser);
if (result != null) {
@ -87,21 +83,14 @@ public class SwiftUtil {
}
public static String putObject(SwiftClientCfg cfg, File srcFile, String container, String fileName) {
String swiftCli = getSwiftCLIPath();
if (fileName == null) {
fileName = srcFile.getName();
}
String srcDirectory = srcFile.getParent();
Script command = new Script("/bin/bash", logger);
long size = srcFile.length();
command.add("-c");
if (size <= SWIFT_MAX_SIZE) {
command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() +
" -K " + cfg.getKey() + " upload " + container + " " + fileName);
} else {
command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() +
" -K " + cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + fileName);
}
command.add(String.format(CD_SRC, srcFile.getParent())+getUploadObjectCommand(cfg, getSwiftCLIPath(), container,fileName, srcFile.length()));
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
@ -120,38 +109,19 @@ public class SwiftUtil {
return container + File.separator + srcFile.getName();
}
private static StringBuilder buildSwiftCmd(SwiftClientCfg swift) {
String swiftCli = getSwiftCLIPath();
StringBuilder sb = new StringBuilder();
sb.append(" /usr/bin/python ");
sb.append(swiftCli);
sb.append(" -A ");
sb.append(swift.getEndPoint());
sb.append(" -U ");
sb.append(swift.getAccount());
sb.append(":");
sb.append(swift.getUserName());
sb.append(" -K ");
sb.append(swift.getKey());
sb.append(" ");
return sb;
}
public static String[] list(SwiftClientCfg swift, String container, String rFilename) {
getSwiftCLIPath();
Script command = new Script("/bin/bash", logger);
command.add("-c");
StringBuilder swiftCmdBuilder = buildSwiftCmd(swift);
swiftCmdBuilder.append(" list ");
swiftCmdBuilder.append(container);
StringBuilder swiftCmdBuilder = new StringBuilder();
swiftCmdBuilder.append(getSwiftContainerCmd(swift, getSwiftCLIPath(), "list", container));
if (rFilename != null) {
swiftCmdBuilder.append(" -p ");
swiftCmdBuilder.append(rFilename);
}
Script command = new Script("/bin/bash", logger);
command.add("-c");
command.add(swiftCmdBuilder.toString());
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase("")) {
@ -178,11 +148,11 @@ public class SwiftUtil {
} else {
destFilePath = destDirectory.getAbsolutePath();
}
String swiftCli = getSwiftCLIPath();
Script command = new Script("/bin/bash", logger);
command.add("-c");
command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() +
" download " + container + " " + srcPath + " -o " + destFilePath);
command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "download", container, srcPath)+" -o " + destFilePath);
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
@ -203,27 +173,6 @@ public class SwiftUtil {
return new File(destFilePath);
}
public static String getContainerName(String type, Long id) {
if (type.startsWith("T")) {
return "T-" + id;
} else if (type.startsWith("S")) {
return "S-" + id;
} else if (type.startsWith("V")) {
return "V-" + id;
}
return null;
}
public static String[] splitSwiftPath(String path) {
int index = path.indexOf(File.separator);
if (index == -1) {
return null;
}
String[] paths = new String[2];
paths[0] = path.substring(0, index);
paths[1] = path.substring(index + 1);
return paths;
}
public static boolean deleteObject(SwiftClientCfg cfg, String path) {
Script command = new Script("/bin/bash", logger);
@ -236,13 +185,8 @@ public class SwiftUtil {
String container = paths[0];
String objectName = paths[1];
StringBuilder swiftCmdBuilder = buildSwiftCmd(cfg);
swiftCmdBuilder.append(" delete ");
swiftCmdBuilder.append(container);
swiftCmdBuilder.append(" ");
swiftCmdBuilder.append(objectName);
command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "delete", container, objectName));
command.add(swiftCmdBuilder.toString());
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
command.execute(parser);
return true;
@ -284,7 +228,7 @@ public class SwiftUtil {
}
public static String calculateRFC2104HMAC(String data, String key)
static String calculateRFC2104HMAC(String data, String key)
throws SignatureException, NoSuchAlgorithmException, InvalidKeyException {
SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM);
@ -294,12 +238,75 @@ public class SwiftUtil {
}
public static String toHexString(byte[] bytes) {
Formatter formatter = new Formatter();
for (byte b : bytes) {
formatter.format("%02x", b);
}
return formatter.toString();
static String toHexString(byte[] bytes) {
return Hex.encodeHexString(bytes);
}
}
/////////////// SWIFT CMD STRING HELPERS ///////////////
protected static String getSwiftCmd(SwiftClientCfg cfg, String swiftCli, String operation){
return String.format(SWIFT_CMD, swiftCli,cfg.getEndPoint(),cfg.getAccount(),cfg.getUserName(),cfg.getKey(),operation);
}
protected static String getSwiftObjectCmd(SwiftClientCfg cfg, String swiftCliPath, String operation,String container, String objectName) {
String cmd = getSwiftCmd(cfg,swiftCliPath, operation) +" "+ container+" "+objectName;
if(StringUtils.isNotBlank(cfg.getStoragePolicy()) && supportsStoragePolicies(operation)){
return cmd + String.format(WITH_STORAGE_POLICY, cfg.getStoragePolicy());
}
return cmd;
}
private static boolean supportsStoragePolicies(String operation) {
for(String supportedOp: OPERATIONS_WITH_STORAGE_POLICIES){
if(supportedOp.equals(operation)){
return true;
}
}
return false;
}
protected static String getSwiftContainerCmd(SwiftClientCfg cfg, String swiftCliPath, String operation, String container) {
return getSwiftCmd(cfg,swiftCliPath, operation) +" "+ container;
}
protected static String getUploadObjectCommand(SwiftClientCfg cfg, String swiftCliPath, String container, String objectName, long size) {
String cmd = getSwiftObjectCmd(cfg, swiftCliPath, "upload", container, objectName);
if(size > SWIFT_MAX_SIZE){
return cmd + WITH_SEGMENTS;
}
return cmd;
}
public static String getContainerName(String type, Long id) {
if (type.startsWith("T")) {
return "T-" + id;
} else if (type.startsWith("S")) {
return "S-" + id;
} else if (type.startsWith("V")) {
return "V-" + id;
}
return null;
}
public static String[] splitSwiftPath(String path) {
int index = path.indexOf(File.separator);
if (index == -1) {
return null;
}
String[] paths = new String[2];
paths[0] = path.substring(0, index);
paths[1] = path.substring(index + 1);
return paths;
}
private static String getMeta(Map<String, String> metas) {
StringBuilder cms = new StringBuilder();
for (Map.Entry<String, String> entry : metas.entrySet()) {
cms.append(" -m ");
cms.append(entry.getKey());
cms.append(":");
cms.append(entry.getValue());
cms.append(" ");
}
return cms.toString();
}
}

150
utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
View File

@ -19,9 +19,15 @@
package com.cloud.utils;
import org.junit.Test;
import org.mockito.Mockito;
import static org.mockito.BDDMockito.given;
import static org.mockito.BDDMockito.mock;
import static org.junit.Assert.assertThat;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import java.io.File;
import java.net.URL;
@ -29,10 +35,10 @@ import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import org.junit.Test;
import org.mockito.Mockito;
import com.cloud.utils.SwiftUtil.SwiftClientCfg;
public class SwiftUtilTest {
@ -90,4 +96,134 @@ public class SwiftUtilTest {
assertEquals(expected, output);
}
@Test
public void testGetSwiftCmd() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn(null);
String cmd = SwiftUtil.getSwiftCmd(cfg, "swift", "stat");
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword stat";
assertThat(cmd, is(equalTo(expected)));
}
@Test
public void testGetSwiftObjectCmd() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn(null);
String objectCmd = SwiftUtil.getSwiftObjectCmd(cfg, "swift", "delete", "T-123", "template.vhd");
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword delete T-123 template.vhd";
assertThat(objectCmd, is(equalTo(expected)));
}
@Test
public void testGetSwiftContainerCmd() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn(null);
String containerCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-123");
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-123";
assertThat(containerCmd, is(equalTo(expected)));
}
@Test
public void testGetUploadCmd() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn(null);
String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 1024);
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd";
assertThat(uploadCmd, is(equalTo(expected)));
}
@Test
public void testGetUploadCmdWithSegmentsBecauseOfSize() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn(null);
String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 5368709121L);
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd -S 5368709120";
assertThat(uploadCmd, is(equalTo(expected)));
}
@Test
public void testGetUploadCmdWithStoragePolicy() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn("policy1");
String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 1024L);
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy \"policy1\"";
assertThat(uploadCmd, is(equalTo(expected)));
}
@Test
public void testGetUploadCmdWithSegmentsAndStoragePolicy() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn("policy1");
String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 5368709121L);
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy \"policy1\" -S 5368709120";
assertThat(uploadCmd, is(equalTo(expected)));
}
@Test
public void testListContainerCmdWithStoragePolicyButNotSupportedByOperation() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn("policy1");
String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-1");
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-1";
assertThat(uploadCmd, is(equalTo(expected)));
}
@Test
public void testListContainerCmdWithoutStoragePolicy() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn(null);
String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-1");
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-1";
assertThat(uploadCmd, is(equalTo(expected)));
}
}