apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-9677: Adding storage policy support for swift as secondary storage (#2412)

Browse Source 
Original-Author: @pdube on PR Fixes #1830.
This commit is contained in:
Khosrow Moossavi 2018-04-25 18:42:15 -04:00 committed by Rohit Yadav
parent 9a13227a78
commit b6d420bec3
11 changed files with 501 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
api/src/com/cloud/agent/api/to/SwiftTO.java
View File

@ -26,17 +26,19 @@ public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg {
String userName;
String key;
String storagePolicy;
private static final String pathSeparator = "/";
public SwiftTO() {
}
public SwiftTO(Long id, String url, String account, String userName, String key) {
public SwiftTO(Long id, String url, String account, String userName, String key, String storagePolicy) {
this.id = id;
this.url = url;
this.account = account;
this.userName = userName;
this.key = key;
this.storagePolicy = storagePolicy;
}
public Long getId() {
@ -63,6 +65,11 @@ public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg {
return key;
}
@Override
public String getStoragePolicy() {
return this.storagePolicy;
}
@Override
public DataStoreRole getRole() {
return DataStoreRole.Image;

1
api/src/org/apache/cloudstack/api/ApiConstants.java
View File

@ -287,6 +287,7 @@ public class ApiConstants {
public static final String STATE = "state";
public static final String STATUS = "status";
public static final String STORAGE_TYPE = "storagetype";
public static final String STORAGE_POLICY = "storagepolicy";
public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
public static final String STORAGE_CAPABILITIES = "storagecapabilities";
public static final String SYSTEM_VM_TYPE = "systemvmtype";

115
plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
View File

@ -18,23 +18,24 @@
*/
package com.cloud.hypervisor.xenserver.resource;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.exception.InternalErrorException;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.resource.StorageProcessor;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.storage.S3.ClientOptions;
import static com.cloud.utils.ReflectUtil.flattenProperties;
import static com.google.common.collect.Lists.newArrayList;
import java.io.File;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.log4j.Logger;
import org.apache.xmlrpc.XmlRpcException;
import com.google.common.annotations.VisibleForTesting;
import com.xensource.xenapi.Connection;
import com.xensource.xenapi.SR;
import com.xensource.xenapi.Types;
@ -44,6 +45,7 @@ import com.xensource.xenapi.Types.XenAPIException;
import com.xensource.xenapi.VBD;
import com.xensource.xenapi.VDI;
import com.xensource.xenapi.VM;
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
@ -65,20 +67,24 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.log4j.Logger;
import org.apache.xmlrpc.XmlRpcException;
import java.io.File;
import java.net.URI;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import static com.cloud.utils.ReflectUtil.flattenProperties;
import static com.google.common.collect.Lists.newArrayList;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.DataTO;
import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.exception.InternalErrorException;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.resource.StorageProcessor;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.storage.S3.ClientOptions;
public class XenServerStorageProcessor implements StorageProcessor {
private static final Logger s_logger = Logger.getLogger(XenServerStorageProcessor.class);
@ -914,20 +920,55 @@ public class XenServerStorageProcessor implements StorageProcessor {
private boolean swiftUpload(final Connection conn, final SwiftTO swift, final String container, final String ldir, final String lfilename, final Boolean isISCSI,
final int wait) {
String result = null;
List<String> params = getSwiftParams(swift, container, ldir, lfilename, isISCSI);
try {
result =
hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, "op", "upload", "url", swift.getUrl(), "account", swift.getAccount(), "username",
swift.getUserName(), "key", swift.getKey(), "container", container, "ldir", ldir, "lfilename", lfilename, "isISCSI", isISCSI.toString());
if (result != null && result.equals("true")) {
return true;
}
String result = hypervisorResource.callHostPluginAsync(conn, "swiftxenserver", "swift", wait, params.toArray(new String[params.size()]));
return BooleanUtils.toBoolean(result);
} catch (final Exception e) {
s_logger.warn("swift upload failed due to " + e.toString(), e);
}
return false;
}
@VisibleForTesting
List<String> getSwiftParams(SwiftTO swift, String container, String ldir, String lfilename, Boolean isISCSI) {
// ORDER IS IMPORTANT
List<String> params = new ArrayList<>();
//operation
params.add("op");
params.add("upload");
//auth
params.add("url");
params.add(swift.getUrl());
params.add("account");
params.add(swift.getAccount());
params.add("username");
params.add(swift.getUserName());
params.add("key");
params.add(swift.getKey());
// object info
params.add("container");
params.add(container);
params.add("ldir");
params.add(ldir);
params.add("lfilename");
params.add(lfilename);
params.add("isISCSI");
params.add(isISCSI.toString());
if (swift.getStoragePolicy() != null) {
params.add("storagepolicy");
params.add(swift.getStoragePolicy());
}
return params;
}
protected String deleteSnapshotBackup(final Connection conn, final String localMountPoint, final String path, final String secondaryStorageMountPath, final String backupUUID) {
// If anybody modifies the formatting below again, I'll skin them

113
plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java Normal file
View File

@ -0,0 +1,113 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.hypervisor.xenserver.resource;
import static org.mockito.Mockito.when;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import com.cloud.agent.api.to.SwiftTO;
public class XenServerStorageProcessorTest {
@Test
public void testOrderOfSwiftUplodScriptParamsWithoutStoragePolicy() {
CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
SwiftTO swift = Mockito.mock(SwiftTO.class);
when(swift.getStoragePolicy()).thenReturn(null);
String container = "sample-container-name";
String ldir = "sample-ldir";
String lfilename = "sample-lfilename";
Boolean isISCSI = true;
List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
// make sure the params not null and has correct number of items in it
Assert.assertNotNull("params is null", params);
Assert.assertTrue("Expected param list size is 18 but it was" + params.size(), params.size() == 18);
// check the order of params
Assert.assertEquals("unexpected param.", "op", params.get(0));
Assert.assertEquals("unexpected param.", "upload", params.get(1));
Assert.assertEquals("unexpected param.", "url", params.get(2));
Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
Assert.assertEquals("unexpected param.", "account", params.get(4));
Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
Assert.assertEquals("unexpected param.", "username", params.get(6));
Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
Assert.assertEquals("unexpected param.", "key", params.get(8));
Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
Assert.assertEquals("unexpected param.", "container", params.get(10));
Assert.assertEquals("unexpected param.", container, params.get(11));
Assert.assertEquals("unexpected param.", "ldir", params.get(12));
Assert.assertEquals("unexpected param.", ldir, params.get(13));
Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
Assert.assertEquals("unexpected param.", lfilename, params.get(15));
Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
}
@Test
public void testOrderOfSwiftUplodScriptParamsWithStoragePolicy() {
CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
XenServerStorageProcessor mock = new XenServerStorageProcessor(resource);
SwiftTO swift = Mockito.mock(SwiftTO.class);
when(swift.getStoragePolicy()).thenReturn("sample-storagepolicy");
String container = "sample-container-name";
String ldir = "sample-ldir";
String lfilename = "sample-lfilename";
Boolean isISCSI = true;
List<String> params = mock.getSwiftParams(swift, container, ldir, lfilename, isISCSI);
// make sure the params not null and has correct number of items in it
Assert.assertNotNull("params is null", params);
Assert.assertTrue("Expected param list size is 20 but it was" + params.size(), params.size() == 20);
// check the order of params
Assert.assertEquals("unexpected param.", "op", params.get(0));
Assert.assertEquals("unexpected param.", "upload", params.get(1));
Assert.assertEquals("unexpected param.", "url", params.get(2));
Assert.assertEquals("unexpected param.", swift.getUrl(), params.get(3));
Assert.assertEquals("unexpected param.", "account", params.get(4));
Assert.assertEquals("unexpected param.", swift.getAccount(), params.get(5));
Assert.assertEquals("unexpected param.", "username", params.get(6));
Assert.assertEquals("unexpected param.", swift.getUserName(), params.get(7));
Assert.assertEquals("unexpected param.", "key", params.get(8));
Assert.assertEquals("unexpected param.", swift.getKey(), params.get(9));
Assert.assertEquals("unexpected param.", "container", params.get(10));
Assert.assertEquals("unexpected param.", container, params.get(11));
Assert.assertEquals("unexpected param.", "ldir", params.get(12));
Assert.assertEquals("unexpected param.", ldir, params.get(13));
Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
Assert.assertEquals("unexpected param.", lfilename, params.get(15));
Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
Assert.assertEquals("unexpected param.", isISCSI.toString(), params.get(17));
Assert.assertEquals("unexpected param.", "storagepolicy", params.get(18));
Assert.assertEquals("unexpected param.", "sample-storagepolicy", params.get(19));
}
}

2
plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
View File

@ -68,7 +68,7 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
public DataStoreTO getStoreTO(DataStore store) {
ImageStoreImpl imgStore = (ImageStoreImpl)store;
Map<String, String> details = _imageStoreDetailsDao.getDetails(imgStore.getId());
return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY));
return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY), details.get(ApiConstants.STORAGE_POLICY));
}
@Override

21
scripts/storage/secondary/swift
View File

@ -1473,8 +1473,9 @@ post [options] [container] [object]
Updates meta information for the account, container, or object depending on
the args given. If the container is not found, it will be created
automatically; but this is not true for accounts and objects. Containers
also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
or --meta option is allowed on all and used to define the user meta data
also allow the -r (or --read-acl) and -w (or --write-acl) options.
The --storage-policy will set a storage policy to the container if the container does not exist.
The -m or --meta option is allowed on all and used to define the user meta data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
@ -1493,6 +1494,8 @@ def st_post(options, args, print_queue, error_queue):
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item with the syntax name:value. This option '
'may be repeated. Example: -m Color:Blue -m Size:Large')
parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if (options.read_acl or options.write_acl or options.sync_to or
@ -1529,6 +1532,8 @@ def st_post(options, args, print_queue, error_queue):
headers['X-Container-Sync-To'] = options.sync_to
if options.sync_key is not None:
headers['X-Container-Sync-Key'] = options.sync_key
if options.storage_policy is not None:
headers['X-Storage-Policy'] = options.storage_policy
try:
conn.post_container(args[0], headers=headers)
except ClientException, err:
@ -1558,7 +1563,8 @@ upload [options] container file_or_directory [file_or_directory] [...]
Uploads to the given container the files and directories specified by the
remaining args. -c or --changed is an option that will only upload files
that have changed since the last upload. -S <size> or --segment-size <size>
and --leave-segments are options as well (see --help for more).
and --leave-segments are options as well (see --help for more). --storage-policy
Sets a storage policy to the container if the container does not exist.
'''.strip('\n')
@ -1576,6 +1582,8 @@ def st_upload(options, args, print_queue, error_queue):
dest='leave_segments', default=False, help='Indicates that you want '
'the older segments of manifest objects left alone (in the case of '
'overwrites)')
parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if len(args) < 2:
@ -1749,9 +1757,12 @@ def st_upload(options, args, print_queue, error_queue):
# permissions, so we'll ignore any error. If there's really a problem,
# it'll surface on the first object PUT.
try:
conn.put_container(args[0])
container_headers = {}
if options.storage_policy is not None:
container_headers['X-Storage-Policy'] = options.storage_policy
conn.put_container(args[0],headers=container_headers)
if options.segment_size is not None:
conn.put_container(args[0] + '_segments')
conn.put_container(args[0] + '_segments',headers=container_headers)
except Exception:
pass
try:

21
scripts/vm/hypervisor/xenserver/swift
View File

@ -1475,8 +1475,9 @@ post [options] [container] [object]
Updates meta information for the account, container, or object depending on
the args given. If the container is not found, it will be created
automatically; but this is not true for accounts and objects. Containers
also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
or --meta option is allowed on all and used to define the user meta data
also allow the -r (or --read-acl) and -w (or --write-acl) options.
The --storage-policy will set a storage policy to the container if the container does not exist.
The -m or --meta option is allowed on all and used to define the user meta data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
@ -1495,6 +1496,8 @@ def st_post(options, args, print_queue, error_queue):
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item with the syntax name:value. This option '
'may be repeated. Example: -m Color:Blue -m Size:Large')
parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if (options.read_acl or options.write_acl or options.sync_to or
@ -1531,6 +1534,8 @@ def st_post(options, args, print_queue, error_queue):
headers['X-Container-Sync-To'] = options.sync_to
if options.sync_key is not None:
headers['X-Container-Sync-Key'] = options.sync_key
if options.storage_policy is not None:
headers['X-Storage-Policy'] = options.storage_policy
try:
conn.post_container(args[0], headers=headers)
except ClientException, err:
@ -1560,7 +1565,8 @@ upload [options] container file_or_directory [file_or_directory] [...]
Uploads to the given container the files and directories specified by the
remaining args. -c or --changed is an option that will only upload files
that have changed since the last upload. -S <size> or --segment-size <size>
and --leave-segments are options as well (see --help for more).
and --leave-segments are options as well (see --help for more). --storage-policy
Sets a storage policy to the container if the container does not exist.
'''.strip('\n')
@ -1578,6 +1584,8 @@ def st_upload(options, args, print_queue, error_queue):
dest='leave_segments', default=False, help='Indicates that you want '
'the older segments of manifest objects left alone (in the case of '
'overwrites)')
parser.add_option('', '--storage-policy', action='store', dest='storage_policy',
help='Sets a storage policy to the container if the container does not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if len(args) < 2:
@ -1751,9 +1759,12 @@ def st_upload(options, args, print_queue, error_queue):
# permissions, so we'll ignore any error. If there's really a problem,
# it'll surface on the first object PUT.
try:
conn.put_container(args[0])
container_headers = {}
if options.storage_policy is not None:
container_headers['X-Storage-Policy'] = options.storage_policy
conn.put_container(args[0],headers=container_headers)
if options.segment_size is not None:
conn.put_container(args[0] + '_segments')
conn.put_container(args[0] + '_segments',headers=container_headers)
except Exception:
pass
try:

1
ui/l10n/en.js
View File

@ -1617,6 +1617,7 @@ var dictionary = {"ICMP.code":"ICMP Code",
"label.storage.tags":"Storage Tags",
"label.storage.traffic":"Storage Traffic",
"label.storage.type":"Storage Type",
"label.storagepolicy":"Storage policy",
"label.subdomain.access":"Subdomain Access",
"label.submit":"Submit",
"label.submitted.by":"[Submitted by: <span id=\"submitted_by\"></span>]",

29
ui/scripts/system.js
View File

@ -19889,6 +19889,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
$form.find('.form-item[rel=storagepolicy]').hide();
} else if ($(this).val() == "SMB") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').css('display', 'inline-block');
@ -19921,6 +19922,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
$form.find('.form-item[rel=storagepolicy]').hide();
} else if ($(this).val() == "S3") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').hide();
@ -19955,6 +19957,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
$form.find('.form-item[rel=storagepolicy]').hide();
} else if ($(this).val() == "Swift") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').hide();
@ -19987,6 +19990,7 @@
$form.find('.form-item[rel=account]').css('display', 'inline-block');
$form.find('.form-item[rel=username]').css('display', 'inline-block');
$form.find('.form-item[rel=key]').css('display', 'inline-block');
$form.find('.form-item[rel=storagepolicy]').css('display', 'inline-block');
}
});
@ -20177,14 +20181,26 @@
}
},
account: {
label: 'label.account'
label: 'label.account',
validation: {
required: true
}
},
username: {
label: 'label.username'
label: 'label.username',
validation: {
required: true
}
},
key: {
label: 'label.key'
}
label: 'label.key',
validation: {
required: true
}
},
storagepolicy: {
label: 'label.storagepolicy'
}
//Swift (end)
}
},
@ -20351,6 +20367,11 @@
data[ 'details[' + index.toString() + '].value'] = args.data.key;
index++;
}
if (args.data.storagepolicy != null && args.data.storagepolicy.length > 0) {
data[ 'details[' + index.toString() + '].key'] = 'storagepolicy';
data[ 'details[' + index.toString() + '].value'] = args.data.storagepolicy;
index++;
}
$.ajax({
url: createURL('addImageStore'),
data: data,

191
utils/src/main/java/com/cloud/utils/SwiftUtil.java
View File

@ -24,26 +24,29 @@ import java.net.URL;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
import java.util.Map;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Formatter;
import java.util.Map;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Hex;
import org.apache.log4j.Logger;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
public class SwiftUtil {
private static Logger logger = Logger.getLogger(SwiftUtil.class);
private static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
protected static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1";
private static final String CD_SRC = "cd %s;";
private static final String SWIFT_CMD= "/usr/bin/python %s -A %s -U %s:%s -K %s %s";
private static final String WITH_STORAGE_POLICY = " --storage-policy \"%s\"";
private static final String WITH_SEGMENTS = " -S "+SWIFT_MAX_SIZE;
private static final String[] OPERATIONS_WITH_STORAGE_POLICIES = {"post","upload"};
public interface SwiftClientCfg {
String getAccount();
@ -53,6 +56,8 @@ public class SwiftUtil {
String getKey();
String getEndPoint();
String getStoragePolicy();
}
private static String getSwiftCLIPath() {
@ -65,19 +70,10 @@ public class SwiftUtil {
}
public static boolean postMeta(SwiftClientCfg cfg, String container, String object, Map<String, String> metas) {
String swiftCli = getSwiftCLIPath();
StringBuilder cms = new StringBuilder();
for (Map.Entry<String, String> entry : metas.entrySet()) {
cms.append(" -m ");
cms.append(entry.getKey());
cms.append(":");
cms.append(entry.getValue());
cms.append(" ");
}
Script command = new Script("/bin/bash", logger);
command.add("-c");
command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() + " post " +
container + " " + object + " " + cms.toString());
command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(),"post", container, object) + getMeta(metas));
OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
String result = command.execute(parser);
if (result != null) {
@ -87,21 +83,14 @@ public class SwiftUtil {
}
public static String putObject(SwiftClientCfg cfg, File srcFile, String container, String fileName) {
String swiftCli = getSwiftCLIPath();
if (fileName == null) {
fileName = srcFile.getName();
}
String srcDirectory = srcFile.getParent();
Script command = new Script("/bin/bash", logger);
long size = srcFile.length();
command.add("-c");
if (size <= SWIFT_MAX_SIZE) {
command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() +
" -K " + cfg.getKey() + " upload " + container + " " + fileName);
} else {
command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() +
" -K " + cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + fileName);
}
command.add(String.format(CD_SRC, srcFile.getParent())+getUploadObjectCommand(cfg, getSwiftCLIPath(), container,fileName, srcFile.length()));
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
@ -120,38 +109,19 @@ public class SwiftUtil {
return container + File.separator + srcFile.getName();
}
private static StringBuilder buildSwiftCmd(SwiftClientCfg swift) {
String swiftCli = getSwiftCLIPath();
StringBuilder sb = new StringBuilder();
sb.append(" /usr/bin/python ");
sb.append(swiftCli);
sb.append(" -A ");
sb.append(swift.getEndPoint());
sb.append(" -U ");
sb.append(swift.getAccount());
sb.append(":");
sb.append(swift.getUserName());
sb.append(" -K ");
sb.append(swift.getKey());
sb.append(" ");
return sb;
}
public static String[] list(SwiftClientCfg swift, String container, String rFilename) {
getSwiftCLIPath();
Script command = new Script("/bin/bash", logger);
command.add("-c");
StringBuilder swiftCmdBuilder = buildSwiftCmd(swift);
swiftCmdBuilder.append(" list ");
swiftCmdBuilder.append(container);
StringBuilder swiftCmdBuilder = new StringBuilder();
swiftCmdBuilder.append(getSwiftContainerCmd(swift, getSwiftCLIPath(), "list", container));
if (rFilename != null) {
swiftCmdBuilder.append(" -p ");
swiftCmdBuilder.append(rFilename);
}
Script command = new Script("/bin/bash", logger);
command.add("-c");
command.add(swiftCmdBuilder.toString());
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result == null && parser.getLines() != null && !parser.getLines().equalsIgnoreCase("")) {
@ -178,11 +148,11 @@ public class SwiftUtil {
} else {
destFilePath = destDirectory.getAbsolutePath();
}
String swiftCli = getSwiftCLIPath();
Script command = new Script("/bin/bash", logger);
command.add("-c");
command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() +
" download " + container + " " + srcPath + " -o " + destFilePath);
command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "download", container, srcPath)+" -o " + destFilePath);
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
@ -203,27 +173,6 @@ public class SwiftUtil {
return new File(destFilePath);
}
public static String getContainerName(String type, Long id) {
if (type.startsWith("T")) {
return "T-" + id;
} else if (type.startsWith("S")) {
return "S-" + id;
} else if (type.startsWith("V")) {
return "V-" + id;
}
return null;
}
public static String[] splitSwiftPath(String path) {
int index = path.indexOf(File.separator);
if (index == -1) {
return null;
}
String[] paths = new String[2];
paths[0] = path.substring(0, index);
paths[1] = path.substring(index + 1);
return paths;
}
public static boolean deleteObject(SwiftClientCfg cfg, String path) {
Script command = new Script("/bin/bash", logger);
@ -236,13 +185,8 @@ public class SwiftUtil {
String container = paths[0];
String objectName = paths[1];
StringBuilder swiftCmdBuilder = buildSwiftCmd(cfg);
swiftCmdBuilder.append(" delete ");
swiftCmdBuilder.append(container);
swiftCmdBuilder.append(" ");
swiftCmdBuilder.append(objectName);
command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "delete", container, objectName));
command.add(swiftCmdBuilder.toString());
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
command.execute(parser);
return true;
@ -284,7 +228,7 @@ public class SwiftUtil {
}
public static String calculateRFC2104HMAC(String data, String key)
static String calculateRFC2104HMAC(String data, String key)
throws SignatureException, NoSuchAlgorithmException, InvalidKeyException {
SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM);
@ -294,12 +238,75 @@ public class SwiftUtil {
}
public static String toHexString(byte[] bytes) {
Formatter formatter = new Formatter();
for (byte b : bytes) {
formatter.format("%02x", b);
}
return formatter.toString();
static String toHexString(byte[] bytes) {
return Hex.encodeHexString(bytes);
}
}
/////////////// SWIFT CMD STRING HELPERS ///////////////
protected static String getSwiftCmd(SwiftClientCfg cfg, String swiftCli, String operation){
return String.format(SWIFT_CMD, swiftCli,cfg.getEndPoint(),cfg.getAccount(),cfg.getUserName(),cfg.getKey(),operation);
}
protected static String getSwiftObjectCmd(SwiftClientCfg cfg, String swiftCliPath, String operation,String container, String objectName) {
String cmd = getSwiftCmd(cfg,swiftCliPath, operation) +" "+ container+" "+objectName;
if(StringUtils.isNotBlank(cfg.getStoragePolicy()) && supportsStoragePolicies(operation)){
return cmd + String.format(WITH_STORAGE_POLICY, cfg.getStoragePolicy());
}
return cmd;
}
private static boolean supportsStoragePolicies(String operation) {
for(String supportedOp: OPERATIONS_WITH_STORAGE_POLICIES){
if(supportedOp.equals(operation)){
return true;
}
}
return false;
}
protected static String getSwiftContainerCmd(SwiftClientCfg cfg, String swiftCliPath, String operation, String container) {
return getSwiftCmd(cfg,swiftCliPath, operation) +" "+ container;
}
protected static String getUploadObjectCommand(SwiftClientCfg cfg, String swiftCliPath, String container, String objectName, long size) {
String cmd = getSwiftObjectCmd(cfg, swiftCliPath, "upload", container, objectName);
if(size > SWIFT_MAX_SIZE){
return cmd + WITH_SEGMENTS;
}
return cmd;
}
public static String getContainerName(String type, Long id) {
if (type.startsWith("T")) {
return "T-" + id;
} else if (type.startsWith("S")) {
return "S-" + id;
} else if (type.startsWith("V")) {
return "V-" + id;
}
return null;
}
public static String[] splitSwiftPath(String path) {
int index = path.indexOf(File.separator);
if (index == -1) {
return null;
}
String[] paths = new String[2];
paths[0] = path.substring(0, index);
paths[1] = path.substring(index + 1);
return paths;
}
private static String getMeta(Map<String, String> metas) {
StringBuilder cms = new StringBuilder();
for (Map.Entry<String, String> entry : metas.entrySet()) {
cms.append(" -m ");
cms.append(entry.getKey());
cms.append(":");
cms.append(entry.getValue());
cms.append(" ");
}
return cms.toString();
}
}

150
utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
View File

@ -19,9 +19,15 @@
package com.cloud.utils;
import org.junit.Test;
import org.mockito.Mockito;
import static org.mockito.BDDMockito.given;
import static org.mockito.BDDMockito.mock;
import static org.junit.Assert.assertThat;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import java.io.File;
import java.net.URL;
@ -29,10 +35,10 @@ import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import org.junit.Test;
import org.mockito.Mockito;
import com.cloud.utils.SwiftUtil.SwiftClientCfg;
public class SwiftUtilTest {
@ -90,4 +96,134 @@ public class SwiftUtilTest {
assertEquals(expected, output);
}
@Test
public void testGetSwiftCmd() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn(null);
String cmd = SwiftUtil.getSwiftCmd(cfg, "swift", "stat");
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword stat";
assertThat(cmd, is(equalTo(expected)));
}
@Test
public void testGetSwiftObjectCmd() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn(null);
String objectCmd = SwiftUtil.getSwiftObjectCmd(cfg, "swift", "delete", "T-123", "template.vhd");
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword delete T-123 template.vhd";
assertThat(objectCmd, is(equalTo(expected)));
}
@Test
public void testGetSwiftContainerCmd() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn(null);
String containerCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-123");
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-123";
assertThat(containerCmd, is(equalTo(expected)));
}
@Test
public void testGetUploadCmd() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn(null);
String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 1024);
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd";
assertThat(uploadCmd, is(equalTo(expected)));
}
@Test
public void testGetUploadCmdWithSegmentsBecauseOfSize() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn(null);
String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 5368709121L);
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd -S 5368709120";
assertThat(uploadCmd, is(equalTo(expected)));
}
@Test
public void testGetUploadCmdWithStoragePolicy() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn("policy1");
String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 1024L);
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy \"policy1\"";
assertThat(uploadCmd, is(equalTo(expected)));
}
@Test
public void testGetUploadCmdWithSegmentsAndStoragePolicy() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn("policy1");
String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift", "T-1", "template.vhd", 5368709121L);
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy \"policy1\" -S 5368709120";
assertThat(uploadCmd, is(equalTo(expected)));
}
@Test
public void testListContainerCmdWithStoragePolicyButNotSupportedByOperation() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn("policy1");
String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-1");
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-1";
assertThat(uploadCmd, is(equalTo(expected)));
}
@Test
public void testListContainerCmdWithoutStoragePolicy() {
SwiftClientCfg cfg = mock(SwiftClientCfg.class);
given(cfg.getEndPoint()).willReturn("swift.endpoint");
given(cfg.getAccount()).willReturn("cs");
given(cfg.getUserName()).willReturn("sec-storage");
given(cfg.getKey()).willReturn("mypassword");
given(cfg.getStoragePolicy()).willReturn(null);
String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift", "list", "T-1");
String expected = "/usr/bin/python swift -A swift.endpoint -U cs:sec-storage -K mypassword list T-1";
assertThat(uploadCmd, is(equalTo(expected)));
}
}