diff --git a/api/src/main/java/com/cloud/agent/api/to/BucketTO.java b/api/src/main/java/com/cloud/agent/api/to/BucketTO.java new file mode 100644 index 00000000000..f7e4bfea80f --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/BucketTO.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.to; + +import org.apache.cloudstack.storage.object.Bucket; + +public final class BucketTO { + + private String name; + + private String accessKey; + + private String secretKey; + + public BucketTO(Bucket bucket) { + this.name = bucket.getName(); + this.accessKey = bucket.getAccessKey(); + this.secretKey = bucket.getSecretKey(); + } + + public BucketTO(String name) { + this.name = name; + } + + public String getName() { + return this.name; + } + + public String getAccessKey() { + return this.accessKey; + } + + public String getSecretKey() { + return this.secretKey; + } +} diff --git a/client/pom.xml b/client/pom.xml index adf25dcbd99..473b711b9e3 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -648,6 +648,11 @@ cloud-plugin-storage-object-minio ${project.version} + + org.apache.cloudstack + cloud-plugin-storage-object-ceph + ${project.version} + org.apache.cloudstack cloud-plugin-storage-object-simulator diff --git a/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java b/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java index 9ee94b083cf..7efb72d23b2 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java +++ b/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java @@ -18,6 +18,7 @@ */ package org.apache.cloudstack.storage.object; +import com.cloud.agent.api.to.BucketTO; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import java.util.List; @@ -30,19 +31,19 @@ public interface ObjectStoreEntity extends DataStore, ObjectStore { boolean createUser(long accountId); - boolean deleteBucket(String name); + boolean deleteBucket(BucketTO bucket); - boolean setBucketEncryption(String name); + boolean setBucketEncryption(BucketTO bucket); - boolean deleteBucketEncryption(String name); + boolean deleteBucketEncryption(BucketTO bucket); - boolean setBucketVersioning(String name); + boolean setBucketVersioning(BucketTO bucket); - boolean deleteBucketVersioning(String name); + boolean deleteBucketVersioning(BucketTO bucket); - void setBucketPolicy(String name, String policy); + void setBucketPolicy(BucketTO bucket, String policy); - void setQuota(String name, int quota); + void setQuota(BucketTO bucket, int quota); Map getAllBucketsUsage(); } diff --git a/engine/schema/src/main/java/com/cloud/storage/BucketVO.java b/engine/schema/src/main/java/com/cloud/storage/BucketVO.java index 181b02e5a1b..53017447c07 100644 --- a/engine/schema/src/main/java/com/cloud/storage/BucketVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/BucketVO.java @@ -97,17 +97,23 @@ public class BucketVO implements Bucket { String uuid; public BucketVO() { + this.uuid = UUID.randomUUID().toString(); + } + + public BucketVO(String name) { + this.uuid = UUID.randomUUID().toString(); + this.name = name; + this.state = State.Allocated; } public BucketVO(long accountId, long domainId, long objectStoreId, String name, Integer quota, boolean versioning, - boolean encryption, boolean objectLock, String policy) - { + boolean encryption, boolean objectLock, String policy) { this.accountId = accountId; this.domainId = domainId; this.objectStoreId = objectStoreId; this.name = name; - state = State.Allocated; - uuid = UUID.randomUUID().toString(); + this.state = State.Allocated; + this.uuid = UUID.randomUUID().toString(); this.quota = quota; this.versioning = versioning; this.encryption = encryption; diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java index 3c525ba9364..f1c27526f52 100644 --- a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java @@ -18,6 +18,7 @@ */ package org.apache.cloudstack.storage.object.store; +import com.cloud.agent.api.to.BucketTO; import com.cloud.agent.api.to.DataStoreTO; import org.apache.cloudstack.storage.object.Bucket; import com.cloud.storage.DataStoreRole; @@ -107,38 +108,38 @@ public class ObjectStoreImpl implements ObjectStoreEntity { } @Override - public boolean deleteBucket(String bucketName) { - return driver.deleteBucket(bucketName, objectStoreVO.getId()); + public boolean deleteBucket(BucketTO bucket) { + return driver.deleteBucket(bucket, objectStoreVO.getId()); } @Override - public boolean setBucketEncryption(String bucketName) { - return driver.setBucketEncryption(bucketName, objectStoreVO.getId()); + public boolean setBucketEncryption(BucketTO bucket) { + return driver.setBucketEncryption(bucket, objectStoreVO.getId()); } @Override - public boolean deleteBucketEncryption(String bucketName) { - return driver.deleteBucketEncryption(bucketName, objectStoreVO.getId()); + public boolean deleteBucketEncryption(BucketTO bucket) { + return driver.deleteBucketEncryption(bucket, objectStoreVO.getId()); } @Override - public boolean setBucketVersioning(String bucketName) { - return driver.setBucketVersioning(bucketName, objectStoreVO.getId()); + public boolean setBucketVersioning(BucketTO bucket) { + return driver.setBucketVersioning(bucket, objectStoreVO.getId()); } @Override - public boolean deleteBucketVersioning(String bucketName) { - return driver.deleteBucketVersioning(bucketName, objectStoreVO.getId()); + public boolean deleteBucketVersioning(BucketTO bucket) { + return driver.deleteBucketVersioning(bucket, objectStoreVO.getId()); } @Override - public void setBucketPolicy(String bucketName, String policy) { - driver.setBucketPolicy(bucketName, policy, objectStoreVO.getId()); + public void setBucketPolicy(BucketTO bucket, String policy) { + driver.setBucketPolicy(bucket, policy, objectStoreVO.getId()); } @Override - public void setQuota(String bucketName, int quota) { - driver.setBucketQuota(bucketName, objectStoreVO.getId(), quota); + public void setQuota(BucketTO bucket, int quota) { + driver.setBucketQuota(bucket, objectStoreVO.getId(), quota); } @Override diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java index 4953b9b0cdf..13aaf7c002e 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.storage.object; import com.amazonaws.services.s3.model.AccessControlList; import com.amazonaws.services.s3.model.BucketPolicy; +import com.cloud.agent.api.to.BucketTO; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; import java.util.List; @@ -30,30 +31,30 @@ public interface ObjectStoreDriver extends DataStoreDriver { List listBuckets(long storeId); - boolean deleteBucket(String bucketName, long storeId); + boolean deleteBucket(BucketTO bucket, long storeId); - AccessControlList getBucketAcl(String bucketName, long storeId); + AccessControlList getBucketAcl(BucketTO bucket, long storeId); - void setBucketAcl(String bucketName, AccessControlList acl, long storeId); + void setBucketAcl(BucketTO bucket, AccessControlList acl, long storeId); - void setBucketPolicy(String bucketName, String policyType, long storeId); + void setBucketPolicy(BucketTO bucket, String policyType, long storeId); - BucketPolicy getBucketPolicy(String bucketName, long storeId); + BucketPolicy getBucketPolicy(BucketTO bucket, long storeId); - void deleteBucketPolicy(String bucketName, long storeId); + void deleteBucketPolicy(BucketTO bucket, long storeId); boolean createUser(long accountId, long storeId); - boolean setBucketEncryption(String bucketName, long storeId); + boolean setBucketEncryption(BucketTO bucket, long storeId); - boolean deleteBucketEncryption(String bucketName, long storeId); + boolean deleteBucketEncryption(BucketTO bucket, long storeId); - boolean setBucketVersioning(String bucketName, long storeId); + boolean setBucketVersioning(BucketTO bucket, long storeId); - boolean deleteBucketVersioning(String bucketName, long storeId); + boolean deleteBucketVersioning(BucketTO bucket, long storeId); - void setBucketQuota(String bucketName, long storeId, long size); + void setBucketQuota(BucketTO bucket, long storeId, long size); Map getAllBucketsUsage(long storeId); } diff --git a/plugins/pom.xml b/plugins/pom.xml index 233fd5ad26d..92fe7951649 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -138,6 +138,7 @@ storage/volume/flasharray storage/volume/primera storage/object/minio + storage/object/ceph storage/object/simulator diff --git a/plugins/storage/object/ceph/pom.xml b/plugins/storage/object/ceph/pom.xml new file mode 100644 index 00000000000..43b3a15731d --- /dev/null +++ b/plugins/storage/object/ceph/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + cloud-plugin-storage-object-ceph + Apache CloudStack Plugin - Ceph RGW object storage provider + + org.apache.cloudstack + cloudstack-plugins + 4.20.0.0-SNAPSHOT + ../../../pom.xml + + + + org.apache.cloudstack + cloud-engine-storage + ${project.version} + + + org.apache.cloudstack + cloud-engine-storage-object + ${project.version} + + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + + + io.github.twonote + radosgw-admin4j + 2.0.9 + + + diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java new file mode 100644 index 00000000000..6fece40e6ac --- /dev/null +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.driver; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.BucketPolicy; +import com.amazonaws.services.s3.model.BucketVersioningConfiguration; +import com.amazonaws.services.s3.model.DeleteBucketPolicyRequest; +import com.amazonaws.services.s3.model.SetBucketPolicyRequest; +import com.amazonaws.services.s3.model.GetBucketPolicyRequest; +import com.amazonaws.services.s3.model.SetBucketVersioningConfigurationRequest; +import com.cloud.agent.api.to.BucketTO; +import com.cloud.agent.api.to.DataStoreTO; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.Account; +import com.cloud.user.AccountDetailsDao; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.BaseObjectStoreDriverImpl; +import org.apache.cloudstack.storage.object.BucketObject; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.twonote.rgwadmin4j.RgwAdmin; +import org.twonote.rgwadmin4j.RgwAdminBuilder; +import org.twonote.rgwadmin4j.model.BucketInfo; +import org.twonote.rgwadmin4j.model.S3Credential; +import org.twonote.rgwadmin4j.model.User; + +import javax.inject.Inject; +import java.util.List; +import java.util.ArrayList; +import java.util.Optional; +import java.util.Map; +import java.util.HashMap; + +public class CephObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { + private static final Logger s_logger = LogManager.getLogger(CephObjectStoreDriverImpl.class); + + @Inject + AccountDao _accountDao; + + @Inject + AccountDetailsDao _accountDetailsDao; + + @Inject + ObjectStoreDao _storeDao; + + @Inject + BucketDao _bucketDao; + + @Inject + ObjectStoreDetailsDao _storeDetailsDao; + + private static final String ACCESS_KEY = "accesskey"; + private static final String SECRET_KEY = "secretkey"; + + private static final String CEPH_ACCESS_KEY = "ceph-rgw-accesskey"; + private static final String CEPH_SECRET_KEY = "ceph-rgw-secretkey"; + + @Override + public DataStoreTO getStoreTO(DataStore store) { + return null; + } + + @Override + public Bucket createBucket(Bucket bucket, boolean objectLock) { + String bucketName = bucket.getName(); + long storeId = bucket.getObjectStoreId(); + long accountId = bucket.getAccountId(); + AmazonS3 s3client = getS3Client(storeId, accountId); + + try { + if (s3client.getBucketAcl(bucketName) != null) { + throw new CloudRuntimeException("Bucket already exists with name " + bucketName); + } + } catch (AmazonS3Exception e) { + if (e.getStatusCode() != 404) { + throw new CloudRuntimeException(e); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + try { + s3client.createBucket(bucketName); + String accessKey = _accountDetailsDao.findDetail(accountId, CEPH_ACCESS_KEY).getValue(); + String secretKey = _accountDetailsDao.findDetail(accountId, CEPH_SECRET_KEY).getValue(); + ObjectStoreVO store = _storeDao.findById(storeId); + BucketVO bucketVO = _bucketDao.findById(bucket.getId()); + bucketVO.setAccessKey(accessKey); + bucketVO.setSecretKey(secretKey); + bucketVO.setBucketURL(store.getUrl() + "/" + bucketName); + _bucketDao.update(bucket.getId(), bucketVO); + return bucketVO; + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public List listBuckets(long storeId) { + RgwAdmin rgwAdmin = getRgwAdminClient(storeId); + List bucketsList = new ArrayList<>(); + try { + List buckets = rgwAdmin.listBucket(); + for(String name : buckets) { + Bucket bucket = new BucketObject(); + bucket.setName(name); + bucketsList.add(bucket); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return bucketsList; + } + + @Override + public boolean deleteBucket(BucketTO bucket, long storeId) { + RgwAdmin rgwAdmin = getRgwAdminClient(storeId); + + try { + rgwAdmin.removeBucket(bucket.getName()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public AccessControlList getBucketAcl(BucketTO bucket, long storeId) { + return null; + } + + @Override + public void setBucketAcl(BucketTO bucket, AccessControlList acl, long storeId) { + + } + + @Override + public void setBucketPolicy(BucketTO bucket, String policy, long storeId) { + String policyConfig; + + if (policy.equalsIgnoreCase("public")) { + s_logger.debug("Setting public policy on bucket " + bucket.getName()); + StringBuilder builder = new StringBuilder(); + builder.append("{\n"); + builder.append(" \"Statement\": [\n"); + builder.append(" {\n"); + builder.append(" \"Action\": [\n"); + builder.append(" \"s3:GetBucketLocation\",\n"); + builder.append(" \"s3:ListBucket\"\n"); + builder.append(" ],\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::" + bucket.getName() + "\"\n"); + builder.append(" },\n"); + builder.append(" {\n"); + builder.append(" \"Action\": \"s3:GetObject\",\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::" + bucket.getName() + "/*\"\n"); + builder.append(" }\n"); + builder.append(" ],\n"); + builder.append(" \"Version\": \"2012-10-17\"\n"); + builder.append("}\n"); + policyConfig = builder.toString(); + } else { + s_logger.debug("Setting private policy on bucket " + bucket.getName()); + policyConfig = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; + } + + AmazonS3 client = getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getAccessKey()); + client.setBucketPolicy(new SetBucketPolicyRequest(bucket.getName(), policyConfig)); + } + + @Override + public BucketPolicy getBucketPolicy(BucketTO bucket, long storeId) { + AmazonS3 client = getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getAccessKey()); + return client.getBucketPolicy(new GetBucketPolicyRequest(bucket.getName())); + } + + @Override + public void deleteBucketPolicy(BucketTO bucket, long storeId) { + AmazonS3 client = getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getAccessKey()); + client.deleteBucketPolicy(new DeleteBucketPolicyRequest(bucket.getName())); + } + + @Override + public boolean createUser(long accountId, long storeId) { + Account account = _accountDao.findById(accountId); + RgwAdmin rgwAdmin = getRgwAdminClient(storeId); + String username = account.getUuid(); + + s_logger.debug("Attempting to create Ceph RGW user for account " + account.getAccountName() + " with UUID " + username); + try { + Optional user = rgwAdmin.getUserInfo(username); + if (user.isPresent()) { + s_logger.info("User already exists in Ceph RGW: " + username); + return true; + } + } catch (Exception e) { + s_logger.debug("User does not exist. Creating user in Ceph RGW: " + username); + } + + try { + rgwAdmin.createUser(username); + User newUser = rgwAdmin.getUserInfo(username).get(); + S3Credential credentials = newUser.getS3Credentials().get(0); + + Map details = new HashMap<>(); + details.put(CEPH_ACCESS_KEY, credentials.getAccessKey()); + details.put(CEPH_SECRET_KEY, credentials.getSecretKey()); + _accountDetailsDao.persist(accountId, details); + return true; + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public boolean setBucketEncryption(BucketTO bucket, long storeId) { + return false; + } + + @Override + public boolean deleteBucketEncryption(BucketTO bucket, long storeId) { + return false; + } + + @Override + public boolean setBucketVersioning(BucketTO bucket, long storeId) { + AmazonS3 client = getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getAccessKey()); + try { + BucketVersioningConfiguration configuration = + new BucketVersioningConfiguration().withStatus("Enabled"); + + SetBucketVersioningConfigurationRequest setBucketVersioningConfigurationRequest = + new SetBucketVersioningConfigurationRequest(bucket.getName(), configuration); + + client.setBucketVersioningConfiguration(setBucketVersioningConfigurationRequest); + return true; + } catch (AmazonS3Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public boolean deleteBucketVersioning(BucketTO bucket, long storeId) { + AmazonS3 client = getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getAccessKey()); + try { + BucketVersioningConfiguration configuration = + new BucketVersioningConfiguration().withStatus("Suspended"); + + SetBucketVersioningConfigurationRequest setBucketVersioningConfigurationRequest = + new SetBucketVersioningConfigurationRequest(bucket.getName(), configuration); + + client.setBucketVersioningConfiguration(setBucketVersioningConfigurationRequest); + return true; + } catch (AmazonS3Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public void setBucketQuota(BucketTO bucket, long storeId, long size) { + RgwAdmin rgwAdmin = getRgwAdminClient(storeId); + + try { + rgwAdmin.setBucketQuota(bucket.getName(), -1, size); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public Map getAllBucketsUsage(long storeId) { + RgwAdmin rgwAdmin = getRgwAdminClient(storeId); + try { + List bucketinfo = rgwAdmin.listBucketInfo(); + Map bucketsusage = new HashMap(); + for (BucketInfo bucket: bucketinfo) { + BucketInfo.Usage usage = bucket.getUsage(); + bucketsusage.put(bucket.getBucket(), usage.getRgwMain().getSize_kb()); + } + return bucketsusage; + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + protected RgwAdmin getRgwAdminClient(long storeId) { + ObjectStoreVO store = _storeDao.findById(storeId); + Map storeDetails = _storeDetailsDao.getDetails(storeId); + String url = store.getUrl(); + String accessKey = storeDetails.get(ACCESS_KEY); + String secretKey = storeDetails.get(SECRET_KEY); + RgwAdmin admin = new RgwAdminBuilder() + .accessKey(accessKey) + .secretKey(secretKey) + .endpoint(url + "/admin") + .build(); + if (admin == null) { + throw new CloudRuntimeException("Error while creating Ceph RGW client"); + } + return admin; + } + + private String getStoreURL(long storeId) { + ObjectStoreVO store = _storeDao.findById(storeId); + String url = store.getUrl(); + return url; + } + + protected AmazonS3 getS3Client(long storeId, long accountId) { + String url = getStoreURL(storeId); + String accessKey = _accountDetailsDao.findDetail(accountId, CEPH_ACCESS_KEY).getValue(); + String secretKey = _accountDetailsDao.findDetail(accountId, CEPH_SECRET_KEY).getValue(); + return this.getS3Client(url, accessKey, secretKey); + } + protected AmazonS3 getS3Client(String url, String accessKey, String secretKey) { + AmazonS3 client = AmazonS3ClientBuilder.standard() + .enablePathStyleAccess() + .withCredentials( + new AWSStaticCredentialsProvider( + new BasicAWSCredentials(accessKey, secretKey))) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(url, "auto")) + .build(); + + if (client == null) { + throw new CloudRuntimeException("Error while creating Ceph RGW S3 client"); + } + return client; + } +} diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java new file mode 100644 index 00000000000..a9b13bf338e --- /dev/null +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.lifecycle; + +import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.twonote.rgwadmin4j.RgwAdmin; +import org.twonote.rgwadmin4j.RgwAdminBuilder; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CephObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle { + + private static final Logger s_logger = LogManager.getLogger(CephObjectStoreLifeCycleImpl.class); + + @Inject + ObjectStoreHelper objectStoreHelper; + @Inject + ObjectStoreProviderManager objectStoreMgr; + + public CephObjectStoreLifeCycleImpl() { + } + + @SuppressWarnings("unchecked") + @Override + public DataStore initialize(Map dsInfos) { + String url = (String)dsInfos.get("url"); + String name = (String)dsInfos.get("name"); + String providerName = (String)dsInfos.get("providerName"); + Map details = (Map)dsInfos.get("details"); + if (details == null) { + throw new CloudRuntimeException("Ceph RGW Admin credentials are missing"); + } + + String accessKey = details.get("accesskey"); + String secretKey = details.get("secretkey"); + + + Map objectStoreParameters = new HashMap(); + objectStoreParameters.put("name", name); + objectStoreParameters.put("url", url); + + objectStoreParameters.put("providerName", providerName); + objectStoreParameters.put("accesskey", accessKey); + objectStoreParameters.put("secretkey", secretKey); + + s_logger.info("Attempting to connect to Ceph RGW at " + url + " with access key " + accessKey); + + RgwAdmin rgwAdmin = new RgwAdminBuilder() + .accessKey(accessKey) + .secretKey(secretKey) + .endpoint(url + "/admin") + .build(); + try { + List buckets = rgwAdmin.listBucket(); + s_logger.debug("Found " + buckets + " buckets at Ceph RGW: " + url); + s_logger.info("Successfully connected to Ceph RGW: " + url); + } catch (Exception e) { + s_logger.debug("Error while initializing Ceph RGW Object Store: " + e.getMessage()); + throw new RuntimeException("Error while initializing Ceph RGW Object Store. Invalid credentials or URL"); + } + + ObjectStoreVO objectStore = objectStoreHelper.createObjectStore(objectStoreParameters, details); + return objectStoreMgr.getObjectStore(objectStore.getId()); + } + + @Override + public boolean attachCluster(DataStore store, ClusterScope scope) { + return false; + } + + @Override + public boolean attachHost(DataStore store, HostScope scope, StoragePoolInfo existingInfo) { + return false; + } + + @Override + public boolean attachZone(DataStore dataStore, ZoneScope scope, HypervisorType hypervisorType) { + return false; + } + + @Override + public boolean maintain(DataStore store) { + return false; + } + + @Override + public boolean cancelMaintain(DataStore store) { + return false; + } + + @Override + public boolean deleteDataStore(DataStore store) { + return false; + } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + +} diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImpl.java new file mode 100644 index 00000000000..e4b0eda42e8 --- /dev/null +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImpl.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.provider; + +import com.cloud.utils.component.ComponentContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; +import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.storage.datastore.driver.CephObjectStoreDriverImpl; +import org.apache.cloudstack.storage.datastore.lifecycle.CephObjectStoreLifeCycleImpl; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Component +public class CephObjectStoreProviderImpl implements ObjectStoreProvider { + + @Inject + ObjectStoreProviderManager storeMgr; + @Inject + ObjectStoreHelper helper; + + private final String providerName = "Ceph"; + protected ObjectStoreLifeCycle lifeCycle; + protected ObjectStoreDriver driver; + + @Override + public DataStoreLifeCycle getDataStoreLifeCycle() { + return lifeCycle; + } + + @Override + public String getName() { + return this.providerName; + } + + @Override + public boolean configure(Map params) { + lifeCycle = ComponentContext.inject(CephObjectStoreLifeCycleImpl.class); + driver = ComponentContext.inject(CephObjectStoreDriverImpl.class); + storeMgr.registerDriver(this.getName(), driver); + return true; + } + + @Override + public DataStoreDriver getDataStoreDriver() { + return this.driver; + } + + @Override + public HypervisorHostListener getHostListener() { + return null; + } + + @Override + public Set getTypes() { + Set types = new HashSet(); + types.add(DataStoreProviderType.OBJECT); + return types; + } +} diff --git a/plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/module.properties b/plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/module.properties new file mode 100644 index 00000000000..2aa3f3e2fa2 --- /dev/null +++ b/plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=storage-object-ceph +parent=storage diff --git a/plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/spring-storage-object-ceph-context.xml b/plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/spring-storage-object-ceph-context.xml new file mode 100644 index 00000000000..c31e652758c --- /dev/null +++ b/plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/spring-storage-object-ceph-context.xml @@ -0,0 +1,31 @@ + + + + diff --git a/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java new file mode 100644 index 00000000000..d0cd2e86a22 --- /dev/null +++ b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.driver; + +import com.amazonaws.services.s3.AmazonS3; +import com.cloud.agent.api.to.BucketTO; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.AccountDetailVO; +import com.cloud.user.AccountDetailsDao; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.Bucket; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.twonote.rgwadmin4j.RgwAdmin; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class CephObjectStoreDriverImplTest { + + @Spy + CephObjectStoreDriverImpl cephObjectStoreDriverImpl = new CephObjectStoreDriverImpl(); + + @Mock + AmazonS3 rgwClient; + @Mock + RgwAdmin rgwAdmin; + @Mock + ObjectStoreDao objectStoreDao; + @Mock + ObjectStoreVO objectStoreVO; + @Mock + ObjectStoreDetailsDao objectStoreDetailsDao; + @Mock + AccountDao accountDao; + @Mock + BucketDao bucketDao; + @Mock + AccountVO account; + @Mock + AccountDetailsDao accountDetailsDao; + + Bucket bucket; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + cephObjectStoreDriverImpl._storeDao = objectStoreDao; + cephObjectStoreDriverImpl._storeDetailsDao = objectStoreDetailsDao; + cephObjectStoreDriverImpl._accountDao = accountDao; + cephObjectStoreDriverImpl._bucketDao = bucketDao; + cephObjectStoreDriverImpl._accountDetailsDao = accountDetailsDao; + bucket = new BucketVO(); + bucket.setName("test-bucket"); + when(objectStoreVO.getUrl()).thenReturn("http://localhost:8000"); + when(objectStoreDao.findById(any())).thenReturn(objectStoreVO); + } + + @Test + public void testCreateBucket() throws Exception { + doReturn(rgwClient).when(cephObjectStoreDriverImpl).getS3Client(anyLong(), anyLong()); + when(accountDetailsDao.findDetail(anyLong(),anyString())). + thenReturn(new AccountDetailVO(1L, "abc","def")); + when(bucketDao.findById(anyLong())).thenReturn(new BucketVO(bucket.getName())); + Bucket bucketRet = cephObjectStoreDriverImpl.createBucket(bucket, false); + assertEquals(bucketRet.getName(), bucket.getName()); + verify(rgwClient, times(1)).getBucketAcl(anyString()); + verify(rgwClient, times(1)).createBucket(anyString()); + } + + @Test + public void testDeleteBucket() throws Exception { + String bucketName = "test-bucket"; + BucketTO bucket = new BucketTO(bucketName); + doReturn(rgwAdmin).when(cephObjectStoreDriverImpl).getRgwAdminClient(anyLong()); + boolean success = cephObjectStoreDriverImpl.deleteBucket(bucket, 1L); + assertTrue(success); + verify(rgwAdmin, times(1)).removeBucket(anyString()); + } +} diff --git a/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImplTest.java b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImplTest.java new file mode 100644 index 00000000000..8b17f52668b --- /dev/null +++ b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImplTest.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.provider; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider.DataStoreProviderType; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockitoAnnotations; + +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +public class CephObjectStoreProviderImplTest { + + private CephObjectStoreProviderImpl cephObjectStoreProviderImpl; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + cephObjectStoreProviderImpl = new CephObjectStoreProviderImpl(); + } + + @Test + public void testGetName() { + String name = cephObjectStoreProviderImpl.getName(); + assertEquals("Ceph", name); + } + + @Test + public void testGetTypes() { + Set types = cephObjectStoreProviderImpl.getTypes(); + assertEquals(1, types.size()); + assertEquals("OBJECT", types.toArray()[0].toString()); + } +} diff --git a/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java index 7effcb78314..9dc4b30414e 100644 --- a/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java +++ b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java @@ -41,6 +41,7 @@ import org.apache.commons.lang3.StringUtils; import com.amazonaws.services.s3.model.AccessControlList; import com.amazonaws.services.s3.model.BucketPolicy; +import com.cloud.agent.api.to.BucketTO; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.storage.BucketVO; import com.cloud.storage.dao.BucketDao; @@ -180,7 +181,8 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean deleteBucket(String bucketName, long storeId) { + public boolean deleteBucket(BucketTO bucket, long storeId) { + String bucketName = bucket.getName(); MinioClient minioClient = getMinIOClient(storeId); try { if(!minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) { @@ -199,17 +201,18 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public AccessControlList getBucketAcl(String bucketName, long storeId) { + public AccessControlList getBucketAcl(BucketTO bucket, long storeId) { return null; } @Override - public void setBucketAcl(String bucketName, AccessControlList acl, long storeId) { + public void setBucketAcl(BucketTO bucket, AccessControlList acl, long storeId) { } @Override - public void setBucketPolicy(String bucketName, String policy, long storeId) { + public void setBucketPolicy(BucketTO bucket, String policy, long storeId) { + String bucketName = bucket.getName(); String privatePolicy = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; StringBuilder builder = new StringBuilder(); @@ -249,12 +252,12 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public BucketPolicy getBucketPolicy(String bucketName, long storeId) { + public BucketPolicy getBucketPolicy(BucketTO bucket, long storeId) { return null; } @Override - public void deleteBucketPolicy(String bucketName, long storeId) { + public void deleteBucketPolicy(BucketTO bucket, long storeId) { } @@ -324,11 +327,11 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean setBucketEncryption(String bucketName, long storeId) { + public boolean setBucketEncryption(BucketTO bucket, long storeId) { MinioClient minioClient = getMinIOClient(storeId); try { minioClient.setBucketEncryption(SetBucketEncryptionArgs.builder() - .bucket(bucketName) + .bucket(bucket.getName()) .config(SseConfiguration.newConfigWithSseS3Rule()) .build() ); @@ -339,11 +342,11 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean deleteBucketEncryption(String bucketName, long storeId) { + public boolean deleteBucketEncryption(BucketTO bucket, long storeId) { MinioClient minioClient = getMinIOClient(storeId); try { minioClient.deleteBucketEncryption(DeleteBucketEncryptionArgs.builder() - .bucket(bucketName) + .bucket(bucket.getName()) .build() ); } catch (Exception e) { @@ -353,11 +356,11 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean setBucketVersioning(String bucketName, long storeId) { + public boolean setBucketVersioning(BucketTO bucket, long storeId) { MinioClient minioClient = getMinIOClient(storeId); try { minioClient.setBucketVersioning(SetBucketVersioningArgs.builder() - .bucket(bucketName) + .bucket(bucket.getName()) .config(new VersioningConfiguration(VersioningConfiguration.Status.ENABLED, null)) .build() ); @@ -368,11 +371,11 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean deleteBucketVersioning(String bucketName, long storeId) { + public boolean deleteBucketVersioning(BucketTO bucket, long storeId) { MinioClient minioClient = getMinIOClient(storeId); try { minioClient.setBucketVersioning(SetBucketVersioningArgs.builder() - .bucket(bucketName) + .bucket(bucket.getName()) .config(new VersioningConfiguration(VersioningConfiguration.Status.SUSPENDED, null)) .build() ); @@ -383,11 +386,11 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public void setBucketQuota(String bucketName, long storeId, long size) { + public void setBucketQuota(BucketTO bucket, long storeId, long size) { MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); try { - minioAdminClient.setBucketQuota(bucketName, size, QuotaUnit.GB); + minioAdminClient.setBucketQuota(bucket.getName(), size, QuotaUnit.GB); } catch (Exception e) { throw new CloudRuntimeException(e); } diff --git a/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java b/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java index 5b2faa8d2b5..1a8b3d9663a 100644 --- a/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java +++ b/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java @@ -37,6 +37,7 @@ import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.agent.api.to.BucketTO; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -128,10 +129,11 @@ public class MinIOObjectStoreDriverImplTest { @Test public void testDeleteBucket() throws Exception { String bucketName = "test-bucket"; + BucketTO bucket = new BucketTO(bucketName); doReturn(minioClient).when(minioObjectStoreDriverImpl).getMinIOClient(anyLong()); when(minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())).thenReturn(true); doNothing().when(minioClient).removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); - boolean success = minioObjectStoreDriverImpl.deleteBucket(bucketName, 1L); + boolean success = minioObjectStoreDriverImpl.deleteBucket(bucket, 1L); assertTrue(success); verify(minioClient, times(1)).bucketExists(any()); verify(minioClient, times(1)).removeBucket(any()); diff --git a/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java index b6912483caa..7b9ac59d5b1 100644 --- a/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java +++ b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.storage.datastore.driver; import com.amazonaws.services.s3.model.AccessControlList; import com.amazonaws.services.s3.model.BucketPolicy; +import com.cloud.agent.api.to.BucketTO; import com.cloud.agent.api.to.DataStoreTO; import org.apache.cloudstack.storage.object.Bucket; import com.cloud.storage.BucketVO; @@ -71,32 +72,32 @@ public class SimulatorObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean deleteBucket(String bucketName, long storeId) { + public boolean deleteBucket(BucketTO bucket, long storeId) { return true; } @Override - public AccessControlList getBucketAcl(String bucketName, long storeId) { + public AccessControlList getBucketAcl(BucketTO bucket, long storeId) { return null; } @Override - public void setBucketAcl(String bucketName, AccessControlList acl, long storeId) { + public void setBucketAcl(BucketTO bucket, AccessControlList acl, long storeId) { } @Override - public void setBucketPolicy(String bucketName, String policy, long storeId) { + public void setBucketPolicy(BucketTO bucket, String policy, long storeId) { } @Override - public BucketPolicy getBucketPolicy(String bucketName, long storeId) { + public BucketPolicy getBucketPolicy(BucketTO bucket, long storeId) { return null; } @Override - public void deleteBucketPolicy(String bucketName, long storeId) { + public void deleteBucketPolicy(BucketTO bucket, long storeId) { } @@ -106,27 +107,27 @@ public class SimulatorObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean setBucketEncryption(String bucketName, long storeId) { + public boolean setBucketEncryption(BucketTO bucket, long storeId) { return true; } @Override - public boolean deleteBucketEncryption(String bucketName, long storeId) { + public boolean deleteBucketEncryption(BucketTO bucket, long storeId) { return true; } @Override - public boolean setBucketVersioning(String bucketName, long storeId) { + public boolean setBucketVersioning(BucketTO bucket, long storeId) { return true; } @Override - public boolean deleteBucketVersioning(String bucketName, long storeId) { + public boolean deleteBucketVersioning(BucketTO bucket, long storeId) { return true; } @Override - public void setBucketQuota(String bucketName, long storeId, long size) { + public void setBucketQuota(BucketTO bucket, long storeId, long size) { } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 5e37fa2cb7b..5a2f4e690fa 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -4110,10 +4110,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } try { - // Check URL UriUtils.validateUrl(url); - } catch (final Exception e) { - throw new InvalidParameterValueException(url + " is not a valid URL"); + } catch (InvalidParameterValueException e) { + throw new InvalidParameterValueException(url + " is not a valid URL:" + e.getMessage()); } // Check Unique object store url diff --git a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java index e6acd180f16..58b41d6a55d 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.storage.object; import com.amazonaws.services.s3.internal.BucketNameUtils; import com.amazonaws.services.s3.model.IllegalBucketNameException; +import com.cloud.agent.api.to.BucketTO; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; @@ -136,29 +137,30 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic ObjectStoreVO objectStoreVO = _objectStoreDao.findById(cmd.getObjectStoragePoolId()); ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); BucketVO bucket = _bucketDao.findById(cmd.getEntityId()); + BucketTO bucketTO = new BucketTO(bucket); boolean objectLock = false; boolean bucketCreated = false; if(cmd.isObjectLocking()) { objectLock = true; } try { - objectStore.createBucket(bucket, objectLock); + bucketTO = new BucketTO(objectStore.createBucket(bucket, objectLock)); bucketCreated = true; if (cmd.isVersioning()) { - objectStore.setBucketVersioning(bucket.getName()); + objectStore.setBucketVersioning(bucketTO); } if (cmd.isEncryption()) { - objectStore.setBucketEncryption(bucket.getName()); + objectStore.setBucketEncryption(bucketTO); } if (cmd.getQuota() != null) { - objectStore.setQuota(bucket.getName(), cmd.getQuota()); + objectStore.setQuota(bucketTO, cmd.getQuota()); } if (cmd.getPolicy() != null) { - objectStore.setBucketPolicy(bucket.getName(), cmd.getPolicy()); + objectStore.setBucketPolicy(bucketTO, cmd.getPolicy()); } bucket.setState(Bucket.State.Created); @@ -166,7 +168,7 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic } catch (Exception e) { logger.debug("Failed to create bucket with name: "+bucket.getName(), e); if(bucketCreated) { - objectStore.deleteBucket(bucket.getName()); + objectStore.deleteBucket(bucketTO); } _bucketDao.remove(bucket.getId()); throw new CloudRuntimeException("Failed to create bucket with name: "+bucket.getName()+". "+e.getMessage()); @@ -178,13 +180,14 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic @ActionEvent(eventType = EventTypes.EVENT_BUCKET_DELETE, eventDescription = "deleting bucket") public boolean deleteBucket(long bucketId, Account caller) { Bucket bucket = _bucketDao.findById(bucketId); + BucketTO bucketTO = new BucketTO(bucket); if (bucket == null) { throw new InvalidParameterValueException("Unable to find bucket with ID: " + bucketId); } _accountMgr.checkAccess(caller, null, true, bucket); ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); - if (objectStore.deleteBucket(bucket.getName())) { + if (objectStore.deleteBucket(bucketTO)) { return _bucketDao.remove(bucketId); } return false; @@ -194,6 +197,7 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic @ActionEvent(eventType = EventTypes.EVENT_BUCKET_UPDATE, eventDescription = "updating bucket") public boolean updateBucket(UpdateBucketCmd cmd, Account caller) { BucketVO bucket = _bucketDao.findById(cmd.getId()); + BucketTO bucketTO = new BucketTO(bucket); if (bucket == null) { throw new InvalidParameterValueException("Unable to find bucket with ID: " + cmd.getId()); } @@ -203,29 +207,29 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic try { if (cmd.getEncryption() != null) { if (cmd.getEncryption()) { - objectStore.setBucketEncryption(bucket.getName()); + objectStore.setBucketEncryption(bucketTO); } else { - objectStore.deleteBucketEncryption(bucket.getName()); + objectStore.deleteBucketEncryption(bucketTO); } bucket.setEncryption(cmd.getEncryption()); } if (cmd.getVersioning() != null) { if (cmd.getVersioning()) { - objectStore.setBucketVersioning(bucket.getName()); + objectStore.setBucketVersioning(bucketTO); } else { - objectStore.deleteBucketVersioning(bucket.getName()); + objectStore.deleteBucketVersioning(bucketTO); } bucket.setVersioning(cmd.getVersioning()); } if (cmd.getPolicy() != null) { - objectStore.setBucketPolicy(bucket.getName(), cmd.getPolicy()); + objectStore.setBucketPolicy(bucketTO, cmd.getPolicy()); bucket.setPolicy(cmd.getPolicy()); } if (cmd.getQuota() != null) { - objectStore.setQuota(bucket.getName(), cmd.getQuota()); + objectStore.setQuota(bucketTO, cmd.getQuota()); bucket.setQuota(cmd.getQuota()); } _bucketDao.update(bucket.getId(), bucket); diff --git a/ui/src/views/infra/AddObjectStorage.vue b/ui/src/views/infra/AddObjectStorage.vue index 4aacd6adc0f..94c4ef0adb9 100644 --- a/ui/src/views/infra/AddObjectStorage.vue +++ b/ui/src/views/infra/AddObjectStorage.vue @@ -82,7 +82,7 @@ export default { inject: ['parentFetchData'], data () { return { - providers: ['MinIO', 'Simulator'], + providers: ['MinIO', 'Ceph', 'Simulator'], zones: [], loading: false } diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index 4964215020d..961c121597f 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -23,8 +23,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; -import java.net.Inet6Address; import java.net.InetAddress; +import java.net.Inet6Address; import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; diff --git a/utils/src/test/java/com/cloud/utils/UriUtilsTest.java b/utils/src/test/java/com/cloud/utils/UriUtilsTest.java index 4ec1f9a9bd9..04a74289122 100644 --- a/utils/src/test/java/com/cloud/utils/UriUtilsTest.java +++ b/utils/src/test/java/com/cloud/utils/UriUtilsTest.java @@ -273,4 +273,13 @@ public class UriUtilsTest { Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.gz")); Assert.assertFalse(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.qcow2")); } + + @Test + public void validateUrl() { + Pair url1 = UriUtils.validateUrl("https://www.cloudstack.org"); + Assert.assertEquals(url1.first(), "www.cloudstack.org"); + + Pair url2 = UriUtils.validateUrl("https://www.apache.org"); + Assert.assertEquals(url2.first(), "www.apache.org"); + } }