From a7beaaf73b819af51294cc35df0830d2a8a2fbc7 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Fri, 7 Feb 2025 16:56:20 +0530 Subject: [PATCH] Add Resource Limits to Backups and Object Storage (#10017) Doc PR : https://github.com/apache/cloudstack-documentation/pull/461 This PR fixes https://github.com/apache/cloudstack/issues/8638 == Description Four new Resource Types have been added. Admin can configure corresponding resource limits for the tenants at different levels (domain, account, project) User dashboard's Storage section will show the new resources, their limits and current usage. 1. backup - No. of backups used by the account 2. backup_storage - Backup storage allocated for the account 3. bucket - No. of buckets used by the accounts 4. object_storage - Object storage allocated for the account. Some other related changes done to BnR framework: 1. Maximum number of Backups to retain can be specified while creating Backup schedules, similar to Scheduled snapshots. 2. Oldest Scheduled backup of the same interval type will be deleted once the number reaches the configured max Backups value. 3. Code refactor: Moved syncBackups method from BackupProvider to the framework BackupManagerImpl, as it is a common functionality and all providers were using duplicated code. Changes done to the Object Storage Framework 1. Quota parameter is made mandatory while creating a bucket. Bucket quota is considered to be the allocated space and will be used to enforce Resource limits. == Schema Changes: 1. New Column `max_backups` added to `backup_schedule` table 4. New Column `backup_interval_type` added to `backups` table == Api Changes: 1. createBackup: new Parameter `scheduleid`. It should be specified whenever a scheduled backup is created. This will translate to the `backup_interval_type` in the `backups` table. 3. createBackupScheduke: new Parameter `max_backups`. To specify maximum number of backups to retain for the given schedule. == Configurations: |Setting |Scope |Default Value |Description| |-------|--------|--------------|-----------| |backup.max.hourly |Global |8 |Maximum recurring hourly backups to be retained for an instance| |backup.max.daily |Global |8 |Maximum recurring daily backups to be retained for an instance| |backup.max.weekly |Global |8 |Maximum recurring weekly backups to be retained for an instance| |backup.max.monthly |Global |8 |Maximum recurring monthly backups to be retained for an instance| |max.account.backups| Global| 20 | The default maximum number of backups that can be created for an account| |max.account.backup.storage| Global| 400 | The default maximum backup storage space (in GiB) that can be used for an account| |max.domain.backups| Global| 40 | The default maximum number of backups that can be created for an domain| |max.domain.backup.storage| Global| 800 | The default maximum backup storage space (in GiB) that can be used for an domain| |max.project.backups| Global| 20 | The default maximum number of backups that can be created for an project| |max.project.backup.storage| Global| 400 | The default maximum backup storage space (in GiB) that can be used for an project| |Setting |Scope |Default Value |Description| |-------|--------|--------------|-----------| |max.account.buckets| Global| 20 | The default maximum number of buckets that can be created for an account| |max.account.object.storage| Global| 400 | The default maximum object storage space (in GiB) that can be used for an account| |max.domain.buckets| Global| 40 | The default maximum number of buckets that can be created for an domain| |max.domain.object.storage| Global| 800 | The default maximum object storage space (in GiB) that can be used for an domain| |max.project.buckets| Global| 20 | The default maximum number of buckets that can be created for an project| |max.project.object.storage| Global| 400 | The default maximum object storage space (in GiB) that can be used for an project| Co-authored-by: Daan Hoogland Co-authored-by: Lucas Martins <56271185+lucas-a-martins@users.noreply.github.com> Co-authored-by: Lucas Martins Co-authored-by: Pearl Dsilva Co-authored-by: Rohit Yadav --- .../com/cloud/configuration/Resource.java | 12 +- .../main/java/com/cloud/event/EventTypes.java | 3 + .../com/cloud/storage/VolumeApiService.java | 2 + .../apache/cloudstack/api/ApiConstants.java | 21 +- .../command/user/backup/CreateBackupCmd.java | 19 +- .../user/backup/CreateBackupScheduleCmd.java | 10 + .../command/user/bucket/CreateBucketCmd.java | 2 +- .../command/user/bucket/UpdateBucketCmd.java | 2 +- .../api/response/AccountResponse.java | 108 ++++++ .../api/response/BackupScheduleResponse.java | 14 +- .../api/response/BucketResponse.java | 2 +- .../api/response/DomainResponse.java | 108 ++++++ .../api/response/ProjectResponse.java | 108 ++++++ .../ResourceLimitAndCountResponse.java | 24 ++ .../org/apache/cloudstack/backup/Backup.java | 22 ++ .../cloudstack/backup/BackupManager.java | 84 +++- .../cloudstack/backup/BackupProvider.java | 15 +- .../cloudstack/backup/BackupSchedule.java | 1 + .../storage/object/BucketApiService.java | 51 ++- .../java/com/cloud/storage/dao/BucketDao.java | 4 + .../com/cloud/storage/dao/BucketDaoImpl.java | 35 ++ .../cloudstack/backup/BackupScheduleVO.java | 14 +- .../apache/cloudstack/backup/BackupVO.java | 11 + .../cloudstack/backup/dao/BackupDao.java | 5 + .../cloudstack/backup/dao/BackupDaoImpl.java | 50 +++ .../backup/dao/BackupScheduleDaoImpl.java | 1 + .../META-INF/db/schema-42010to42100.sql | 4 + .../META-INF/db/views/cloud.account_view.sql | 32 ++ .../META-INF/db/views/cloud.domain_view.sql | 36 +- .../backup/DummyBackupProvider.java | 24 +- .../cloudstack/backup/NASBackupProvider.java | 27 +- .../backup/NetworkerBackupProvider.java | 138 +++---- .../backup/VeeamBackupProvider.java | 102 ++--- .../cloudstack/backup/veeam/VeeamClient.java | 4 +- .../driver/CephObjectStoreDriverImpl.java | 2 +- .../java/com/cloud/api/ApiResponseHelper.java | 2 +- .../cloud/api/query/ViewResponseHelper.java | 28 ++ .../api/query/dao/AccountJoinDaoImpl.java | 38 +- .../api/query/dao/DomainJoinDaoImpl.java | 36 ++ .../com/cloud/api/query/vo/AccountJoinVO.java | 56 +++ .../com/cloud/api/query/vo/DomainJoinVO.java | 69 +++- .../java/com/cloud/configuration/Config.java | 2 +- .../ResourceLimitManagerImpl.java | 74 +++- .../cloud/storage/VolumeApiServiceImpl.java | 19 +- .../cloudstack/backup/BackupManagerImpl.java | 246 +++++++++++- .../storage/object/BucketApiServiceImpl.java | 38 +- .../ResourceLimitManagerImplTest.java | 121 +++++- .../cloudstack/backup/BackupManagerTest.java | 365 +++++++++++++++++- .../object/BucketApiServiceImplTest.java | 182 +++++++++ ui/public/locales/en.json | 12 +- ui/src/components/view/ListResourceTable.vue | 2 +- ui/src/components/view/ResourceCountUsage.vue | 2 +- ui/src/components/view/ResourceLimitTab.vue | 6 +- .../views/compute/backup/BackupSchedule.vue | 5 + ui/src/views/compute/backup/FormSchedule.vue | 13 + ui/src/views/dashboard/UsageDashboard.vue | 32 +- ui/src/views/storage/CreateBucket.vue | 5 +- ui/src/views/storage/UpdateBucket.vue | 2 +- 58 files changed, 2182 insertions(+), 270 deletions(-) create mode 100644 server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java diff --git a/api/src/main/java/com/cloud/configuration/Resource.java b/api/src/main/java/com/cloud/configuration/Resource.java index bf8fca9d905..c7bf44de76c 100644 --- a/api/src/main/java/com/cloud/configuration/Resource.java +++ b/api/src/main/java/com/cloud/configuration/Resource.java @@ -21,7 +21,7 @@ public interface Resource { short RESOURCE_UNLIMITED = -1; String UNLIMITED = "Unlimited"; - enum ResourceType { // Primary and Secondary storage are allocated_storage and not the physical storage. + enum ResourceType { // All storage type resources are allocated_storage and not the physical storage. user_vm("user_vm", 0), public_ip("public_ip", 1), volume("volume", 2), @@ -33,7 +33,11 @@ public interface Resource { cpu("cpu", 8), memory("memory", 9), primary_storage("primary_storage", 10), - secondary_storage("secondary_storage", 11); + secondary_storage("secondary_storage", 11), + backup("backup", 12), + backup_storage("backup_storage", 13), + bucket("bucket", 14), + object_storage("object_storage", 15); private String name; private int ordinal; @@ -62,6 +66,10 @@ public interface Resource { } return null; } + + public static Boolean isStorageType(ResourceType type) { + return (type == primary_storage || type == secondary_storage || type == backup_storage || type == object_storage); + } } public static class ResourceOwnerType { diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 81ed185dae5..862a6e21fa8 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -785,6 +785,9 @@ public class EventTypes { public static final String EVENT_SHAREDFS_EXPUNGE = "SHAREDFS.EXPUNGE"; public static final String EVENT_SHAREDFS_RECOVER = "SHAREDFS.RECOVER"; + // Resource Limit + public static final String EVENT_RESOURCE_LIMIT_UPDATE = "RESOURCE.LIMIT.UPDATE"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 6f4c7aa09e2..3a72f8bddc4 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -190,4 +190,6 @@ public interface VolumeApiService { boolean stateTransitTo(Volume vol, Volume.Event event) throws NoTransitionException; Pair checkAndRepairVolume(CheckAndRepairVolumeCmd cmd) throws ResourceAllocationException; + + Long getVolumePhysicalSize(Storage.ImageFormat format, String path, String chainInfo); } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 03de07c37da..6f4780a7b1d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -51,9 +51,15 @@ public class ApiConstants { public static final String AVAILABLE = "available"; public static final String AVAILABLE_SUBNETS = "availablesubnets"; public static final String AVAILABLE_VIRTUAL_MACHINE_COUNT = "availablevirtualmachinecount"; + public static final String BACKUP_AVAILABLE = "backupavailable"; public static final String BACKUP_ID = "backupid"; + public static final String BACKUP_LIMIT = "backuplimit"; public static final String BACKUP_OFFERING_NAME = "backupofferingname"; public static final String BACKUP_OFFERING_ID = "backupofferingid"; + public static final String BACKUP_STORAGE_AVAILABLE = "backupstorageavailable"; + public static final String BACKUP_STORAGE_LIMIT = "backupstoragelimit"; + public static final String BACKUP_STORAGE_TOTAL = "backupstoragetotal"; + public static final String BACKUP_TOTAL = "backuptotal"; public static final String BASE64_IMAGE = "base64image"; public static final String BGP_PEERS = "bgppeers"; public static final String BGP_PEER_IDS = "bgppeerids"; @@ -322,6 +328,7 @@ public class ApiConstants { public static final String MAC_ADDRESS = "macaddress"; public static final String MAX = "max"; public static final String MAX_SNAPS = "maxsnaps"; + public static final String MAX_BACKUPS = "maxbackups"; public static final String MAX_CPU_NUMBER = "maxcpunumber"; public static final String MAX_MEMORY = "maxmemory"; public static final String MIN_CPU_NUMBER = "mincpunumber"; @@ -436,6 +443,7 @@ public class ApiConstants { public static final String QUALIFIERS = "qualifiers"; public static final String QUERY_FILTER = "queryfilter"; public static final String SCHEDULE = "schedule"; + public static final String SCHEDULE_ID = "scheduleid"; public static final String SCOPE = "scope"; public static final String SEARCH_BASE = "searchbase"; public static final String SECONDARY_IP = "secondaryip"; @@ -1148,7 +1156,6 @@ public class ApiConstants { public static final String MTU = "mtu"; public static final String AUTO_ENABLE_KVM_HOST = "autoenablekvmhost"; public static final String LIST_APIS = "listApis"; - public static final String OBJECT_STORAGE_ID = "objectstorageid"; public static final String VERSIONING = "versioning"; public static final String OBJECT_LOCKING = "objectlocking"; public static final String ENCRYPTION = "encryption"; @@ -1162,7 +1169,6 @@ public class ApiConstants { public static final String DISK_PATH = "diskpath"; public static final String IMPORT_SOURCE = "importsource"; public static final String TEMP_PATH = "temppath"; - public static final String OBJECT_STORAGE = "objectstore"; public static final String HEURISTIC_RULE = "heuristicrule"; public static final String HEURISTIC_TYPE_VALID_OPTIONS = "Valid options are: ISO, SNAPSHOT, TEMPLATE and VOLUME."; public static final String MANAGEMENT = "management"; @@ -1190,11 +1196,20 @@ public class ApiConstants { public static final String SHAREDFSVM_MIN_CPU_COUNT = "sharedfsvmmincpucount"; public static final String SHAREDFSVM_MIN_RAM_SIZE = "sharedfsvmminramsize"; + // Object Storage related + public static final String BUCKET_AVAILABLE = "bucketavailable"; + public static final String BUCKET_LIMIT = "bucketlimit"; + public static final String BUCKET_TOTAL = "buckettotal"; + public static final String OBJECT_STORAGE_ID = "objectstorageid"; + public static final String OBJECT_STORAGE = "objectstore"; + public static final String OBJECT_STORAGE_AVAILABLE = "objectstorageavailable"; + public static final String OBJECT_STORAGE_LIMIT = "objectstoragelimit"; + public static final String OBJECT_STORAGE_TOTAL = "objectstoragetotal"; + public static final String PARAMETER_DESCRIPTION_ACTIVATION_RULE = "Quota tariff's activation rule. It can receive a JS script that results in either " + "a boolean or a numeric value: if it results in a boolean value, the tariff value will be applied according to the result; if it results in a numeric value, the " + "numeric value will be applied; if the result is neither a boolean nor a numeric value, the tariff will not be applied. If the rule is not informed, the tariff " + "value will be applied."; - public static final String PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS = "The recommended format is \"yyyy-MM-dd'T'HH:mm:ssZ\" (e.g.: \"2023-01-01T12:00:00+0100\"); " + "however, the following formats are also accepted: \"yyyy-MM-dd HH:mm:ss\" (e.g.: \"2023-01-01 12:00:00\") and \"yyyy-MM-dd\" (e.g.: \"2023-01-01\" - if the time is not " + "added, it will be interpreted as \"00:00:00\"). If the recommended format is not used, the date will be considered in the server timezone."; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java index 558f92e4006..2d387788243 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.api.command.user.backup; import javax.inject.Inject; +import com.cloud.storage.Snapshot; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -27,6 +28,7 @@ import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCreateCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupScheduleResponse; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.backup.BackupManager; @@ -60,6 +62,13 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd { description = "ID of the VM") private Long vmId; + @Parameter(name = ApiConstants.SCHEDULE_ID, + type = CommandType.LONG, + entityType = BackupScheduleResponse.class, + description = "backup schedule ID of the VM, if this is null, it indicates that it is a manual backup.", + since = "4.21.0") + private Long scheduleId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -68,6 +77,14 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd { return vmId; } + public Long getScheduleId() { + if (scheduleId != null) { + return scheduleId; + } else { + return Snapshot.MANUAL_POLICY_ID; + } + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -75,7 +92,7 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { try { - boolean result = backupManager.createBackup(getVmId()); + boolean result = backupManager.createBackup(getVmId(), getScheduleId()); if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java index 5dc06af2123..1d0741e6217 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java @@ -75,6 +75,12 @@ public class CreateBackupScheduleCmd extends BaseCmd { description = "Specifies a timezone for this command. For more information on the timezone parameter, see TimeZone Format.") private String timezone; + @Parameter(name = ApiConstants.MAX_BACKUPS, + type = CommandType.INTEGER, + description = "maximum number of backups to retain", + since = "4.21.0") + private Integer maxBackups; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -95,6 +101,10 @@ public class CreateBackupScheduleCmd extends BaseCmd { return timezone; } + public Integer getMaxBackups() { + return maxBackups; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java index d2c91e57871..722556b8e2d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java @@ -72,7 +72,7 @@ public class CreateBucketCmd extends BaseAsyncCreateCmd implements UserCmd { description = "Id of the Object Storage Pool where bucket is created") private long objectStoragePoolId; - @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB") + @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER, required = true, description = "Bucket Quota in GiB") private Integer quota; @Parameter(name = ApiConstants.ENCRYPTION, type = CommandType.BOOLEAN, description = "Enable bucket encryption") diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java index 8e281b20e91..f913373c04b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java @@ -56,7 +56,7 @@ public class UpdateBucketCmd extends BaseCmd { @Parameter(name = ApiConstants.POLICY, type = CommandType.STRING, description = "Bucket Access Policy") private String policy; - @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB") + @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER, description = "Bucket Quota in GiB") private Integer quota; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java index 6fc098295f6..aaad7f985fc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java @@ -127,6 +127,30 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the total number of snapshots available for this account") private String snapshotAvailable; + @SerializedName(ApiConstants.BACKUP_LIMIT) + @Param(description = "the total number of backups which can be stored by this account", since = "4.21.0") + private String backupLimit; + + @SerializedName(ApiConstants.BACKUP_TOTAL) + @Param(description = "the total number of backups stored by this account", since = "4.21.0") + private Long backupTotal; + + @SerializedName(ApiConstants.BACKUP_AVAILABLE) + @Param(description = "the total number of backups available to this account", since = "4.21.0") + private String backupAvailable; + + @SerializedName(ApiConstants.BACKUP_STORAGE_LIMIT) + @Param(description = "the total backup storage space (in GiB) the account can own", since = "4.21.0") + private String backupStorageLimit; + + @SerializedName(ApiConstants.BACKUP_STORAGE_TOTAL) + @Param(description = "the total backup storage space (in GiB) owned by the account", since = "4.21.0") + private Long backupStorageTotal; + + @SerializedName(ApiConstants.BACKUP_STORAGE_AVAILABLE) + @Param(description = "the total backup storage space (in GiB) available to the account", since = "4.21.0") + private String backupStorageAvailable; + @SerializedName("templatelimit") @Param(description = "the total number of templates which can be created by this account") private String templateLimit; @@ -231,6 +255,30 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the total secondary storage space (in GiB) available to be used for this account", since = "4.2.0") private String secondaryStorageAvailable; + @SerializedName(ApiConstants.BUCKET_LIMIT) + @Param(description = "the total number of buckets which can be stored by this account", since = "4.21.0") + private String bucketLimit; + + @SerializedName(ApiConstants.BUCKET_TOTAL) + @Param(description = "the total number of buckets stored by this account", since = "4.21.0") + private Long bucketTotal; + + @SerializedName(ApiConstants.BUCKET_AVAILABLE) + @Param(description = "the total number of buckets available to this account", since = "4.21.0") + private String bucketAvailable; + + @SerializedName(ApiConstants.OBJECT_STORAGE_LIMIT) + @Param(description = "the total object storage space (in GiB) the account can own", since = "4.21.0") + private String objectStorageLimit; + + @SerializedName(ApiConstants.OBJECT_STORAGE_TOTAL) + @Param(description = "the total object storage space (in GiB) owned by the account", since = "4.21.0") + private Long objectStorageTotal; + + @SerializedName(ApiConstants.OBJECT_STORAGE_AVAILABLE) + @Param(description = "the total object storage space (in GiB) available to the account", since = "4.21.0") + private String objectStorageAvailable; + @SerializedName(ApiConstants.STATE) @Param(description = "the state of the account") private String state; @@ -386,6 +434,36 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou this.snapshotAvailable = snapshotAvailable; } + @Override + public void setBackupLimit(String backupLimit) { + this.backupLimit = backupLimit; + } + + @Override + public void setBackupTotal(Long backupTotal) { + this.backupTotal = backupTotal; + } + + @Override + public void setBackupAvailable(String backupAvailable) { + this.backupAvailable = backupAvailable; + } + + @Override + public void setBackupStorageLimit(String backupStorageLimit) { + this.backupStorageLimit = backupStorageLimit; + } + + @Override + public void setBackupStorageTotal(Long backupStorageTotal) { + this.backupStorageTotal = backupStorageTotal; + } + + @Override + public void setBackupStorageAvailable(String backupStorageAvailable) { + this.backupStorageAvailable = backupStorageAvailable; + } + @Override public void setTemplateLimit(String templateLimit) { this.templateLimit = templateLimit; @@ -537,6 +615,36 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou this.secondaryStorageAvailable = secondaryStorageAvailable; } + @Override + public void setBucketLimit(String bucketLimit) { + this.bucketLimit = bucketLimit; + } + + @Override + public void setBucketTotal(Long bucketTotal) { + this.bucketTotal = bucketTotal; + } + + @Override + public void setBucketAvailable(String bucketAvailable) { + this.bucketAvailable = bucketAvailable; + } + + @Override + public void setObjectStorageLimit(String objectStorageLimit) { + this.objectStorageLimit = objectStorageLimit; + } + + @Override + public void setObjectStorageTotal(Long objectStorageTotal) { + this.objectStorageTotal = objectStorageTotal; + } + + @Override + public void setObjectStorageAvailable(String objectStorageAvailable) { + this.objectStorageAvailable = objectStorageAvailable; + } + public void setDefaultZone(String defaultZoneId) { this.defaultZoneId = defaultZoneId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java index ba44f1e024f..d7c6f96add5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java @@ -37,18 +37,22 @@ public class BackupScheduleResponse extends BaseResponse { @Param(description = "ID of the VM") private String vmId; - @SerializedName("schedule") + @SerializedName(ApiConstants.SCHEDULE) @Param(description = "time the backup is scheduled to be taken.") private String schedule; - @SerializedName("intervaltype") + @SerializedName(ApiConstants.INTERVAL_TYPE) @Param(description = "the interval type of the backup schedule") private DateUtil.IntervalType intervalType; - @SerializedName("timezone") + @SerializedName(ApiConstants.TIMEZONE) @Param(description = "the time zone of the backup schedule") private String timezone; + @SerializedName(ApiConstants.MAX_BACKUPS) + @Param(description = "maximum number of backups retained") + private Integer maxBakups; + public String getVmName() { return vmName; } @@ -88,4 +92,8 @@ public class BackupScheduleResponse extends BaseResponse { public void setTimezone(String timezone) { this.timezone = timezone; } + + public void setMaxBakups(Integer maxBakups) { + this.maxBakups = maxBakups; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java index f2dd365452c..cde140839ec 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java @@ -75,7 +75,7 @@ public class BucketResponse extends BaseResponseWithTagInformation implements Co private String state; @SerializedName(ApiConstants.QUOTA) - @Param(description = "Bucket Quota in GB") + @Param(description = "Bucket Quota in GiB") private Integer quota; @SerializedName(ApiConstants.ENCRYPTION) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java index 7c6ad3a91c3..74fa2cbb1e4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java @@ -105,6 +105,30 @@ public class DomainResponse extends BaseResponseWithAnnotations implements Resou @SerializedName("snapshotavailable") @Param(description="the total number of snapshots available for this domain") private String snapshotAvailable; + @SerializedName(ApiConstants.BACKUP_LIMIT) + @Param(description = "the total number of backups which can be stored by this domain", since = "4.21.0") + private String backupLimit; + + @SerializedName(ApiConstants.BACKUP_TOTAL) + @Param(description = "the total number of backups stored by this domain", since = "4.21.0") + private Long backupTotal; + + @SerializedName(ApiConstants.BACKUP_AVAILABLE) + @Param(description = "the total number of backups available to this domain", since = "4.21.0") + private String backupAvailable; + + @SerializedName(ApiConstants.BACKUP_STORAGE_LIMIT) + @Param(description = "the total backup storage space (in GiB) the domain can own", since = "4.21.0") + private String backupStorageLimit; + + @SerializedName(ApiConstants.BACKUP_STORAGE_TOTAL) + @Param(description = "the total backup storage space (in GiB) owned by the domain", since = "4.21.0") + private Long backupStorageTotal; + + @SerializedName(ApiConstants.BACKUP_STORAGE_AVAILABLE) + @Param(description = "the total backup storage space (in GiB) available to the domain", since = "4.21.0") + private String backupStorageAvailable; + @SerializedName("templatelimit") @Param(description="the total number of templates which can be created by this domain") private String templateLimit; @@ -177,6 +201,30 @@ public class DomainResponse extends BaseResponseWithAnnotations implements Resou @SerializedName("secondarystorageavailable") @Param(description="the total secondary storage space (in GiB) available to be used for this domain", since="4.2.0") private String secondaryStorageAvailable; + @SerializedName(ApiConstants.BUCKET_LIMIT) + @Param(description = "the total number of buckets which can be stored by this domain", since = "4.21.0") + private String bucketLimit; + + @SerializedName(ApiConstants.BUCKET_TOTAL) + @Param(description = "the total number of buckets stored by this domain", since = "4.21.0") + private Long bucketTotal; + + @SerializedName(ApiConstants.BUCKET_AVAILABLE) + @Param(description = "the total number of buckets available to this domain", since = "4.21.0") + private String bucketAvailable; + + @SerializedName(ApiConstants.OBJECT_STORAGE_LIMIT) + @Param(description = "the total object storage space (in GiB) the domain can own", since = "4.21.0") + private String objectStorageLimit; + + @SerializedName(ApiConstants.OBJECT_STORAGE_TOTAL) + @Param(description = "the total object storage space (in GiB) owned by the domain", since = "4.21.0") + private Long objectStorageTotal; + + @SerializedName(ApiConstants.OBJECT_STORAGE_AVAILABLE) + @Param(description = "the total object storage space (in GiB) available to the domain", since = "4.21.0") + private String objectStorageAvailable; + @SerializedName(ApiConstants.RESOURCE_ICON) @Param(description = "Base64 string representation of the resource icon", since = "4.16.0.0") ResourceIconResponse icon; @@ -313,6 +361,36 @@ public class DomainResponse extends BaseResponseWithAnnotations implements Resou this.snapshotAvailable = snapshotAvailable; } + @Override + public void setBackupLimit(String backupLimit) { + this.backupLimit = backupLimit; + } + + @Override + public void setBackupTotal(Long backupTotal) { + this.backupTotal = backupTotal; + } + + @Override + public void setBackupAvailable(String backupAvailable) { + this.backupAvailable = backupAvailable; + } + + @Override + public void setBackupStorageLimit(String backupStorageLimit) { + this.backupStorageLimit = backupStorageLimit; + } + + @Override + public void setBackupStorageTotal(Long backupStorageTotal) { + this.backupStorageTotal = backupStorageTotal; + } + + @Override + public void setBackupStorageAvailable(String backupStorageAvailable) { + this.backupStorageAvailable = backupStorageAvailable; + } + @Override public void setTemplateLimit(String templateLimit) { this.templateLimit = templateLimit; @@ -430,6 +508,36 @@ public class DomainResponse extends BaseResponseWithAnnotations implements Resou this.secondaryStorageAvailable = secondaryStorageAvailable; } + @Override + public void setBucketLimit(String bucketLimit) { + this.bucketLimit = bucketLimit; + } + + @Override + public void setBucketTotal(Long bucketTotal) { + this.bucketTotal = bucketTotal; + } + + @Override + public void setBucketAvailable(String bucketAvailable) { + this.bucketAvailable = bucketAvailable; + } + + @Override + public void setObjectStorageLimit(String objectStorageLimit) { + this.objectStorageLimit = objectStorageLimit; + } + + @Override + public void setObjectStorageTotal(Long objectStorageTotal) { + this.objectStorageTotal = objectStorageTotal; + } + + @Override + public void setObjectStorageAvailable(String objectStorageAvailable) { + this.objectStorageAvailable = objectStorageAvailable; + } + public void setState(String state) { this.state = state; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java index 1c63697559b..8bdf042add0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java @@ -140,6 +140,30 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the total secondary storage space (in GiB) available to be used for this project", since = "4.2.0") private String secondaryStorageAvailable; + @SerializedName(ApiConstants.BUCKET_LIMIT) + @Param(description = "the total number of buckets which can be stored by this project", since = "4.21.0") + private String bucketLimit; + + @SerializedName(ApiConstants.BUCKET_TOTAL) + @Param(description = "the total number of buckets stored by this project", since = "4.21.0") + private Long bucketTotal; + + @SerializedName(ApiConstants.BUCKET_AVAILABLE) + @Param(description = "the total number of buckets available to this project", since = "4.21.0") + private String bucketAvailable; + + @SerializedName(ApiConstants.OBJECT_STORAGE_LIMIT) + @Param(description = "the total object storage space (in GiB) the project can own", since = "4.21.0") + private String objectStorageLimit; + + @SerializedName(ApiConstants.OBJECT_STORAGE_TOTAL) + @Param(description = "the total object storage space (in GiB) owned by the project", since = "4.21.0") + private Long objectStorageTotal; + + @SerializedName(ApiConstants.OBJECT_STORAGE_AVAILABLE) + @Param(description = "the total object storage space (in GiB) available to the project", since = "4.21.0") + private String objectStorageAvailable; + @SerializedName(ApiConstants.VM_LIMIT) @Param(description = "the total number of virtual machines that can be deployed by this project", since = "4.2.0") private String vmLimit; @@ -188,6 +212,30 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the total number of snapshots available for this project", since = "4.2.0") private String snapshotAvailable; + @SerializedName(ApiConstants.BACKUP_LIMIT) + @Param(description = "the total number of backups which can be stored by this project", since = "4.21.0") + private String backupLimit; + + @SerializedName(ApiConstants.BACKUP_TOTAL) + @Param(description = "the total number of backups stored by this project", since = "4.21.0") + private Long backupTotal; + + @SerializedName(ApiConstants.BACKUP_AVAILABLE) + @Param(description = "the total number of backups available to this project", since = "4.21.0") + private String backupAvailable; + + @SerializedName(ApiConstants.BACKUP_STORAGE_LIMIT) + @Param(description = "the total backup storage space (in GiB) the project can own", since = "4.21.0") + private String backupStorageLimit; + + @SerializedName(ApiConstants.BACKUP_STORAGE_TOTAL) + @Param(description = "the total backup storage space (in GiB) owned by the project", since = "4.21.0") + private Long backupStorageTotal; + + @SerializedName(ApiConstants.BACKUP_STORAGE_AVAILABLE) + @Param(description = "the total backup storage space (in GiB) available to the project", since = "4.21.0") + private String backupStorageAvailable; + @SerializedName("templatelimit") @Param(description = "the total number of templates which can be created by this project", since = "4.2.0") private String templateLimit; @@ -320,6 +368,36 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou this.snapshotAvailable = snapshotAvailable; } + @Override + public void setBackupLimit(String backupLimit) { + this.backupLimit = backupLimit; + } + + @Override + public void setBackupTotal(Long backupTotal) { + this.backupTotal = backupTotal; + } + + @Override + public void setBackupAvailable(String backupAvailable) { + this.backupAvailable = backupAvailable; + } + + @Override + public void setBackupStorageLimit(String backupStorageLimit) { + this.backupStorageLimit = backupStorageLimit; + } + + @Override + public void setBackupStorageTotal(Long backupStorageTotal) { + this.backupStorageTotal = backupStorageTotal; + } + + @Override + public void setBackupStorageAvailable(String backupStorageAvailable) { + this.backupStorageAvailable = backupStorageAvailable; + } + @Override public void setTemplateLimit(String templateLimit) { this.templateLimit = templateLimit; @@ -435,6 +513,36 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou this.secondaryStorageAvailable = secondaryStorageAvailable; } + @Override + public void setBucketLimit(String bucketLimit) { + this.bucketLimit = bucketLimit; + } + + @Override + public void setBucketTotal(Long bucketTotal) { + this.bucketTotal = bucketTotal; + } + + @Override + public void setBucketAvailable(String bucketAvailable) { + this.bucketAvailable = bucketAvailable; + } + + @Override + public void setObjectStorageLimit(String objectStorageLimit) { + this.objectStorageLimit = objectStorageLimit; + } + + @Override + public void setObjectStorageTotal(Long objectStorageTotal) { + this.objectStorageTotal = objectStorageTotal; + } + + @Override + public void setObjectStorageAvailable(String objectStorageAvailable) { + this.objectStorageAvailable = objectStorageAvailable; + } + public void setOwners(List> owners) { this.owners = owners; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java index f9e6df3a038..b86723b36c4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java @@ -84,6 +84,30 @@ public interface ResourceLimitAndCountResponse { public void setSnapshotAvailable(String snapshotAvailable); + public void setBackupLimit(String backupLimit); + + public void setBackupTotal(Long backupTotal); + + public void setBackupAvailable(String backupAvailable); + + public void setBackupStorageLimit(String backupStorageLimit); + + public void setBackupStorageTotal(Long backupStorageTotal); + + public void setBackupStorageAvailable(String backupStorageAvailable); + + void setBucketLimit(String bucketLimit); + + void setBucketTotal(Long bucketTotal); + + void setBucketAvailable(String bucketAvailable); + + void setObjectStorageLimit(String objectStorageLimit); + + void setObjectStorageTotal(Long objectStorageTotal); + + void setObjectStorageAvailable(String objectStorageAvailable); + public void setTemplateLimit(String templateLimit); public void setTemplateTotal(Long templateTotal); diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index f21f20adb33..dffe8a03213 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -33,6 +33,28 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged } + public enum Type { + MANUAL, HOURLY, DAILY, WEEKLY, MONTHLY; + private int max = 8; + + public void setMax(int max) { + this.max = max; + } + + public int getMax() { + return max; + } + + @Override + public String toString() { + return this.name(); + } + + public boolean equals(String snapshotType) { + return this.toString().equalsIgnoreCase(snapshotType); + } + } + class Metric { private Long backupSize = 0L; private Long dataSize = 0L; diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 8b45bb4ee5e..cbd4b7e0596 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.backup; import java.util.List; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; @@ -56,6 +57,86 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer "false", "Enable volume attach/detach operations for VMs that are assigned to Backup Offerings.", true); + ConfigKey BackupHourlyMax = new ConfigKey("Advanced", Integer.class, + "backup.max.hourly", + "8", + "Maximum recurring hourly backups to be retained for an instance. If the limit is reached, early backups from the start of the hour are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring hourly backups can not be scheduled.", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey BackupDailyMax = new ConfigKey("Advanced", Integer.class, + "backup.max.daily", + "8", + "Maximum recurring daily backups to be retained for an instance. If the limit is reached, backups from the start of the day are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring daily backups can not be scheduled.", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey BackupWeeklyMax = new ConfigKey("Advanced", Integer.class, + "backup.max.weekly", + "8", + "Maximum recurring weekly backups to be retained for an instance. If the limit is reached, backups from the beginning of the week are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring weekly backups can not be scheduled.", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey BackupMonthlyMax = new ConfigKey("Advanced", Integer.class, + "backup.max.monthly", + "8", + "Maximum recurring monthly backups to be retained for an instance. If the limit is reached, backups from the beginning of the month are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring monthly backups can not be scheduled.", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxAccountBackups = new ConfigKey("Account Defaults", Long.class, + "max.account.backups", + "20", + "The default maximum number of backups that can be created for an account", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxAccountBackupStorage = new ConfigKey("Account Defaults", Long.class, + "max.account.backup.storage", + "400", + "The default maximum backup storage space (in GiB) that can be used for an account", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxProjectBackups = new ConfigKey("Project Defaults", Long.class, + "max.project.backups", + "20", + "The default maximum number of backups that can be created for a project", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxProjectBackupStorage = new ConfigKey("Project Defaults", Long.class, + "max.project.backup.storage", + "400", + "The default maximum backup storage space (in GiB) that can be used for a project", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxDomainBackups = new ConfigKey("Domain Defaults", Long.class, + "max.domain.backups", + "40", + "The default maximum number of backups that can be created for a domain", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxDomainBackupStorage = new ConfigKey("Domain Defaults", Long.class, + "max.domain.backup.storage", + "800", + "The default maximum backup storage space (in GiB) that can be used for a domain", + false, + ConfigKey.Scope.Global, + null); + /** * List backup provider offerings * @param zoneId zone id @@ -119,9 +200,10 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer /** * Creates backup of a VM * @param vmId Virtual Machine ID + * @param scheduleId Virtual Machine Backup Schedule ID * @return returns operation success */ - boolean createBackup(final Long vmId); + boolean createBackup(final Long vmId, final Long scheduleId) throws ResourceAllocationException; /** * List existing backups for a VM diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index d36dfb7360f..e3a6c3a62bd 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -75,7 +75,7 @@ public interface BackupProvider { * @param backup * @return */ - boolean takeBackup(VirtualMachine vm); + Pair takeBackup(VirtualMachine vm); /** * Delete an existing backup @@ -104,9 +104,16 @@ public interface BackupProvider { Map getBackupMetrics(Long zoneId, List vms); /** - * This method should reconcile and create backup entries for any backups created out-of-band - * @param vm + * This method should TODO + * @param + */ + public List listRestorePoints(VirtualMachine vm); + + /** + * This method should TODO + * @param + * @param * @param metric */ - void syncBackups(VirtualMachine vm, Backup.Metric metric); + Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java b/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java index d81dd731b1f..4ff946be9cd 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java @@ -30,4 +30,5 @@ public interface BackupSchedule extends InternalIdentity { String getTimezone(); Date getScheduledTimestamp(); Long getAsyncJobId(); + Integer getMaxBackups(); } diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java index 7e1361d1e71..e27ef308d7f 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java @@ -22,10 +22,59 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.user.Account; import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; +import org.apache.cloudstack.framework.config.ConfigKey; public interface BucketApiService { + ConfigKey DefaultMaxAccountBuckets = new ConfigKey("Account Defaults", Long.class, + "max.account.buckets", + "20", + "The default maximum number of buckets that can be created for an account", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxAccountObjectStorage = new ConfigKey("Account Defaults", Long.class, + "max.account.object.storage", + "400", + "The default maximum object storage space (in GiB) that can be used for an account", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxProjectBuckets = new ConfigKey("Project Defaults", Long.class, + "max.project.buckets", + "20", + "The default maximum number of buckets that can be created for a project", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxProjectObjectStorage = new ConfigKey("Project Defaults", Long.class, + "max.project.object.storage", + "400", + "The default maximum object storage space (in GiB) that can be used for a project", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxDomainBuckets = new ConfigKey("Domain Defaults", Long.class, + "max.domain.buckets", + "20", + "The default maximum number of buckets that can be created for a domain", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxDomainObjectStorage = new ConfigKey("Domain Defaults", Long.class, + "max.domain.object.storage", + "400", + "The default maximum object storage space (in GiB) that can be used for a domain", + false, + ConfigKey.Scope.Global, + null); + /** * Creates the database object for a Bucket based on the given criteria * @@ -48,7 +97,7 @@ public interface BucketApiService { boolean deleteBucket(long bucketId, Account caller); - boolean updateBucket(UpdateBucketCmd cmd, Account caller); + boolean updateBucket(UpdateBucketCmd cmd, Account caller) throws ResourceAllocationException; void getBucketUsage(); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java index f45f28b5c2c..2511df49807 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java @@ -27,4 +27,8 @@ public interface BucketDao extends GenericDao { List listByObjectStoreIdAndAccountId(long objectStoreId, long accountId); List searchByIds(Long[] ids); + + Long countBucketsForAccount(long accountId); + + Long calculateObjectStorageAllocationForAccount(long accountId); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java index 98bef6201a1..473879d933d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java @@ -16,8 +16,10 @@ // under the License. package com.cloud.storage.dao; +import com.cloud.configuration.Resource; import com.cloud.storage.BucketVO; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import org.springframework.stereotype.Component; @@ -31,6 +33,8 @@ public class BucketDaoImpl extends GenericDaoBase implements Buc private SearchBuilder searchFilteringStoreId; private SearchBuilder bucketSearch; + private GenericSearchBuilder CountBucketsByAccount; + private GenericSearchBuilder CalculateBucketsQuotaByAccount; private static final String STORE_ID = "store_id"; private static final String STATE = "state"; @@ -54,6 +58,20 @@ public class BucketDaoImpl extends GenericDaoBase implements Buc bucketSearch.and("idIN", bucketSearch.entity().getId(), SearchCriteria.Op.IN); bucketSearch.done(); + CountBucketsByAccount = createSearchBuilder(Long.class); + CountBucketsByAccount.select(null, SearchCriteria.Func.COUNT, null); + CountBucketsByAccount.and(ACCOUNT_ID, CountBucketsByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CountBucketsByAccount.and(STATE, CountBucketsByAccount.entity().getState(), SearchCriteria.Op.NIN); + CountBucketsByAccount.and("removed", CountBucketsByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); + CountBucketsByAccount.done(); + + CalculateBucketsQuotaByAccount = createSearchBuilder(SumCount.class); + CalculateBucketsQuotaByAccount.select("sum", SearchCriteria.Func.SUM, CalculateBucketsQuotaByAccount.entity().getQuota()); + CalculateBucketsQuotaByAccount.and(ACCOUNT_ID, CalculateBucketsQuotaByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CalculateBucketsQuotaByAccount.and(STATE, CalculateBucketsQuotaByAccount.entity().getState(), SearchCriteria.Op.NIN); + CalculateBucketsQuotaByAccount.and("removed", CalculateBucketsQuotaByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); + CalculateBucketsQuotaByAccount.done(); + return true; } @Override @@ -79,4 +97,21 @@ public class BucketDaoImpl extends GenericDaoBase implements Buc sc.setParameters("idIN", ids); return search(sc, null, null, false); } + + @Override + public Long countBucketsForAccount(long accountId) { + SearchCriteria sc = CountBucketsByAccount.create(); + sc.setParameters(ACCOUNT_ID, accountId); + sc.setParameters(STATE, BucketVO.State.Destroyed); + return customSearch(sc, null).get(0); + } + + @Override + public Long calculateObjectStorageAllocationForAccount(long accountId) { + SearchCriteria sc = CalculateBucketsQuotaByAccount.create(); + sc.setParameters(ACCOUNT_ID, accountId); + sc.setParameters(STATE, BucketVO.State.Destroyed); + Long totalQuota = customSearch(sc, null).get(0).sum; + return (totalQuota * Resource.ResourceType.bytesToGiB); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java index fd3c0be18d2..0258c42c52b 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java @@ -58,15 +58,19 @@ public class BackupScheduleVO implements BackupSchedule { @Column(name = "async_job_id") Long asyncJobId; + @Column(name = "max_backups") + Integer maxBackups = 0; + public BackupScheduleVO() { } - public BackupScheduleVO(Long vmId, DateUtil.IntervalType scheduleType, String schedule, String timezone, Date scheduledTimestamp) { + public BackupScheduleVO(Long vmId, DateUtil.IntervalType scheduleType, String schedule, String timezone, Date scheduledTimestamp, Integer maxBackups) { this.vmId = vmId; this.scheduleType = (short) scheduleType.ordinal(); this.schedule = schedule; this.timezone = timezone; this.scheduledTimestamp = scheduledTimestamp; + this.maxBackups = maxBackups; } @Override @@ -128,4 +132,12 @@ public class BackupScheduleVO implements BackupSchedule { public void setAsyncJobId(Long asyncJobId) { this.asyncJobId = asyncJobId; } + + public Integer getMaxBackups() { + return maxBackups; + } + + public void setMaxBackups(Integer maxBackups) { + this.maxBackups = maxBackups; + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index b4cd2f7bada..9ef442baff9 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -88,6 +88,9 @@ public class BackupVO implements Backup { @Column(name = "zone_id") private long zoneId; + @Column(name = "backup_interval_type") + private short backupIntervalType; + @Column(name = "backed_volumes", length = 65535) protected String backedUpVolumes; @@ -208,6 +211,14 @@ public class BackupVO implements Backup { this.zoneId = zoneId; } + public short getBackupIntervalType() { + return backupIntervalType; + } + + public void setBackupIntervalType(short backupIntervalType) { + this.backupIntervalType = backupIntervalType; + } + @Override public Class getEntityType() { return Backup.class; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java index 89a13245b0a..ffd5e5a4a66 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java @@ -35,5 +35,10 @@ public interface BackupDao extends GenericDao { List syncBackups(Long zoneId, Long vmId, List externalBackups); BackupVO getBackupVO(Backup backup); List listByOfferingId(Long backupOfferingId); + + List listBackupsByVMandIntervalType(Long vmId, Backup.Type backupType); + BackupResponse newBackupResponse(Backup backup); + public Long countBackupsForAccount(long accountId); + public Long calculateBackupStorageForAccount(long accountId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java index 5a9cd062037..b4e1a760282 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java @@ -24,6 +24,7 @@ import java.util.Objects; import javax.annotation.PostConstruct; import javax.inject.Inject; +import com.cloud.utils.db.GenericSearchBuilder; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupOffering; @@ -60,6 +61,9 @@ public class BackupDaoImpl extends GenericDaoBase implements Bac BackupOfferingDao backupOfferingDao; private SearchBuilder backupSearch; + private GenericSearchBuilder CountBackupsByAccount; + private GenericSearchBuilder CalculateBackupStorageByAccount; + private SearchBuilder ListBackupsByVMandIntervalType; public BackupDaoImpl() { } @@ -72,6 +76,27 @@ public class BackupDaoImpl extends GenericDaoBase implements Bac backupSearch.and("backup_offering_id", backupSearch.entity().getBackupOfferingId(), SearchCriteria.Op.EQ); backupSearch.and("zone_id", backupSearch.entity().getZoneId(), SearchCriteria.Op.EQ); backupSearch.done(); + + CountBackupsByAccount = createSearchBuilder(Long.class); + CountBackupsByAccount.select(null, SearchCriteria.Func.COUNT, null); + CountBackupsByAccount.and("account", CountBackupsByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CountBackupsByAccount.and("status", CountBackupsByAccount.entity().getStatus(), SearchCriteria.Op.NIN); + CountBackupsByAccount.and("removed", CountBackupsByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); + CountBackupsByAccount.done(); + + CalculateBackupStorageByAccount = createSearchBuilder(SumCount.class); + CalculateBackupStorageByAccount.select("sum", SearchCriteria.Func.SUM, CalculateBackupStorageByAccount.entity().getSize()); + CalculateBackupStorageByAccount.and("account", CalculateBackupStorageByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CalculateBackupStorageByAccount.and("status", CalculateBackupStorageByAccount.entity().getStatus(), SearchCriteria.Op.NIN); + CalculateBackupStorageByAccount.and("removed", CalculateBackupStorageByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); + CalculateBackupStorageByAccount.done(); + + ListBackupsByVMandIntervalType = createSearchBuilder(); + ListBackupsByVMandIntervalType.and("vmId", ListBackupsByVMandIntervalType.entity().getVmId(), SearchCriteria.Op.EQ); + ListBackupsByVMandIntervalType.and("intervalType", ListBackupsByVMandIntervalType.entity().getBackupIntervalType(), SearchCriteria.Op.EQ); + ListBackupsByVMandIntervalType.and("status", ListBackupsByVMandIntervalType.entity().getStatus(), SearchCriteria.Op.EQ); + ListBackupsByVMandIntervalType.and("removed", ListBackupsByVMandIntervalType.entity().getRemoved(), SearchCriteria.Op.NULL); + ListBackupsByVMandIntervalType.done(); } @Override @@ -142,6 +167,31 @@ public class BackupDaoImpl extends GenericDaoBase implements Bac return listByVmId(zoneId, vmId); } + @Override + public Long countBackupsForAccount(long accountId) { + SearchCriteria sc = CountBackupsByAccount.create(); + sc.setParameters("account", accountId); + sc.setParameters("status", Backup.Status.Error, Backup.Status.Failed, Backup.Status.Removed, Backup.Status.Expunged); + return customSearch(sc, null).get(0); + } + + @Override + public Long calculateBackupStorageForAccount(long accountId) { + SearchCriteria sc = CalculateBackupStorageByAccount.create(); + sc.setParameters("account", accountId); + sc.setParameters("status", Backup.Status.Error, Backup.Status.Failed, Backup.Status.Removed, Backup.Status.Expunged); + return customSearch(sc, null).get(0).sum; + } + + @Override + public List listBackupsByVMandIntervalType(Long vmId, Backup.Type backupType) { + SearchCriteria sc = ListBackupsByVMandIntervalType.create(); + sc.setParameters("vmId", vmId); + sc.setParameters("type", backupType.ordinal()); + sc.setParameters("status", Backup.Status.BackedUp); + return listBy(sc, null); + } + @Override public BackupResponse newBackupResponse(Backup backup) { VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java index e00ccc5abd7..aac2e3bf232 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java @@ -97,6 +97,7 @@ public class BackupScheduleDaoImpl extends GenericDaoBase listRestorePoints(VirtualMachine vm) { + return null; + } + + @Override + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + return null; + } + @Override public boolean removeVMFromBackupOffering(VirtualMachine vm) { logger.debug(String.format("Removing VM %s from backup offering by the Dummy Backup Provider", vm)); @@ -111,7 +122,7 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { } @Override - public boolean takeBackup(VirtualMachine vm) { + public Pair takeBackup(VirtualMachine vm) { logger.debug(String.format("Starting backup for VM %s on Dummy provider", vm)); BackupVO backup = new BackupVO(); @@ -119,23 +130,20 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { backup.setExternalId("dummy-external-id"); backup.setType("FULL"); backup.setDate(new Date()); - backup.setSize(1024L); - backup.setProtectedSize(1024000L); + backup.setSize(1024000L); + backup.setProtectedSize(1 * Resource.ResourceType.bytesToGiB); backup.setStatus(Backup.Status.BackedUp); backup.setBackupOfferingId(vm.getBackupOfferingId()); backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); - return backupDao.persist(backup) != null; + backup = backupDao.persist(backup); + return new Pair<>(true, backup); } @Override public boolean deleteBackup(Backup backup, boolean forced) { return true; } - - @Override - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - } } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 5d3d1a91933..f148c53e614 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -46,6 +46,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; + import javax.inject.Inject; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -141,7 +142,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } @Override - public boolean takeBackup(final VirtualMachine vm) { + public Pair takeBackup(final VirtualMachine vm) { final Host host = getVMHypervisorHost(vm); final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); @@ -179,12 +180,16 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co backupVO.setSize(answer.getSize()); backupVO.setStatus(Backup.Status.BackedUp); backupVO.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); - return backupDao.update(backupVO.getId(), backupVO); + if (backupDao.update(backupVO.getId(), backupVO)) { + return new Pair<>(true, backupVO); + } else { + throw new CloudRuntimeException("Failed to update backup"); + } } else { backupVO.setStatus(Backup.Status.Failed); backupDao.remove(backupVO.getId()); + return new Pair<>(false, null); } - return Objects.nonNull(answer) && answer.getResult(); } private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { @@ -358,6 +363,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co return backupDao.remove(backup.getId()); } + logger.debug("There was an error removing the backup with id " + backup.getId()); return false; } @@ -383,6 +389,16 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co return metrics; } + @Override + public List listRestorePoints(VirtualMachine vm) { + return null; + } + + @Override + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + return null; + } + @Override public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { return Hypervisor.HypervisorType.KVM.equals(vm.getHypervisorType()); @@ -398,11 +414,6 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co return false; } - @Override - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - // TODO: check and sum/return backups metrics on per VM basis - } - @Override public List listBackupOfferings(Long zoneId) { final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 393e2911ac3..822688a86a3 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -29,15 +29,11 @@ import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.component.AdapterBase; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDaoImpl; import org.apache.cloudstack.backup.networker.NetworkerClient; @@ -462,7 +458,7 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid } @Override - public boolean takeBackup(VirtualMachine vm) { + public Pair takeBackup(VirtualMachine vm) { String networkerServer; String clusterName; @@ -514,11 +510,11 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid if (backup != null) { backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); backupDao.persist(backup); - return true; + return new Pair<>(true, backup); } else { LOG.error("Could not register backup for vm {} with saveset Time: {}", vm, saveTime); // We need to handle this rare situation where backup is successful but can't be registered properly. - return false; + return new Pair<>(false, null); } } @@ -532,7 +528,7 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid LOG.debug("EMC Networker successfully deleted backup with id " + externalBackupId); return true; } else { - LOG.debug("There was an error removing the backup with id " + externalBackupId + " from EMC NEtworker"); + LOG.debug("There was an error removing the backup with id " + externalBackupId + " from EMC Networker"); } return false; } @@ -550,12 +546,12 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid for (final VirtualMachine vm : vms) { for ( Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { - vmBackupSize += (thisVMVol.getSize() / 1024L / 1024L); + vmBackupProtectedSize += (thisVMVol.getSize() / 1024L / 1024L); } final ArrayList vmBackups = getClient(zoneId).getBackupsForVm(vm); for ( String vmBackup : vmBackups ) { NetworkerBackup vmNwBackup = getClient(zoneId).getNetworkerBackupInfo(vmBackup); - vmBackupProtectedSize+= vmNwBackup.getSize().getValue() / 1024L; + vmBackupSize += vmNwBackup.getSize().getValue() / 1024L; } Backup.Metric vmBackupMetric = new Backup.Metric(vmBackupSize,vmBackupProtectedSize); LOG.debug(String.format("Metrics for VM [%s] is [backup size: %s, data size: %s].", vm, vmBackupMetric.getBackupSize(), vmBackupMetric.getDataSize())); @@ -565,83 +561,53 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid } @Override - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - final Long zoneId = vm.getDataCenterId(); - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - final List backupsInDb = backupDao.listByVmId(null, vm.getId()); - final ArrayList backupsInNetworker = getClient(zoneId).getBackupsForVm(vm); - final List removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); - for (final String networkerBackupId : backupsInNetworker ) { - Long vmBackupSize=0L; - boolean backupExists = false; - for (final Backup backupInDb : backupsInDb) { - LOG.debug(String.format("Checking if Backup %s with external ID %s for VM %s is valid", backupsInDb, backupInDb.getName(), vm)); - if ( networkerBackupId.equals(backupInDb.getExternalId()) ) { - LOG.debug(String.format("Found Backup %s in both Database and Networker", backupInDb)); - backupExists = true; - removeList.remove(backupInDb.getId()); - if (metric != null) { - LOG.debug(String.format("Update backup [%s] from [size: %s, protected size: %s] to [size: %s, protected size: %s].", - backupInDb, backupInDb.getSize(), backupInDb.getProtectedSize(), - metric.getBackupSize(), metric.getDataSize())); - ((BackupVO) backupInDb).setSize(metric.getBackupSize()); - ((BackupVO) backupInDb).setProtectedSize(metric.getDataSize()); - backupDao.update(backupInDb.getId(), ((BackupVO) backupInDb)); - } - break; - } - } - if (backupExists) { - continue; - } - // Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts - // with the proper parameters. So we will register any backups taken on the Networker side from - // outside Cloudstack. If ever Networker will support KVM out of the box this functionality also will - // ensure that SLA like backups will be found and registered. - NetworkerBackup strayNetworkerBackup = getClient(vm.getDataCenterId()).getNetworkerBackupInfo(networkerBackupId); - // Since running backups are already present in Networker Server but not completed - // make sure the backup is not in progress at this time. - if ( strayNetworkerBackup.getCompletionTime() != null) { - BackupVO strayBackup = new BackupVO(); - strayBackup.setVmId(vm.getId()); - strayBackup.setExternalId(strayNetworkerBackup.getId()); - strayBackup.setType(strayNetworkerBackup.getType()); - SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); - try { - strayBackup.setDate(formatterDateTime.parse(strayNetworkerBackup.getSaveTime())); - } catch (ParseException e) { - String msg = String.format("Unable to parse date [%s].", strayNetworkerBackup.getSaveTime()); - LOG.error(msg, e); - throw new CloudRuntimeException(msg, e); - } - strayBackup.setStatus(Backup.Status.BackedUp); - for ( Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { - vmBackupSize += (thisVMVol.getSize() / 1024L /1024L); - } - strayBackup.setSize(vmBackupSize); - strayBackup.setProtectedSize(strayNetworkerBackup.getSize().getValue() / 1024L ); - strayBackup.setBackupOfferingId(vm.getBackupOfferingId()); - strayBackup.setAccountId(vm.getAccountId()); - strayBackup.setDomainId(vm.getDomainId()); - strayBackup.setZoneId(vm.getDataCenterId()); - LOG.debug(String.format("Creating a new entry in backups: [id: %s, uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " - + "domain_id: %s, zone_id: %s].", strayBackup.getId(), strayBackup.getUuid(), strayBackup.getVmId(), strayBackup.getExternalId(), - strayBackup.getType(), strayBackup.getDate(), strayBackup.getBackupOfferingId(), strayBackup.getAccountId(), - strayBackup.getDomainId(), strayBackup.getZoneId())); - backupDao.persist(strayBackup); - LOG.warn("Added backup found in provider [" + strayBackup + "]"); - } else { - LOG.debug ("Backup is in progress, skipping addition for this run"); - } - } - for (final Long backupIdToRemove : removeList) { - LOG.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); - backupDao.remove(backupIdToRemove); - } + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + // Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts + // with the proper parameters. So we will register any backups taken on the Networker side from + // outside Cloudstack. If ever Networker will support KVM out of the box this functionality also will + // ensure that SLA like backups will be found and registered. + NetworkerBackup strayNetworkerBackup = getClient(vm.getDataCenterId()).getNetworkerBackupInfo(restorePoint.getId()); + + // Since running backups are already present in Networker Server but not completed + // make sure the backup is not in progress at this time. + if (strayNetworkerBackup.getCompletionTime() != null) { + BackupVO backup = new BackupVO(); + backup.setVmId(vm.getId()); + backup.setExternalId(strayNetworkerBackup.getId()); + backup.setType(strayNetworkerBackup.getType()); + SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + try { + backup.setDate(formatterDateTime.parse(strayNetworkerBackup.getSaveTime())); + } catch (ParseException e) { + String msg = String.format("Unable to parse date [%s].", strayNetworkerBackup.getSaveTime()); + LOG.error(msg, e); + throw new CloudRuntimeException(msg, e); } - }); + backup.setStatus(Backup.Status.BackedUp); + Long vmBackupProtectedSize=0L; + for (Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { + vmBackupProtectedSize += (thisVMVol.getSize() / 1024L / 1024L); + } + backup.setSize(strayNetworkerBackup.getSize().getValue() / 1024L); + backup.setProtectedSize(vmBackupProtectedSize); + backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setAccountId(vm.getAccountId()); + backup.setDomainId(vm.getDomainId()); + backup.setZoneId(vm.getDataCenterId()); + backupDao.persist(backup); + return backup; + } + LOG.debug ("Backup is in progress, skipping addition for this run"); + return null; + } + + @Override + public List listRestorePoints(VirtualMachine vm) { + final Long zoneId = vm.getDataCenterId(); + final ArrayList backupIds = getClient(zoneId).getBackupsForVm(vm); + List restorePoints = + backupIds.stream().map(id -> new Backup.RestorePoint(id, null, null)).collect(Collectors.toList()); + return restorePoints; } @Override diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index c120d8bd599..0735136d15d 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -29,8 +29,6 @@ import java.util.stream.Collectors; import javax.inject.Inject; -import org.apache.cloudstack.api.ApiCommandResourceType; -import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.backup.Backup.Metric; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.veeam.VeeamClient; @@ -42,20 +40,13 @@ import org.apache.commons.lang3.BooleanUtils; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; -import com.cloud.event.ActionEventUtils; -import com.cloud.event.EventTypes; -import com.cloud.event.EventVO; import com.cloud.hypervisor.Hypervisor; import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; -import com.cloud.user.User; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; @@ -220,9 +211,10 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, } @Override - public boolean takeBackup(final VirtualMachine vm) { + public Pair takeBackup(final VirtualMachine vm) { final VeeamClient client = getClient(vm.getDataCenterId()); - return client.startBackupJob(vm.getBackupExternalId()); + Boolean result = client.startBackupJob(vm.getBackupExternalId()); + return new Pair<>(result, null); } @Override @@ -322,78 +314,30 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, return metrics; } - private List listRestorePoints(VirtualMachine vm) { - String backupName = getGuestBackupName(vm.getInstanceName(), vm.getUuid()); - return getClient(vm.getDataCenterId()).listRestorePoints(backupName, vm.getInstanceName()); - } - - private Backup checkAndUpdateIfBackupEntryExistsForRestorePoint(List backupsInDb, Backup.RestorePoint restorePoint, Backup.Metric metric) { - for (final Backup backup : backupsInDb) { - if (restorePoint.getId().equals(backup.getExternalId())) { - if (metric != null) { - logger.debug("Update backup with [id: {}, uuid: {}, name: {}, external id: {}] from [size: {}, protected size: {}] to [size: {}, protected size: {}].", - backup.getId(), backup.getUuid(), backup.getName(), backup.getExternalId(), backup.getSize(), backup.getProtectedSize(), metric.getBackupSize(), metric.getDataSize()); - - ((BackupVO) backup).setSize(metric.getBackupSize()); - ((BackupVO) backup).setProtectedSize(metric.getDataSize()); - backupDao.update(backup.getId(), ((BackupVO) backup)); - } - return backup; - } + @Override + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + BackupVO backup = new BackupVO(); + backup.setVmId(vm.getId()); + backup.setExternalId(restorePoint.getId()); + backup.setType(restorePoint.getType()); + backup.setDate(restorePoint.getCreated()); + backup.setStatus(Backup.Status.BackedUp); + if (metric != null) { + backup.setSize(metric.getBackupSize()); + backup.setProtectedSize(metric.getDataSize()); } - return null; + backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setAccountId(vm.getAccountId()); + backup.setDomainId(vm.getDomainId()); + backup.setZoneId(vm.getDataCenterId()); + backupDao.persist(backup); + return backup; } @Override - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - List restorePoints = listRestorePoints(vm); - if (CollectionUtils.isEmpty(restorePoints)) { - logger.debug("Can't find any restore point to VM: {}", vm); - return; - } - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - final List backupsInDb = backupDao.listByVmId(null, vm.getId()); - final List removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); - for (final Backup.RestorePoint restorePoint : restorePoints) { - if (!(restorePoint.getId() == null || restorePoint.getType() == null || restorePoint.getCreated() == null)) { - Backup existingBackupEntry = checkAndUpdateIfBackupEntryExistsForRestorePoint(backupsInDb, restorePoint, metric); - if (existingBackupEntry != null) { - removeList.remove(existingBackupEntry.getId()); - continue; - } - - BackupVO backup = new BackupVO(); - backup.setVmId(vm.getId()); - backup.setExternalId(restorePoint.getId()); - backup.setType(restorePoint.getType()); - backup.setDate(restorePoint.getCreated()); - backup.setStatus(Backup.Status.BackedUp); - if (metric != null) { - backup.setSize(metric.getBackupSize()); - backup.setProtectedSize(metric.getDataSize()); - } - backup.setBackupOfferingId(vm.getBackupOfferingId()); - backup.setAccountId(vm.getAccountId()); - backup.setDomainId(vm.getDomainId()); - backup.setZoneId(vm.getDataCenterId()); - - logger.debug("Creating a new entry in backups: [id: {}, uuid: {}, name: {}, vm_id: {}, external_id: {}, type: {}, date: {}, backup_offering_id: {}, account_id: {}, " - + "domain_id: {}, zone_id: {}].", backup.getId(), backup.getUuid(), backup.getName(), backup.getVmId(), backup.getExternalId(), backup.getType(), backup.getDate(), backup.getBackupOfferingId(), backup.getAccountId(), backup.getDomainId(), backup.getZoneId()); - backupDao.persist(backup); - - ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_CREATE, - String.format("Created backup %s for VM ID: %s", backup.getUuid(), vm.getUuid()), - vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); - } - } - for (final Long backupIdToRemove : removeList) { - logger.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); - backupDao.remove(backupIdToRemove); - } - } - }); + public List listRestorePoints(VirtualMachine vm) { + String backupName = getGuestBackupName(vm.getInstanceName(), vm.getUuid()); + return getClient(vm.getDataCenterId()).listRestorePoints(backupName, vm.getInstanceName()); } @Override diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index d911736090c..9accc0714de 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -844,11 +844,11 @@ public class VeeamClient { "if ($restore) { $restore ^| Format-List } }" ); Pair response = executePowerShellCommands(cmds); - final List restorePoints = new ArrayList<>(); if (response == null || !response.first()) { - return restorePoints; + return null; } + final List restorePoints = new ArrayList<>(); for (final String block : response.second().split("\r\n\r\n")) { if (block.isEmpty()) { continue; diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index 551d96eab9a..7eb350a0643 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -290,7 +290,7 @@ public class CephObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { RgwAdmin rgwAdmin = getRgwAdminClient(storeId); try { - rgwAdmin.setBucketQuota(bucket.getName(), -1, size); + rgwAdmin.setIndividualBucketQuota(null, bucket.getName(), -1, size * 1024 * 1024); } catch (Exception e) { throw new CloudRuntimeException(e); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index fcc4444670c..92892118d81 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -590,7 +590,7 @@ public class ApiResponseHelper implements ResponseGenerator { } resourceLimitResponse.setResourceType(limit.getType()); - if ((limit.getType() == ResourceType.primary_storage || limit.getType() == ResourceType.secondary_storage) && limit.getMax() >= 0) { + if (ResourceType.isStorageType(limit.getType()) && limit.getMax() >= 0) { resourceLimitResponse.setMax((long)Math.ceil((double)limit.getMax() / ResourceType.bytesToGiB)); } else { resourceLimitResponse.setMax(limit.getMax()); diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index 7d5658f6782..bf5c4666984 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -453,6 +453,10 @@ public class ViewResponseHelper { resourceLimitMap.put(Resource.ResourceType.primary_storage, domainJoinVO.getPrimaryStorageLimit()); resourceLimitMap.put(Resource.ResourceType.secondary_storage, domainJoinVO.getSecondaryStorageLimit()); resourceLimitMap.put(Resource.ResourceType.project, domainJoinVO.getProjectLimit()); + resourceLimitMap.put(Resource.ResourceType.backup, domainJoinVO.getBackupLimit()); + resourceLimitMap.put(Resource.ResourceType.backup_storage, domainJoinVO.getBackupStorageLimit()); + resourceLimitMap.put(Resource.ResourceType.bucket, domainJoinVO.getBucketLimit()); + resourceLimitMap.put(Resource.ResourceType.object_storage, domainJoinVO.getObjectStorageLimit()); } private static void copyResourceLimitsFromMap(Map resourceLimitMap, DomainJoinVO domainJoinVO){ @@ -468,6 +472,10 @@ public class ViewResponseHelper { domainJoinVO.setPrimaryStorageLimit(resourceLimitMap.get(Resource.ResourceType.primary_storage)); domainJoinVO.setSecondaryStorageLimit(resourceLimitMap.get(Resource.ResourceType.secondary_storage)); domainJoinVO.setProjectLimit(resourceLimitMap.get(Resource.ResourceType.project)); + domainJoinVO.setBackupLimit(resourceLimitMap.get(Resource.ResourceType.backup)); + domainJoinVO.setBackupStorageLimit(resourceLimitMap.get(Resource.ResourceType.backup_storage)); + domainJoinVO.setBucketLimit(resourceLimitMap.get(Resource.ResourceType.bucket)); + domainJoinVO.setObjectStorageLimit(resourceLimitMap.get(Resource.ResourceType.object_storage)); } private static void setParentResourceLimitIfNeeded(Map resourceLimitMap, DomainJoinVO domainJoinVO, List domainsCopy) { @@ -486,6 +494,10 @@ public class ViewResponseHelper { Long primaryStorageLimit = resourceLimitMap.get(Resource.ResourceType.primary_storage); Long secondaryStorageLimit = resourceLimitMap.get(Resource.ResourceType.secondary_storage); Long projectLimit = resourceLimitMap.get(Resource.ResourceType.project); + Long backupLimit = resourceLimitMap.get(Resource.ResourceType.backup); + Long backupStorageLimit = resourceLimitMap.get(Resource.ResourceType.backup_storage); + Long bucketLimit = resourceLimitMap.get(Resource.ResourceType.bucket); + Long objectStorageLimit = resourceLimitMap.get(Resource.ResourceType.object_storage); if (vmLimit == null) { vmLimit = parentDomainJoinVO.getVmLimit(); @@ -535,6 +547,22 @@ public class ViewResponseHelper { projectLimit = parentDomainJoinVO.getProjectLimit(); resourceLimitMap.put(Resource.ResourceType.project, projectLimit); } + if (backupLimit == null) { + backupLimit = parentDomainJoinVO.getBackupLimit(); + resourceLimitMap.put(Resource.ResourceType.backup, backupLimit); + } + if (backupStorageLimit == null) { + backupStorageLimit = parentDomainJoinVO.getBackupStorageLimit(); + resourceLimitMap.put(Resource.ResourceType.backup_storage, backupStorageLimit); + } + if (bucketLimit == null) { + bucketLimit = parentDomainJoinVO.getBucketLimit(); + resourceLimitMap.put(Resource.ResourceType.bucket, bucketLimit); + } + if (objectStorageLimit == null) { + objectStorageLimit = parentDomainJoinVO.getObjectStorageLimit(); + resourceLimitMap.put(Resource.ResourceType.object_storage, objectStorageLimit); + } //-- try till parent present if (parentDomainJoinVO.getParent() != null && parentDomainJoinVO.getParent() != Domain.ROOT_DOMAIN) { setParentResourceLimitIfNeeded(resourceLimitMap, parentDomainJoinVO, domainsCopy); diff --git a/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java index 07b5c27438b..c81481dec74 100644 --- a/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java @@ -220,7 +220,7 @@ public class AccountJoinDaoImpl extends GenericDaoBase impl response.setMemoryTotal(memoryTotal); response.setMemoryAvailable(memoryAvail); - //get resource limits for primary storage space and convert it from Bytes to GiB + //get resource limits for primary storage space and convert it from Bytes to GiB long primaryStorageLimit = ApiDBUtils.findCorrectResourceLimit(account.getPrimaryStorageLimit(), account.getId(), ResourceType.primary_storage); String primaryStorageLimitDisplay = (fullView || primaryStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf(primaryStorageLimit / ResourceType.bytesToGiB); long primaryStorageTotal = (account.getPrimaryStorageTotal() == null) ? 0 : (account.getPrimaryStorageTotal() / ResourceType.bytesToGiB); @@ -240,6 +240,42 @@ public class AccountJoinDaoImpl extends GenericDaoBase impl response.setSecondaryStorageLimit(secondaryStorageLimitDisplay); response.setSecondaryStorageTotal(secondaryStorageTotal); response.setSecondaryStorageAvailable(secondaryStorageAvail); + + //get resource limits for backups + long backupLimit = ApiDBUtils.findCorrectResourceLimit(account.getBackupLimit(), account.getId(), ResourceType.backup); + String backupLimitDisplay = (fullView || backupLimit == -1) ? Resource.UNLIMITED : String.valueOf(backupLimit); + long backupTotal = (account.getBackupTotal() == null) ? 0 : account.getBackupTotal(); + String backupAvail = (fullView || backupLimit == -1) ? Resource.UNLIMITED : String.valueOf(backupLimit - backupTotal); + response.setBackupLimit(backupLimitDisplay); + response.setBackupTotal(backupTotal); + response.setBackupAvailable(backupAvail); + + //get resource limits for backup storage space and convert it from Bytes to GiB + long backupStorageLimit = ApiDBUtils.findCorrectResourceLimit(account.getBackupStorageLimit(), account.getId(), ResourceType.backup_storage); + String backupStorageLimitDisplay = (fullView || backupStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf(backupStorageLimit / ResourceType.bytesToGiB); + long backupStorageTotal = (account.getBackupStorageTotal() == null) ? 0 : (account.getBackupStorageTotal() / ResourceType.bytesToGiB); + String backupStorageAvail = (fullView || backupStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf((backupStorageLimit / ResourceType.bytesToGiB) - backupStorageTotal); + response.setBackupStorageLimit(backupStorageLimitDisplay); + response.setBackupStorageTotal(backupStorageTotal); + response.setBackupStorageAvailable(backupStorageAvail); + + //get resource limits for buckets + long bucketLimit = ApiDBUtils.findCorrectResourceLimit(account.getBucketLimit(), account.getId(), ResourceType.bucket); + String bucketLimitDisplay = (fullView || bucketLimit == -1) ? Resource.UNLIMITED : String.valueOf(bucketLimit); + long bucketTotal = (account.getBucketTotal() == null) ? 0 : account.getBucketTotal(); + String bucketAvail = (fullView || bucketLimit == -1) ? Resource.UNLIMITED : String.valueOf(bucketLimit - bucketTotal); + response.setBucketLimit(bucketLimitDisplay); + response.setBucketTotal(bucketTotal); + response.setBucketAvailable(bucketAvail); + + //get resource limits for object storage space and convert it from Bytes to GiB + long objectStorageLimit = ApiDBUtils.findCorrectResourceLimit(account.getObjectStorageLimit(), account.getId(), ResourceType.object_storage); + String objectStorageLimitDisplay = (fullView || objectStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf(objectStorageLimit / ResourceType.bytesToGiB); + long objectStorageTotal = (account.getObjectStorageTotal() == null) ? 0 : (account.getObjectStorageTotal() / ResourceType.bytesToGiB); + String objectStorageAvail = (fullView || objectStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf((objectStorageLimit / ResourceType.bytesToGiB) - objectStorageTotal); + response.setObjectStorageLimit(objectStorageLimitDisplay); + response.setObjectStorageTotal(objectStorageTotal); + response.setObjectStorageAvailable(objectStorageAvail); } @Override diff --git a/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java index 9ad05d216a9..79376a37b97 100644 --- a/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java @@ -212,6 +212,42 @@ public class DomainJoinDaoImpl extends GenericDaoBase implem response.setSecondaryStorageLimit(secondaryStorageLimitDisplay); response.setSecondaryStorageTotal(secondaryStorageTotal); response.setSecondaryStorageAvailable(secondaryStorageAvail); + + //get resource limits for backups + long backupLimit = ApiDBUtils.findCorrectResourceLimitForDomain(domain.getBackupLimit(), ResourceType.backup, domain.getId()); + String backupLimitDisplay = (fullView || snapshotLimit == -1) ? Resource.UNLIMITED : String.valueOf(backupLimit); + long backupTotal = (domain.getBackupTotal() == null) ? 0 : domain.getBackupTotal(); + String backupAvail = (fullView || snapshotLimit == -1) ? Resource.UNLIMITED : String.valueOf(backupLimit - backupTotal); + response.setBackupLimit(backupLimitDisplay); + response.setBackupTotal(backupTotal); + response.setBackupAvailable(backupAvail); + + //get resource limits for backup storage space and convert it from Bytes to GiB + long backupStorageLimit = ApiDBUtils.findCorrectResourceLimitForDomain(domain.getBackupStorageLimit(), ResourceType.backup_storage, domain.getId()); + String backupStorageLimitDisplay = (fullView || backupLimit == -1) ? Resource.UNLIMITED : String.valueOf(backupStorageLimit / ResourceType.bytesToGiB); + long backupStorageTotal = (domain.getBackupStorageTotal() == null) ? 0 : (domain.getBackupStorageTotal() / ResourceType.bytesToGiB); + String backupStorageAvail = (fullView || backupStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf((backupStorageLimit / ResourceType.bytesToGiB) - backupStorageTotal); + response.setBackupStorageLimit(backupStorageLimitDisplay); + response.setBackupStorageTotal(backupStorageTotal); + response.setBackupStorageAvailable(backupStorageAvail); + + //get resource limits for buckets + long bucketLimit = ApiDBUtils.findCorrectResourceLimit(domain.getBucketLimit(), domain.getId(), ResourceType.bucket); + String bucketLimitDisplay = (fullView || bucketLimit == -1) ? Resource.UNLIMITED : String.valueOf(bucketLimit); + long bucketTotal = (domain.getBucketTotal() == null) ? 0 : domain.getBucketTotal(); + String bucketAvail = (fullView || bucketLimit == -1) ? Resource.UNLIMITED : String.valueOf(bucketLimit - bucketTotal); + response.setBucketLimit(bucketLimitDisplay); + response.setBucketTotal(bucketTotal); + response.setBucketAvailable(bucketAvail); + + //get resource limits for object storage space and convert it from Bytes to GiB + long objectStorageLimit = ApiDBUtils.findCorrectResourceLimit(domain.getObjectStorageLimit(), domain.getId(), ResourceType.object_storage); + String objectStorageLimitDisplay = (fullView || objectStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf(objectStorageLimit / ResourceType.bytesToGiB); + long objectStorageTotal = (domain.getObjectStorageTotal() == null) ? 0 : (domain.getObjectStorageTotal() / ResourceType.bytesToGiB); + String objectStorageAvail = (fullView || objectStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf((objectStorageLimit / ResourceType.bytesToGiB) - objectStorageTotal); + response.setObjectStorageLimit(objectStorageLimitDisplay); + response.setObjectStorageTotal(objectStorageTotal); + response.setObjectStorageAvailable(objectStorageAvail); } @Override diff --git a/server/src/main/java/com/cloud/api/query/vo/AccountJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/AccountJoinVO.java index 0bd28d2af32..2e39816ed41 100644 --- a/server/src/main/java/com/cloud/api/query/vo/AccountJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/AccountJoinVO.java @@ -123,6 +123,18 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident @Column(name = "snapshotTotal") private Long snapshotTotal; + @Column(name = "backupLimit") + private Long backupLimit; + + @Column(name = "backupTotal") + private Long backupTotal; + + @Column(name = "backupStorageLimit") + private Long backupStorageLimit; + + @Column(name = "backupStorageTotal") + private Long backupStorageTotal; + @Column(name = "templateLimit") private Long templateLimit; @@ -177,6 +189,18 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident @Column(name = "secondaryStorageTotal") private Long secondaryStorageTotal; + @Column(name = "bucketLimit") + private Long bucketLimit; + + @Column(name = "bucketTotal") + private Long bucketTotal; + + @Column(name = "objectStorageLimit") + private Long objectStorageLimit; + + @Column(name = "objectStorageTotal") + private Long objectStorageTotal; + @Column(name = "job_id") private Long jobId; @@ -293,6 +317,14 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident return snapshotTotal; } + public Long getBackupTotal() { + return backupTotal; + } + + public Long getBackupStorageTotal() { + return backupStorageTotal; + } + public Long getTemplateTotal() { return templateTotal; } @@ -333,6 +365,14 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident return secondaryStorageTotal; } + public Long getBucketTotal() { + return bucketTotal; + } + + public Long getObjectStorageTotal() { + return objectStorageTotal; + } + public Long getVmLimit() { return vmLimit; } @@ -349,6 +389,14 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident return snapshotLimit; } + public Long getBackupLimit() { + return backupLimit; + } + + public Long getBackupStorageLimit() { + return backupStorageLimit; + } + public Long getTemplateLimit() { return templateLimit; } @@ -381,6 +429,14 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident return secondaryStorageLimit; } + public Long getBucketLimit() { + return bucketLimit; + } + + public Long getObjectStorageLimit() { + return objectStorageLimit; + } + public Long getJobId() { return jobId; } diff --git a/server/src/main/java/com/cloud/api/query/vo/DomainJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/DomainJoinVO.java index e17eacd68fa..3e623690f10 100644 --- a/server/src/main/java/com/cloud/api/query/vo/DomainJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/DomainJoinVO.java @@ -100,6 +100,18 @@ public class DomainJoinVO extends BaseViewVO implements InternalIdentity, Identi @Column(name="snapshotTotal") private Long snapshotTotal; + @Column(name="backupLimit") + private Long backupLimit; + + @Column(name = "backupStorageLimit") + private Long backupStorageLimit; + + @Column(name = "backupStorageTotal") + private Long backupStorageTotal; + + @Column(name="backupTotal") + private Long backupTotal; + @Column(name="templateLimit") private Long templateLimit; @@ -154,6 +166,18 @@ public class DomainJoinVO extends BaseViewVO implements InternalIdentity, Identi @Column(name="secondaryStorageTotal") private Long secondaryStorageTotal; + @Column(name = "bucketLimit") + private Long bucketLimit; + + @Column(name = "bucketTotal") + private Long bucketTotal; + + @Column(name = "objectStorageLimit") + private Long objectStorageLimit; + + @Column(name = "objectStorageTotal") + private Long objectStorageTotal; + @Transient private String parentName; @@ -311,8 +335,13 @@ public class DomainJoinVO extends BaseViewVO implements InternalIdentity, Identi this.snapshotTotal = snapshotTotal; } + public Long getBackupTotal() { + return backupTotal; + } - + public Long getBackupStorageTotal() { + return backupStorageTotal; + } public Long getTemplateTotal() { return templateTotal; @@ -393,6 +422,13 @@ public class DomainJoinVO extends BaseViewVO implements InternalIdentity, Identi this.secondaryStorageTotal = secondaryStorageTotal; } + public Long getBucketTotal() { + return bucketTotal; + } + + public Long getObjectStorageTotal() { + return objectStorageTotal; + } public Long getVmLimit() { return vmLimit; @@ -433,6 +469,21 @@ public class DomainJoinVO extends BaseViewVO implements InternalIdentity, Identi this.snapshotLimit = snapshotLimit; } + public Long getBackupLimit() { + return backupLimit; + } + + public void setBackupLimit(Long backupLimit) { + this.backupLimit = backupLimit; + } + + public Long getBackupStorageLimit() { + return backupStorageLimit; + } + + public void setBackupStorageLimit(Long backupStorageLimit) { + this.backupStorageLimit = backupStorageLimit; + } public Long getTemplateLimit() { return templateLimit; @@ -513,6 +564,22 @@ public class DomainJoinVO extends BaseViewVO implements InternalIdentity, Identi this.secondaryStorageLimit = secondaryStorageLimit; } + public Long getBucketLimit() { + return bucketLimit; + } + + public void setBucketLimit(Long bucketLimit) { + this.bucketLimit = bucketLimit; + } + + public Long getObjectStorageLimit() { + return objectStorageLimit; + } + + public void setObjectStorageLimit(Long objectStorageLimit) { + this.objectStorageLimit = objectStorageLimit; + } + public String getParentName() { return parentName; } diff --git a/server/src/main/java/com/cloud/configuration/Config.java b/server/src/main/java/com/cloud/configuration/Config.java index b9de906ba46..1cef1cbdb78 100644 --- a/server/src/main/java/com/cloud/configuration/Config.java +++ b/server/src/main/java/com/cloud/configuration/Config.java @@ -1365,7 +1365,7 @@ public enum Config { "200", "The default maximum primary storage space (in GiB) that can be used for an account", null), -DefaultMaxAccountProjects( + DefaultMaxAccountProjects( "Account Defaults", ManagementServer.class, Long.class, diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index b8c6e29c278..6023ae2154b 100644 --- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -36,12 +36,17 @@ import java.util.stream.Stream; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; import com.cloud.utils.Ternary; import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ResourceLimitAndCountResponse; import org.apache.cloudstack.api.response.TaggedResourceLimitAndCountResponse; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.framework.config.ConfigKey; @@ -55,6 +60,7 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.object.BucketApiService; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; @@ -100,6 +106,7 @@ import com.cloud.storage.SnapshotVO; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.BucketDao; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VMTemplateDao; @@ -170,6 +177,8 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim @Inject protected SnapshotDao _snapshotDao; @Inject + protected BackupDao backupDao; + @Inject private SnapshotDataStoreDao _snapshotDataStoreDao; @Inject private TemplateDataStoreDao _vmTemplateStoreDao; @@ -193,6 +202,8 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim ServiceOfferingDao serviceOfferingDao; @Inject DiskOfferingDao diskOfferingDao; + @Inject + BucketDao bucketDao; protected GenericSearchBuilder templateSizeSearch; protected GenericSearchBuilder snapshotSizeSearch; @@ -288,6 +299,10 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim projectResourceLimitMap.put(Resource.ResourceType.memory.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectMemory.key()))); projectResourceLimitMap.put(Resource.ResourceType.primary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectPrimaryStorage.key()))); projectResourceLimitMap.put(Resource.ResourceType.secondary_storage.name(), MaxProjectSecondaryStorage.value()); + projectResourceLimitMap.put(Resource.ResourceType.backup.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxProjectBackups.key()))); + projectResourceLimitMap.put(Resource.ResourceType.backup_storage.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxProjectBackupStorage.key()))); + projectResourceLimitMap.put(Resource.ResourceType.bucket.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxProjectBuckets.key()))); + projectResourceLimitMap.put(Resource.ResourceType.object_storage.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxProjectObjectStorage.key()))); accountResourceLimitMap.put(Resource.ResourceType.public_ip.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountPublicIPs.key()))); accountResourceLimitMap.put(Resource.ResourceType.snapshot.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountSnapshots.key()))); @@ -301,6 +316,10 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim accountResourceLimitMap.put(Resource.ResourceType.primary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountPrimaryStorage.key()))); accountResourceLimitMap.put(Resource.ResourceType.secondary_storage.name(), MaxAccountSecondaryStorage.value()); accountResourceLimitMap.put(Resource.ResourceType.project.name(), DefaultMaxAccountProjects.value()); + accountResourceLimitMap.put(Resource.ResourceType.backup.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxAccountBackups.key()))); + accountResourceLimitMap.put(Resource.ResourceType.backup_storage.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxAccountBackupStorage.key()))); + accountResourceLimitMap.put(Resource.ResourceType.bucket.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxAccountBuckets.key()))); + accountResourceLimitMap.put(Resource.ResourceType.object_storage.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxAccountObjectStorage.key()))); domainResourceLimitMap.put(Resource.ResourceType.public_ip.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainPublicIPs.key()))); domainResourceLimitMap.put(Resource.ResourceType.snapshot.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainSnapshots.key()))); @@ -314,6 +333,10 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim domainResourceLimitMap.put(Resource.ResourceType.primary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainPrimaryStorage.key()))); domainResourceLimitMap.put(Resource.ResourceType.secondary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainSecondaryStorage.key()))); domainResourceLimitMap.put(Resource.ResourceType.project.name(), DefaultMaxDomainProjects.value()); + domainResourceLimitMap.put(Resource.ResourceType.backup.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxDomainBackups.key()))); + domainResourceLimitMap.put(Resource.ResourceType.backup_storage.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxDomainBackupStorage.key()))); + domainResourceLimitMap.put(Resource.ResourceType.bucket.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxDomainBuckets.key()))); + domainResourceLimitMap.put(Resource.ResourceType.object_storage.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxDomainObjectStorage.key()))); } catch (NumberFormatException e) { logger.error("NumberFormatException during configuration", e); throw new ConfigurationException("Configuration failed due to NumberFormatException, see log for the stacktrace"); @@ -390,8 +413,8 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim if (value < 0) { // return unlimit if value is set to negative return max; } - // convert the value from GiB to bytes in case of primary or secondary storage. - if (type == ResourceType.primary_storage || type == ResourceType.secondary_storage) { + // convert the value from GiB to bytes in case of storage type resource. + if (ResourceType.isStorageType(type)) { value = value * ResourceType.bytesToGiB; } return value; @@ -431,7 +454,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim if (value < 0) { // return unlimit if value is set to negative return max; } - if (type == ResourceType.primary_storage || type == ResourceType.secondary_storage) { + if (ResourceType.isStorageType(type)) { value = value * ResourceType.bytesToGiB; } return value; @@ -478,7 +501,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim if (value < 0) { // return unlimit if value is set to negative return max; } - if (type == ResourceType.primary_storage || type == ResourceType.secondary_storage) { + if (ResourceType.isStorageType(type)) { value = value * ResourceType.bytesToGiB; } return value; @@ -512,7 +535,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim String convCurrentResourceReservation = String.valueOf(currentResourceReservation); String convNumResources = String.valueOf(numResources); - if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + if (ResourceType.isStorageType(type)) { convDomainResourceLimit = toHumanReadableSize(domainResourceLimit); convCurrentDomainResourceCount = toHumanReadableSize(currentDomainResourceCount); convCurrentResourceReservation = toHumanReadableSize(currentResourceReservation); @@ -557,7 +580,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim String convertedCurrentResourceReservation = String.valueOf(currentResourceReservation); String convertedNumResources = String.valueOf(numResources); - if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + if (ResourceType.isStorageType(type)) { convertedAccountResourceLimit = toHumanReadableSize(accountResourceLimit); convertedCurrentResourceCount = toHumanReadableSize(currentResourceCount); convertedCurrentResourceReservation = toHumanReadableSize(currentResourceReservation); @@ -594,7 +617,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim public long findDefaultResourceLimitForDomain(ResourceType resourceType) { Long resourceLimit = null; resourceLimit = domainResourceLimitMap.get(resourceType.getName()); - if (resourceLimit != null && (resourceType == ResourceType.primary_storage || resourceType == ResourceType.secondary_storage)) { + if (resourceLimit != null && ResourceType.isStorageType(resourceType)) { if (! Long.valueOf(Resource.RESOURCE_UNLIMITED).equals(resourceLimit)) { resourceLimit = resourceLimit * ResourceType.bytesToGiB; } @@ -908,12 +931,14 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } //Convert max storage size from GiB to bytes - if ((resourceType == ResourceType.primary_storage || resourceType == ResourceType.secondary_storage) && max >= 0) { + if (ResourceType.isStorageType(resourceType) && max >= 0) { max *= ResourceType.bytesToGiB; } ResourceOwnerType ownerType = null; Long ownerId = null; + ApiCommandResourceType ownerResourceType = null; + Long ownerResourceId = null; if (accountId != null) { Account account = _entityMgr.findById(Account.class, accountId); @@ -942,6 +967,16 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim ownerType = ResourceOwnerType.Account; ownerId = accountId; + + if (account.getType() == Account.Type.PROJECT) { + ownerResourceType = ApiCommandResourceType.Project; + Project project = _projectDao.findByProjectAccountId(accountId); + ownerResourceId = project.getId(); + } else { + ownerResourceType = ApiCommandResourceType.Account; + ownerResourceId = ownerId; + } + if (StringUtils.isNotEmpty(tag)) { long untaggedLimit = findCorrectResourceLimitForAccount(account, resourceType, null); if (untaggedLimit > 0 && max > untaggedLimit) { @@ -980,6 +1015,8 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } ownerType = ResourceOwnerType.Domain; ownerId = domainId; + ownerResourceType = ApiCommandResourceType.Domain; + ownerResourceId = ownerId; } if (ownerId == null) { @@ -987,6 +1024,12 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } ResourceLimitVO limit = _resourceLimitDao.findByOwnerIdAndTypeAndTag(ownerId, ownerType, resourceType, tag); + + ActionEventUtils.onActionEvent(caller.getId(), caller.getAccountId(), + caller.getDomainId(), EventTypes.EVENT_RESOURCE_LIMIT_UPDATE, + "Resource limit updated. Resource Type: " + resourceType.toString() + ", New Value: " + max, + ownerResourceId, ownerResourceType.toString()); + if (limit != null) { // Update the existing limit _resourceLimitDao.update(limit.getId(), max); @@ -1135,7 +1178,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } if (logger.isDebugEnabled()) { String convertedDelta = String.valueOf(delta); - if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + if (ResourceType.isStorageType(type)) { convertedDelta = toHumanReadableSize(delta); } String typeStr = StringUtils.isNotEmpty(tag) ? String.format("%s (tag: %s)", type, tag) : type.getName(); @@ -1236,6 +1279,10 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim newCount = calculateVolumeCountForAccount(accountId, tag); } else if (type == Resource.ResourceType.snapshot) { newCount = _snapshotDao.countSnapshotsForAccount(accountId); + } else if (type == Resource.ResourceType.backup) { + newCount = backupDao.countBackupsForAccount(accountId); + } else if (type == Resource.ResourceType.backup_storage) { + newCount = backupDao.calculateBackupStorageForAccount(accountId); } else if (type == Resource.ResourceType.public_ip) { newCount = calculatePublicIpForAccount(accountId); } else if (type == Resource.ResourceType.template) { @@ -1254,6 +1301,10 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim newCount = calculatePrimaryStorageForAccount(accountId, tag); } else if (type == Resource.ResourceType.secondary_storage) { newCount = calculateSecondaryStorageForAccount(accountId); + } else if (type == Resource.ResourceType.bucket) { + newCount = bucketDao.countBucketsForAccount(accountId); + } else if (type == ResourceType.object_storage) { + newCount = bucketDao.calculateObjectStorageAllocationForAccount(accountId); } else { throw new InvalidParameterValueException("Unsupported resource type " + type); } @@ -1270,10 +1321,9 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim _resourceCountDao.persist(new ResourceCountVO(type, newCount, accountId, ResourceOwnerType.Account, tag)); } - // No need to log message for primary and secondary storage because both are recalculating the + // No need to log message for storage type resources because both are recalculating the // resource count which will not lead to any discrepancy. - if (newCount != null && !newCount.equals(oldCount) && - type != Resource.ResourceType.primary_storage && type != Resource.ResourceType.secondary_storage) { + if (newCount != null && !newCount.equals(oldCount) && !ResourceType.isStorageType(type)) { logger.warn("Discrepancy in the resource count " + "(original count=" + oldCount + " correct count = " + newCount + ") for type " + type + " for account ID " + accountId + " is fixed during resource count recalculation."); } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index a371a064701..b066402313c 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -164,6 +164,7 @@ import com.cloud.resource.ResourceState; import com.cloud.serializer.GsonHelper; import com.cloud.server.ManagementService; import com.cloud.server.ResourceTag; +import com.cloud.server.StatsCollector; import com.cloud.server.TaggedResourceService; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; @@ -353,9 +354,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Inject private BackupDao backupDao; @Inject + private StatsCollector statsCollector; + @Inject HostPodDao podDao; - protected Gson _gson; private static final List SupportedHypervisorsForVolResize = Arrays.asList(HypervisorType.KVM, HypervisorType.XenServer, @@ -5204,6 +5206,21 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return workJob; } + @Override + public Long getVolumePhysicalSize(ImageFormat format, String path, String chainInfo) { + VolumeStats vs = null; + if (format == ImageFormat.VHD || format == ImageFormat.QCOW2 || format == ImageFormat.RAW) { + if (path != null) { + vs = statsCollector.getVolumeStats(path); + } + } else if (format == ImageFormat.OVA) { + if (chainInfo != null) { + vs = statsCollector.getVolumeStats(chainInfo); + } + } + return (vs == null) ? null : vs.getPhysicalSize(); + } + @Override public String getConfigComponentName() { return VolumeApiService.class.getSimpleName(); diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 2e52d1ccc44..816d8fd66c4 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -29,7 +29,13 @@ import java.util.TimerTask; import java.util.stream.Collectors; import com.amazonaws.util.CollectionUtils; +import com.cloud.alert.AlertManager; +import com.cloud.configuration.Resource; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.storage.Snapshot; import com.cloud.storage.VolumeApiService; +import com.cloud.user.DomainManager; +import com.cloud.user.ResourceLimitService; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.VirtualMachineManager; import javax.inject.Inject; @@ -37,6 +43,7 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.api.command.admin.backup.DeleteBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.ListBackupProviderOfferingsCmd; @@ -115,6 +122,7 @@ import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; @@ -139,6 +147,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { @Inject private AccountManager accountManager; @Inject + private DomainManager domainManager; + @Inject private VolumeDao volumeDao; @Inject private DataCenterDao dataCenterDao; @@ -164,6 +174,10 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { private VolumeApiService volumeApiService; @Inject private VolumeOrchestrationService volumeOrchestrationService; + @Inject + private ResourceLimitService resourceLimitMgr; + @Inject + private AlertManager alertManager; private AsyncJobDispatcher asyncJobDispatcher; private Timer backupTimer; @@ -396,8 +410,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(), vm.getBackupOfferingId(), null, null, Backup.class.getSimpleName(), vm.getUuid()); - final BackupSchedule backupSchedule = backupScheduleDao.findByVM(vm.getId()); - if (backupSchedule != null) { + final List backupSchedules = backupScheduleDao.listByVM(vm.getId()); + for(BackupSchedule backupSchedule: backupSchedules) { backupScheduleDao.remove(backupSchedule.getId()); } result = true; @@ -415,6 +429,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { final DateUtil.IntervalType intervalType = cmd.getIntervalType(); final String scheduleString = cmd.getSchedule(); final TimeZone timeZone = TimeZone.getTimeZone(cmd.getTimezone()); + final Integer maxBackups = cmd.getMaxBackups(); if (intervalType == null) { throw new CloudRuntimeException("Invalid interval type provided"); @@ -427,12 +442,41 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { if (vm.getBackupOfferingId() == null) { throw new CloudRuntimeException("Cannot configure backup schedule for the VM without having any backup offering"); } + if (maxBackups != null && maxBackups <= 0) { + throw new InvalidParameterValueException(String.format("maxBackups [%s] for instance %s should be greater than 0.", maxBackups, vm.getName())); + } + + Backup.Type backupType = Backup.Type.valueOf(intervalType.name()); + int intervalMaxBackups = backupType.getMax(); + if (maxBackups != null && maxBackups > intervalMaxBackups) { + throw new InvalidParameterValueException(String.format("maxBackups [%s] for instance %s exceeds limit [%s] for interval type [%s].", maxBackups, vm.getName(), + intervalMaxBackups, intervalType)); + } + + Account owner = accountManager.getAccount(vm.getAccountId()); + + long accountLimit = resourceLimitMgr.findCorrectResourceLimitForAccount(owner, Resource.ResourceType.backup, null); + long domainLimit = resourceLimitMgr.findCorrectResourceLimitForDomain(domainManager.getDomain(owner.getDomainId()), Resource.ResourceType.backup, null); + if (maxBackups != null && !accountManager.isRootAdmin(owner.getId()) && ((accountLimit != -1 && maxBackups > accountLimit) || (domainLimit != -1 && maxBackups > domainLimit))) { + String message = "domain/account"; + if (owner.getType() == Account.Type.PROJECT) { + message = "domain/project"; + } + throw new InvalidParameterValueException("Max number of backups shouldn't exceed the " + message + " level backup limit"); + } final BackupOffering offering = backupOfferingDao.findById(vm.getBackupOfferingId()); if (offering == null || !offering.isUserDrivenBackupAllowed()) { throw new CloudRuntimeException("The selected backup offering does not allow user-defined backup schedule"); } + if (maxBackups == null && !"veeam".equals(offering.getProvider())) { + throw new CloudRuntimeException("Please specify the maximum number of buckets to retain."); + } + if (maxBackups != null && "veeam".equals(offering.getProvider())) { + throw new CloudRuntimeException("The maximum backups to retain cannot be configured through CloudStack for Veeam. Retention is managed directly in Veeam based on the settings specified when creating the backup job."); + } + final String timezoneId = timeZone.getID(); if (!timezoneId.equals(cmd.getTimezone())) { logger.warn("Using timezone: " + timezoneId + " for running this snapshot policy as an equivalent of " + cmd.getTimezone()); @@ -447,15 +491,16 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { final BackupScheduleVO schedule = backupScheduleDao.findByVMAndIntervalType(vmId, intervalType); if (schedule == null) { - return backupScheduleDao.persist(new BackupScheduleVO(vmId, intervalType, scheduleString, timezoneId, nextDateTime)); + return backupScheduleDao.persist(new BackupScheduleVO(vmId, intervalType, scheduleString, timezoneId, nextDateTime, maxBackups)); } schedule.setScheduleType((short) intervalType.ordinal()); schedule.setSchedule(scheduleString); schedule.setTimezone(timezoneId); schedule.setScheduledTimestamp(nextDateTime); + schedule.setMaxBackups(maxBackups); backupScheduleDao.update(schedule.getId(), schedule); - return backupScheduleDao.findByVM(vmId); + return backupScheduleDao.findById(schedule.getId()); } @Override @@ -469,7 +514,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_SCHEDULE_DELETE, eventDescription = "deleting VM backup schedule") - public boolean deleteBackupSchedule(final Long vmId) { + public boolean deleteBackupSchedule(Long vmId) { final VMInstanceVO vm = findVmById(vmId); validateForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -481,9 +526,30 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { return backupScheduleDao.remove(schedule.getId()); } + private void postCreateScheduledBackup(Backup.Type backupType, Long vmId) { + DateUtil.IntervalType intervalType = DateUtil.IntervalType.valueOf(backupType.name()); + final BackupScheduleVO schedule = backupScheduleDao.findByVMAndIntervalType(vmId, intervalType); + if (schedule == null) { + return; + } + Integer maxBackups = schedule.getMaxBackups(); + if (maxBackups == null) { + return; + } + List backups = backupDao.listBackupsByVMandIntervalType(vmId, backupType); + while (backups.size() > maxBackups) { + BackupVO oldestBackup = backups.get(0); + if (deleteBackup(oldestBackup.getId(), false)) { + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, oldestBackup.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_DELETE, + "Successfully deleted oldest backup: " + oldestBackup.getId(), oldestBackup.getId(), ApiCommandResourceType.Backup.toString(), 0); + } + backups.remove(oldestBackup); + } + } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "creating VM backup", async = true) - public boolean createBackup(final Long vmId) { + public boolean createBackup(final Long vmId, final Long scheduleId) throws ResourceAllocationException { final VMInstanceVO vm = findVmById(vmId); validateForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -501,13 +567,65 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { throw new CloudRuntimeException("The assigned backup offering does not allow ad-hoc user backup"); } + Backup.Type type = getBackupType(scheduleId); + Account owner = accountManager.getAccount(vm.getAccountId()); + try { + resourceLimitMgr.checkResourceLimit(owner, Resource.ResourceType.backup); + } catch (ResourceAllocationException e) { + if (type != Backup.Type.MANUAL) { + String msg = "Backup resource limit exceeded for account id : " + owner.getId() + ". Failed to create backup"; + logger.warn(msg); + alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup resource limit exceeded for account id : " + owner.getId() + + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + } + throw e; + } + + Long backupSize = 0L; + for (final Volume volume: volumeDao.findByInstance(vmId)) { + if (Volume.State.Ready.equals(volume.getState())) { + Long volumeSize = volumeApiService.getVolumePhysicalSize(volume.getFormat(), volume.getPath(), volume.getChainInfo()); + if (volumeSize == null) { + volumeSize = volume.getSize(); + } + backupSize += volumeSize; + } + } + try { + resourceLimitMgr.checkResourceLimit(owner, Resource.ResourceType.backup_storage, backupSize); + } catch (ResourceAllocationException e) { + if (type != Backup.Type.MANUAL) { + String msg = "Backup storage space resource limit exceeded for account id : " + owner.getId() + ". Failed to create backup"; + logger.warn(msg); + alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup storage space resource limit exceeded for account id : " + owner.getId() + + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + } + throw e; + } + ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventTypes.EVENT_VM_BACKUP_CREATE, "creating backup for VM ID:" + vm.getUuid(), vmId, ApiCommandResourceType.VirtualMachine.toString(), true, 0); + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); - if (backupProvider != null && backupProvider.takeBackup(vm)) { + if (backupProvider != null) { + Pair result = backupProvider.takeBackup(vm); + if (!result.first()) { + throw new CloudRuntimeException("Failed to create VM backup"); + } + Backup backup = result.second(); + if (backup != null) { + BackupVO vmBackup = backupDao.findById(result.second().getId()); + vmBackup.setBackupIntervalType((short) type.ordinal()); + backupDao.update(vmBackup.getId(), vmBackup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + } + if (type != Backup.Type.MANUAL) { + postCreateScheduledBackup(type, vm.getId()); + } return true; } throw new CloudRuntimeException("Failed to create VM backup"); @@ -682,6 +800,29 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { } } + private Backup.Type getBackupType(Long scheduleId) { + if (scheduleId.equals(Snapshot.MANUAL_POLICY_ID)) { + return Backup.Type.MANUAL; + } else { + BackupScheduleVO scheduleVO = backupScheduleDao.findById(scheduleId); + DateUtil.IntervalType intvType = scheduleVO.getScheduleType(); + return getBackupType(intvType); + } + } + + private Backup.Type getBackupType(DateUtil.IntervalType intvType) { + if (intvType.equals(DateUtil.IntervalType.HOURLY)) { + return Backup.Type.HOURLY; + } else if (intvType.equals(DateUtil.IntervalType.DAILY)) { + return Backup.Type.DAILY; + } else if (intvType.equals(DateUtil.IntervalType.WEEKLY)) { + return Backup.Type.WEEKLY; + } else if (intvType.equals(DateUtil.IntervalType.MONTHLY)) { + return Backup.Type.MONTHLY; + } + return null; + } + /** * Tries to update the state of given VM, given specified event * @param vm The VM to update its state @@ -858,6 +999,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); boolean result = backupProvider.deleteBackup(backup, forced); if (result) { + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); return backupDao.remove(backup.getId()); } throw new CloudRuntimeException("Failed to delete the backup"); @@ -925,6 +1068,10 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); backgroundPollManager.submitTask(new BackupSyncTask(this)); + Backup.Type.HOURLY.setMax(BackupHourlyMax.value()); + Backup.Type.DAILY.setMax(BackupDailyMax.value()); + Backup.Type.WEEKLY.setMax(BackupWeeklyMax.value()); + Backup.Type.MONTHLY.setMax(BackupMonthlyMax.value()); return true; } @@ -1004,7 +1151,17 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { BackupFrameworkEnabled, BackupProviderPlugin, BackupSyncPollingInterval, - BackupEnableAttachDetachVolumes + BackupEnableAttachDetachVolumes, + BackupHourlyMax, + BackupDailyMax, + BackupWeeklyMax, + BackupMonthlyMax, + DefaultMaxAccountBackups, + DefaultMaxAccountBackupStorage, + DefaultMaxProjectBackups, + DefaultMaxProjectBackupStorage, + DefaultMaxDomainBackups, + DefaultMaxDomainBackupStorage }; } @@ -1137,6 +1294,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { true, 0); final Map params = new HashMap(); params.put(ApiConstants.VIRTUAL_MACHINE_ID, "" + vmId); + params.put(ApiConstants.SCHEDULE_ID, "" + backupScheduleId); params.put("ctxUserId", "1"); params.put("ctxAccountId", "" + vm.getAccountId()); params.put("ctxStartEventId", String.valueOf(eventId)); @@ -1205,7 +1363,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { * This background task syncs backups from providers side in CloudStack db * along with creation of usage records */ - private final class BackupSyncTask extends ManagedContextRunnable implements BackgroundPollTask { + protected final class BackupSyncTask extends ManagedContextRunnable implements BackgroundPollTask { private BackupManager backupManager; public BackupSyncTask(final BackupManager backupManager) { @@ -1253,13 +1411,81 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { } } + private Backup checkAndUpdateIfBackupEntryExistsForRestorePoint(Backup.RestorePoint restorePoint, List backupsInDb, VirtualMachine vm, Backup.Metric metric) { + for (final Backup backupInDb : backupsInDb) { + logger.debug(String.format("Checking if Backup %s with external ID %s for VM %s is valid", backupsInDb, backupInDb.getName(), vm)); + if (restorePoint.getId().equals(backupInDb.getExternalId())) { + logger.debug(String.format("Found Backup %s in both Database and Networker", backupInDb)); + if (metric != null) { + logger.debug(String.format("Update backup [%s] from [size: %s, protected size: %s] to [size: %s, protected size: %s].", + backupInDb, backupInDb.getSize(), backupInDb.getProtectedSize(), metric.getBackupSize(), metric.getDataSize())); + + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backupInDb.getSize()); + ((BackupVO) backupInDb).setSize(metric.getBackupSize()); + ((BackupVO) backupInDb).setProtectedSize(metric.getDataSize()); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backupInDb.getSize()); + + backupDao.update(backupInDb.getId(), ((BackupVO) backupInDb)); + } + return backupInDb; + } + } + return null; + } + + private void syncBackups(BackupProvider backupProvider, VirtualMachine vm, Backup.Metric metric) { + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + final List backupsInDb = backupDao.listByVmId(null, vm.getId()); + List restorePoints = backupProvider.listRestorePoints(vm); + if (restorePoints == null) { + return; + } + + final List removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); + for (final Backup.RestorePoint restorePoint : restorePoints) { + if (!(restorePoint.getId() == null || restorePoint.getType() == null || restorePoint.getCreated() == null)) { + Backup existingBackupEntry = checkAndUpdateIfBackupEntryExistsForRestorePoint(restorePoint, backupsInDb, vm, metric); + if (existingBackupEntry != null) { + removeList.remove(existingBackupEntry.getId()); + continue; + } + } + + Backup backup = backupProvider.createNewBackupEntryForRestorePoint(restorePoint, vm, metric); + if (backup != null) { + logger.warn("Added backup found in provider [" + backup + "]"); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + + logger.debug(String.format("Creating a new entry in backups: [id: %s, uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " + + "domain_id: %s, zone_id: %s].", backup.getId(), backup.getUuid(), backup.getVmId(), backup.getExternalId(), backup.getType(), backup.getDate(), + backup.getBackupOfferingId(), backup.getAccountId(), backup.getDomainId(), backup.getZoneId())); + + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_CREATE, + String.format("Created backup %s for VM ID: %s", backup.getUuid(), vm.getUuid()), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); + } + } + for (final Long backupIdToRemove : removeList) { + logger.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); + Backup backup = backupDao.findById(backupIdToRemove); + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + backupDao.remove(backupIdToRemove); + } + } + }); + } + private void tryToSyncVMBackups(BackupProvider backupProvider, Map metrics, VirtualMachine vm) { try { final Backup.Metric metric = metrics.get(vm); if (metric != null) { logger.debug(String.format("Trying to sync backups of VM [%s] using backup provider [%s].", vm, backupProvider.getName())); // Sync out-of-band backups - backupProvider.syncBackups(vm, metric); + syncBackups(backupProvider, vm, metric); // Emit a usage event, update usage metric for the VM by the usage server UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(), diff --git a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java index 389ca52b03b..ca0e6291e52 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java @@ -19,9 +19,12 @@ package org.apache.cloudstack.storage.object; import com.amazonaws.services.s3.internal.BucketNameUtils; import com.amazonaws.services.s3.model.IllegalBucketNameException; import com.cloud.agent.api.to.BucketTO; +import com.cloud.configuration.Resource; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.resourcelimit.ResourceLimitManagerImpl; import com.cloud.storage.BucketVO; import com.cloud.storage.DataStoreRole; import com.cloud.storage.dao.BucketDao; @@ -60,6 +63,8 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic private BucketDao _bucketDao; @Inject private AccountManager _accountMgr; + @Inject + private ResourceLimitManagerImpl resourceLimitManager; @Inject private BucketStatisticsDao _bucketStatisticsDao; @@ -98,12 +103,18 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] { + DefaultMaxAccountBuckets, + DefaultMaxAccountObjectStorage, + DefaultMaxProjectBuckets, + DefaultMaxProjectObjectStorage, + DefaultMaxDomainBuckets, + DefaultMaxDomainObjectStorage }; } @Override @ActionEvent(eventType = EventTypes.EVENT_BUCKET_CREATE, eventDescription = "creating bucket", create = true) - public Bucket allocBucket(CreateBucketCmd cmd) { + public Bucket allocBucket(CreateBucketCmd cmd) throws ResourceAllocationException { try { BucketNameUtils.validateBucketName(cmd.getBucketName()); } catch (IllegalBucketNameException e) { @@ -125,6 +136,9 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic return null; } + resourceLimitManager.checkResourceLimit(owner, Resource.ResourceType.bucket); + resourceLimitManager.checkResourceLimit(owner, Resource.ResourceType.object_storage, (cmd.getQuota() * Resource.ResourceType.bytesToGiB)); + BucketVO bucket = new BucketVO(ownerId, owner.getDomainId(), cmd.getObjectStoragePoolId(), cmd.getBucketName(), cmd.getQuota(), cmd.isVersioning(), cmd.isEncryption(), cmd.isObjectLocking(), cmd.getPolicy()); _bucketDao.persist(bucket); @@ -146,6 +160,7 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic try { bucketTO = new BucketTO(objectStore.createBucket(bucket, objectLock)); bucketCreated = true; + resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.bucket); if (cmd.isVersioning()) { objectStore.setBucketVersioning(bucketTO); @@ -157,6 +172,7 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic if (cmd.getQuota() != null) { objectStore.setQuota(bucketTO, cmd.getQuota()); + resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (cmd.getQuota() * Resource.ResourceType.bytesToGiB)); } if (cmd.getPolicy() != null) { @@ -188,6 +204,8 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); if (objectStore.deleteBucket(bucketTO)) { + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.bucket); + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (bucket.getQuota() * Resource.ResourceType.bytesToGiB)); return _bucketDao.remove(bucketId); } return false; @@ -195,7 +213,7 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic @Override @ActionEvent(eventType = EventTypes.EVENT_BUCKET_UPDATE, eventDescription = "updating bucket") - public boolean updateBucket(UpdateBucketCmd cmd, Account caller) { + public boolean updateBucket(UpdateBucketCmd cmd, Account caller) throws ResourceAllocationException { BucketVO bucket = _bucketDao.findById(cmd.getId()); BucketTO bucketTO = new BucketTO(bucket); if (bucket == null) { @@ -204,6 +222,17 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic _accountMgr.checkAccess(caller, null, true, bucket); ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + Integer quota = cmd.getQuota(); + Integer quotaDelta = null; + + if (quota != null) { + quotaDelta = quota - bucket.getQuota(); + if (quotaDelta > 0) { + Account owner = _accountMgr.getActiveAccountById(bucket.getAccountId()); + resourceLimitManager.checkResourceLimit(owner, Resource.ResourceType.object_storage, (quotaDelta * Resource.ResourceType.bytesToGiB)); + } + } + try { if (cmd.getEncryption() != null) { if (cmd.getEncryption()) { @@ -231,6 +260,11 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic if (cmd.getQuota() != null) { objectStore.setQuota(bucketTO, cmd.getQuota()); bucket.setQuota(cmd.getQuota()); + if (quotaDelta > 0) { + resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (quotaDelta * Resource.ResourceType.bytesToGiB)); + } else { + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, ((-quotaDelta) * Resource.ResourceType.bytesToGiB)); + } } _bucketDao.update(bucket.getId(), bucket); } catch (Exception e) { diff --git a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java index 34030626d22..34e4632d24a 100644 --- a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java +++ b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java @@ -23,9 +23,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; +import com.cloud.utils.db.EntityManager; + +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.TaggedResourceLimitAndCountResponse; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.commons.collections.CollectionUtils; @@ -39,6 +45,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @@ -67,6 +74,7 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; +import com.cloud.user.User; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; @@ -80,6 +88,9 @@ import com.cloud.vpc.MockResourceLimitManagerImpl; import junit.framework.TestCase; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + @RunWith(MockitoJUnitRunner.class) public class ResourceLimitManagerImplTest extends TestCase { private Logger logger = LogManager.getLogger(ResourceLimitManagerImplTest.class); @@ -118,7 +129,10 @@ public class ResourceLimitManagerImplTest extends TestCase { VolumeDao volumeDao; @Mock UserVmDao userVmDao; + @Mock + EntityManager entityManager; + private CallContext callContext; private List hostTags = List.of("htag1", "htag2", "htag3"); private List storageTags = List.of("stag1", "stag2"); @@ -136,10 +150,15 @@ public class ResourceLimitManagerImplTest extends TestCase { } catch (IllegalAccessException | NoSuchFieldException e) { logger.error("Failed to update configurations"); } + + Account account = mock(Account.class); + User user = mock(User.class); + CallContext.register(user, account); } @After public void tearDown() throws Exception { + CallContext.unregister(); } @Test @@ -415,6 +434,9 @@ public class ResourceLimitManagerImplTest extends TestCase { Mockito.when(resourceLimitDao.findByOwnerIdAndTypeAndTag(1L, Resource.ResourceOwnerType.Account, Resource.ResourceType.cpu, hostTags.get(0))).thenReturn(null); result = resourceLimitManager.findCorrectResourceLimitForAccount(account, Resource.ResourceType.cpu, hostTags.get(0)); Assert.assertEquals(defaultAccountCpuMax, result); + + result = resourceLimitManager.findCorrectResourceLimitForAccount(account, Resource.ResourceType.cpu, hostTags.get(0)); + Assert.assertEquals(defaultAccountCpuMax, result); } @Test @@ -449,24 +471,26 @@ public class ResourceLimitManagerImplTest extends TestCase { @Test public void testFindCorrectResourceLimitForAccountId1() { -// long accountId = 1L; -// Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(true); -// long result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, null, Resource.ResourceType.cpu); -// Assert.assertEquals(Resource.RESOURCE_UNLIMITED, result); -// -// accountId = 2L; -// Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(false); -// Long limit = 100L; -// long result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, limit, Resource.ResourceType.cpu); -// Assert.assertEquals(limit.longValue(), result); -// -// long defaultAccountCpuMax = 25L; -// Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(false); -// Map accountResourceLimitMap = new HashMap<>(); -// accountResourceLimitMap.put(Resource.ResourceType.cpu.name(), defaultAccountCpuMax); -// resourceLimitManager.accountResourceLimitMap = accountResourceLimitMap; -// result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, null, Resource.ResourceType.cpu); -// Assert.assertEquals(defaultAccountCpuMax, result); + long accountId = 1L; + Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(true); + long result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, null, Resource.ResourceType.cpu); + Assert.assertEquals(Resource.RESOURCE_UNLIMITED, result); + + accountId = 2L; + AccountVO account = mock(AccountVO.class); + Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(false); + Mockito.when(accountDao.findById(accountId)).thenReturn(account); + Long limit = 100L; + result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, limit, Resource.ResourceType.cpu); + Assert.assertEquals(limit.longValue(), result); + + long defaultAccountCpuMax = 25L; + Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(false); + Map accountResourceLimitMap = new HashMap<>(); + accountResourceLimitMap.put(Resource.ResourceType.cpu.name(), defaultAccountCpuMax); + resourceLimitManager.accountResourceLimitMap = accountResourceLimitMap; + result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, null, Resource.ResourceType.cpu); + Assert.assertEquals(defaultAccountCpuMax, result); } @Test @@ -1223,4 +1247,65 @@ public class ResourceLimitManagerImplTest extends TestCase { Mockito.verify(resourceLimitManager, Mockito.times(1)) .decrementResourceCountWithTag(accountId, Resource.ResourceType.memory, tag, Long.valueOf(memory)); } + + @Test + public void testUpdateResourceLimitForAccount() { + Long accountId = 1L; + Long resourceLimitId = 3L; + Integer typeId = 13; + Long maxGB = 10L; + Long maxBytes = maxGB * Resource.ResourceType.bytesToGiB; + + Account account = mock(Account.class); + when(entityManager.findById(Account.class, accountId)).thenReturn(account); + ResourceLimitVO resourceLimitVO = mock(ResourceLimitVO.class); + when(resourceLimitVO.getId()).thenReturn(resourceLimitId); + when(resourceLimitDao.findByOwnerIdAndTypeAndTag(accountId, Resource.ResourceOwnerType.Account, Resource.ResourceType.backup_storage, null)).thenReturn(resourceLimitVO); + + try (MockedStatic actionEventUtilsMockedStatic = Mockito.mockStatic(ActionEventUtils.class)) { + Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString(), + Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); + + resourceLimitManager.updateResourceLimit(accountId, null, typeId, maxGB, null); + + Mockito.verify(resourceLimitDao, Mockito.times(1)).update(resourceLimitId, maxBytes); + Mockito.verify(resourceLimitDao, Mockito.never()).persist(Mockito.any()); + actionEventUtilsMockedStatic.verify(() -> ActionEventUtils.onActionEvent(0L, 0L, 0L, EventTypes.EVENT_RESOURCE_LIMIT_UPDATE, + "Resource limit updated. Resource Type: " + Resource.ResourceType.backup_storage.toString() + ", New Value: " + maxBytes, + accountId, ApiCommandResourceType.Account.toString())); + } + } + + @Test + public void testUpdateResourceLimitForDomain() { + Long domainId = 2L; + Long resourceLimitId = 3L; + Integer typeId = 13; + Long maxGB = 10L; + Long maxBytes = maxGB * Resource.ResourceType.bytesToGiB; + + Domain domain = mock(Domain.class); + when(domain.getParent()).thenReturn(null); + when(entityManager.findById(Domain.class, domainId)).thenReturn(domain); + ResourceLimitVO resourceLimitVO = mock(ResourceLimitVO.class); + when(resourceLimitVO.getId()).thenReturn(resourceLimitId); + when(resourceLimitDao.findByOwnerIdAndTypeAndTag(domainId, Resource.ResourceOwnerType.Domain, Resource.ResourceType.backup_storage, null)).thenReturn(resourceLimitVO); + + try (MockedStatic actionEventUtilsMockedStatic = Mockito.mockStatic(ActionEventUtils.class)) { + Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString(), + Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); + + resourceLimitManager.updateResourceLimit(null, domainId, typeId, maxGB, null); + + Mockito.verify(resourceLimitDao, Mockito.times(1)).update(resourceLimitId, maxBytes); + Mockito.verify(resourceLimitDao, Mockito.never()).persist(Mockito.any()); + actionEventUtilsMockedStatic.verify(() -> ActionEventUtils.onActionEvent(0L, 0L, 0L, EventTypes.EVENT_RESOURCE_LIMIT_UPDATE, + "Resource limit updated. Resource Type: " + Resource.ResourceType.backup_storage.toString() + ", New Value: " + maxBytes, + domainId, ApiCommandResourceType.Domain.toString())); + } + } } diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 3bf1fb97e4d..808511e6fdf 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -16,22 +16,45 @@ // under the License. package org.apache.cloudstack.backup; +import com.cloud.alert.AlertManager; +import com.cloud.configuration.Resource; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.domain.Domain; import com.cloud.event.ActionEventUtils; +import com.cloud.event.UsageEventUtils; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.DomainManager; +import com.cloud.user.ResourceLimitService; +import com.cloud.user.User; +import com.cloud.user.UserVO; +import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; +import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; +import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.backup.dao.BackupScheduleDao; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -42,13 +65,21 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.Spy; +import org.springframework.test.util.ReflectionTestUtils; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -72,9 +103,38 @@ public class BackupManagerTest { @Mock VolumeDao volumeDao; + @Mock + VMInstanceDao vmInstanceDao; + + @Mock + AccountManager accountManager; + + @Mock + DomainManager domainManager; + + @Mock + ResourceLimitService resourceLimitMgr; + + @Mock + BackupScheduleDao backupScheduleDao; + + @Mock + BackupDao backupDao; + + @Mock + DataCenterDao dataCenterDao; + + @Mock + AlertManager alertManager; + + private AccountVO account; + private UserVO user; + private String[] hostPossibleValues = {"127.0.0.1", "hostname"}; private String[] datastoresPossibleValues = {"e9804933-8609-4de3-bccc-6278072a496c", "datastore-name"}; private AutoCloseable closeable; + private ConfigDepotImpl configDepotImpl; + private boolean updatedConfigKeyDepot = false; @Before public void setup() throws Exception { @@ -97,11 +157,19 @@ public class BackupManagerTest { offering.setUserDrivenBackupAllowed(true); return true; }); + + Account account = mock(Account.class); + User user = mock(User.class); + CallContext.register(user, account); } @After public void tearDown() throws Exception { closeable.close(); + if (updatedConfigKeyDepot) { + ReflectionTestUtils.setField(BackupManager.BackupFrameworkEnabled, "s_depot", configDepotImpl); + } + CallContext.unregister(); } @Test @@ -169,7 +237,7 @@ public class BackupManagerTest { Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), Mockito.eq("127.0.0.1"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success", restoreBackedUpVolume.second()); @@ -190,7 +258,7 @@ public class BackupManagerTest { Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), Mockito.eq("127.0.0.1"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success2")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success2", restoreBackedUpVolume.second()); @@ -211,8 +279,8 @@ public class BackupManagerTest { Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("hostname"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState) )).thenReturn(new Pair(Boolean.TRUE, "Success3")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Mockito.eq("hostname"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success3")); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success3", restoreBackedUpVolume.second()); @@ -233,8 +301,8 @@ public class BackupManagerTest { Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("hostname"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success4")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Mockito.eq("hostname"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success4")); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success4", restoreBackedUpVolume.second()); @@ -304,4 +372,289 @@ public class BackupManagerTest { } } } + + private void overrideBackupFrameworkConfigValue() { + ConfigKey configKey = BackupManager.BackupFrameworkEnabled; + this.configDepotImpl = (ConfigDepotImpl) ReflectionTestUtils.getField(configKey, "s_depot"); + ConfigDepotImpl configDepot = Mockito.mock(ConfigDepotImpl.class); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupFrameworkEnabled.key()), + Mockito.eq(ConfigKey.Scope.Global), Mockito.isNull())).thenReturn("true"); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupFrameworkEnabled.key()), + Mockito.eq(ConfigKey.Scope.Zone), Mockito.anyLong())).thenReturn("true"); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupProviderPlugin.key()), + Mockito.eq(ConfigKey.Scope.Zone), Mockito.anyLong())).thenReturn("testbackupprovider"); + ReflectionTestUtils.setField(configKey, "s_depot", configDepot); + updatedConfigKeyDepot = true; + } + + @Test + public void testConfigureBackupSchedule() { + Long vmId = 1L; + Long zoneId = 2L; + Long accountId = 3L; + Long domainId = 4L; + Long backupOfferingId = 5L; + + CreateBackupScheduleCmd cmd = Mockito.mock(CreateBackupScheduleCmd.class); + when(cmd.getVmId()).thenReturn(vmId); + when(cmd.getTimezone()).thenReturn("GMT"); + when(cmd.getIntervalType()).thenReturn(DateUtil.IntervalType.DAILY); + when(cmd.getMaxBackups()).thenReturn(8); + when(cmd.getSchedule()).thenReturn("00:00:00"); + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getAccountId()).thenReturn(accountId); + when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); + + overrideBackupFrameworkConfigValue(); + + Account account = Mockito.mock(Account.class); + when(accountManager.getAccount(accountId)).thenReturn(account); + when(account.getDomainId()).thenReturn(domainId); + Domain domain = Mockito.mock(Domain.class); + when(domainManager.getDomain(domainId)).thenReturn(domain); + when(resourceLimitMgr.findCorrectResourceLimitForAccount(account, Resource.ResourceType.backup, null)).thenReturn(8L); + when(resourceLimitMgr.findCorrectResourceLimitForDomain(domain, Resource.ResourceType.backup, null)).thenReturn(8L); + + BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); + when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); + when(offering.isUserDrivenBackupAllowed()).thenReturn(true); + when(offering.getProvider()).thenReturn("test"); + + BackupScheduleVO schedule = mock(BackupScheduleVO.class); + when(backupScheduleDao.findByVMAndIntervalType(vmId, DateUtil.IntervalType.DAILY)).thenReturn(schedule); + + backupManager.configureBackupSchedule(cmd); + + verify(schedule, times(1)).setScheduleType((short) DateUtil.IntervalType.DAILY.ordinal()); + verify(schedule, times(1)).setSchedule("00:00:00"); + verify(schedule, times(1)).setTimezone(TimeZone.getTimeZone("GMT").getID()); + verify(schedule, times(1)).setMaxBackups(8); + } + + @Test + public void testConfigureBackupScheduleLimitReached() { + Long vmId = 1L; + Long zoneId = 2L; + Long accountId = 3L; + Long domainId = 4L; + + CreateBackupScheduleCmd cmd = Mockito.mock(CreateBackupScheduleCmd.class); + when(cmd.getVmId()).thenReturn(vmId); + when(cmd.getTimezone()).thenReturn("GMT"); + when(cmd.getIntervalType()).thenReturn(DateUtil.IntervalType.DAILY); + when(cmd.getMaxBackups()).thenReturn(8); + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getAccountId()).thenReturn(accountId); + + overrideBackupFrameworkConfigValue(); + + Account account = Mockito.mock(Account.class); + when(accountManager.getAccount(accountId)).thenReturn(account); + when(account.getDomainId()).thenReturn(domainId); + Domain domain = Mockito.mock(Domain.class); + when(domainManager.getDomain(domainId)).thenReturn(domain); + when(resourceLimitMgr.findCorrectResourceLimitForAccount(account, Resource.ResourceType.backup, null)).thenReturn(10L); + when(resourceLimitMgr.findCorrectResourceLimitForDomain(domain, Resource.ResourceType.backup, null)).thenReturn(1L); + + InvalidParameterValueException exception = Assert.assertThrows(InvalidParameterValueException.class, + () -> backupManager.configureBackupSchedule(cmd)); + Assert.assertEquals(exception.getMessage(), "Max number of backups shouldn't exceed the domain/account level backup limit"); + } + + @Test + public void testCreateScheduledBackup() throws ResourceAllocationException { + Long vmId = 1L; + Long zoneId = 2L; + Long scheduleId = 3L; + Long backupOfferingId = 4L; + Long accountId = 5L; + Long backupId = 6L; + Long oldestBackupId = 7L; + Long newBackupSize = 1000000000L; + Long oldBackupSize = 400000000L; + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + when(vm.getId()).thenReturn(vmId); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); + when(vm.getAccountId()).thenReturn(accountId); + + overrideBackupFrameworkConfigValue(); + BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); + when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); + when(offering.isUserDrivenBackupAllowed()).thenReturn(true); + when(offering.getProvider()).thenReturn("test"); + + Account account = Mockito.mock(Account.class); + when(accountManager.getAccount(accountId)).thenReturn(account); + + BackupScheduleVO schedule = mock(BackupScheduleVO.class); + when(schedule.getScheduleType()).thenReturn(DateUtil.IntervalType.DAILY); + when(schedule.getMaxBackups()).thenReturn(0); + when(backupScheduleDao.findById(scheduleId)).thenReturn(schedule); + when(backupScheduleDao.findByVMAndIntervalType(vmId, DateUtil.IntervalType.DAILY)).thenReturn(schedule); + + BackupProvider backupProvider = mock(BackupProvider.class); + Backup backup = mock(Backup.class); + when(backup.getId()).thenReturn(backupId); + when(backup.getSize()).thenReturn(newBackupSize); + when(backupProvider.getName()).thenReturn("test"); + when(backupProvider.takeBackup(vm)).thenReturn(new Pair<>(true, backup)); + Map backupProvidersMap = new HashMap<>(); + backupProvidersMap.put(backupProvider.getName().toLowerCase(), backupProvider); + ReflectionTestUtils.setField(backupManager, "backupProvidersMap", backupProvidersMap); + + BackupVO backupVO = mock(BackupVO.class); + when(backupVO.getId()).thenReturn(backupId); + BackupVO oldestBackupVO = mock(BackupVO.class); + when(oldestBackupVO.getSize()).thenReturn(oldBackupSize); + when(oldestBackupVO.getId()).thenReturn(oldestBackupId); + when(oldestBackupVO.getVmId()).thenReturn(vmId); + when(oldestBackupVO.getBackupOfferingId()).thenReturn(backupOfferingId); + + when(backupDao.findById(backupId)).thenReturn(backupVO); + List backups = new ArrayList<>(List.of(oldestBackupVO)); + when(backupDao.listBackupsByVMandIntervalType(vmId, Backup.Type.DAILY)).thenReturn(backups); + when(backupDao.findByIdIncludingRemoved(oldestBackupId)).thenReturn(oldestBackupVO); + when(backupOfferingDao.findByIdIncludingRemoved(backupOfferingId)).thenReturn(offering); + when(backupProvider.deleteBackup(oldestBackupVO, false)).thenReturn(true); + when(backupDao.remove(oldestBackupVO.getId())).thenReturn(true); + + try (MockedStatic ignored = Mockito.mockStatic(ActionEventUtils.class)) { + Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString(), + Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); + + Assert.assertEquals(backupManager.createBackup(vmId, scheduleId), true); + + Mockito.verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup); + Mockito.verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, newBackupSize); + Mockito.verify(backupDao, times(1)).update(backupVO.getId(), backupVO); + + Mockito.verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup); + Mockito.verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, oldBackupSize); + Mockito.verify(backupDao, times(1)).remove(oldestBackupId); + } + } + + @Test (expected = ResourceAllocationException.class) + public void testCreateBackupLimitReached() throws ResourceAllocationException { + Long vmId = 1L; + Long zoneId = 2L; + Long scheduleId = 3L; + Long backupOfferingId = 4L; + Long accountId = 5L; + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); + when(vm.getAccountId()).thenReturn(accountId); + + overrideBackupFrameworkConfigValue(); + BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); + when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); + when(offering.isUserDrivenBackupAllowed()).thenReturn(true); + + BackupScheduleVO schedule = mock(BackupScheduleVO.class); + when(schedule.getScheduleType()).thenReturn(DateUtil.IntervalType.DAILY); + when(backupScheduleDao.findById(scheduleId)).thenReturn(schedule); + + Account account = Mockito.mock(Account.class); + when(account.getId()).thenReturn(accountId); + when(accountManager.getAccount(accountId)).thenReturn(account); + Mockito.doThrow(new ResourceAllocationException("", Resource.ResourceType.backup_storage)).when(resourceLimitMgr).checkResourceLimit(account, Resource.ResourceType.backup_storage, 0L); + + backupManager.createBackup(vmId, scheduleId); + + String msg = "Backup storage space resource limit exceeded for account id : " + accountId + ". Failed to create backup"; + Mockito.verify(alertManager, times(1)).sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup storage space resource limit exceeded for account id : " + accountId + + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + } + + @Test + public void testBackupSyncTask() { + Long dataCenterId = 1L; + Long vmId = 2L; + Long accountId = 3L; + Long backup2Id = 4L; + String restorePoint1ExternalId = "1234"; + Long backup1Size = 1 * Resource.ResourceType.bytesToGiB; + Long backup2Size = 2 * Resource.ResourceType.bytesToGiB; + Long newBackupSize = 3 * Resource.ResourceType.bytesToGiB; + Long metricSize = 4 * Resource.ResourceType.bytesToGiB; + + overrideBackupFrameworkConfigValue(); + + DataCenterVO dataCenter = mock(DataCenterVO.class); + when(dataCenter.getId()).thenReturn(dataCenterId); + when(dataCenterDao.listAllZones()).thenReturn(List.of(dataCenter)); + + BackupProvider backupProvider = mock(BackupProvider.class); + when(backupProvider.getName()).thenReturn("testbackupprovider"); + backupManager.setBackupProviders(List.of(backupProvider)); + backupManager.start(); + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vm.getId()).thenReturn(vmId); + when(vm.getAccountId()).thenReturn(accountId); + when(vmInstanceDao.listByZoneWithBackups(dataCenterId, null)).thenReturn(List.of(vm)); + Backup.Metric metric = new Backup.Metric(metricSize, null); + Map metricMap = new HashMap<>(); + metricMap.put(vm, metric); + when(backupProvider.getBackupMetrics(Mockito.anyLong(), Mockito.anyList())).thenReturn(metricMap); + + Backup.RestorePoint restorePoint1 = new Backup.RestorePoint(restorePoint1ExternalId, DateUtil.now(), "Root"); + Backup.RestorePoint restorePoint2 = new Backup.RestorePoint("12345", DateUtil.now(), "Root"); + List restorePoints = new ArrayList<>(List.of(restorePoint1, restorePoint2)); + when(backupProvider.listRestorePoints(vm)).thenReturn(restorePoints); + + BackupVO backupInDb1 = new BackupVO(); + backupInDb1.setSize(backup1Size); + backupInDb1.setExternalId(restorePoint1ExternalId); + + BackupVO backupInDb2 = new BackupVO(); + backupInDb2.setSize(backup2Size); + backupInDb2.setExternalId(null); + ReflectionTestUtils.setField(backupInDb2, "id", backup2Id); + when(backupDao.findById(backup2Id)).thenReturn(backupInDb2); + + when(backupDao.listByVmId(null, vmId)).thenReturn(List.of(backupInDb1, backupInDb2)); + + BackupVO newBackupEntry = new BackupVO(); + newBackupEntry.setSize(newBackupSize); + when(backupProvider.createNewBackupEntryForRestorePoint(restorePoint2, vm, metric)).thenReturn(newBackupEntry); + + try (MockedStatic ignored = Mockito.mockStatic(ActionEventUtils.class)) { + Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString(), + Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); + + try (MockedStatic ignored2 = Mockito.mockStatic(UsageEventUtils.class)) { + + BackupManagerImpl.BackupSyncTask backupSyncTask = backupManager.new BackupSyncTask(backupManager); + backupSyncTask.runInContext(); + + verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, backup1Size); + verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, metricSize); + Assert.assertEquals(backupInDb1.getSize(), metricSize); + + verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup); + verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, newBackupSize); + + verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup); + verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, backup2Size); + } + } + } } diff --git a/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java new file mode 100644 index 00000000000..3ce855b504b --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.agent.api.to.BucketTO; +import com.cloud.configuration.Resource; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.resourcelimit.ResourceLimitManagerImpl; +import com.cloud.storage.BucketVO; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; + +@RunWith(MockitoJUnitRunner.class) +public class BucketApiServiceImplTest { + @Spy + @InjectMocks + BucketApiServiceImpl bucketApiService; + + @Mock + AccountManager accountManager; + + @Mock + ObjectStoreDao objectStoreDao; + + @Mock + DataStoreManager dataStoreMgr; + + @Mock + private ResourceLimitManagerImpl resourceLimitManager; + + @Mock + private BucketDao bucketDao; + + @Test + public void testAllocBucket() throws ResourceAllocationException { + String bucketName = "bucket1"; + Long accountId = 1L; + Long poolId = 2L; + Long objectStoreId = 3L; + + CreateBucketCmd cmd = Mockito.mock(CreateBucketCmd.class); + Mockito.when(cmd.getBucketName()).thenReturn(bucketName); + Mockito.when(cmd.getEntityOwnerId()).thenReturn(accountId); + Mockito.when(cmd.getObjectStoragePoolId()).thenReturn(poolId); + Mockito.when(cmd.getQuota()).thenReturn(1); + + Account account = Mockito.mock(Account.class); + Mockito.when(accountManager.getActiveAccountById(accountId)).thenReturn(account); + + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreDao.findById(poolId)).thenReturn(objectStoreVO); + ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); + Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); + Mockito.when(objectStore.createUser(accountId)).thenReturn(true); + + bucketApiService.allocBucket(cmd); + + Mockito.verify(resourceLimitManager, Mockito.times(1)).checkResourceLimit(account, Resource.ResourceType.bucket); + Mockito.verify(resourceLimitManager, Mockito.times(1)).checkResourceLimit(account, Resource.ResourceType.object_storage, 1 * Resource.ResourceType.bytesToGiB); + } + + @Test + public void testCreateBucket() { + Long objectStoreId = 1L; + Long poolId = 2L; + Long bucketId = 3L; + Long accountId = 4L; + String bucketName = "bucket1"; + + CreateBucketCmd cmd = Mockito.mock(CreateBucketCmd.class); + Mockito.when(cmd.getObjectStoragePoolId()).thenReturn(poolId); + Mockito.when(cmd.getEntityId()).thenReturn(bucketId); + Mockito.when(cmd.getQuota()).thenReturn(1); + + BucketVO bucket = new BucketVO(bucketName); + Mockito.when(bucketDao.findById(bucketId)).thenReturn(bucket); + ReflectionTestUtils.setField(bucket, "accountId", accountId); + + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreDao.findById(poolId)).thenReturn(objectStoreVO); + ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); + Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); + Mockito.when(objectStore.createBucket(bucket, false)).thenReturn(bucket); + + bucketApiService.createBucket(cmd); + + Mockito.verify(resourceLimitManager, Mockito.times(1)).incrementResourceCount(accountId, Resource.ResourceType.bucket); + Mockito.verify(resourceLimitManager, Mockito.times(1)).incrementResourceCount(accountId, Resource.ResourceType.object_storage, 1 * Resource.ResourceType.bytesToGiB); + Assert.assertEquals(bucket.getState(), Bucket.State.Created); + } + + @Test + public void testDeleteBucket() { + Long bucketId = 1L; + Long accountId = 2L; + Long objectStoreId = 3L; + String bucketName = "bucket1"; + + BucketVO bucket = new BucketVO(bucketName); + Mockito.when(bucketDao.findById(bucketId)).thenReturn(bucket); + ReflectionTestUtils.setField(bucket, "objectStoreId", objectStoreId); + ReflectionTestUtils.setField(bucket, "quota", 1); + ReflectionTestUtils.setField(bucket, "accountId", accountId); + + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreDao.findById(objectStoreId)).thenReturn(objectStoreVO); + ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); + Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); + Mockito.when(objectStore.deleteBucket(Mockito.any(BucketTO.class))).thenReturn(true); + + bucketApiService.deleteBucket(bucketId, null); + + Mockito.verify(resourceLimitManager, Mockito.times(1)).decrementResourceCount(accountId, Resource.ResourceType.bucket); + Mockito.verify(resourceLimitManager, Mockito.times(1)).decrementResourceCount(accountId, Resource.ResourceType.object_storage, 1 * Resource.ResourceType.bytesToGiB); + } + + @Test + public void testUpdateBucket() throws ResourceAllocationException { + Long bucketId = 1L; + Long objectStoreId = 2L; + Long accountId = 3L; + Integer bucketQuota = 2; + Integer cmdQuota = 1; + String bucketName = "bucket1"; + + UpdateBucketCmd cmd = Mockito.mock(UpdateBucketCmd.class); + Mockito.when(cmd.getId()).thenReturn(bucketId); + Mockito.when(cmd.getQuota()).thenReturn(cmdQuota); + + BucketVO bucket = new BucketVO(bucketName); + ReflectionTestUtils.setField(bucket, "quota", bucketQuota); + ReflectionTestUtils.setField(bucket, "accountId", accountId); + ReflectionTestUtils.setField(bucket, "objectStoreId", objectStoreId); + Mockito.when(bucketDao.findById(bucketId)).thenReturn(bucket); + + Account account = Mockito.mock(Account.class); + + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreDao.findById(objectStoreId)).thenReturn(objectStoreVO); + ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); + Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); + + bucketApiService.updateBucket(cmd, null); + + Mockito.verify(resourceLimitManager, Mockito.times(1)).decrementResourceCount(accountId, Resource.ResourceType.object_storage, (bucketQuota - cmdQuota) * Resource.ResourceType.bytesToGiB); + } +} diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index e07962d63d0..7fdd8fab414 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -432,6 +432,9 @@ "label.backupofferingname": "Backup offering", "label.backup.repository.add": "Add backup repository", "label.backup.repository.remove": "Remove backup repository", +"label.backuplimit": "Backup Limits", +"label.backup.storage": "Backup Storage", +"label.backupstoragelimit": "Backup Storage Limits (GiB)", "label.balance": "Balance", "label.bandwidth": "Bandwidth", "label.baremetal.dhcp.devices": "Bare metal DHCP devices", @@ -460,6 +463,7 @@ "label.brocade.vcs.address": "Vcs switch address", "label.browser": "Browser", "label.bucket": "Bucket", +"label.bucketlimit": "Bucket Limits", "label.by.account": "By Account", "label.by.domain": "By domain", "label.by.level": "By level", @@ -1394,6 +1398,10 @@ "label.max.primary.storage": "Max. primary (GiB)", "label.max.secondary.storage": "Max. secondary (GiB)", "label.max.migrations": "Max. migrations", +"label.maxbackup": "Max. Backups", +"label.maxbackupstorage": "Max. Backup Storage (GiB)", +"label.maxbackups.to.retain": "Max. Backups to retain", +"label.maxbucket": "Max. Buckets", "label.maxcpu": "Max. CPU cores", "label.maxcpunumber": "Max CPU cores", "label.maxdatavolumeslimit": "Max data volumes limit", @@ -1406,6 +1414,7 @@ "label.maxmembers": "Max members", "label.maxmemory": "Max. memory (MiB)", "label.maxnetwork": "Max. Networks", +"label.maxobjectstorage": "Max. Object Storage (GiB)", "label.maxprimarystorage": "Max. primary storage (GiB)", "label.maxproject": "Max. projects", "label.maxpublicip": "Max. public IPs", @@ -1592,6 +1601,7 @@ "label.oauth.verification": "OAuth verification", "label.ocfs2": "OCFS2", "label.object.storage" : "Object Storage", +"label.objectstoragelimit": "Object Storage Limits (GiB)", "label.object.presigned.url": "Presigned URL", "label.object.presigned.url.description" : "Presigned URL of the object in order to access it without authentication.", "label.object.url.description" : "URL of the object", @@ -2626,7 +2636,7 @@ "label.objectstorageid": "Object Storage Pool", "label.bucket.update": "Update Bucket", "label.bucket.delete": "Delete Bucket", -"label.quotagb": "Quota in GB", +"label.quotagib": "Quota in GiB", "label.encryption": "Encryption", "label.versioning": "Versioning", "label.objectlocking": "Object Lock", diff --git a/ui/src/components/view/ListResourceTable.vue b/ui/src/components/view/ListResourceTable.vue index a95927f00cf..f8fdc5b5ada 100644 --- a/ui/src/components/view/ListResourceTable.vue +++ b/ui/src/components/view/ListResourceTable.vue @@ -51,7 +51,7 @@