diff --git a/api/src/main/java/com/cloud/storage/StorageService.java b/api/src/main/java/com/cloud/storage/StorageService.java index 3c6c22f0ea2..c3609cfd8ee 100644 --- a/api/src/main/java/com/cloud/storage/StorageService.java +++ b/api/src/main/java/com/cloud/storage/StorageService.java @@ -36,6 +36,10 @@ import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceInUseException; import com.cloud.exception.ResourceUnavailableException; +import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; import org.apache.cloudstack.storage.object.ObjectStore; public interface StorageService { @@ -112,6 +116,12 @@ public interface StorageService { StoragePool syncStoragePool(SyncStoragePoolCmd cmd); + Heuristic createSecondaryStorageHeuristic(CreateSecondaryStorageSelectorCmd cmd); + + Heuristic updateSecondaryStorageHeuristic(UpdateSecondaryStorageSelectorCmd cmd); + + void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd); + ObjectStore discoverObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException; boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index fa0a24670bd..ff188684557 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1061,6 +1061,9 @@ public class ApiConstants { public static final String HAS_RULES = "hasrules"; public static final String OBJECT_STORAGE = "objectstore"; + public static final String HEURISTIC_RULE = "heuristicrule"; + public static final String HEURISTIC_TYPE_VALID_OPTIONS = "Valid options are: ISO, SNAPSHOT, TEMPLATE and VOLUME."; + public static final String MANAGEMENT = "management"; public static final String IS_VNF = "isvnf"; public static final String VNF_NICS = "vnfnics"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index 944bb2194a5..0bddf6d2994 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -104,6 +104,7 @@ import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.RollingMaintenanceResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; import org.apache.cloudstack.api.response.SSHKeyPairResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.ServiceResponse; @@ -148,6 +149,7 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule; import org.apache.cloudstack.region.PortableIp; import org.apache.cloudstack.region.PortableIpRange; import org.apache.cloudstack.region.Region; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.cloudstack.usage.Usage; @@ -536,6 +538,8 @@ public interface ResponseGenerator { FirewallResponse createIpv6FirewallRuleResponse(FirewallRule acl); + SecondaryStorageHeuristicsResponse createSecondaryStorageSelectorResponse(Heuristic heuristic); + IpQuarantineResponse createQuarantinedIpsResponse(PublicIpQuarantine publicIp); ObjectStoreResponse createObjectStoreResponse(ObjectStore os); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java new file mode 100644 index 00000000000..a0e99b55971 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS; + +@APICommand(name = "createSecondaryStorageSelector", description = "Creates a secondary storage selector, described by the heuristic rule.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, entityType = {Heuristic.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class CreateSecondaryStorageSelectorCmd extends BaseCmd{ + + @Parameter(name = ApiConstants.NAME, required = true, type = CommandType.STRING, description = "The name identifying the heuristic rule.") + private String name; + + @Parameter(name = ApiConstants.DESCRIPTION, required = true, type = BaseCmd.CommandType.STRING, description = "The description of the heuristic rule.") + private String description; + + @Parameter(name = ApiConstants.ZONE_ID, required = true, entityType = ZoneResponse.class, type = BaseCmd.CommandType.UUID, description = "The zone in which the heuristic " + + "rule will be applied.") + private Long zoneId; + + @Parameter(name = ApiConstants.TYPE, required = true, type = BaseCmd.CommandType.STRING, description = + "The resource type directed to a specific secondary storage by the selector. " + HEURISTIC_TYPE_VALID_OPTIONS) + private String type; + + @Parameter(name = ApiConstants.HEURISTIC_RULE, required = true, type = BaseCmd.CommandType.STRING, description = "The heuristic rule, in JavaScript language. It is required " + + "that it returns the UUID of a secondary storage pool. An example of a rule is `if (snapshot.hypervisorType === 'KVM') { '7832f261-c602-4e8e-8580-2496ffbbc45d'; " + + "}` would allocate all snapshots with the KVM hypervisor to the specified secondary storage UUID.", length = 65535) + private String heuristicRule; + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public Long getZoneId() { + return zoneId; + } + + public String getType() { + return type; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + @Override + public void execute() { + Heuristic heuristic = _storageService.createSecondaryStorageHeuristic(this); + + if (heuristic == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create a secondary storage selector."); + } + + SecondaryStorageHeuristicsResponse response = _responseGenerator.createSecondaryStorageSelectorResponse(heuristic); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java new file mode 100644 index 00000000000..c437cd660e7 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS; + +@APICommand(name = "listSecondaryStorageSelectors", description = "Lists the secondary storage selectors and their rules.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class ListSecondaryStorageSelectorsCmd extends BaseListCmd { + + @Parameter(name = ApiConstants.ZONE_ID, required = true, entityType = ZoneResponse.class, type = CommandType.UUID, description = "The zone ID to be used in the search filter.") + private Long zoneId; + + @Parameter(name = ApiConstants.TYPE, type = CommandType.STRING, description = + "Whether to filter the selectors by type and, if so, which one. " + HEURISTIC_TYPE_VALID_OPTIONS) + private String type; + + @Parameter(name = ApiConstants.SHOW_REMOVED, type = CommandType.BOOLEAN, description = "Show removed heuristics.") + private boolean showRemoved = false; + + public Long getZoneId() { + return zoneId; + } + + public String getType() { + return type; + } + + public boolean isShowRemoved() { + return showRemoved; + } + + @Override + public void execute() { + ListResponse response = _queryService.listSecondaryStorageSelectors(this); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java new file mode 100644 index 00000000000..79554f44782 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +@APICommand(name = "removeSecondaryStorageSelector", description = "Removes an existing secondary storage selector.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class RemoveSecondaryStorageSelectorCmd extends BaseCmd { + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, entityType = SecondaryStorageHeuristicsResponse.class, required = true, + description = "The unique identifier of the secondary storage selector to be removed.") + private Long id; + + public Long getId() { + return id; + } + + @Override + public void execute() { + _storageService.removeSecondaryStorageHeuristic(this); + final SuccessResponse response = new SuccessResponse(); + response.setResponseName(getCommandName()); + response.setSuccess(true); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java new file mode 100644 index 00000000000..e63d1cbfa27 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +@APICommand(name = "updateSecondaryStorageSelector", description = "Updates an existing secondary storage selector.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class UpdateSecondaryStorageSelectorCmd extends BaseCmd { + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, entityType = SecondaryStorageHeuristicsResponse.class, required = true, + description = "The unique identifier of the secondary storage selector.") + private Long id; + + @Parameter(name = ApiConstants.HEURISTIC_RULE, required = true, type = BaseCmd.CommandType.STRING, description = "The heuristic rule, in JavaScript language. It is required " + + "that it returns the UUID of a secondary storage pool. An example of a rule is `if (snapshot.hypervisorType === 'KVM') { '7832f261-c602-4e8e-8580-2496ffbbc45d'; " + + "}` would allocate all snapshots with the KVM hypervisor to the specified secondary storage UUID.", length = 65535) + private String heuristicRule; + + public Long getId() { + return id; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + @Override + public void execute() { + Heuristic heuristic = _storageService.updateSecondaryStorageHeuristic(this); + + if (heuristic == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update the secondary storage selector."); + } + + SecondaryStorageHeuristicsResponse response = _responseGenerator.createSecondaryStorageSelectorResponse(heuristic); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java new file mode 100644 index 00000000000..25a6b2eca75 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java @@ -0,0 +1,141 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +import java.util.Date; + +import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS; + +@EntityReference(value = {Heuristic.class}) +public class SecondaryStorageHeuristicsResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "ID of the heuristic.") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the heuristic.") + private String name; + + @SerializedName(ApiConstants.DESCRIPTION) + @Param(description = "Description of the heuristic.") + private String description; + + @SerializedName(ApiConstants.ZONE_ID) + @Param(description = "The zone which the heuristic is valid upon.") + private String zoneId; + + @SerializedName(ApiConstants.TYPE) + @Param(description = "The resource type directed to a specific secondary storage by the selector. " + HEURISTIC_TYPE_VALID_OPTIONS) + private String type; + + @SerializedName(ApiConstants.HEURISTIC_RULE) + @Param(description = "The heuristic rule, in JavaScript language, used to select a secondary storage to be directed.") + private String heuristicRule; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "When the heuristic was created.") + private Date created; + + @SerializedName(ApiConstants.REMOVED) + @Param(description = "When the heuristic was removed.") + private Date removed; + + + public SecondaryStorageHeuristicsResponse(String id, String name, String description, String zoneId, String type, String heuristicRule, Date created, Date removed) { + super("heuristics"); + this.id = id; + this.name = name; + this.description = description; + this.zoneId = zoneId; + this.type = type; + this.heuristicRule = heuristicRule; + this.created = created; + this.removed = removed; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getZoneId() { + return zoneId; + } + + public void setZoneId(String zoneId) { + this.zoneId = zoneId; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + public void setHeuristicRule(String heuristicRule) { + this.heuristicRule = heuristicRule; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 17dd14da84c..3299e7537a2 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd; import org.apache.cloudstack.api.command.admin.user.ListUsersCmd; import org.apache.cloudstack.api.command.user.account.ListAccountsCmd; import org.apache.cloudstack.api.command.user.account.ListProjectAccountsCmd; @@ -79,6 +80,7 @@ import org.apache.cloudstack.api.response.ResourceDetailResponse; import org.apache.cloudstack.api.response.ResourceIconResponse; import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.SnapshotResponse; @@ -189,6 +191,8 @@ public interface QueryService { List listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd); + ListResponse listSecondaryStorageSelectors(ListSecondaryStorageSelectorsCmd cmd); + ListResponse listQuarantinedIps(ListQuarantinedIpsCmd cmd); ListResponse listSnapshots(ListSnapshotsCmd cmd); diff --git a/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java new file mode 100644 index 00000000000..2a0b8d6ea24 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.heuristics; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +public interface Heuristic extends InternalIdentity, Identity { + + String getName(); + + String getDescription(); + + Long getZoneId(); + + String getType(); + + String getHeuristicRule(); + + Date getCreated(); + + Date getRemoved(); + +} diff --git a/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java new file mode 100644 index 00000000000..f23e4b0b633 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.heuristics; + +/** + * The type of the heuristic used in the allocation process of secondary storage resources. + * Valid options are: {@link #ISO}, {@link #SNAPSHOT}, {@link #TEMPLATE} and {@link #VOLUME} + */ +public enum HeuristicType { + ISO, SNAPSHOT, TEMPLATE, VOLUME +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java index 3ee5803a91a..de26a09c6e5 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java @@ -35,6 +35,8 @@ public interface DataStoreManager { List getImageStoresByScopeExcludingReadOnly(ZoneScope scope); + List getImageStoresByZoneIds(Long ... zoneIds); + DataStore getRandomImageStore(long zoneId); DataStore getRandomUsableImageStore(long zoneId); @@ -55,5 +57,7 @@ public interface DataStoreManager { boolean isRegionStore(DataStore store); + DataStore getImageStoreByUuid(String uuid); + Long getStoreZoneId(long storeId, DataStoreRole role); } diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 28e7c89419a..1437457725a 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -192,6 +192,9 @@ public interface StorageManager extends StorageService { ConfigKey.Scope.Global, null); + ConfigKey HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000", + "The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true); + /** * should we execute in sequence not involving any storages? * @return tru if commands should execute in sequence diff --git a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java index 2c129dfd6a5..3b3537d5488 100644 --- a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java +++ b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java @@ -34,6 +34,7 @@ import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; import com.cloud.utils.Pair; import com.cloud.vm.VirtualMachineProfile; @@ -116,7 +117,7 @@ public interface TemplateManager { Long getTemplateSize(long templateId, long zoneId); - DataStore getImageStore(String storeUuid, Long zoneId); + DataStore getImageStore(String storeUuid, Long zoneId, VolumeVO volume); String getChecksum(DataStore store, String templatePath, String algorithm); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java new file mode 100644 index 00000000000..b0da0c5e747 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import java.util.Date; +import java.util.UUID; + +@Entity +@Table(name = "heuristics") +public class HeuristicVO implements Heuristic { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id", nullable = false) + private Long id; + + @Column(name = "uuid", nullable = false) + private String uuid = UUID.randomUUID().toString(); + + @Column(name = "name") + private String name; + + @Column(name = "description") + private String description; + + @Column(name = "zone_id", nullable = false) + private Long zoneId; + + @Column(name = "type", nullable = false) + private String type; + + @Column(name = "heuristic_rule", nullable = false, length = 65535) + private String heuristicRule; + + @Column(name = GenericDao.CREATED_COLUMN, nullable = false) + @Temporal(value = TemporalType.TIMESTAMP) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed = null; + + public HeuristicVO() { + } + + public HeuristicVO(String name, String description, Long zoneId, String type, String heuristicRule) { + this.name = name; + this.description = description; + this.zoneId = zoneId; + this.type = type; + this.heuristicRule = heuristicRule; + } + + @Override + public long getId() { + return id; + } + + public String getUuid() { + return uuid; + } + + @Override + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public Long getZoneId() { + return zoneId; + } + + public String getType() { + return type; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + public void setHeuristicRule(String heuristicRule) { + this.heuristicRule = heuristicRule; + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "name", "heuristicRule", "type"); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java new file mode 100644 index 00000000000..1c4057a7beb --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; + +public interface SecondaryStorageHeuristicDao extends GenericDao { + + HeuristicVO findByZoneIdAndType(long zoneId, HeuristicType type); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java new file mode 100644 index 00000000000..0b51b2aec6b --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.dao; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +public class SecondaryStorageHeuristicDaoImpl extends GenericDaoBase implements SecondaryStorageHeuristicDao { + private SearchBuilder zoneAndTypeSearch; + + @PostConstruct + public void init() { + zoneAndTypeSearch = createSearchBuilder(); + zoneAndTypeSearch.and("zoneId", zoneAndTypeSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + zoneAndTypeSearch.and("type", zoneAndTypeSearch.entity().getType(), SearchCriteria.Op.IN); + zoneAndTypeSearch.done(); + } + + @Override + public HeuristicVO findByZoneIdAndType(long zoneId, HeuristicType type) { + SearchCriteria searchCriteria = zoneAndTypeSearch.create(); + searchCriteria.setParameters("zoneId", zoneId); + searchCriteria.setParameters("type", type.toString()); + final Filter filter = new Filter(HeuristicVO.class, "created", false); + + return findOneBy(searchCriteria, filter); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java index ba9825c3c86..1c31b3e0cc4 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java @@ -49,4 +49,6 @@ public interface ImageStoreDao extends GenericDao { List findByProtocol(String protocol); ImageStoreVO findOneByZoneAndProtocol(long zoneId, String protocol); + + List listImageStoresByZoneIds(Long... zoneIds); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java index 3468b6008d9..a4827a1beae 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java @@ -43,6 +43,8 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem private SearchBuilder protocolSearch; private SearchBuilder zoneProtocolSearch; + private SearchBuilder zonesInSearch; + public ImageStoreDaoImpl() { super(); protocolSearch = createSearchBuilder(); @@ -55,6 +57,11 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem zoneProtocolSearch.and("protocol", zoneProtocolSearch.entity().getProtocol(), SearchCriteria.Op.EQ); zoneProtocolSearch.and("role", zoneProtocolSearch.entity().getRole(), SearchCriteria.Op.EQ); zoneProtocolSearch.done(); + + zonesInSearch = createSearchBuilder(); + zonesInSearch.and("zonesIn", zonesInSearch.entity().getDcId(), SearchCriteria.Op.IN); + zonesInSearch.done(); + } @Override public boolean configure(String name, Map params) throws ConfigurationException { @@ -191,4 +198,12 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem List results = listBy(sc, filter); return results.size() == 0 ? null : results.get(0); } + + + @Override + public List listImageStoresByZoneIds(Long... zoneIds) { + SearchCriteria sc = zonesInSearch.create(); + sc.setParametersIfNotNull("zonesIn", zoneIds); + return listBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 97da7e83e7b..a8c239b8f6b 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -276,6 +276,8 @@ + + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index b36814528c0..27170fcac14 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -44,6 +44,21 @@ CREATE TABLE IF NOT EXISTS `cloud`.`quarantined_ips` ( -- create_public_parameter_on_roles. #6960 ALTER TABLE `cloud`.`roles` ADD COLUMN `public_role` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'Indicates whether the role will be visible to all users (public) or only to root admins (private). If this parameter is not specified during the creation of the role its value will be defaulted to true (public).'; +-- Create heuristic table for dynamic allocating resources to the secondary storage +CREATE TABLE IF NOT EXISTS `cloud`.`heuristics` ( + `id` bigint(20) unsigned NOT NULL auto_increment, + `uuid` varchar(255) UNIQUE NOT NULL, + `name` text NOT NULL, + `description` text DEFAULT NULL, + `zone_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the zone to apply the heuristic, foreign key to `data_center` table', + `type` varchar(255) NOT NULL, + `heuristic_rule` text NOT NULL COMMENT 'JS script that defines to which secondary storage the resource will be allocated.', + `created` datetime NOT NULL, + `removed` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + CONSTRAINT `fk_heuristics__zone_id` FOREIGN KEY(`zone_id`) REFERENCES `cloud`.`data_center`(`id`) +); + -- Add tables for VM Scheduler DROP TABLE IF EXISTS `cloud`.`vm_schedule`; CREATE TABLE `cloud`.`vm_schedule` ( diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java index 6d25c481537..5bb0d19be74 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java @@ -94,7 +94,7 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager, @Override public ImageStoreEntity getImageStore(String uuid) { ImageStoreVO dataStore = dataStoreDao.findByUuid(uuid); - return getImageStore(dataStore.getId()); + return dataStore == null ? null : getImageStore(dataStore.getId()); } @Override @@ -248,6 +248,16 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager, return stores; } + @Override + public List listImageStoresFilteringByZoneIds(Long... zoneIds) { + List stores = dataStoreDao.listImageStoresByZoneIds(zoneIds); + List imageStores = new ArrayList<>(); + for (ImageStoreVO store : stores) { + imageStores.add(getImageStore(store.getId())); + } + return imageStores; + } + @Override public String getConfigComponentName() { return ImageStoreProviderManager.class.getSimpleName(); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 4268cf6446f..9c7ee983474 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -47,12 +47,14 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.log4j.Logger; @@ -99,6 +101,9 @@ public class SnapshotServiceImpl implements SnapshotService { @Inject ConfigurationDao _configDao; + @Inject + private HeuristicRuleHelper heuristicRuleHelper; + static private class CreateSnapshotContext extends AsyncRpcContext { final SnapshotInfo snapshot; final AsyncCallFuture future; @@ -297,7 +302,7 @@ public class SnapshotServiceImpl implements SnapshotService { fullSnapshot = snapshotFullBackup; } if (fullSnapshot) { - return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); + return getImageStoreForSnapshot(snapshot.getDataCenterId(), snapshot); } else { SnapshotInfo parentSnapshot = snapshot.getParent(); // Note that DataStore information in parentSnapshot is for primary @@ -314,12 +319,25 @@ public class SnapshotServiceImpl implements SnapshotService { } } if (parentSnapshotOnBackupStore == null) { - return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); + return getImageStoreForSnapshot(snapshot.getDataCenterId(), snapshot); } return dataStoreMgr.getDataStore(parentSnapshotOnBackupStore.getDataStoreId(), parentSnapshotOnBackupStore.getRole()); } } + /** + * Verify if the data center has heuristic rules for allocating snapshots; if there is then returns the {@link DataStore} returned by the JS script. + * Otherwise, returns {@link DataStore}s with free capacity. + */ + protected DataStore getImageStoreForSnapshot(Long dataCenterId, SnapshotInfo snapshot) { + DataStore imageStore = heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(dataCenterId, HeuristicType.SNAPSHOT, snapshot); + + if (imageStore == null) { + imageStore = dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); + } + return imageStore; + } + @Override public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { SnapshotObject snapObj = (SnapshotObject)snapshot; diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java index 917fb2d9c75..6d59b6f36e0 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java @@ -18,6 +18,9 @@ */ package org.apache.cloudstack.storage.snapshot; +import com.cloud.storage.DataStoreRole; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; @@ -26,8 +29,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCallFuture; -import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; -import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -40,8 +43,6 @@ import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.support.AnnotationConfigContextLoader; -import com.cloud.storage.DataStoreRole; - @RunWith(MockitoJUnitRunner.class) @ContextConfiguration(loader = AnnotationConfigContextLoader.class) public class SnapshotServiceImplTest { @@ -57,20 +58,28 @@ public class SnapshotServiceImplTest { SnapshotDataFactory _snapshotFactory; @Mock - AsyncCallbackDispatcher caller; + HeuristicRuleHelper heuristicRuleHelperMock; + + @Mock + SnapshotInfo snapshotMock; + + @Mock + VolumeInfo volumeInfoMock; + + @Mock + DataStoreManager dataStoreManagerMock; + + private static final long DUMMY_ID = 1L; @Test public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception { - SnapshotInfo snapshot = Mockito.mock(SnapshotInfo.class); - VolumeInfo volumeInfo = Mockito.mock(VolumeInfo.class); - - Mockito.when(snapshot.getId()).thenReturn(1L); - Mockito.when(snapshot.getVolumeId()).thenReturn(1L); + Mockito.when(snapshotMock.getId()).thenReturn(DUMMY_ID); + Mockito.when(snapshotMock.getVolumeId()).thenReturn(DUMMY_ID); Mockito.when(_snapshotFactory.getSnapshotOnPrimaryStore(1L)).thenReturn(null); - Mockito.when(volFactory.getVolume(1L, DataStoreRole.Primary)).thenReturn(volumeInfo); + Mockito.when(volFactory.getVolume(DUMMY_ID, DataStoreRole.Primary)).thenReturn(volumeInfoMock); PrimaryDataStore store = Mockito.mock(PrimaryDataStore.class); - Mockito.when(volumeInfo.getDataStore()).thenReturn(store); + Mockito.when(volumeInfoMock.getDataStore()).thenReturn(store); PrimaryDataStoreDriver driver = Mockito.mock(PrimaryDataStoreDriver.class); Mockito.when(store.getDriver()).thenReturn(driver); @@ -80,7 +89,29 @@ public class SnapshotServiceImplTest { Mockito.when(mock.get()).thenReturn(result); Mockito.when(result.isFailed()).thenReturn(false); })) { - Assert.assertTrue(snapshotService.revertSnapshot(snapshot)); + Assert.assertTrue(snapshotService.revertSnapshot(snapshotMock)); } } + + @Test + public void getImageStoreForSnapshotTestShouldListFreeImageStoresWithNoHeuristicRule() { + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))). + thenReturn(null); + Mockito.when(snapshotMock.getDataCenterId()).thenReturn(DUMMY_ID); + + snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock); + + Mockito.verify(dataStoreManagerMock, Mockito.times(1)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } + + @Test + public void getImageStoreForSnapshotTestShouldReturnImageStoreReturnedByTheHeuristicRule() { + DataStore dataStore = Mockito.mock(DataStore.class); + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))). + thenReturn(dataStore); + + snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock); + + Mockito.verify(dataStoreManagerMock, Mockito.times(0)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java index 60d01119302..757623e3d04 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java @@ -84,6 +84,16 @@ public class DataStoreManagerImpl implements DataStoreManager { return imageDataStoreMgr.listImageStoresByScopeExcludingReadOnly(scope); } + @Override + public List getImageStoresByZoneIds(Long... zoneIds) { + return imageDataStoreMgr.listImageStoresFilteringByZoneIds(zoneIds); + } + + @Override + public DataStore getImageStoreByUuid(String uuid) { + return imageDataStoreMgr.getImageStore(uuid); + } + @Override public DataStore getRandomImageStore(long zoneId) { List stores = getImageStoresByScope(new ZoneScope(zoneId)); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java index 47e2ee38307..39f42e842c1 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java @@ -80,5 +80,8 @@ public interface ImageStoreProviderManager { List listImageStoresWithFreeCapacity(List imageStores); List orderImageStoresOnFreeCapacity(List imageStores); + + List listImageStoresFilteringByZoneIds(Long... zoneIds); + long getImageStoreZoneId(long dataStoreId); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index a6dc017074b..3259835f1df 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -144,6 +144,7 @@ import org.apache.cloudstack.api.response.RollingMaintenanceHostUpdatedResponse; import org.apache.cloudstack.api.response.RollingMaintenanceResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; import org.apache.cloudstack.api.response.SSHKeyPairResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.SecurityGroupRuleResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; @@ -200,6 +201,7 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule; import org.apache.cloudstack.region.PortableIp; import org.apache.cloudstack.region.PortableIpRange; import org.apache.cloudstack.region.Region; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -5105,6 +5107,16 @@ public class ApiResponseHelper implements ResponseGenerator { return response; } + @Override + public SecondaryStorageHeuristicsResponse createSecondaryStorageSelectorResponse(Heuristic heuristic) { + String zoneUuid = ApiDBUtils.findZoneById(heuristic.getZoneId()).getUuid(); + SecondaryStorageHeuristicsResponse secondaryStorageHeuristicsResponse = new SecondaryStorageHeuristicsResponse(heuristic.getUuid(), heuristic.getName(), + heuristic.getDescription(), zoneUuid, heuristic.getType(), heuristic.getHeuristicRule(), heuristic.getCreated(), heuristic.getRemoved()); + secondaryStorageHeuristicsResponse.setResponseName("secondarystorageheuristics"); + + return secondaryStorageHeuristicsResponse; + } + @Override public IpQuarantineResponse createQuarantinedIpsResponse(PublicIpQuarantine quarantinedIp) { IpQuarantineResponse quarantinedIpsResponse = new IpQuarantineResponse(); diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index fd3e3298338..6b0144045a2 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -87,6 +87,7 @@ import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd; import org.apache.cloudstack.api.command.admin.template.ListTemplatesCmdByAdmin; import org.apache.cloudstack.api.command.admin.user.ListUsersCmd; import org.apache.cloudstack.api.command.admin.zone.ListZonesCmdByAdmin; @@ -135,6 +136,7 @@ import org.apache.cloudstack.api.response.ResourceDetailResponse; import org.apache.cloudstack.api.response.ResourceIconResponse; import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.SnapshotResponse; @@ -158,6 +160,9 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -509,6 +514,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private VirtualMachineManager virtualMachineManager; + @Inject private VolumeDao volumeDao; @@ -518,6 +524,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private ManagementServerHostDao msHostDao; + @Inject + private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + @Inject private NetworkDao networkDao; @@ -4799,6 +4808,36 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q return responseGenerator.createHealthCheckResponse(_routerDao.findById(routerId), result); } + @Override + public ListResponse listSecondaryStorageSelectors(ListSecondaryStorageSelectorsCmd cmd) { + ListResponse response = new ListResponse<>(); + Pair, Integer> result = listSecondaryStorageSelectorsInternal(cmd.getZoneId(), cmd.getType(), cmd.isShowRemoved()); + List listOfSecondaryStorageHeuristicsResponses = new ArrayList<>(); + + for (Heuristic heuristic : result.first()) { + SecondaryStorageHeuristicsResponse secondaryStorageHeuristicsResponse = responseGenerator.createSecondaryStorageSelectorResponse(heuristic); + listOfSecondaryStorageHeuristicsResponses.add(secondaryStorageHeuristicsResponse); + } + + response.setResponses(listOfSecondaryStorageHeuristicsResponses); + return response; + } + + private Pair, Integer> listSecondaryStorageSelectorsInternal(Long zoneId, String type, boolean showRemoved) { + SearchBuilder searchBuilder = secondaryStorageHeuristicDao.createSearchBuilder(); + + searchBuilder.and("zoneId", searchBuilder.entity().getZoneId(), SearchCriteria.Op.EQ); + searchBuilder.and("type", searchBuilder.entity().getType(), SearchCriteria.Op.EQ); + + searchBuilder.done(); + + SearchCriteria searchCriteria = searchBuilder.create(); + searchCriteria.setParameters("zoneId", zoneId); + searchCriteria.setParametersIfNotNull("type", type); + + return secondaryStorageHeuristicDao.searchAndCount(searchCriteria, null, showRemoved); + } + @Override public ListResponse listQuarantinedIps(ListQuarantinedIpsCmd cmd) { ListResponse response = new ListResponse<>(); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 37cdbc8c033..9e88b6f1a1a 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -233,6 +233,10 @@ import org.apache.cloudstack.api.command.admin.storage.UpdateImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStorageCapabilitiesCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd; import org.apache.cloudstack.api.command.admin.swift.AddSwiftCmd; import org.apache.cloudstack.api.command.admin.swift.ListSwiftsCmd; import org.apache.cloudstack.api.command.admin.systemvm.DestroySystemVmCmd; @@ -3897,6 +3901,11 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(PatchSystemVMCmd.class); cmdList.add(ListGuestVlansCmd.class); cmdList.add(AssignVolumeCmd.class); + cmdList.add(ListSecondaryStorageSelectorsCmd.class); + cmdList.add(CreateSecondaryStorageSelectorCmd.class); + cmdList.add(UpdateSecondaryStorageSelectorCmd.class); + cmdList.add(RemoveSecondaryStorageSelectorCmd.class); + // Out-of-band management APIs for admins cmdList.add(EnableOutOfBandManagementForHostCmd.class); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index adbb43c03f5..ec432c96782 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -27,6 +27,7 @@ import java.nio.file.Files; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -60,6 +61,9 @@ import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingSto import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; @@ -97,6 +101,10 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.SyncVolumePathAnswer; @@ -124,6 +132,7 @@ import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.cloudstack.storage.object.ObjectStoreEntity; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang.time.DateUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -359,6 +368,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C @Inject private AnnotationDao annotationDao; + @Inject + private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + @Inject protected UserVmManager userVmManager; @Inject @@ -2006,6 +2018,65 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C return (PrimaryDataStoreInfo) _dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); } + + @Override + public Heuristic createSecondaryStorageHeuristic(CreateSecondaryStorageSelectorCmd cmd) { + String name = cmd.getName(); + String description = cmd.getDescription(); + long zoneId = cmd.getZoneId(); + String heuristicRule = cmd.getHeuristicRule(); + String type = cmd.getType(); + HeuristicType formattedType = EnumUtils.getEnumIgnoreCase(HeuristicType.class, type); + + if (formattedType == null) { + throw new IllegalArgumentException(String.format("The given heuristic type [%s] is not valid for creating a new secondary storage selector." + + " The valid options are %s.", type, Arrays.asList(HeuristicType.values()))); + } + + HeuristicVO heuristic = secondaryStorageHeuristicDao.findByZoneIdAndType(zoneId, formattedType); + + if (heuristic != null) { + DataCenterVO dataCenter = _dcDao.findById(zoneId); + throw new CloudRuntimeException(String.format("There is already a heuristic rule in the specified %s with the type [%s].", + dataCenter, type)); + } + + validateHeuristicRule(heuristicRule); + + HeuristicVO heuristicVO = new HeuristicVO(name, description, zoneId, formattedType.toString(), heuristicRule); + return secondaryStorageHeuristicDao.persist(heuristicVO); + } + + @Override + public Heuristic updateSecondaryStorageHeuristic(UpdateSecondaryStorageSelectorCmd cmd) { + long heuristicId = cmd.getId(); + String heuristicRule = cmd.getHeuristicRule(); + + HeuristicVO heuristicVO = secondaryStorageHeuristicDao.findById(heuristicId); + validateHeuristicRule(heuristicRule); + heuristicVO.setHeuristicRule(heuristicRule); + + return secondaryStorageHeuristicDao.persist(heuristicVO); + } + + @Override + public void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd) { + Long heuristicId = cmd.getId(); + HeuristicVO heuristicVO = secondaryStorageHeuristicDao.findById(heuristicId); + + if (heuristicVO != null) { + secondaryStorageHeuristicDao.remove(heuristicId); + } else { + throw new CloudRuntimeException("Unable to find an active heuristic with the specified UUID."); + } + } + + protected void validateHeuristicRule(String heuristicRule) { + if (StringUtils.isBlank(heuristicRule)) { + throw new IllegalArgumentException("Unable to create a new secondary storage selector as the given heuristic rule is blank."); + } + } + public void syncDatastoreClusterStoragePool(long datastoreClusterPoolId, List childDatastoreAnswerList, long hostId) { StoragePoolVO datastoreClusterPool = _storagePoolDao.findById(datastoreClusterPoolId); List storageTags = _storagePoolTagsDao.findStoragePoolTags(datastoreClusterPoolId); diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index b4faf2ee479..69b5f984081 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -409,7 +409,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic String format = sanitizeFormat(cmd.getFormat()); Long diskOfferingId = cmd.getDiskOfferingId(); String imageStoreUuid = cmd.getImageStoreUuid(); - DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId); validateVolume(caller, ownerId, zoneId, volumeName, url, format, diskOfferingId); @@ -419,6 +418,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic RegisterVolumePayload payload = new RegisterVolumePayload(cmd.getUrl(), cmd.getChecksum(), format); vol.addPayload(payload); + DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId, volume); volService.registerVolume(vol, store); return volume; @@ -451,7 +451,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic String format = sanitizeFormat(cmd.getFormat()); final Long diskOfferingId = cmd.getDiskOfferingId(); String imageStoreUuid = cmd.getImageStoreUuid(); - final DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId); validateVolume(caller, ownerId, zoneId, volumeName, null, format, diskOfferingId); @@ -461,6 +460,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic VolumeVO volume = persistVolume(owner, zoneId, volumeName, null, format, diskOfferingId, Volume.State.NotUploaded); + final DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId, volume); + VolumeInfo vol = volFactory.getVolume(volume.getId()); RegisterVolumePayload payload = new RegisterVolumePayload(null, cmd.getChecksum(), format); @@ -655,6 +656,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic //url can be null incase of postupload if (url != null) { _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); + volume.setSize(UriUtils.getRemoteSize(url)); } return volume; diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index d958cc550ba..b886f0868f6 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -59,9 +59,11 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.collections.CollectionUtils; @@ -107,7 +109,7 @@ import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; public class HypervisorTemplateAdapter extends TemplateAdapterBase { - private final static Logger s_logger = Logger.getLogger(HypervisorTemplateAdapter.class); + protected final static Logger s_logger = Logger.getLogger(HypervisorTemplateAdapter.class); @Inject DownloadMonitor _downloadMonitor; @Inject @@ -145,6 +147,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { @Inject private AnnotationDao annotationDao; @Inject + private HeuristicRuleHelper heuristicRuleHelper; + @Inject VMInstanceDao _vmInstanceDao; @Override @@ -271,17 +275,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } if (!profile.isDirectDownload()) { - List zones = profile.getZoneIdList(); - - //zones is null when this template is to be registered to all zones - if (zones == null){ - createTemplateWithinZone(null, profile, template); - } - else { - for (Long zId : zones) { - createTemplateWithinZone(zId, profile, template); - } - } + createTemplateWithinZones(profile, template); } else { //KVM direct download templates bypassing Secondary Storage persistDirectDownloadTemplate(template.getId(), profile.getSize()); @@ -291,46 +285,67 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { return template; } - private void createTemplateWithinZone(Long zId, TemplateProfile profile, VMTemplateVO template) { - // find all eligible image stores for this zone scope - List imageStores = storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(zId)); - if (imageStores == null || imageStores.size() == 0) { - throw new CloudRuntimeException("Unable to find image store to download template " + profile.getTemplate()); + /** + * For each zone ID in {@link TemplateProfile#zoneIdList}, verifies if there is active heuristic rules for allocating template and returns the + * {@link DataStore} returned by the heuristic rule. If there is not an active heuristic rule, then allocate it to a random {@link DataStore}, if the ISO/template is private + * or allocate it to all {@link DataStore} in the zone, if it is public. + * @param profile + * @param template + */ + protected void createTemplateWithinZones(TemplateProfile profile, VMTemplateVO template) { + List zonesIds = profile.getZoneIdList(); + + if (zonesIds == null) { + zonesIds = _dcDao.listAllZones().stream().map(DataCenterVO::getId).collect(Collectors.toList()); } + List imageStores = getImageStoresThrowsExceptionIfNotFound(zonesIds, profile); + + for (long zoneId : zonesIds) { + DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId); + + if (imageStore == null) { + standardImageStoreAllocation(imageStores, template); + } else { + validateSecondaryStorageAndCreateTemplate(List.of(imageStore), template, null); + } + } + } + + protected List getImageStoresThrowsExceptionIfNotFound(List zonesIds, TemplateProfile profile) { + List imageStores = storeMgr.getImageStoresByZoneIds(zonesIds.toArray(new Long[0])); + if (imageStores == null || imageStores.size() == 0) { + throw new CloudRuntimeException(String.format("Unable to find image store to download the template [%s].", profile.getTemplate())); + } + return imageStores; + } + + protected DataStore verifyHeuristicRulesForZone(VMTemplateVO template, Long zoneId) { + HeuristicType heuristicType; + if (ImageFormat.ISO.equals(template.getFormat())) { + heuristicType = HeuristicType.ISO; + } else { + heuristicType = HeuristicType.TEMPLATE; + } + return heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, heuristicType, template); + } + + protected void standardImageStoreAllocation(List imageStores, VMTemplateVO template) { Set zoneSet = new HashSet(); Collections.shuffle(imageStores); - // For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc. + validateSecondaryStorageAndCreateTemplate(imageStores, template, zoneSet); + } + + protected void validateSecondaryStorageAndCreateTemplate(List imageStores, VMTemplateVO template, Set zoneSet) { for (DataStore imageStore : imageStores) { - // skip data stores for a disabled zone Long zoneId = imageStore.getScope().getScopeId(); - if (zoneId != null) { - DataCenterVO zone = _dcDao.findById(zoneId); - if (zone == null) { - s_logger.warn("Unable to find zone by id " + zoneId + ", so skip downloading template to its image store " + imageStore.getId()); - continue; - } - // Check if zone is disabled - if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { - s_logger.info("Zone " + zoneId + " is disabled. Skip downloading template to its image store " + imageStore.getId()); - continue; - } - - // Check if image store has enough capacity for template - if (!_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { - s_logger.info("Image store doesn't have enough capacity. Skip downloading template to this image store " + imageStore.getId()); - continue; - } - // We want to download private template to one of the image store in a zone - if (isPrivateTemplate(template) && zoneSet.contains(zoneId)) { - continue; - } else { - zoneSet.add(zoneId); - } + if (!isZoneAndImageStoreAvailable(imageStore, zoneId, zoneSet, isPrivateTemplate(template))) { + continue; } + TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore); - CreateTemplateContext context = new CreateTemplateContext(null, tmpl); + CreateTemplateContext context = new CreateTemplateContext<>(null, tmpl); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createTemplateAsyncCallBack(null, null)); caller.setContext(context); @@ -338,6 +353,44 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } } + protected boolean isZoneAndImageStoreAvailable(DataStore imageStore, Long zoneId, Set zoneSet, boolean isTemplatePrivate) { + if (zoneId == null) { + s_logger.warn(String.format("Zone ID is null, cannot allocate ISO/template in image store [%s].", imageStore)); + return false; + } + + DataCenterVO zone = _dcDao.findById(zoneId); + if (zone == null) { + s_logger.warn(String.format("Unable to find zone by id [%s], so skip downloading template to its image store [%s].", zoneId, imageStore.getId())); + return false; + } + + if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { + s_logger.info(String.format("Zone [%s] is disabled. Skip downloading template to its image store [%s].", zoneId, imageStore.getId())); + return false; + } + + if (!_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { + s_logger.info(String.format("Image store doesn't have enough capacity. Skip downloading template to this image store [%s].", imageStore.getId())); + return false; + } + + if (zoneSet == null) { + s_logger.info(String.format("Zone set is null; therefore, the ISO/template should be allocated in every secondary storage of zone [%s].", zone)); + return true; + } + + if (isTemplatePrivate && zoneSet.contains(zoneId)) { + s_logger.info(String.format("The template is private and it is already allocated in a secondary storage in zone [%s]; therefore, image store [%s] will be skipped.", + zone, imageStore)); + return false; + } + + s_logger.info(String.format("Private template will be allocated in image store [%s] in zone [%s].", imageStore, zone)); + zoneSet.add(zoneId); + return true; + } + @Override public List createTemplateForPostUpload(final TemplateProfile profile) { // persist entry in vm_template, vm_template_details and template_zone_ref tables, not that entry at template_store_ref is not created here, and created in createTemplateAsync. @@ -352,80 +405,27 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { throw new CloudRuntimeException("Unable to persist the template " + profile.getTemplate()); } - if (profile.getZoneIdList() != null && profile.getZoneIdList().size() > 1) - throw new CloudRuntimeException("Operation is not supported for more than one zone id at a time"); + List zoneIdList = profile.getZoneIdList(); - Long zoneId = null; - if (profile.getZoneIdList() != null) - zoneId = profile.getZoneIdList().get(0); - - // find all eligible image stores for this zone scope - List imageStores = storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(zoneId)); - if (imageStores == null || imageStores.size() == 0) { - throw new CloudRuntimeException("Unable to find image store to download template " + profile.getTemplate()); + if (zoneIdList == null) { + throw new CloudRuntimeException("Zone ID is null, cannot upload ISO/template."); } + if (zoneIdList.size() > 1) + throw new CloudRuntimeException("Operation is not supported for more than one zone id at a time."); + + Long zoneId = zoneIdList.get(0); + DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId); List payloads = new LinkedList<>(); - Set zoneSet = new HashSet(); - Collections.shuffle(imageStores); // For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc. - for (DataStore imageStore : imageStores) { - // skip data stores for a disabled zone - Long zoneId_is = imageStore.getScope().getScopeId(); - if (zoneId != null) { - DataCenterVO zone = _dcDao.findById(zoneId_is); - if (zone == null) { - s_logger.warn("Unable to find zone by id " + zoneId_is + - ", so skip downloading template to its image store " + imageStore.getId()); - continue; - } - // Check if zone is disabled - if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { - s_logger.info("Zone " + zoneId_is + - " is disabled, so skip downloading template to its image store " + imageStore.getId()); - continue; - } + if (imageStore == null) { + List imageStores = getImageStoresThrowsExceptionIfNotFound(List.of(zoneId), profile); + postUploadAllocation(imageStores, template, payloads); + } else { + postUploadAllocation(List.of(imageStore), template, payloads); - // We want to download private template to one of the image store in a zone - if (isPrivateTemplate(template) && zoneSet.contains(zoneId_is)) { - continue; - } else { - zoneSet.add(zoneId_is); - } - - } - - TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore); - //imageService.createTemplateAsync(tmpl, imageStore, caller); - - // persist template_store_ref entry - DataObject templateOnStore = imageStore.create(tmpl); - // update template_store_ref and template state - - EndPoint ep = _epSelector.select(templateOnStore); - if (ep == null) { - String errMsg = "There is no secondary storage VM for downloading template to image store " + imageStore.getName(); - s_logger.warn(errMsg); - throw new CloudRuntimeException(errMsg); - } - - TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), template.getUuid(), tmpl.getInstallPath(), tmpl - .getChecksum(), tmpl.getType().toString(), template.getUniqueName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(), - templateOnStore.getDataStore().getRole().toString()); - //using the existing max template size configuration - payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key())); - - Long accountId = template.getAccountId(); - Account account = _accountDao.findById(accountId); - Domain domain = _domainDao.findById(account.getDomainId()); - - payload.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage)); - payload.setAccountId(accountId); - payload.setRemoteEndPoint(ep.getPublicAddr()); - payload.setRequiresHvm(template.requiresHvm()); - payload.setDescription(template.getDisplayText()); - payloads.add(payload); } + if(payloads.isEmpty()) { throw new CloudRuntimeException("unable to find zone or an image store with enough capacity"); } @@ -435,6 +435,52 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { }); } + /** + * If the template/ISO is marked as private, then it is allocated to a random secondary storage; otherwise, allocates to every storage pool in every zone given by the + * {@link TemplateProfile#zoneIdList}. + */ + private void postUploadAllocation(List imageStores, VMTemplateVO template, List payloads) { + Set zoneSet = new HashSet(); + Collections.shuffle(imageStores); + for (DataStore imageStore : imageStores) { + Long zoneId_is = imageStore.getScope().getScopeId(); + + if (!isZoneAndImageStoreAvailable(imageStore, zoneId_is, zoneSet, isPrivateTemplate(template))) { + continue; + } + + TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore); + + // persist template_store_ref entry + DataObject templateOnStore = imageStore.create(tmpl); + + // update template_store_ref and template state + EndPoint ep = _epSelector.select(templateOnStore); + if (ep == null) { + String errMsg = "There is no secondary storage VM for downloading template to image store " + imageStore.getName(); + s_logger.warn(errMsg); + throw new CloudRuntimeException(errMsg); + } + + TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), template.getUuid(), tmpl.getInstallPath(), tmpl + .getChecksum(), tmpl.getType().toString(), template.getUniqueName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(), + templateOnStore.getDataStore().getRole().toString()); + //using the existing max template size configuration + payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key())); + + Long accountId = template.getAccountId(); + Account account = _accountDao.findById(accountId); + Domain domain = _domainDao.findById(account.getDomainId()); + + payload.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage)); + payload.setAccountId(accountId); + payload.setRemoteEndPoint(ep.getPublicAddr()); + payload.setRequiresHvm(template.requiresHvm()); + payload.setDescription(template.getDisplayText()); + payloads.add(payload); + } + } + private boolean isPrivateTemplate(VMTemplateVO template){ // if public OR featured OR system template diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index f5d385b4e10..2ed42087020 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -86,6 +86,8 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.snapshot.SnapshotHelper; import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.storage.command.CommandResult; @@ -98,6 +100,7 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.template.VnfTemplateManager; import org.apache.cloudstack.storage.template.VnfTemplateUtils; @@ -311,6 +314,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject VnfTemplateManager vnfTemplateManager; + @Inject + private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + + @Inject + private HeuristicRuleHelper heuristicRuleHelper; + private TemplateAdapter getAdapter(HypervisorType type) { TemplateAdapter adapter = null; if (type == HypervisorType.BareMetal) { @@ -451,17 +460,21 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } @Override - public DataStore getImageStore(String storeUuid, Long zoneId) { + public DataStore getImageStore(String storeUuid, Long zoneId, VolumeVO volume) { DataStore imageStore = null; if (storeUuid != null) { imageStore = _dataStoreMgr.getDataStore(storeUuid, DataStoreRole.Image); } else { - imageStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); + imageStore = heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.VOLUME, volume); if (imageStore == null) { - throw new CloudRuntimeException("cannot find an image store for zone " + zoneId); + imageStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); } } + if (imageStore == null) { + throw new CloudRuntimeException(String.format("Cannot find an image store for zone [%s].", zoneId)); + } + return imageStore; } diff --git a/server/src/main/java/com/cloud/test/TestAppender.java b/server/src/main/java/com/cloud/test/TestAppender.java index d8e97faef7c..9a6ec62efc6 100644 --- a/server/src/main/java/com/cloud/test/TestAppender.java +++ b/server/src/main/java/com/cloud/test/TestAppender.java @@ -46,6 +46,7 @@ import static org.apache.log4j.Level.ERROR; import static org.apache.log4j.Level.FATAL; import static org.apache.log4j.Level.INFO; import static org.apache.log4j.Level.OFF; +import static org.apache.log4j.Level.WARN; /** * @@ -152,6 +153,7 @@ public final class TestAppender extends AppenderSkeleton { expectedPatterns.put(FATAL, new HashSet()); expectedPatterns.put(INFO, new HashSet()); expectedPatterns.put(OFF, new HashSet()); + expectedPatterns.put(WARN, new HashSet()); } public TestAppenderBuilder addExpectedPattern(final Level level, final String pattern) { checkArgument(level != null, "addExpectedPattern requires a non-null level"); diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java new file mode 100644 index 00000000000..9dfc75e06f2 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java @@ -0,0 +1,278 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics; + +import com.cloud.api.ApiDBUtils; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StorageStats; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.heuristics.presetvariables.Account; +import org.apache.cloudstack.storage.heuristics.presetvariables.Domain; +import org.apache.cloudstack.storage.heuristics.presetvariables.PresetVariables; +import org.apache.cloudstack.storage.heuristics.presetvariables.SecondaryStorage; +import org.apache.cloudstack.storage.heuristics.presetvariables.Snapshot; +import org.apache.cloudstack.storage.heuristics.presetvariables.Template; +import org.apache.cloudstack.storage.heuristics.presetvariables.Volume; +import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Helper class for building and injecting the heuristics preset variables into the JS script. + */ +public class HeuristicRuleHelper { + + protected static final Logger LOGGER = Logger.getLogger(HeuristicRuleHelper.class); + + private static final Long HEURISTICS_SCRIPT_TIMEOUT = StorageManager.HEURISTICS_SCRIPT_TIMEOUT.value(); + + @Inject + private DataStoreManager dataStoreManager; + + @Inject + private ImageStoreDao imageStoreDao; + + @Inject + private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + + @Inject + private DomainDao domainDao; + + @Inject + private AccountDao accountDao; + + /** + * Returns the {@link DataStore} object if the zone, specified by the ID, has an active heuristic rule for the given {@link HeuristicType}. + * It returns null otherwise. + * @param zoneId used to search for the heuristic rules. + * @param heuristicType used for checking if there is a heuristic rule for the given {@link HeuristicType}. + * @param obj can be from the following classes: {@link VMTemplateVO}, {@link SnapshotInfo} and {@link VolumeVO}. + * They are used to retrieve attributes for injecting in the JS rule. + * @return the corresponding {@link DataStore} if there is a heuristic rule, returns null otherwise. + */ + public DataStore getImageStoreIfThereIsHeuristicRule(Long zoneId, HeuristicType heuristicType, Object obj) { + HeuristicVO heuristicsVO = secondaryStorageHeuristicDao.findByZoneIdAndType(zoneId, heuristicType); + + if (heuristicsVO == null) { + LOGGER.debug(String.format("No heuristic rules found for zone with ID [%s] and heuristic type [%s]. Returning null.", zoneId, heuristicType)); + return null; + } else { + LOGGER.debug(String.format("Found the heuristic rule %s to apply for zone with ID [%s].", heuristicsVO, zoneId)); + return interpretHeuristicRule(heuristicsVO.getHeuristicRule(), heuristicType, obj, zoneId); + } + } + + /** + * Build the preset variables ({@link Template}, {@link Snapshot} and {@link Volume}) for the JS script. + */ + protected void buildPresetVariables(JsInterpreter jsInterpreter, HeuristicType heuristicType, long zoneId, Object obj) { + PresetVariables presetVariables = new PresetVariables(); + Long accountId = null; + + switch (heuristicType) { + case TEMPLATE: + case ISO: + presetVariables.setTemplate(setTemplatePresetVariable((VMTemplateVO) obj)); + accountId = ((VMTemplateVO) obj).getAccountId(); + break; + case SNAPSHOT: + presetVariables.setSnapshot(setSnapshotPresetVariable((SnapshotInfo) obj)); + accountId = ((SnapshotInfo) obj).getAccountId(); + break; + case VOLUME: + presetVariables.setVolume(setVolumePresetVariable((VolumeVO) obj)); + accountId = ((VolumeVO) obj).getAccountId(); + break; + } + presetVariables.setAccount(setAccountPresetVariable(accountId)); + presetVariables.setSecondaryStorages(setSecondaryStoragesVariable(zoneId)); + + injectPresetVariables(jsInterpreter, presetVariables); + } + + /** + * Inject the {@link PresetVariables} ({@link Template}, {@link Snapshot} and {@link Volume}) into the JS {@link JsInterpreter}. + * For each type, they can be accessed using the following variables in the script: + *
    + *
  • ISO/Template: using the variable iso or template, e.g. iso.format
    • + *
  • Snapshot: using the variable snapshot, e.g. snapshot.size
    • + *
  • Volume: using the variable volume, e.g. volume.format
    • + *
+ * @param jsInterpreter the JS script + * @param presetVariables used for injecting in the JS interpreter. + */ + protected void injectPresetVariables(JsInterpreter jsInterpreter, PresetVariables presetVariables) { + jsInterpreter.injectVariable("secondaryStorages", presetVariables.getSecondaryStorages().toString()); + + if (presetVariables.getTemplate() != null) { + jsInterpreter.injectVariable("template", presetVariables.getTemplate().toString()); + jsInterpreter.injectVariable("iso", presetVariables.getTemplate().toString()); + } + + if (presetVariables.getSnapshot() != null) { + jsInterpreter.injectVariable("snapshot", presetVariables.getSnapshot().toString()); + } + + if (presetVariables.getVolume() != null) { + jsInterpreter.injectVariable("volume", presetVariables.getVolume().toString()); + } + + if (presetVariables.getAccount() != null) { + jsInterpreter.injectVariable("account", presetVariables.getAccount().toString()); + } + } + + protected List setSecondaryStoragesVariable(long zoneId) { + List secondaryStorageList = new ArrayList<>(); + List imageStoreVOS = imageStoreDao.listStoresByZoneId(zoneId); + + for (ImageStoreVO imageStore : imageStoreVOS) { + SecondaryStorage secondaryStorage = new SecondaryStorage(); + + secondaryStorage.setName(imageStore.getName()); + secondaryStorage.setId(imageStore.getUuid()); + secondaryStorage.setProtocol(imageStore.getProtocol()); + StorageStats storageStats = ApiDBUtils.getSecondaryStorageStatistics(imageStore.getId()); + + if (storageStats != null) { + secondaryStorage.setUsedDiskSize(storageStats.getByteUsed()); + secondaryStorage.setTotalDiskSize(storageStats.getCapacityBytes()); + } + + secondaryStorageList.add(secondaryStorage); + } + return secondaryStorageList; + } + + protected Template setTemplatePresetVariable(VMTemplateVO templateVO) { + Template template = new Template(); + + template.setName(templateVO.getName()); + template.setFormat(templateVO.getFormat()); + template.setHypervisorType(templateVO.getHypervisorType()); + + return template; + } + + protected Volume setVolumePresetVariable(VolumeVO volumeVO) { + Volume volume = new Volume(); + + volume.setName(volumeVO.getName()); + volume.setFormat(volumeVO.getFormat()); + volume.setSize(volumeVO.getSize()); + + return volume; + } + + protected Snapshot setSnapshotPresetVariable(SnapshotInfo snapshotInfo) { + Snapshot snapshot = new Snapshot(); + + snapshot.setName(snapshotInfo.getName()); + snapshot.setSize(snapshotInfo.getSize()); + snapshot.setHypervisorType(snapshotInfo.getHypervisorType()); + + return snapshot; + } + + protected Account setAccountPresetVariable(Long accountId) { + if (accountId == null) { + return null; + } + + AccountVO account = accountDao.findById(accountId); + if (account == null) { + return null; + } + + Account accountVariable = new Account(); + accountVariable.setName(account.getName()); + accountVariable.setId(account.getUuid()); + + accountVariable.setDomain(setDomainPresetVariable(account.getDomainId())); + + return accountVariable; + } + + protected Domain setDomainPresetVariable(long domainId) { + DomainVO domain = domainDao.findById(domainId); + if (domain == null) { + return null; + } + Domain domainVariable = new Domain(); + domainVariable.setName(domain.getName()); + domainVariable.setId(domain.getUuid()); + + return domainVariable; + } + + /** + * This method calls {@link HeuristicRuleHelper#buildPresetVariables(JsInterpreter, HeuristicType, long, Object)} for building the preset variables and + * execute the JS script specified in the rule ({@link String}) parameter. The script is pre-injected with the preset variables, to allow the JS script to reference them + * in the code scope. + *
+ *
+ * The JS script needs to return a valid UUID ({@link String}) of a secondary storage, otherwise a {@link CloudRuntimeException} is thrown. + * @param rule the {@link String} representing the JS script. + * @param heuristicType used for building the preset variables accordingly to the {@link HeuristicType} specified. + * @param obj can be from the following classes: {@link VMTemplateVO}, {@link SnapshotInfo} and {@link VolumeVO}. + * They are used to retrieve attributes for injecting in the JS rule. + * @param zoneId used for injecting the {@link SecondaryStorage} preset variables. + * @return the {@link DataStore} returned by the script. + */ + public DataStore interpretHeuristicRule(String rule, HeuristicType heuristicType, Object obj, long zoneId) { + try (JsInterpreter jsInterpreter = new JsInterpreter(HEURISTICS_SCRIPT_TIMEOUT)) { + buildPresetVariables(jsInterpreter, heuristicType, zoneId, obj); + Object scriptReturn = jsInterpreter.executeScript(rule); + + if (!(scriptReturn instanceof String)) { + throw new CloudRuntimeException(String.format("Error while interpreting heuristic rule [%s], the rule did not return a String.", rule)); + } + + DataStore dataStore = dataStoreManager.getImageStoreByUuid((String) scriptReturn); + + if (dataStore == null) { + throw new CloudRuntimeException(String.format("Unable to find a secondary storage with the UUID [%s] returned by the heuristic rule [%s]. Check if the rule is " + + "returning a valid UUID.", scriptReturn, rule)); + } + + return dataStore; + } catch (IOException ex) { + String message = String.format("Error while executing script [%s].", rule); + LOGGER.error(message, ex); + throw new CloudRuntimeException(message, ex); + } + } + +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Account.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Account.java new file mode 100644 index 00000000000..67750e8ec5c --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Account.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +public class Account extends GenericHeuristicPresetVariable { + private String id; + + private Domain domain; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + fieldNamesToIncludeInToString.add("id"); + } + + public Domain getDomain() { + return domain; + } + + public void setDomain(Domain domain) { + this.domain = domain; + fieldNamesToIncludeInToString.add("domain"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java new file mode 100644 index 00000000000..6565c06bfbb --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +public class Domain extends GenericHeuristicPresetVariable{ + private String id; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + fieldNamesToIncludeInToString.add("id"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java new file mode 100644 index 00000000000..f8ded3a864a --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import java.util.HashSet; +import java.util.Set; + +public class GenericHeuristicPresetVariable { + + protected transient Set fieldNamesToIncludeInToString = new HashSet<>(); + + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + fieldNamesToIncludeInToString.add("name"); + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, fieldNamesToIncludeInToString.toArray(new String[0])); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/PresetVariables.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/PresetVariables.java new file mode 100644 index 00000000000..d0487495327 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/PresetVariables.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import java.util.List; + +public class PresetVariables { + + private Account account; + + private List secondaryStorages; + + private Template template; + + private Snapshot snapshot; + + private Volume volume; + + public List getSecondaryStorages() { + return secondaryStorages; + } + + public void setSecondaryStorages(List secondaryStorages) { + this.secondaryStorages = secondaryStorages; + } + + public Template getTemplate() { + return template; + } + + public void setTemplate(Template template) { + this.template = template; + } + + public Snapshot getSnapshot() { + return snapshot; + } + + public void setSnapshot(Snapshot snapshot) { + this.snapshot = snapshot; + } + + public Volume getVolume() { + return volume; + } + + public void setVolume(Volume volume) { + this.volume = volume; + } + + public Account getAccount() { + return account; + } + + public void setAccount(Account account) { + this.account = account; + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorage.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorage.java new file mode 100644 index 00000000000..ad7058d8336 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorage.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +public class SecondaryStorage extends GenericHeuristicPresetVariable { + + private String id; + + private Long usedDiskSize; + + private Long totalDiskSize; + + private String protocol; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + fieldNamesToIncludeInToString.add("id"); + } + + public Long getUsedDiskSize() { + return usedDiskSize; + } + + public void setUsedDiskSize(Long usedDiskSize) { + this.usedDiskSize = usedDiskSize; + fieldNamesToIncludeInToString.add("usedDiskSize"); + } + + public Long getTotalDiskSize() { + return totalDiskSize; + } + + public void setTotalDiskSize(Long totalDiskSize) { + this.totalDiskSize = totalDiskSize; + fieldNamesToIncludeInToString.add("totalDiskSize"); + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + fieldNamesToIncludeInToString.add("protocol"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Snapshot.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Snapshot.java new file mode 100644 index 00000000000..34acd394dbe --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Snapshot.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import com.cloud.hypervisor.Hypervisor; + +public class Snapshot extends GenericHeuristicPresetVariable { + + private Long size; + + private Hypervisor.HypervisorType hypervisorType; + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + fieldNamesToIncludeInToString.add("size"); + } + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + public void setHypervisorType(Hypervisor.HypervisorType hypervisorType) { + this.hypervisorType = hypervisorType; + fieldNamesToIncludeInToString.add("hypervisorType"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Template.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Template.java new file mode 100644 index 00000000000..297c95fad9a --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Template.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Storage; + +public class Template extends GenericHeuristicPresetVariable { + + private Hypervisor.HypervisorType hypervisorType; + + private Storage.ImageFormat format; + + private Storage.TemplateType templateType; + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + public void setHypervisorType(Hypervisor.HypervisorType hypervisorType) { + this.hypervisorType = hypervisorType; + fieldNamesToIncludeInToString.add("hypervisorType"); + } + + public Storage.ImageFormat getFormat() { + return format; + } + + public void setFormat(Storage.ImageFormat format) { + this.format = format; + fieldNamesToIncludeInToString.add("format"); + } + + public Storage.TemplateType getTemplateType() { + return templateType; + } + + public void setTemplateType(Storage.TemplateType templateType) { + this.templateType = templateType; + fieldNamesToIncludeInToString.add("templateType"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Volume.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Volume.java new file mode 100644 index 00000000000..4e5e81b117f --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Volume.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import com.cloud.storage.Storage; + +public class Volume extends GenericHeuristicPresetVariable { + + private Long size; + + private Storage.ImageFormat format; + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + fieldNamesToIncludeInToString.add("size"); + } + + public Storage.ImageFormat getFormat() { + return format; + } + + public void setFormat(Storage.ImageFormat format) { + this.format = format; + fieldNamesToIncludeInToString.add("format"); + } +} diff --git a/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java b/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java index 9d4d34f4ca6..8657c07b5ef 100644 --- a/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java +++ b/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java @@ -18,21 +18,26 @@ package com.cloud.template; +import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; import com.cloud.event.UsageEventVO; import com.cloud.event.dao.UsageEventDao; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.org.Grouping; +import com.cloud.server.StatsCollector; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.TemplateProfile; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.test.TestAppender; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; import com.cloud.user.dao.AccountDao; import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; @@ -45,9 +50,12 @@ import org.apache.cloudstack.framework.events.Event; import org.apache.cloudstack.framework.events.EventBus; import org.apache.cloudstack.framework.events.EventBusException; import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.log4j.Level; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -58,6 +66,7 @@ import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import org.mockito.Spy; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; @@ -68,9 +77,12 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.regex.Pattern; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyLong; @@ -119,10 +131,17 @@ public class HypervisorTemplateAdapterTest { ConfigurationDao _configDao; @Mock - DataStoreManager storeMgr; + DataStoreManager dataStoreManagerMock; + @Mock + HeuristicRuleHelper heuristicRuleHelperMock; + + @Mock + StatsCollector statsCollectorMock; + + @Spy @InjectMocks - HypervisorTemplateAdapter _adapter; + HypervisorTemplateAdapter _adapter = new HypervisorTemplateAdapter(); //UsageEventUtils reflection abuse helpers private Map oldFields = new HashMap<>(); @@ -298,6 +317,282 @@ public class HypervisorTemplateAdapterTest { cleanupUsageUtils(); } + @Test + public void createTemplateWithinZonesTestZoneIdsNullShouldCallListAllZones() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(null); + Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class)); + + _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); + + Mockito.verify(_dcDao, Mockito.times(1)).listAllZones(); + } + + @Test + public void createTemplateWithinZonesTestZoneIdsNotNullShouldNotCallListAllZones() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + List zoneIds = List.of(1L); + + Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); + Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class)); + Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); + + _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); + + Mockito.verify(_dcDao, Mockito.times(0)).listAllZones(); + } + + @Test + public void createTemplateWithinZonesTestZoneDoesNotHaveActiveHeuristicRulesShouldCallStandardImageStoreAllocation() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + List zoneIds = List.of(1L); + + Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); + Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class)); + Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); + + _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); + + Mockito.verify(_adapter, Mockito.times(1)).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); + } + + @Test + public void createTemplateWithinZonesTestZoneWithHeuristicRuleShouldCallValidateSecondaryStorageAndCreateTemplate() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + DataStore dataStoreMock = Mockito.mock(DataStore.class); + List zoneIds = List.of(1L); + + Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); + Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class)); + Mockito.doReturn(dataStoreMock).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doNothing().when(_adapter).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class), Mockito.any(VMTemplateVO.class), Mockito.isNull()); + + _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); + + Mockito.verify(_adapter, Mockito.times(1)).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class), Mockito.any(VMTemplateVO.class), Mockito.isNull()); + } + + @Test(expected = CloudRuntimeException.class) + public void getImageStoresThrowsExceptionIfNotFoundTestNullImageStoreShouldThrowCloudRuntimeException() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + List zoneIds = List.of(1L); + + Mockito.when(dataStoreManagerMock.getImageStoresByZoneIds(Mockito.anyLong())).thenReturn(null); + + _adapter.getImageStoresThrowsExceptionIfNotFound(zoneIds, templateProfileMock); + } + + @Test(expected = CloudRuntimeException.class) + public void getImageStoresThrowsExceptionIfNotFoundTestEmptyImageStoreShouldThrowCloudRuntimeException() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + List zoneIds = List.of(1L); + List imageStoresList = new ArrayList<>(); + + Mockito.when(dataStoreManagerMock.getImageStoresByZoneIds(Mockito.anyLong())).thenReturn(imageStoresList); + + _adapter.getImageStoresThrowsExceptionIfNotFound(zoneIds, templateProfileMock); + } + + @Test + public void getImageStoresThrowsExceptionIfNotFoundTestNonEmptyImageStoreShouldNotThrowCloudRuntimeException() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + List zoneIds = List.of(1L); + DataStore dataStoreMock = Mockito.mock(DataStore.class); + List imageStoresList = List.of(dataStoreMock); + + Mockito.when(dataStoreManagerMock.getImageStoresByZoneIds(Mockito.anyLong())).thenReturn(imageStoresList); + + _adapter.getImageStoresThrowsExceptionIfNotFound(zoneIds, templateProfileMock); + } + + @Test + public void verifyHeuristicRulesForZoneTestTemplateIsISOFormatShouldCheckForISOHeuristicType() { + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.ISO); + _adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); + + Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.ISO, vmTemplateVOMock); + } + + @Test + public void verifyHeuristicRulesForZoneTestTemplateNotISOFormatShouldCheckForTemplateHeuristicType() { + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.QCOW2); + _adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); + + Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.TEMPLATE, vmTemplateVOMock); + } + + @Test + public void isZoneAndImageStoreAvailableTestZoneIdIsNullShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = null; + Set zoneSet = null; + boolean isTemplatePrivate = false; + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.WARN, Pattern.quote(String.format("Zone ID is null, cannot allocate ISO/template in image store [%s].", dataStoreMock))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestZoneIsNullShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = null; + boolean isTemplatePrivate = false; + DataCenterVO dataCenterVOMock = null; + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataStoreMock.getId()).thenReturn(2L); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.WARN, Pattern.quote(String.format("Unable to find zone by id [%s], so skip downloading template to its image store [%s].", + zoneId, dataStoreMock.getId()))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestZoneIsDisabledShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = null; + boolean isTemplatePrivate = false; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Disabled); + Mockito.when(dataStoreMock.getId()).thenReturn(2L); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Zone [%s] is disabled. Skip downloading template to its image store [%s].", zoneId, dataStoreMock.getId()))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestImageStoreDoesNotHaveEnoughCapacityShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = null; + boolean isTemplatePrivate = false; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(dataStoreMock.getId()).thenReturn(2L); + Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(false); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Image store doesn't have enough capacity. Skip downloading template to this image store [%s].", + dataStoreMock.getId()))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestImageStoreHasEnoughCapacityAndZoneSetIsNullShouldReturnTrue() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = null; + boolean isTemplatePrivate = false; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Zone set is null; therefore, the ISO/template should be allocated in every secondary storage " + + "of zone [%s].", dataCenterVOMock))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertTrue(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestTemplateIsPrivateAndItIsAlreadyAllocatedToTheSameZoneShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = Set.of(1L); + boolean isTemplatePrivate = true; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("The template is private and it is already allocated in a secondary storage in zone [%s]; " + + "therefore, image store [%s] will be skipped.", dataCenterVOMock, dataStoreMock))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestTemplateIsPrivateAndItIsNotAlreadyAllocatedToTheSameZoneShouldReturnTrue() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = new HashSet<>(); + boolean isTemplatePrivate = true; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Private template will be allocated in image store [%s] in zone [%s].", + dataStoreMock, dataCenterVOMock))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertTrue(result); + } + @Test public void testCheckZoneImageStoresDirectDownloadTemplate() { VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); @@ -309,7 +604,7 @@ public class HypervisorTemplateAdapterTest { public void testCheckZoneImageStoresRegularTemplateWithStore() { VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); Mockito.when(templateVO.isDirectDownload()).thenReturn(false); - Mockito.when(storeMgr.getImageStoresByScope(Mockito.any())).thenReturn(List.of(Mockito.mock(DataStore.class))); + Mockito.when(dataStoreManagerMock.getImageStoresByScope(Mockito.any())).thenReturn(List.of(Mockito.mock(DataStore.class))); _adapter.checkZoneImageStores(templateVO, List.of(1L)); } @@ -317,7 +612,7 @@ public class HypervisorTemplateAdapterTest { public void testCheckZoneImageStoresRegularTemplateNoStore() { VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); Mockito.when(templateVO.isDirectDownload()).thenReturn(false); - Mockito.when(storeMgr.getImageStoresByScope(Mockito.any())).thenReturn(new ArrayList<>()); + Mockito.when(dataStoreManagerMock.getImageStoresByScope(Mockito.any())).thenReturn(new ArrayList<>()); _adapter.checkZoneImageStores(templateVO, List.of(1L)); } } diff --git a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java index cb4d701d8a5..a69795cf2c8 100755 --- a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java +++ b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java @@ -19,80 +19,6 @@ package com.cloud.template; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import javax.inject.Inject; - -import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; -import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; -import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; -import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd; -import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; -import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd; -import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.messagebus.MessageBus; -import org.apache.cloudstack.snapshot.SnapshotHelper; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.cloudstack.storage.template.VnfTemplateManager; -import org.apache.cloudstack.test.utils.SpringUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; -import org.springframework.core.type.classreading.MetadataReader; -import org.springframework.core.type.classreading.MetadataReaderFactory; -import org.springframework.core.type.filter.TypeFilter; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.support.AnnotationConfigContextLoader; - import com.cloud.agent.AgentManager; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.configuration.Resource; @@ -106,6 +32,7 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.projects.ProjectManager; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.GuestOSVO; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; @@ -117,6 +44,7 @@ import com.cloud.storage.TemplateProfile; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.LaunchPermissionDao; import com.cloud.storage.dao.SnapshotDao; @@ -140,6 +68,82 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; +import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; +import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd; +import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; +import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd; +import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.snapshot.SnapshotHelper; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; +import org.apache.cloudstack.storage.template.VnfTemplateManager; +import org.apache.cloudstack.test.utils.SpringUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.core.type.classreading.MetadataReader; +import org.springframework.core.type.classreading.MetadataReaderFactory; +import org.springframework.core.type.filter.TypeFilter; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(loader = AnnotationConfigContextLoader.class) @@ -201,6 +205,9 @@ public class TemplateManagerImplTest { @Inject VnfTemplateManager vnfTemplateManager; + @Inject + HeuristicRuleHelper heuristicRuleHelperMock; + public class CustomThreadPoolExecutor extends ThreadPoolExecutor { AtomicInteger ai = new AtomicInteger(0); public CustomThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, @@ -596,6 +603,50 @@ public class TemplateManagerImplTest { Assert.assertEquals(template, resultTemplate); } + @Test + public void getImageStoreTestStoreUuidIsNotNullShouldReturnAValidImageStoreIfValidUuid() { + DataStore dataStore = Mockito.mock(DataStore.class); + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + + Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(dataStore); + + templateManager.getImageStore("UUID", 1L, volumeVO); + } + + @Test(expected = CloudRuntimeException.class) + public void getImageStoreTestStoreUuidIsNotNullShouldThrowCloudRuntimeExceptionIfInvalidUuid() { + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + + Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(null); + + templateManager.getImageStore("UUID", 1L, volumeVO); + } + + @Test + public void getImageStoreTestStoreUuidIsNullAndThereIsNoActiveHeuristicRulesShouldCallGetImageStoreWithFreeCapacity() { + DataStore dataStore = Mockito.mock(DataStore.class); + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + + Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(null); + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(VolumeVO.class))).thenReturn(null); + Mockito.when(dataStoreManager.getImageStoreWithFreeCapacity(Mockito.anyLong())).thenReturn(dataStore); + + templateManager.getImageStore(null, 1L, volumeVO); + Mockito.verify(dataStoreManager, Mockito.times(1)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } + + @Test + public void getImageStoreTestStoreUuidIsNullAndThereIsActiveHeuristicRulesShouldNotCallGetImageStoreWithFreeCapacity() { + DataStore dataStore = Mockito.mock(DataStore.class); + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + + Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(null); + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(VolumeVO.class))).thenReturn(dataStore); + + templateManager.getImageStore(null, 1L, volumeVO); + Mockito.verify(dataStoreManager, Mockito.times(0)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } + @Test public void testRegisterTemplateWithTemplateType() { RegisterTemplateCmd cmd = Mockito.mock(RegisterTemplateCmd.class); @@ -915,6 +966,16 @@ public class TemplateManagerImplTest { return Mockito.mock(SnapshotService.class); } + @Bean + public SecondaryStorageHeuristicDao secondaryStorageHeuristicDao() { + return Mockito.mock(SecondaryStorageHeuristicDao.class); + } + + @Bean + public HeuristicRuleHelper heuristicRuleHelper() { + return Mockito.mock(HeuristicRuleHelper.class); + } + public static class Library implements TypeFilter { @Override public boolean match(MetadataReader mdr, MetadataReaderFactory arg1) throws IOException { diff --git a/server/src/test/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelperTest.java b/server/src/test/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelperTest.java new file mode 100644 index 00000000000..6bf7eef2844 --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelperTest.java @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics; + +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.test.TestAppender; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.heuristics.presetvariables.PresetVariables; +import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; +import org.apache.log4j.Level; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.regex.Pattern; + +@RunWith(MockitoJUnitRunner.class) +public class HeuristicRuleHelperTest { + + @Mock + SecondaryStorageHeuristicDao secondaryStorageHeuristicDaoMock; + + @Mock + HeuristicVO heuristicVOMock; + + @Mock + VMTemplateVO vmTemplateVOMock; + + @Mock + SnapshotInfo snapshotInfoMock; + + @Mock + VolumeVO volumeVOMock; + + @Mock + DataStoreManager dataStoreManagerMock; + + @Mock + DataStore dataStoreMock; + + @Spy + @InjectMocks + HeuristicRuleHelper heuristicRuleHelperSpy = new HeuristicRuleHelper(); + + @Test + public void getImageStoreIfThereIsHeuristicRuleTestZoneDoesNotHaveHeuristicRuleShouldReturnNull() { + Long zoneId = 1L; + + Mockito.when(secondaryStorageHeuristicDaoMock.findByZoneIdAndType(Mockito.anyLong(), Mockito.any(HeuristicType.class))).thenReturn(null); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.DEBUG, Pattern.quote(String.format("No heuristic rules found for zone with ID [%s] and heuristic type [%s]. Returning null.", + zoneId, HeuristicType.TEMPLATE))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HeuristicRuleHelper.LOGGER, testLogAppender); + + DataStore result = heuristicRuleHelperSpy.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.TEMPLATE, null); + + testLogAppender.assertMessagesLogged(); + Assert.assertNull(result); + } + + @Test + public void getImageStoreIfThereIsHeuristicRuleTestZoneHasHeuristicRuleShouldCallInterpretHeuristicRule() { + Long zoneId = 1L; + + Mockito.when(secondaryStorageHeuristicDaoMock.findByZoneIdAndType(Mockito.anyLong(), Mockito.any(HeuristicType.class))).thenReturn(heuristicVOMock); + Mockito.when(heuristicVOMock.getHeuristicRule()).thenReturn("rule"); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).interpretHeuristicRule(Mockito.anyString(), Mockito.any(HeuristicType.class), Mockito.isNull(), + Mockito.anyLong()); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.DEBUG, Pattern.quote(String.format("Found the heuristic rule %s to apply for zone with ID [%s].", heuristicVOMock, zoneId))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HeuristicRuleHelper.LOGGER, testLogAppender); + + DataStore result = heuristicRuleHelperSpy.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.TEMPLATE, null); + + testLogAppender.assertMessagesLogged(); + Assert.assertNull(result); + } + + @Test + public void buildPresetVariablesTestWithTemplateHeuristicTypeShouldSetTemplateAndSecondaryStorageAndAccountPresetVariables() { + Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong()); + + heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.TEMPLATE, 1L, vmTemplateVOMock); + + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setTemplatePresetVariable(Mockito.any(VMTemplateVO.class)); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + } + + @Test + public void buildPresetVariablesTestWithIsoHeuristicTypeShouldSetTemplateAndSecondaryStorageAndAccountPresetVariables() { + Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong()); + + heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.ISO, 1L, vmTemplateVOMock); + + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setTemplatePresetVariable(Mockito.any(VMTemplateVO.class)); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + } + + @Test + public void buildPresetVariablesTestWithSnapshotHeuristicTypeShouldSetSnapshotAndSecondaryStorageAndAccountPresetVariables() { + Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong()); + + heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.SNAPSHOT, 1L, snapshotInfoMock); + + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSnapshotPresetVariable(Mockito.any(SnapshotInfo.class)); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + } + + @Test + public void buildPresetVariablesTestWithSnapshotHeuristicTypeShouldSetVolumeAndSecondaryStorageAndAccountPresetVariables() { + Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong()); + + heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.VOLUME, 1L, volumeVOMock); + + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setVolumePresetVariable(Mockito.any(VolumeVO.class)); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + } + + @Test + public void interpretHeuristicRuleTestHeuristicRuleDoesNotReturnAStringShouldThrowCloudRuntimeException() { + String heuristicRule = "1"; + + Mockito.doNothing().when(heuristicRuleHelperSpy).buildPresetVariables(Mockito.any(JsInterpreter.class), Mockito.any(HeuristicType.class), Mockito.anyLong(), + Mockito.any()); + + String expectedMessage = String.format("Error while interpreting heuristic rule [%s], the rule did not return a String.", heuristicRule); + CloudRuntimeException assertThrows = Assert.assertThrows(CloudRuntimeException.class, + () -> heuristicRuleHelperSpy.interpretHeuristicRule(heuristicRule, HeuristicType.TEMPLATE, volumeVOMock, 1L)); + Assert.assertEquals(expectedMessage, assertThrows.getMessage()); + } + + @Test + public void interpretHeuristicRuleTestHeuristicRuleReturnAStringWithInvalidUuidShouldThrowCloudRuntimeException() { + String heuristicRule = "'uuid'"; + + Mockito.doNothing().when(heuristicRuleHelperSpy).buildPresetVariables(Mockito.any(JsInterpreter.class), Mockito.any(HeuristicType.class), Mockito.anyLong(), + Mockito.any()); + Mockito.doReturn(null).when(dataStoreManagerMock).getImageStoreByUuid(Mockito.anyString()); + + String expectedMessage = String.format("Unable to find a secondary storage with the UUID [%s] returned by the heuristic rule [%s]. Check if the rule is " + + "returning a valid UUID.", "uuid", heuristicRule); + CloudRuntimeException assertThrows = Assert.assertThrows(CloudRuntimeException.class, + () -> heuristicRuleHelperSpy.interpretHeuristicRule(heuristicRule, HeuristicType.TEMPLATE, volumeVOMock, 1L)); + Assert.assertEquals(expectedMessage, assertThrows.getMessage()); + } + + @Test + public void interpretHeuristicRuleTestHeuristicRuleReturnAStringWithAValidUuidShouldReturnAImageStore() { + String heuristicRule = "'uuid'"; + + Mockito.doNothing().when(heuristicRuleHelperSpy).buildPresetVariables(Mockito.any(JsInterpreter.class), Mockito.any(HeuristicType.class), Mockito.anyLong(), + Mockito.any()); + Mockito.doReturn(dataStoreMock).when(dataStoreManagerMock).getImageStoreByUuid(Mockito.anyString()); + + DataStore result = heuristicRuleHelperSpy.interpretHeuristicRule(heuristicRule, HeuristicType.TEMPLATE, volumeVOMock, 1L); + + Assert.assertNotNull(result); + } +}