apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add dynamic secondary storage selection (#7659)

Browse Source
This commit is contained in:
Bryan Lima 2023-12-04 05:52:32 -03:00 committed by GitHub
parent 108651ad40
commit b0910fc61d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 2456 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
api/src/main/java/com/cloud/storage/StorageService.java
View File

@ -36,6 +36,10 @@ import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.ResourceInUseException;
import com.cloud.exception.ResourceUnavailableException;
import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd;
import org.apache.cloudstack.secstorage.heuristics.Heuristic;
import org.apache.cloudstack.storage.object.ObjectStore;
public interface StorageService {
@ -112,6 +116,12 @@ public interface StorageService {
StoragePool syncStoragePool(SyncStoragePoolCmd cmd);
Heuristic createSecondaryStorageHeuristic(CreateSecondaryStorageSelectorCmd cmd);
Heuristic updateSecondaryStorageHeuristic(UpdateSecondaryStorageSelectorCmd cmd);
void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd);
ObjectStore discoverObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException;
boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd);

3
api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
View File

@ -1061,6 +1061,9 @@ public class ApiConstants {
public static final String HAS_RULES = "hasrules";
public static final String OBJECT_STORAGE = "objectstore";
public static final String HEURISTIC_RULE = "heuristicrule";
public static final String HEURISTIC_TYPE_VALID_OPTIONS = "Valid options are: ISO, SNAPSHOT, TEMPLATE and VOLUME.";
public static final String MANAGEMENT = "management";
public static final String IS_VNF = "isvnf";
public static final String VNF_NICS = "vnfnics";

4
api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java
View File

@ -104,6 +104,7 @@ import org.apache.cloudstack.api.response.ResourceTagResponse;
import org.apache.cloudstack.api.response.RollingMaintenanceResponse;
import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse;
import org.apache.cloudstack.api.response.SSHKeyPairResponse;
import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse;
import org.apache.cloudstack.api.response.SecurityGroupResponse;
import org.apache.cloudstack.api.response.ServiceOfferingResponse;
import org.apache.cloudstack.api.response.ServiceResponse;
@ -148,6 +149,7 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule;
import org.apache.cloudstack.region.PortableIp;
import org.apache.cloudstack.region.PortableIpRange;
import org.apache.cloudstack.region.Region;
import org.apache.cloudstack.secstorage.heuristics.Heuristic;
import org.apache.cloudstack.storage.object.ObjectStore;
import org.apache.cloudstack.usage.Usage;
@ -536,6 +538,8 @@ public interface ResponseGenerator {
FirewallResponse createIpv6FirewallRuleResponse(FirewallRule acl);
SecondaryStorageHeuristicsResponse createSecondaryStorageSelectorResponse(Heuristic heuristic);
IpQuarantineResponse createQuarantinedIpsResponse(PublicIpQuarantine publicIp);
ObjectStoreResponse createObjectStoreResponse(ObjectStore os);

93
api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java Normal file
View File

@ -0,0 +1,93 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.storage.heuristics;
import com.cloud.user.Account;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse;
import org.apache.cloudstack.api.response.ZoneResponse;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.secstorage.heuristics.Heuristic;
import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS;
@APICommand(name = "createSecondaryStorageSelector", description = "Creates a secondary storage selector, described by the heuristic rule.", since = "4.19.0", responseObject =
SecondaryStorageHeuristicsResponse.class, entityType = {Heuristic.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin})
public class CreateSecondaryStorageSelectorCmd extends BaseCmd{
@Parameter(name = ApiConstants.NAME, required = true, type = CommandType.STRING, description = "The name identifying the heuristic rule.")
private String name;
@Parameter(name = ApiConstants.DESCRIPTION, required = true, type = BaseCmd.CommandType.STRING, description = "The description of the heuristic rule.")
private String description;
@Parameter(name = ApiConstants.ZONE_ID, required = true, entityType = ZoneResponse.class, type = BaseCmd.CommandType.UUID, description = "The zone in which the heuristic " +
"rule will be applied.")
private Long zoneId;
@Parameter(name = ApiConstants.TYPE, required = true, type = BaseCmd.CommandType.STRING, description =
"The resource type directed to a specific secondary storage by the selector. " + HEURISTIC_TYPE_VALID_OPTIONS)
private String type;
@Parameter(name = ApiConstants.HEURISTIC_RULE, required = true, type = BaseCmd.CommandType.STRING, description = "The heuristic rule, in JavaScript language. It is required " +
"that it returns the UUID of a secondary storage pool. An example of a rule is `if (snapshot.hypervisorType === 'KVM') { '7832f261-c602-4e8e-8580-2496ffbbc45d'; " +
"}` would allocate all snapshots with the KVM hypervisor to the specified secondary storage UUID.", length = 65535)
private String heuristicRule;
public String getName() {
return name;
}
public String getDescription() {
return description;
}
public Long getZoneId() {
return zoneId;
}
public String getType() {
return type;
}
public String getHeuristicRule() {
return heuristicRule;
}
@Override
public void execute() {
Heuristic heuristic = _storageService.createSecondaryStorageHeuristic(this);
if (heuristic == null) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create a secondary storage selector.");
}
SecondaryStorageHeuristicsResponse response = _responseGenerator.createSecondaryStorageSelectorResponse(heuristic);
response.setResponseName(getCommandName());
this.setResponseObject(response);
}
@Override
public long getEntityOwnerId() {
return Account.ACCOUNT_ID_SYSTEM;
}
}

63
api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java Normal file
View File

@ -0,0 +1,63 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.storage.heuristics;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseListCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.response.ListResponse;
import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse;
import org.apache.cloudstack.api.response.ZoneResponse;
import org.apache.cloudstack.secstorage.heuristics.Heuristic;
import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS;
@APICommand(name = "listSecondaryStorageSelectors", description = "Lists the secondary storage selectors and their rules.", since = "4.19.0", responseObject =
SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin})
public class ListSecondaryStorageSelectorsCmd extends BaseListCmd {
@Parameter(name = ApiConstants.ZONE_ID, required = true, entityType = ZoneResponse.class, type = CommandType.UUID, description = "The zone ID to be used in the search filter.")
private Long zoneId;
@Parameter(name = ApiConstants.TYPE, type = CommandType.STRING, description =
"Whether to filter the selectors by type and, if so, which one. " + HEURISTIC_TYPE_VALID_OPTIONS)
private String type;
@Parameter(name = ApiConstants.SHOW_REMOVED, type = CommandType.BOOLEAN, description = "Show removed heuristics.")
private boolean showRemoved = false;
public Long getZoneId() {
return zoneId;
}
public String getType() {
return type;
}
public boolean isShowRemoved() {
return showRemoved;
}
@Override
public void execute() {
ListResponse<SecondaryStorageHeuristicsResponse> response = _queryService.listSecondaryStorageSelectors(this);
response.setResponseName(getCommandName());
this.setResponseObject(response);
}
}

54
api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java Normal file
View File

@ -0,0 +1,54 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.storage.heuristics;
import com.cloud.user.Account;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse;
import org.apache.cloudstack.api.response.SuccessResponse;
import org.apache.cloudstack.secstorage.heuristics.Heuristic;
@APICommand(name = "removeSecondaryStorageSelector", description = "Removes an existing secondary storage selector.", since = "4.19.0", responseObject =
SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin})
public class RemoveSecondaryStorageSelectorCmd extends BaseCmd {
@Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, entityType = SecondaryStorageHeuristicsResponse.class, required = true,
description = "The unique identifier of the secondary storage selector to be removed.")
private Long id;
public Long getId() {
return id;
}
@Override
public void execute() {
_storageService.removeSecondaryStorageHeuristic(this);
final SuccessResponse response = new SuccessResponse();
response.setResponseName(getCommandName());
response.setSuccess(true);
setResponseObject(response);
}
@Override
public long getEntityOwnerId() {
return Account.ACCOUNT_ID_SYSTEM;
}
}

67
api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java Normal file
View File

@ -0,0 +1,67 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.storage.heuristics;
import com.cloud.user.Account;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse;
import org.apache.cloudstack.secstorage.heuristics.Heuristic;
@APICommand(name = "updateSecondaryStorageSelector", description = "Updates an existing secondary storage selector.", since = "4.19.0", responseObject =
SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin})
public class UpdateSecondaryStorageSelectorCmd extends BaseCmd {
@Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, entityType = SecondaryStorageHeuristicsResponse.class, required = true,
description = "The unique identifier of the secondary storage selector.")
private Long id;
@Parameter(name = ApiConstants.HEURISTIC_RULE, required = true, type = BaseCmd.CommandType.STRING, description = "The heuristic rule, in JavaScript language. It is required " +
"that it returns the UUID of a secondary storage pool. An example of a rule is `if (snapshot.hypervisorType === 'KVM') { '7832f261-c602-4e8e-8580-2496ffbbc45d'; " +
"}` would allocate all snapshots with the KVM hypervisor to the specified secondary storage UUID.", length = 65535)
private String heuristicRule;
public Long getId() {
return id;
}
public String getHeuristicRule() {
return heuristicRule;
}
@Override
public void execute() {
Heuristic heuristic = _storageService.updateSecondaryStorageHeuristic(this);
if (heuristic == null) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update the secondary storage selector.");
}
SecondaryStorageHeuristicsResponse response = _responseGenerator.createSecondaryStorageSelectorResponse(heuristic);
response.setResponseName(getCommandName());
this.setResponseObject(response);
}
@Override
public long getEntityOwnerId() {
return Account.ACCOUNT_ID_SYSTEM;
}
}

141
api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java Normal file
View File

@ -0,0 +1,141 @@
//Licensed to the Apache Software Foundation (ASF) under one
//or more contributor license agreements. See the NOTICE file
//distributed with this work for additional information
//regarding copyright ownership. The ASF licenses this file
//to you under the Apache License, Version 2.0 (the
//"License"); you may not use this file except in compliance
//with the License. You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing,
//software distributed under the License is distributed on an
//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
//KIND, either express or implied. See the License for the
//specific language governing permissions and limitations
//under the License.
package org.apache.cloudstack.api.response;
import com.cloud.serializer.Param;
import com.google.gson.annotations.SerializedName;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseResponse;
import org.apache.cloudstack.api.EntityReference;
import org.apache.cloudstack.secstorage.heuristics.Heuristic;
import java.util.Date;
import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS;
@EntityReference(value = {Heuristic.class})
public class SecondaryStorageHeuristicsResponse extends BaseResponse {
@SerializedName(ApiConstants.ID)
@Param(description = "ID of the heuristic.")
private String id;
@SerializedName(ApiConstants.NAME)
@Param(description = "Name of the heuristic.")
private String name;
@SerializedName(ApiConstants.DESCRIPTION)
@Param(description = "Description of the heuristic.")
private String description;
@SerializedName(ApiConstants.ZONE_ID)
@Param(description = "The zone which the heuristic is valid upon.")
private String zoneId;
@SerializedName(ApiConstants.TYPE)
@Param(description = "The resource type directed to a specific secondary storage by the selector. " + HEURISTIC_TYPE_VALID_OPTIONS)
private String type;
@SerializedName(ApiConstants.HEURISTIC_RULE)
@Param(description = "The heuristic rule, in JavaScript language, used to select a secondary storage to be directed.")
private String heuristicRule;
@SerializedName(ApiConstants.CREATED)
@Param(description = "When the heuristic was created.")
private Date created;
@SerializedName(ApiConstants.REMOVED)
@Param(description = "When the heuristic was removed.")
private Date removed;
public SecondaryStorageHeuristicsResponse(String id, String name, String description, String zoneId, String type, String heuristicRule, Date created, Date removed) {
super("heuristics");
this.id = id;
this.name = name;
this.description = description;
this.zoneId = zoneId;
this.type = type;
this.heuristicRule = heuristicRule;
this.created = created;
this.removed = removed;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public String getZoneId() {
return zoneId;
}
public void setZoneId(String zoneId) {
this.zoneId = zoneId;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getHeuristicRule() {
return heuristicRule;
}
public void setHeuristicRule(String heuristicRule) {
this.heuristicRule = heuristicRule;
}
public Date getCreated() {
return created;
}
public void setCreated(Date created) {
this.created = created;
}
public Date getRemoved() {
return removed;
}
public void setRemoved(Date removed) {
this.removed = removed;
}
}

4
api/src/main/java/org/apache/cloudstack/query/QueryService.java
View File

@ -32,6 +32,7 @@ import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd
import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd;
import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd;
import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd;
import org.apache.cloudstack.api.command.admin.user.ListUsersCmd;
import org.apache.cloudstack.api.command.user.account.ListAccountsCmd;
import org.apache.cloudstack.api.command.user.account.ListProjectAccountsCmd;
@ -79,6 +80,7 @@ import org.apache.cloudstack.api.response.ResourceDetailResponse;
import org.apache.cloudstack.api.response.ResourceIconResponse;
import org.apache.cloudstack.api.response.ResourceTagResponse;
import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse;
import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse;
import org.apache.cloudstack.api.response.SecurityGroupResponse;
import org.apache.cloudstack.api.response.ServiceOfferingResponse;
import org.apache.cloudstack.api.response.SnapshotResponse;
@ -189,6 +191,8 @@ public interface QueryService {
List<RouterHealthCheckResultResponse> listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd);
ListResponse<SecondaryStorageHeuristicsResponse> listSecondaryStorageSelectors(ListSecondaryStorageSelectorsCmd cmd);
ListResponse<IpQuarantineResponse> listQuarantinedIps(ListQuarantinedIpsCmd cmd);
ListResponse<SnapshotResponse> listSnapshots(ListSnapshotsCmd cmd);

40
api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java Normal file
View File

@ -0,0 +1,40 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.secstorage.heuristics;
import org.apache.cloudstack.api.Identity;
import org.apache.cloudstack.api.InternalIdentity;
import java.util.Date;
public interface Heuristic extends InternalIdentity, Identity {
String getName();
String getDescription();
Long getZoneId();
String getType();
String getHeuristicRule();
Date getCreated();
Date getRemoved();
}

25
api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java Normal file
View File

@ -0,0 +1,25 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.secstorage.heuristics;
/**
* The type of the heuristic used in the allocation process of secondary storage resources.
* Valid options are: {@link #ISO}, {@link #SNAPSHOT}, {@link #TEMPLATE} and {@link #VOLUME}
*/
public enum HeuristicType {
ISO, SNAPSHOT, TEMPLATE, VOLUME
}

4
engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java
View File

@ -35,6 +35,8 @@ public interface DataStoreManager {
List<DataStore> getImageStoresByScopeExcludingReadOnly(ZoneScope scope);
List<DataStore> getImageStoresByZoneIds(Long ... zoneIds);
DataStore getRandomImageStore(long zoneId);
DataStore getRandomUsableImageStore(long zoneId);
@ -55,5 +57,7 @@ public interface DataStoreManager {
boolean isRegionStore(DataStore store);
DataStore getImageStoreByUuid(String uuid);
Long getStoreZoneId(long storeId, DataStoreRole role);
}

3
engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
View File

@ -192,6 +192,9 @@ public interface StorageManager extends StorageService {
ConfigKey.Scope.Global,
null);
ConfigKey<Long> HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000",
"The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true);
/**
* should we execute in sequence not involving any storages?
* @return tru if commands should execute in sequence

3
engine/components-api/src/main/java/com/cloud/template/TemplateManager.java
View File

@ -34,6 +34,7 @@ import com.cloud.storage.Storage.TemplateType;
import com.cloud.storage.StoragePool;
import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VolumeVO;
import com.cloud.utils.Pair;
import com.cloud.vm.VirtualMachineProfile;
@ -116,7 +117,7 @@ public interface TemplateManager {
Long getTemplateSize(long templateId, long zoneId);
DataStore getImageStore(String storeUuid, Long zoneId);
DataStore getImageStore(String storeUuid, Long zoneId, VolumeVO volume);
String getChecksum(DataStore store, String templatePath, String algorithm);

125
engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java Normal file
View File

@ -0,0 +1,125 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.secstorage;
import com.cloud.utils.db.GenericDao;
import org.apache.cloudstack.secstorage.heuristics.Heuristic;
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import java.util.Date;
import java.util.UUID;
@Entity
@Table(name = "heuristics")
public class HeuristicVO implements Heuristic {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@Column(name = "id", nullable = false)
private Long id;
@Column(name = "uuid", nullable = false)
private String uuid = UUID.randomUUID().toString();
@Column(name = "name")
private String name;
@Column(name = "description")
private String description;
@Column(name = "zone_id", nullable = false)
private Long zoneId;
@Column(name = "type", nullable = false)
private String type;
@Column(name = "heuristic_rule", nullable = false, length = 65535)
private String heuristicRule;
@Column(name = GenericDao.CREATED_COLUMN, nullable = false)
@Temporal(value = TemporalType.TIMESTAMP)
private Date created;
@Column(name = GenericDao.REMOVED_COLUMN)
@Temporal(value = TemporalType.TIMESTAMP)
private Date removed = null;
public HeuristicVO() {
}
public HeuristicVO(String name, String description, Long zoneId, String type, String heuristicRule) {
this.name = name;
this.description = description;
this.zoneId = zoneId;
this.type = type;
this.heuristicRule = heuristicRule;
}
@Override
public long getId() {
return id;
}
public String getUuid() {
return uuid;
}
@Override
public String getName() {
return name;
}
public String getDescription() {
return description;
}
public Long getZoneId() {
return zoneId;
}
public String getType() {
return type;
}
public String getHeuristicRule() {
return heuristicRule;
}
public Date getCreated() {
return created;
}
public Date getRemoved() {
return removed;
}
public void setHeuristicRule(String heuristicRule) {
this.heuristicRule = heuristicRule;
}
@Override
public String toString() {
return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "name", "heuristicRule", "type");
}
}

26
engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java Normal file
View File

@ -0,0 +1,26 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.secstorage.dao;
import com.cloud.utils.db.GenericDao;
import org.apache.cloudstack.secstorage.HeuristicVO;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
public interface SecondaryStorageHeuristicDao extends GenericDao<HeuristicVO, Long> {
HeuristicVO findByZoneIdAndType(long zoneId, HeuristicType type);
}

50
engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java Normal file
View File

@ -0,0 +1,50 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.secstorage.dao;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import org.apache.cloudstack.secstorage.HeuristicVO;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
@Component
public class SecondaryStorageHeuristicDaoImpl extends GenericDaoBase<HeuristicVO, Long> implements SecondaryStorageHeuristicDao {
private SearchBuilder<HeuristicVO> zoneAndTypeSearch;
@PostConstruct
public void init() {
zoneAndTypeSearch = createSearchBuilder();
zoneAndTypeSearch.and("zoneId", zoneAndTypeSearch.entity().getZoneId(), SearchCriteria.Op.EQ);
zoneAndTypeSearch.and("type", zoneAndTypeSearch.entity().getType(), SearchCriteria.Op.IN);
zoneAndTypeSearch.done();
}
@Override
public HeuristicVO findByZoneIdAndType(long zoneId, HeuristicType type) {
SearchCriteria<HeuristicVO> searchCriteria = zoneAndTypeSearch.create();
searchCriteria.setParameters("zoneId", zoneId);
searchCriteria.setParameters("type", type.toString());
final Filter filter = new Filter(HeuristicVO.class, "created", false);
return findOneBy(searchCriteria, filter);
}
}

2
engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java
View File

@ -49,4 +49,6 @@ public interface ImageStoreDao extends GenericDao<ImageStoreVO, Long> {
List<ImageStoreVO> findByProtocol(String protocol);
ImageStoreVO findOneByZoneAndProtocol(long zoneId, String protocol);
List<ImageStoreVO> listImageStoresByZoneIds(Long... zoneIds);
}

15
engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java
View File

@ -43,6 +43,8 @@ public class ImageStoreDaoImpl extends GenericDaoBase<ImageStoreVO, Long> implem
private SearchBuilder<ImageStoreVO> protocolSearch;
private SearchBuilder<ImageStoreVO> zoneProtocolSearch;
private SearchBuilder<ImageStoreVO> zonesInSearch;
public ImageStoreDaoImpl() {
super();
protocolSearch = createSearchBuilder();
@ -55,6 +57,11 @@ public class ImageStoreDaoImpl extends GenericDaoBase<ImageStoreVO, Long> implem
zoneProtocolSearch.and("protocol", zoneProtocolSearch.entity().getProtocol(), SearchCriteria.Op.EQ);
zoneProtocolSearch.and("role", zoneProtocolSearch.entity().getRole(), SearchCriteria.Op.EQ);
zoneProtocolSearch.done();
zonesInSearch = createSearchBuilder();
zonesInSearch.and("zonesIn", zonesInSearch.entity().getDcId(), SearchCriteria.Op.IN);
zonesInSearch.done();
}
@Override
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
@ -191,4 +198,12 @@ public class ImageStoreDaoImpl extends GenericDaoBase<ImageStoreVO, Long> implem
List<ImageStoreVO> results = listBy(sc, filter);
return results.size() == 0 ? null : results.get(0);
}
@Override
public List<ImageStoreVO> listImageStoresByZoneIds(Long... zoneIds) {
SearchCriteria<ImageStoreVO> sc = zonesInSearch.create();
sc.setParametersIfNotNull("zonesIn", zoneIds);
return listBy(sc);
}
}

2
engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml
View File

@ -276,6 +276,8 @@
<bean id="UserVmDeployAsIsDetailsDaoImpl" class="com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDaoImpl" />
<bean id="NetworkPermissionDaoImpl" class="org.apache.cloudstack.network.dao.NetworkPermissionDaoImpl" />
<bean id="PassphraseDaoImpl" class="org.apache.cloudstack.secret.dao.PassphraseDaoImpl" />
<bean id="secondaryStorageHeuristicDaoImpl" class="org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDaoImpl" />
<bean id="heuristicRuleHelper" class="org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper" />
<bean id="publicIpQuarantineDaoImpl" class="com.cloud.network.dao.PublicIpQuarantineDaoImpl" />
<bean id="VMScheduleDaoImpl" class="org.apache.cloudstack.vm.schedule.dao.VMScheduleDaoImpl" />
<bean id="VMScheduledJobDaoImpl" class="org.apache.cloudstack.vm.schedule.dao.VMScheduledJobDaoImpl" />

15
engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql
View File

@ -44,6 +44,21 @@ CREATE TABLE IF NOT EXISTS `cloud`.`quarantined_ips` (
-- create_public_parameter_on_roles. #6960
ALTER TABLE `cloud`.`roles` ADD COLUMN `public_role` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'Indicates whether the role will be visible to all users (public) or only to root admins (private). If this parameter is not specified during the creation of the role its value will be defaulted to true (public).';
-- Create heuristic table for dynamic allocating resources to the secondary storage
CREATE TABLE IF NOT EXISTS `cloud`.`heuristics` (
`id` bigint(20) unsigned NOT NULL auto_increment,
`uuid` varchar(255) UNIQUE NOT NULL,
`name` text NOT NULL,
`description` text DEFAULT NULL,
`zone_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the zone to apply the heuristic, foreign key to `data_center` table',
`type` varchar(255) NOT NULL,
`heuristic_rule` text NOT NULL COMMENT 'JS script that defines to which secondary storage the resource will be allocated.',
`created` datetime NOT NULL,
`removed` datetime DEFAULT NULL,
PRIMARY KEY (`id`),
CONSTRAINT `fk_heuristics__zone_id` FOREIGN KEY(`zone_id`) REFERENCES `cloud`.`data_center`(`id`)
);
-- Add tables for VM Scheduler
DROP TABLE IF EXISTS `cloud`.`vm_schedule`;
CREATE TABLE `cloud`.`vm_schedule` (

12
engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java
View File

@ -94,7 +94,7 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager,
@Override
public ImageStoreEntity getImageStore(String uuid) {
ImageStoreVO dataStore = dataStoreDao.findByUuid(uuid);
return getImageStore(dataStore.getId());
return dataStore == null ? null : getImageStore(dataStore.getId());
}
@Override
@ -248,6 +248,16 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager,
return stores;
}
@Override
public List<DataStore> listImageStoresFilteringByZoneIds(Long... zoneIds) {
List<ImageStoreVO> stores = dataStoreDao.listImageStoresByZoneIds(zoneIds);
List<DataStore> imageStores = new ArrayList<>();
for (ImageStoreVO store : stores) {
imageStores.add(getImageStore(store.getId()));
}
return imageStores;
}
@Override
public String getConfigComponentName() {
return ImageStoreProviderManager.class.getSimpleName();

22
engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java
View File

@ -47,12 +47,14 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
import org.apache.cloudstack.framework.async.AsyncRpcContext;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.jobs.AsyncJob;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
import org.apache.cloudstack.storage.command.CommandResult;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer;
import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.log4j.Logger;
@ -99,6 +101,9 @@ public class SnapshotServiceImpl implements SnapshotService {
@Inject
ConfigurationDao _configDao;
@Inject
private HeuristicRuleHelper heuristicRuleHelper;
static private class CreateSnapshotContext<T> extends AsyncRpcContext<T> {
final SnapshotInfo snapshot;
final AsyncCallFuture<SnapshotResult> future;
@ -297,7 +302,7 @@ public class SnapshotServiceImpl implements SnapshotService {
fullSnapshot = snapshotFullBackup;
}
if (fullSnapshot) {
return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId());
return getImageStoreForSnapshot(snapshot.getDataCenterId(), snapshot);
} else {
SnapshotInfo parentSnapshot = snapshot.getParent();
// Note that DataStore information in parentSnapshot is for primary
@ -314,12 +319,25 @@ public class SnapshotServiceImpl implements SnapshotService {
}
}
if (parentSnapshotOnBackupStore == null) {
return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId());
return getImageStoreForSnapshot(snapshot.getDataCenterId(), snapshot);
}
return dataStoreMgr.getDataStore(parentSnapshotOnBackupStore.getDataStoreId(), parentSnapshotOnBackupStore.getRole());
}
}
/**
* Verify if the data center has heuristic rules for allocating snapshots; if there is then returns the {@link DataStore} returned by the JS script.
* Otherwise, returns {@link DataStore}s with free capacity.
*/
protected DataStore getImageStoreForSnapshot(Long dataCenterId, SnapshotInfo snapshot) {
DataStore imageStore = heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(dataCenterId, HeuristicType.SNAPSHOT, snapshot);
if (imageStore == null) {
imageStore = dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId());
}
return imageStore;
}
@Override
public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) {
SnapshotObject snapObj = (SnapshotObject)snapshot;

57
engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java
View File

@ -18,6 +18,9 @@
*/
package org.apache.cloudstack.storage.snapshot;
import com.cloud.storage.DataStoreRole;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
@ -26,8 +29,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher;
import org.apache.cloudstack.storage.command.CommandResult;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -40,8 +43,6 @@ import org.mockito.junit.MockitoJUnitRunner;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.support.AnnotationConfigContextLoader;
import com.cloud.storage.DataStoreRole;
@RunWith(MockitoJUnitRunner.class)
@ContextConfiguration(loader = AnnotationConfigContextLoader.class)
public class SnapshotServiceImplTest {
@ -57,20 +58,28 @@ public class SnapshotServiceImplTest {
SnapshotDataFactory _snapshotFactory;
@Mock
AsyncCallbackDispatcher<SnapshotServiceImpl, CommandResult> caller;
HeuristicRuleHelper heuristicRuleHelperMock;
@Mock
SnapshotInfo snapshotMock;
@Mock
VolumeInfo volumeInfoMock;
@Mock
DataStoreManager dataStoreManagerMock;
private static final long DUMMY_ID = 1L;
@Test
public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception {
SnapshotInfo snapshot = Mockito.mock(SnapshotInfo.class);
VolumeInfo volumeInfo = Mockito.mock(VolumeInfo.class);
Mockito.when(snapshot.getId()).thenReturn(1L);
Mockito.when(snapshot.getVolumeId()).thenReturn(1L);
Mockito.when(snapshotMock.getId()).thenReturn(DUMMY_ID);
Mockito.when(snapshotMock.getVolumeId()).thenReturn(DUMMY_ID);
Mockito.when(_snapshotFactory.getSnapshotOnPrimaryStore(1L)).thenReturn(null);
Mockito.when(volFactory.getVolume(1L, DataStoreRole.Primary)).thenReturn(volumeInfo);
Mockito.when(volFactory.getVolume(DUMMY_ID, DataStoreRole.Primary)).thenReturn(volumeInfoMock);
PrimaryDataStore store = Mockito.mock(PrimaryDataStore.class);
Mockito.when(volumeInfo.getDataStore()).thenReturn(store);
Mockito.when(volumeInfoMock.getDataStore()).thenReturn(store);
PrimaryDataStoreDriver driver = Mockito.mock(PrimaryDataStoreDriver.class);
Mockito.when(store.getDriver()).thenReturn(driver);
@ -80,7 +89,29 @@ public class SnapshotServiceImplTest {
Mockito.when(mock.get()).thenReturn(result);
Mockito.when(result.isFailed()).thenReturn(false);
})) {
Assert.assertTrue(snapshotService.revertSnapshot(snapshot));
Assert.assertTrue(snapshotService.revertSnapshot(snapshotMock));
}
}
@Test
public void getImageStoreForSnapshotTestShouldListFreeImageStoresWithNoHeuristicRule() {
Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))).
thenReturn(null);
Mockito.when(snapshotMock.getDataCenterId()).thenReturn(DUMMY_ID);
snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock);
Mockito.verify(dataStoreManagerMock, Mockito.times(1)).getImageStoreWithFreeCapacity(Mockito.anyLong());
}
@Test
public void getImageStoreForSnapshotTestShouldReturnImageStoreReturnedByTheHeuristicRule() {
DataStore dataStore = Mockito.mock(DataStore.class);
Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))).
thenReturn(dataStore);
snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock);
Mockito.verify(dataStoreManagerMock, Mockito.times(0)).getImageStoreWithFreeCapacity(Mockito.anyLong());
}
}

10
engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java
View File

@ -84,6 +84,16 @@ public class DataStoreManagerImpl implements DataStoreManager {
return imageDataStoreMgr.listImageStoresByScopeExcludingReadOnly(scope);
}
@Override
public List<DataStore> getImageStoresByZoneIds(Long... zoneIds) {
return imageDataStoreMgr.listImageStoresFilteringByZoneIds(zoneIds);
}
@Override
public DataStore getImageStoreByUuid(String uuid) {
return imageDataStoreMgr.getImageStore(uuid);
}
@Override
public DataStore getRandomImageStore(long zoneId) {
List<DataStore> stores = getImageStoresByScope(new ZoneScope(zoneId));

3
engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java
View File

@ -80,5 +80,8 @@ public interface ImageStoreProviderManager {
List<DataStore> listImageStoresWithFreeCapacity(List<DataStore> imageStores);
List<DataStore> orderImageStoresOnFreeCapacity(List<DataStore> imageStores);
List<DataStore> listImageStoresFilteringByZoneIds(Long... zoneIds);
long getImageStoreZoneId(long dataStoreId);
}

12
server/src/main/java/com/cloud/api/ApiResponseHelper.java
View File

@ -144,6 +144,7 @@ import org.apache.cloudstack.api.response.RollingMaintenanceHostUpdatedResponse;
import org.apache.cloudstack.api.response.RollingMaintenanceResponse;
import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse;
import org.apache.cloudstack.api.response.SSHKeyPairResponse;
import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse;
import org.apache.cloudstack.api.response.SecurityGroupResponse;
import org.apache.cloudstack.api.response.SecurityGroupRuleResponse;
import org.apache.cloudstack.api.response.ServiceOfferingResponse;
@ -200,6 +201,7 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule;
import org.apache.cloudstack.region.PortableIp;
import org.apache.cloudstack.region.PortableIpRange;
import org.apache.cloudstack.region.Region;
import org.apache.cloudstack.secstorage.heuristics.Heuristic;
import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao;
import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
@ -5105,6 +5107,16 @@ public class ApiResponseHelper implements ResponseGenerator {
return response;
}
@Override
public SecondaryStorageHeuristicsResponse createSecondaryStorageSelectorResponse(Heuristic heuristic) {
String zoneUuid = ApiDBUtils.findZoneById(heuristic.getZoneId()).getUuid();
SecondaryStorageHeuristicsResponse secondaryStorageHeuristicsResponse = new SecondaryStorageHeuristicsResponse(heuristic.getUuid(), heuristic.getName(),
heuristic.getDescription(), zoneUuid, heuristic.getType(), heuristic.getHeuristicRule(), heuristic.getCreated(), heuristic.getRemoved());
secondaryStorageHeuristicsResponse.setResponseName("secondarystorageheuristics");
return secondaryStorageHeuristicsResponse;
}
@Override
public IpQuarantineResponse createQuarantinedIpsResponse(PublicIpQuarantine quarantinedIp) {
IpQuarantineResponse quarantinedIpsResponse = new IpQuarantineResponse();

39
server/src/main/java/com/cloud/api/query/QueryManagerImpl.java
View File

@ -87,6 +87,7 @@ import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd
import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd;
import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd;
import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd;
import org.apache.cloudstack.api.command.admin.template.ListTemplatesCmdByAdmin;
import org.apache.cloudstack.api.command.admin.user.ListUsersCmd;
import org.apache.cloudstack.api.command.admin.zone.ListZonesCmdByAdmin;
@ -135,6 +136,7 @@ import org.apache.cloudstack.api.response.ResourceDetailResponse;
import org.apache.cloudstack.api.response.ResourceIconResponse;
import org.apache.cloudstack.api.response.ResourceTagResponse;
import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse;
import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse;
import org.apache.cloudstack.api.response.SecurityGroupResponse;
import org.apache.cloudstack.api.response.ServiceOfferingResponse;
import org.apache.cloudstack.api.response.SnapshotResponse;
@ -158,6 +160,9 @@ import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
import org.apache.cloudstack.query.QueryService;
import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao;
import org.apache.cloudstack.secstorage.HeuristicVO;
import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao;
import org.apache.cloudstack.secstorage.heuristics.Heuristic;
import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao;
import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
@ -509,6 +514,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
@Inject
private VirtualMachineManager virtualMachineManager;
@Inject
private VolumeDao volumeDao;
@ -518,6 +524,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
@Inject
private ManagementServerHostDao msHostDao;
@Inject
private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao;
@Inject
private NetworkDao networkDao;
@ -4799,6 +4808,36 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
return responseGenerator.createHealthCheckResponse(_routerDao.findById(routerId), result);
}
@Override
public ListResponse<SecondaryStorageHeuristicsResponse> listSecondaryStorageSelectors(ListSecondaryStorageSelectorsCmd cmd) {
ListResponse<SecondaryStorageHeuristicsResponse> response = new ListResponse<>();
Pair<List<HeuristicVO>, Integer> result = listSecondaryStorageSelectorsInternal(cmd.getZoneId(), cmd.getType(), cmd.isShowRemoved());
List<SecondaryStorageHeuristicsResponse> listOfSecondaryStorageHeuristicsResponses = new ArrayList<>();
for (Heuristic heuristic : result.first()) {
SecondaryStorageHeuristicsResponse secondaryStorageHeuristicsResponse = responseGenerator.createSecondaryStorageSelectorResponse(heuristic);
listOfSecondaryStorageHeuristicsResponses.add(secondaryStorageHeuristicsResponse);
}
response.setResponses(listOfSecondaryStorageHeuristicsResponses);
return response;
}
private Pair<List<HeuristicVO>, Integer> listSecondaryStorageSelectorsInternal(Long zoneId, String type, boolean showRemoved) {
SearchBuilder<HeuristicVO> searchBuilder = secondaryStorageHeuristicDao.createSearchBuilder();
searchBuilder.and("zoneId", searchBuilder.entity().getZoneId(), SearchCriteria.Op.EQ);
searchBuilder.and("type", searchBuilder.entity().getType(), SearchCriteria.Op.EQ);
searchBuilder.done();
SearchCriteria<HeuristicVO> searchCriteria = searchBuilder.create();
searchCriteria.setParameters("zoneId", zoneId);
searchCriteria.setParametersIfNotNull("type", type);
return secondaryStorageHeuristicDao.searchAndCount(searchCriteria, null, showRemoved);
}
@Override
public ListResponse<IpQuarantineResponse> listQuarantinedIps(ListQuarantinedIpsCmd cmd) {
ListResponse<IpQuarantineResponse> response = new ListResponse<>();

9
server/src/main/java/com/cloud/server/ManagementServerImpl.java
View File

@ -233,6 +233,10 @@ import org.apache.cloudstack.api.command.admin.storage.UpdateImageStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateStorageCapabilitiesCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd;
import org.apache.cloudstack.api.command.admin.swift.AddSwiftCmd;
import org.apache.cloudstack.api.command.admin.swift.ListSwiftsCmd;
import org.apache.cloudstack.api.command.admin.systemvm.DestroySystemVmCmd;
@ -3897,6 +3901,11 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
cmdList.add(PatchSystemVMCmd.class);
cmdList.add(ListGuestVlansCmd.class);
cmdList.add(AssignVolumeCmd.class);
cmdList.add(ListSecondaryStorageSelectorsCmd.class);
cmdList.add(CreateSecondaryStorageSelectorCmd.class);
cmdList.add(UpdateSecondaryStorageSelectorCmd.class);
cmdList.add(RemoveSecondaryStorageSelectorCmd.class);
// Out-of-band management APIs for admins
cmdList.add(EnableOutOfBandManagementForHostCmd.class);

71
server/src/main/java/com/cloud/storage/StorageManagerImpl.java
View File

@ -27,6 +27,7 @@ import java.nio.file.Files;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
@ -60,6 +61,9 @@ import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingSto
import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@ -97,6 +101,10 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.management.ManagementServerHost;
import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao;
import org.apache.cloudstack.secstorage.HeuristicVO;
import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao;
import org.apache.cloudstack.secstorage.heuristics.Heuristic;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand;
import org.apache.cloudstack.storage.command.DettachCommand;
import org.apache.cloudstack.storage.command.SyncVolumePathAnswer;
@ -124,6 +132,7 @@ import org.apache.cloudstack.storage.object.ObjectStore;
import org.apache.cloudstack.storage.object.ObjectStoreEntity;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.EnumUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
@ -359,6 +368,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
@Inject
private AnnotationDao annotationDao;
@Inject
private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao;
@Inject
protected UserVmManager userVmManager;
@Inject
@ -2006,6 +2018,65 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
return (PrimaryDataStoreInfo) _dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary);
}
@Override
public Heuristic createSecondaryStorageHeuristic(CreateSecondaryStorageSelectorCmd cmd) {
String name = cmd.getName();
String description = cmd.getDescription();
long zoneId = cmd.getZoneId();
String heuristicRule = cmd.getHeuristicRule();
String type = cmd.getType();
HeuristicType formattedType = EnumUtils.getEnumIgnoreCase(HeuristicType.class, type);
if (formattedType == null) {
throw new IllegalArgumentException(String.format("The given heuristic type [%s] is not valid for creating a new secondary storage selector." +
" The valid options are %s.", type, Arrays.asList(HeuristicType.values())));
}
HeuristicVO heuristic = secondaryStorageHeuristicDao.findByZoneIdAndType(zoneId, formattedType);
if (heuristic != null) {
DataCenterVO dataCenter = _dcDao.findById(zoneId);
throw new CloudRuntimeException(String.format("There is already a heuristic rule in the specified %s with the type [%s].",
dataCenter, type));
}
validateHeuristicRule(heuristicRule);
HeuristicVO heuristicVO = new HeuristicVO(name, description, zoneId, formattedType.toString(), heuristicRule);
return secondaryStorageHeuristicDao.persist(heuristicVO);
}
@Override
public Heuristic updateSecondaryStorageHeuristic(UpdateSecondaryStorageSelectorCmd cmd) {
long heuristicId = cmd.getId();
String heuristicRule = cmd.getHeuristicRule();
HeuristicVO heuristicVO = secondaryStorageHeuristicDao.findById(heuristicId);
validateHeuristicRule(heuristicRule);
heuristicVO.setHeuristicRule(heuristicRule);
return secondaryStorageHeuristicDao.persist(heuristicVO);
}
@Override
public void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd) {
Long heuristicId = cmd.getId();
HeuristicVO heuristicVO = secondaryStorageHeuristicDao.findById(heuristicId);
if (heuristicVO != null) {
secondaryStorageHeuristicDao.remove(heuristicId);
} else {
throw new CloudRuntimeException("Unable to find an active heuristic with the specified UUID.");
}
}
protected void validateHeuristicRule(String heuristicRule) {
if (StringUtils.isBlank(heuristicRule)) {
throw new IllegalArgumentException("Unable to create a new secondary storage selector as the given heuristic rule is blank.");
}
}
public void syncDatastoreClusterStoragePool(long datastoreClusterPoolId, List<ModifyStoragePoolAnswer> childDatastoreAnswerList, long hostId) {
StoragePoolVO datastoreClusterPool = _storagePoolDao.findById(datastoreClusterPoolId);
List<StoragePoolTagVO> storageTags = _storagePoolTagsDao.findStoragePoolTags(datastoreClusterPoolId);

6
server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java
View File

@ -409,7 +409,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
String format = sanitizeFormat(cmd.getFormat());
Long diskOfferingId = cmd.getDiskOfferingId();
String imageStoreUuid = cmd.getImageStoreUuid();
DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId);
validateVolume(caller, ownerId, zoneId, volumeName, url, format, diskOfferingId);
@ -419,6 +418,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
RegisterVolumePayload payload = new RegisterVolumePayload(cmd.getUrl(), cmd.getChecksum(), format);
vol.addPayload(payload);
DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId, volume);
volService.registerVolume(vol, store);
return volume;
@ -451,7 +451,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
String format = sanitizeFormat(cmd.getFormat());
final Long diskOfferingId = cmd.getDiskOfferingId();
String imageStoreUuid = cmd.getImageStoreUuid();
final DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId);
validateVolume(caller, ownerId, zoneId, volumeName, null, format, diskOfferingId);
@ -461,6 +460,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
VolumeVO volume = persistVolume(owner, zoneId, volumeName, null, format, diskOfferingId, Volume.State.NotUploaded);
final DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId, volume);
VolumeInfo vol = volFactory.getVolume(volume.getId());
RegisterVolumePayload payload = new RegisterVolumePayload(null, cmd.getChecksum(), format);
@ -655,6 +656,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
//url can be null incase of postupload
if (url != null) {
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
volume.setSize(UriUtils.getRemoteSize(url));
}
return volume;

266
server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
View File

@ -59,9 +59,11 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
import org.apache.cloudstack.framework.async.AsyncRpcContext;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.PublishScope;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.cloudstack.utils.security.DigestHelper;
import org.apache.commons.collections.CollectionUtils;
@ -107,7 +109,7 @@ import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
public class HypervisorTemplateAdapter extends TemplateAdapterBase {
private final static Logger s_logger = Logger.getLogger(HypervisorTemplateAdapter.class);
protected final static Logger s_logger = Logger.getLogger(HypervisorTemplateAdapter.class);
@Inject
DownloadMonitor _downloadMonitor;
@Inject
@ -145,6 +147,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
@Inject
private AnnotationDao annotationDao;
@Inject
private HeuristicRuleHelper heuristicRuleHelper;
@Inject
VMInstanceDao _vmInstanceDao;
@Override
@ -271,17 +275,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
}
if (!profile.isDirectDownload()) {
List<Long> zones = profile.getZoneIdList();
//zones is null when this template is to be registered to all zones
if (zones == null){
createTemplateWithinZone(null, profile, template);
}
else {
for (Long zId : zones) {
createTemplateWithinZone(zId, profile, template);
}
}
createTemplateWithinZones(profile, template);
} else {
//KVM direct download templates bypassing Secondary Storage
persistDirectDownloadTemplate(template.getId(), profile.getSize());
@ -291,46 +285,67 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
return template;
}
private void createTemplateWithinZone(Long zId, TemplateProfile profile, VMTemplateVO template) {
// find all eligible image stores for this zone scope
List<DataStore> imageStores = storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(zId));
if (imageStores == null || imageStores.size() == 0) {
throw new CloudRuntimeException("Unable to find image store to download template " + profile.getTemplate());
/**
* For each zone ID in {@link TemplateProfile#zoneIdList}, verifies if there is active heuristic rules for allocating template and returns the
* {@link DataStore} returned by the heuristic rule. If there is not an active heuristic rule, then allocate it to a random {@link DataStore}, if the ISO/template is private
* or allocate it to all {@link DataStore} in the zone, if it is public.
* @param profile
* @param template
*/
protected void createTemplateWithinZones(TemplateProfile profile, VMTemplateVO template) {
List<Long> zonesIds = profile.getZoneIdList();
if (zonesIds == null) {
zonesIds = _dcDao.listAllZones().stream().map(DataCenterVO::getId).collect(Collectors.toList());
}
List<DataStore> imageStores = getImageStoresThrowsExceptionIfNotFound(zonesIds, profile);
for (long zoneId : zonesIds) {
DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId);
if (imageStore == null) {
standardImageStoreAllocation(imageStores, template);
} else {
validateSecondaryStorageAndCreateTemplate(List.of(imageStore), template, null);
}
}
}
protected List<DataStore> getImageStoresThrowsExceptionIfNotFound(List<Long> zonesIds, TemplateProfile profile) {
List<DataStore> imageStores = storeMgr.getImageStoresByZoneIds(zonesIds.toArray(new Long[0]));
if (imageStores == null || imageStores.size() == 0) {
throw new CloudRuntimeException(String.format("Unable to find image store to download the template [%s].", profile.getTemplate()));
}
return imageStores;
}
protected DataStore verifyHeuristicRulesForZone(VMTemplateVO template, Long zoneId) {
HeuristicType heuristicType;
if (ImageFormat.ISO.equals(template.getFormat())) {
heuristicType = HeuristicType.ISO;
} else {
heuristicType = HeuristicType.TEMPLATE;
}
return heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, heuristicType, template);
}
protected void standardImageStoreAllocation(List<DataStore> imageStores, VMTemplateVO template) {
Set<Long> zoneSet = new HashSet<Long>();
Collections.shuffle(imageStores);
// For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc.
validateSecondaryStorageAndCreateTemplate(imageStores, template, zoneSet);
}
protected void validateSecondaryStorageAndCreateTemplate(List<DataStore> imageStores, VMTemplateVO template, Set<Long> zoneSet) {
for (DataStore imageStore : imageStores) {
// skip data stores for a disabled zone
Long zoneId = imageStore.getScope().getScopeId();
if (zoneId != null) {
DataCenterVO zone = _dcDao.findById(zoneId);
if (zone == null) {
s_logger.warn("Unable to find zone by id " + zoneId + ", so skip downloading template to its image store " + imageStore.getId());
continue;
}
// Check if zone is disabled
if (Grouping.AllocationState.Disabled == zone.getAllocationState()) {
s_logger.info("Zone " + zoneId + " is disabled. Skip downloading template to its image store " + imageStore.getId());
continue;
}
// Check if image store has enough capacity for template
if (!_statsCollector.imageStoreHasEnoughCapacity(imageStore)) {
s_logger.info("Image store doesn't have enough capacity. Skip downloading template to this image store " + imageStore.getId());
continue;
}
// We want to download private template to one of the image store in a zone
if (isPrivateTemplate(template) && zoneSet.contains(zoneId)) {
continue;
} else {
zoneSet.add(zoneId);
}
if (!isZoneAndImageStoreAvailable(imageStore, zoneId, zoneSet, isPrivateTemplate(template))) {
continue;
}
TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore);
CreateTemplateContext<TemplateApiResult> context = new CreateTemplateContext<TemplateApiResult>(null, tmpl);
CreateTemplateContext<TemplateApiResult> context = new CreateTemplateContext<>(null, tmpl);
AsyncCallbackDispatcher<HypervisorTemplateAdapter, TemplateApiResult> caller = AsyncCallbackDispatcher.create(this);
caller.setCallback(caller.getTarget().createTemplateAsyncCallBack(null, null));
caller.setContext(context);
@ -338,6 +353,44 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
}
}
protected boolean isZoneAndImageStoreAvailable(DataStore imageStore, Long zoneId, Set<Long> zoneSet, boolean isTemplatePrivate) {
if (zoneId == null) {
s_logger.warn(String.format("Zone ID is null, cannot allocate ISO/template in image store [%s].", imageStore));
return false;
}
DataCenterVO zone = _dcDao.findById(zoneId);
if (zone == null) {
s_logger.warn(String.format("Unable to find zone by id [%s], so skip downloading template to its image store [%s].", zoneId, imageStore.getId()));
return false;
}
if (Grouping.AllocationState.Disabled == zone.getAllocationState()) {
s_logger.info(String.format("Zone [%s] is disabled. Skip downloading template to its image store [%s].", zoneId, imageStore.getId()));
return false;
}
if (!_statsCollector.imageStoreHasEnoughCapacity(imageStore)) {
s_logger.info(String.format("Image store doesn't have enough capacity. Skip downloading template to this image store [%s].", imageStore.getId()));
return false;
}
if (zoneSet == null) {
s_logger.info(String.format("Zone set is null; therefore, the ISO/template should be allocated in every secondary storage of zone [%s].", zone));
return true;
}
if (isTemplatePrivate && zoneSet.contains(zoneId)) {
s_logger.info(String.format("The template is private and it is already allocated in a secondary storage in zone [%s]; therefore, image store [%s] will be skipped.",
zone, imageStore));
return false;
}
s_logger.info(String.format("Private template will be allocated in image store [%s] in zone [%s].", imageStore, zone));
zoneSet.add(zoneId);
return true;
}
@Override
public List<TemplateOrVolumePostUploadCommand> createTemplateForPostUpload(final TemplateProfile profile) {
// persist entry in vm_template, vm_template_details and template_zone_ref tables, not that entry at template_store_ref is not created here, and created in createTemplateAsync.
@ -352,80 +405,27 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
throw new CloudRuntimeException("Unable to persist the template " + profile.getTemplate());
}
if (profile.getZoneIdList() != null && profile.getZoneIdList().size() > 1)
throw new CloudRuntimeException("Operation is not supported for more than one zone id at a time");
List<Long> zoneIdList = profile.getZoneIdList();
Long zoneId = null;
if (profile.getZoneIdList() != null)
zoneId = profile.getZoneIdList().get(0);
// find all eligible image stores for this zone scope
List<DataStore> imageStores = storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(zoneId));
if (imageStores == null || imageStores.size() == 0) {
throw new CloudRuntimeException("Unable to find image store to download template " + profile.getTemplate());
if (zoneIdList == null) {
throw new CloudRuntimeException("Zone ID is null, cannot upload ISO/template.");
}
if (zoneIdList.size() > 1)
throw new CloudRuntimeException("Operation is not supported for more than one zone id at a time.");
Long zoneId = zoneIdList.get(0);
DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId);
List<TemplateOrVolumePostUploadCommand> payloads = new LinkedList<>();
Set<Long> zoneSet = new HashSet<Long>();
Collections.shuffle(imageStores); // For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc.
for (DataStore imageStore : imageStores) {
// skip data stores for a disabled zone
Long zoneId_is = imageStore.getScope().getScopeId();
if (zoneId != null) {
DataCenterVO zone = _dcDao.findById(zoneId_is);
if (zone == null) {
s_logger.warn("Unable to find zone by id " + zoneId_is +
", so skip downloading template to its image store " + imageStore.getId());
continue;
}
// Check if zone is disabled
if (Grouping.AllocationState.Disabled == zone.getAllocationState()) {
s_logger.info("Zone " + zoneId_is +
" is disabled, so skip downloading template to its image store " + imageStore.getId());
continue;
}
if (imageStore == null) {
List<DataStore> imageStores = getImageStoresThrowsExceptionIfNotFound(List.of(zoneId), profile);
postUploadAllocation(imageStores, template, payloads);
} else {
postUploadAllocation(List.of(imageStore), template, payloads);
// We want to download private template to one of the image store in a zone
if (isPrivateTemplate(template) && zoneSet.contains(zoneId_is)) {
continue;
} else {
zoneSet.add(zoneId_is);
}
}
TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore);
//imageService.createTemplateAsync(tmpl, imageStore, caller);
// persist template_store_ref entry
DataObject templateOnStore = imageStore.create(tmpl);
// update template_store_ref and template state
EndPoint ep = _epSelector.select(templateOnStore);
if (ep == null) {
String errMsg = "There is no secondary storage VM for downloading template to image store " + imageStore.getName();
s_logger.warn(errMsg);
throw new CloudRuntimeException(errMsg);
}
TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), template.getUuid(), tmpl.getInstallPath(), tmpl
.getChecksum(), tmpl.getType().toString(), template.getUniqueName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(),
templateOnStore.getDataStore().getRole().toString());
//using the existing max template size configuration
payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key()));
Long accountId = template.getAccountId();
Account account = _accountDao.findById(accountId);
Domain domain = _domainDao.findById(account.getDomainId());
payload.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage));
payload.setAccountId(accountId);
payload.setRemoteEndPoint(ep.getPublicAddr());
payload.setRequiresHvm(template.requiresHvm());
payload.setDescription(template.getDisplayText());
payloads.add(payload);
}
if(payloads.isEmpty()) {
throw new CloudRuntimeException("unable to find zone or an image store with enough capacity");
}
@ -435,6 +435,52 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
});
}
/**
* If the template/ISO is marked as private, then it is allocated to a random secondary storage; otherwise, allocates to every storage pool in every zone given by the
* {@link TemplateProfile#zoneIdList}.
*/
private void postUploadAllocation(List<DataStore> imageStores, VMTemplateVO template, List<TemplateOrVolumePostUploadCommand> payloads) {
Set<Long> zoneSet = new HashSet<Long>();
Collections.shuffle(imageStores);
for (DataStore imageStore : imageStores) {
Long zoneId_is = imageStore.getScope().getScopeId();
if (!isZoneAndImageStoreAvailable(imageStore, zoneId_is, zoneSet, isPrivateTemplate(template))) {
continue;
}
TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore);
// persist template_store_ref entry
DataObject templateOnStore = imageStore.create(tmpl);
// update template_store_ref and template state
EndPoint ep = _epSelector.select(templateOnStore);
if (ep == null) {
String errMsg = "There is no secondary storage VM for downloading template to image store " + imageStore.getName();
s_logger.warn(errMsg);
throw new CloudRuntimeException(errMsg);
}
TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), template.getUuid(), tmpl.getInstallPath(), tmpl
.getChecksum(), tmpl.getType().toString(), template.getUniqueName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(),
templateOnStore.getDataStore().getRole().toString());
//using the existing max template size configuration
payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key()));
Long accountId = template.getAccountId();
Account account = _accountDao.findById(accountId);
Domain domain = _domainDao.findById(account.getDomainId());
payload.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage));
payload.setAccountId(accountId);
payload.setRemoteEndPoint(ep.getPublicAddr());
payload.setRequiresHvm(template.requiresHvm());
payload.setDescription(template.getDisplayText());
payloads.add(payload);
}
}
private boolean isPrivateTemplate(VMTemplateVO template){
// if public OR featured OR system template

19
server/src/main/java/com/cloud/template/TemplateManagerImpl.java
View File

@ -86,6 +86,8 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.PublishScope;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
import org.apache.cloudstack.snapshot.SnapshotHelper;
import org.apache.cloudstack.storage.command.AttachCommand;
import org.apache.cloudstack.storage.command.CommandResult;
@ -98,6 +100,7 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.cloudstack.storage.template.VnfTemplateManager;
import org.apache.cloudstack.storage.template.VnfTemplateUtils;
@ -311,6 +314,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
@Inject
VnfTemplateManager vnfTemplateManager;
@Inject
private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao;
@Inject
private HeuristicRuleHelper heuristicRuleHelper;
private TemplateAdapter getAdapter(HypervisorType type) {
TemplateAdapter adapter = null;
if (type == HypervisorType.BareMetal) {
@ -451,17 +460,21 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
}
@Override
public DataStore getImageStore(String storeUuid, Long zoneId) {
public DataStore getImageStore(String storeUuid, Long zoneId, VolumeVO volume) {
DataStore imageStore = null;
if (storeUuid != null) {
imageStore = _dataStoreMgr.getDataStore(storeUuid, DataStoreRole.Image);
} else {
imageStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId);
imageStore = heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.VOLUME, volume);
if (imageStore == null) {
throw new CloudRuntimeException("cannot find an image store for zone " + zoneId);
imageStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId);
}
}
if (imageStore == null) {
throw new CloudRuntimeException(String.format("Cannot find an image store for zone [%s].", zoneId));
}
return imageStore;
}

2
server/src/main/java/com/cloud/test/TestAppender.java
View File

@ -46,6 +46,7 @@ import static org.apache.log4j.Level.ERROR;
import static org.apache.log4j.Level.FATAL;
import static org.apache.log4j.Level.INFO;
import static org.apache.log4j.Level.OFF;
import static org.apache.log4j.Level.WARN;
/**
*
@ -152,6 +153,7 @@ public final class TestAppender extends AppenderSkeleton {
expectedPatterns.put(FATAL, new HashSet<PatternResult>());
expectedPatterns.put(INFO, new HashSet<PatternResult>());
expectedPatterns.put(OFF, new HashSet<PatternResult>());
expectedPatterns.put(WARN, new HashSet<PatternResult>());
}
public TestAppenderBuilder addExpectedPattern(final Level level, final String pattern) {
checkArgument(level != null, "addExpectedPattern requires a non-null level");

278
server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java Normal file
View File

@ -0,0 +1,278 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.heuristics;
import com.cloud.api.ApiDBUtils;
import com.cloud.domain.DomainVO;
import com.cloud.domain.dao.DomainDao;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StorageStats;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VolumeVO;
import com.cloud.user.AccountVO;
import com.cloud.user.dao.AccountDao;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.secstorage.HeuristicVO;
import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.apache.cloudstack.storage.heuristics.presetvariables.Account;
import org.apache.cloudstack.storage.heuristics.presetvariables.Domain;
import org.apache.cloudstack.storage.heuristics.presetvariables.PresetVariables;
import org.apache.cloudstack.storage.heuristics.presetvariables.SecondaryStorage;
import org.apache.cloudstack.storage.heuristics.presetvariables.Snapshot;
import org.apache.cloudstack.storage.heuristics.presetvariables.Template;
import org.apache.cloudstack.storage.heuristics.presetvariables.Volume;
import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* Helper class for building and injecting the heuristics preset variables into the JS script.
*/
public class HeuristicRuleHelper {
protected static final Logger LOGGER = Logger.getLogger(HeuristicRuleHelper.class);
private static final Long HEURISTICS_SCRIPT_TIMEOUT = StorageManager.HEURISTICS_SCRIPT_TIMEOUT.value();
@Inject
private DataStoreManager dataStoreManager;
@Inject
private ImageStoreDao imageStoreDao;
@Inject
private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao;
@Inject
private DomainDao domainDao;
@Inject
private AccountDao accountDao;
/**
* Returns the {@link DataStore} object if the zone, specified by the ID, has an active heuristic rule for the given {@link HeuristicType}.
* It returns null otherwise.
* @param zoneId used to search for the heuristic rules.
* @param heuristicType used for checking if there is a heuristic rule for the given {@link HeuristicType}.
* @param obj can be from the following classes: {@link VMTemplateVO}, {@link SnapshotInfo} and {@link VolumeVO}.
* They are used to retrieve attributes for injecting in the JS rule.
* @return the corresponding {@link DataStore} if there is a heuristic rule, returns null otherwise.
*/
public DataStore getImageStoreIfThereIsHeuristicRule(Long zoneId, HeuristicType heuristicType, Object obj) {
HeuristicVO heuristicsVO = secondaryStorageHeuristicDao.findByZoneIdAndType(zoneId, heuristicType);
if (heuristicsVO == null) {
LOGGER.debug(String.format("No heuristic rules found for zone with ID [%s] and heuristic type [%s]. Returning null.", zoneId, heuristicType));
return null;
} else {
LOGGER.debug(String.format("Found the heuristic rule %s to apply for zone with ID [%s].", heuristicsVO, zoneId));
return interpretHeuristicRule(heuristicsVO.getHeuristicRule(), heuristicType, obj, zoneId);
}
}
/**
* Build the preset variables ({@link Template}, {@link Snapshot} and {@link Volume}) for the JS script.
*/
protected void buildPresetVariables(JsInterpreter jsInterpreter, HeuristicType heuristicType, long zoneId, Object obj) {
PresetVariables presetVariables = new PresetVariables();
Long accountId = null;
switch (heuristicType) {
case TEMPLATE:
case ISO:
presetVariables.setTemplate(setTemplatePresetVariable((VMTemplateVO) obj));
accountId = ((VMTemplateVO) obj).getAccountId();
break;
case SNAPSHOT:
presetVariables.setSnapshot(setSnapshotPresetVariable((SnapshotInfo) obj));
accountId = ((SnapshotInfo) obj).getAccountId();
break;
case VOLUME:
presetVariables.setVolume(setVolumePresetVariable((VolumeVO) obj));
accountId = ((VolumeVO) obj).getAccountId();
break;
}
presetVariables.setAccount(setAccountPresetVariable(accountId));
presetVariables.setSecondaryStorages(setSecondaryStoragesVariable(zoneId));
injectPresetVariables(jsInterpreter, presetVariables);
}
/**
* Inject the {@link PresetVariables} ({@link Template}, {@link Snapshot} and {@link Volume}) into the JS {@link JsInterpreter}.
* For each type, they can be accessed using the following variables in the script:
* <ul>
* <li>ISO/Template: using the variable <b>iso</b> or <b>template</b>, e.g. <b>iso.format</b> </li>
* <li>Snapshot: using the variable <b>snapshot</b>, e.g. <b>snapshot.size</b></li>
* <li>Volume: using the variable <b>volume</b>, e.g. <b>volume.format</b></li>
* </ul>
* @param jsInterpreter the JS script
* @param presetVariables used for injecting in the JS interpreter.
*/
protected void injectPresetVariables(JsInterpreter jsInterpreter, PresetVariables presetVariables) {
jsInterpreter.injectVariable("secondaryStorages", presetVariables.getSecondaryStorages().toString());
if (presetVariables.getTemplate() != null) {
jsInterpreter.injectVariable("template", presetVariables.getTemplate().toString());
jsInterpreter.injectVariable("iso", presetVariables.getTemplate().toString());
}
if (presetVariables.getSnapshot() != null) {
jsInterpreter.injectVariable("snapshot", presetVariables.getSnapshot().toString());
}
if (presetVariables.getVolume() != null) {
jsInterpreter.injectVariable("volume", presetVariables.getVolume().toString());
}
if (presetVariables.getAccount() != null) {
jsInterpreter.injectVariable("account", presetVariables.getAccount().toString());
}
}
protected List<SecondaryStorage> setSecondaryStoragesVariable(long zoneId) {
List<SecondaryStorage> secondaryStorageList = new ArrayList<>();
List<ImageStoreVO> imageStoreVOS = imageStoreDao.listStoresByZoneId(zoneId);
for (ImageStoreVO imageStore : imageStoreVOS) {
SecondaryStorage secondaryStorage = new SecondaryStorage();
secondaryStorage.setName(imageStore.getName());
secondaryStorage.setId(imageStore.getUuid());
secondaryStorage.setProtocol(imageStore.getProtocol());
StorageStats storageStats = ApiDBUtils.getSecondaryStorageStatistics(imageStore.getId());
if (storageStats != null) {
secondaryStorage.setUsedDiskSize(storageStats.getByteUsed());
secondaryStorage.setTotalDiskSize(storageStats.getCapacityBytes());
}
secondaryStorageList.add(secondaryStorage);
}
return secondaryStorageList;
}
protected Template setTemplatePresetVariable(VMTemplateVO templateVO) {
Template template = new Template();
template.setName(templateVO.getName());
template.setFormat(templateVO.getFormat());
template.setHypervisorType(templateVO.getHypervisorType());
return template;
}
protected Volume setVolumePresetVariable(VolumeVO volumeVO) {
Volume volume = new Volume();
volume.setName(volumeVO.getName());
volume.setFormat(volumeVO.getFormat());
volume.setSize(volumeVO.getSize());
return volume;
}
protected Snapshot setSnapshotPresetVariable(SnapshotInfo snapshotInfo) {
Snapshot snapshot = new Snapshot();
snapshot.setName(snapshotInfo.getName());
snapshot.setSize(snapshotInfo.getSize());
snapshot.setHypervisorType(snapshotInfo.getHypervisorType());
return snapshot;
}
protected Account setAccountPresetVariable(Long accountId) {
if (accountId == null) {
return null;
}
AccountVO account = accountDao.findById(accountId);
if (account == null) {
return null;
}
Account accountVariable = new Account();
accountVariable.setName(account.getName());
accountVariable.setId(account.getUuid());
accountVariable.setDomain(setDomainPresetVariable(account.getDomainId()));
return accountVariable;
}
protected Domain setDomainPresetVariable(long domainId) {
DomainVO domain = domainDao.findById(domainId);
if (domain == null) {
return null;
}
Domain domainVariable = new Domain();
domainVariable.setName(domain.getName());
domainVariable.setId(domain.getUuid());
return domainVariable;
}
/**
* This method calls {@link HeuristicRuleHelper#buildPresetVariables(JsInterpreter, HeuristicType, long, Object)} for building the preset variables and
* execute the JS script specified in the <b>rule</b> ({@link String}) parameter. The script is pre-injected with the preset variables, to allow the JS script to reference them
* in the code scope.
* <br>
* <br>
* The JS script needs to return a valid UUID ({@link String}) of a secondary storage, otherwise a {@link CloudRuntimeException} is thrown.
* @param rule the {@link String} representing the JS script.
* @param heuristicType used for building the preset variables accordingly to the {@link HeuristicType} specified.
* @param obj can be from the following classes: {@link VMTemplateVO}, {@link SnapshotInfo} and {@link VolumeVO}.
* They are used to retrieve attributes for injecting in the JS rule.
* @param zoneId used for injecting the {@link SecondaryStorage} preset variables.
* @return the {@link DataStore} returned by the script.
*/
public DataStore interpretHeuristicRule(String rule, HeuristicType heuristicType, Object obj, long zoneId) {
try (JsInterpreter jsInterpreter = new JsInterpreter(HEURISTICS_SCRIPT_TIMEOUT)) {
buildPresetVariables(jsInterpreter, heuristicType, zoneId, obj);
Object scriptReturn = jsInterpreter.executeScript(rule);
if (!(scriptReturn instanceof String)) {
throw new CloudRuntimeException(String.format("Error while interpreting heuristic rule [%s], the rule did not return a String.", rule));
}
DataStore dataStore = dataStoreManager.getImageStoreByUuid((String) scriptReturn);
if (dataStore == null) {
throw new CloudRuntimeException(String.format("Unable to find a secondary storage with the UUID [%s] returned by the heuristic rule [%s]. Check if the rule is " +
"returning a valid UUID.", scriptReturn, rule));
}
return dataStore;
} catch (IOException ex) {
String message = String.format("Error while executing script [%s].", rule);
LOGGER.error(message, ex);
throw new CloudRuntimeException(message, ex);
}
}
}

41
server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Account.java Normal file
View File

@ -0,0 +1,41 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.heuristics.presetvariables;
public class Account extends GenericHeuristicPresetVariable {
private String id;
private Domain domain;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
fieldNamesToIncludeInToString.add("id");
}
public Domain getDomain() {
return domain;
}
public void setDomain(Domain domain) {
this.domain = domain;
fieldNamesToIncludeInToString.add("domain");
}
}

30
server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java Normal file
View File

@ -0,0 +1,30 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.heuristics.presetvariables;
public class Domain extends GenericHeuristicPresetVariable{
private String id;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
fieldNamesToIncludeInToString.add("id");
}
}

43
server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java Normal file
View File

@ -0,0 +1,43 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.heuristics.presetvariables;
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
import java.util.HashSet;
import java.util.Set;
public class GenericHeuristicPresetVariable {
protected transient Set<String> fieldNamesToIncludeInToString = new HashSet<>();
private String name;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
fieldNamesToIncludeInToString.add("name");
}
@Override
public String toString() {
return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, fieldNamesToIncludeInToString.toArray(new String[0]));
}
}

72
server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/PresetVariables.java Normal file
View File

@ -0,0 +1,72 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.heuristics.presetvariables;
import java.util.List;
public class PresetVariables {
private Account account;
private List<SecondaryStorage> secondaryStorages;
private Template template;
private Snapshot snapshot;
private Volume volume;
public List<SecondaryStorage> getSecondaryStorages() {
return secondaryStorages;
}
public void setSecondaryStorages(List<SecondaryStorage> secondaryStorages) {
this.secondaryStorages = secondaryStorages;
}
public Template getTemplate() {
return template;
}
public void setTemplate(Template template) {
this.template = template;
}
public Snapshot getSnapshot() {
return snapshot;
}
public void setSnapshot(Snapshot snapshot) {
this.snapshot = snapshot;
}
public Volume getVolume() {
return volume;
}
public void setVolume(Volume volume) {
this.volume = volume;
}
public Account getAccount() {
return account;
}
public void setAccount(Account account) {
this.account = account;
}
}

64
server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorage.java Normal file
View File

@ -0,0 +1,64 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.heuristics.presetvariables;
public class SecondaryStorage extends GenericHeuristicPresetVariable {
private String id;
private Long usedDiskSize;
private Long totalDiskSize;
private String protocol;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
fieldNamesToIncludeInToString.add("id");
}
public Long getUsedDiskSize() {
return usedDiskSize;
}
public void setUsedDiskSize(Long usedDiskSize) {
this.usedDiskSize = usedDiskSize;
fieldNamesToIncludeInToString.add("usedDiskSize");
}
public Long getTotalDiskSize() {
return totalDiskSize;
}
public void setTotalDiskSize(Long totalDiskSize) {
this.totalDiskSize = totalDiskSize;
fieldNamesToIncludeInToString.add("totalDiskSize");
}
public String getProtocol() {
return protocol;
}
public void setProtocol(String protocol) {
this.protocol = protocol;
fieldNamesToIncludeInToString.add("protocol");
}
}

44
server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Snapshot.java Normal file
View File

@ -0,0 +1,44 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.heuristics.presetvariables;
import com.cloud.hypervisor.Hypervisor;
public class Snapshot extends GenericHeuristicPresetVariable {
private Long size;
private Hypervisor.HypervisorType hypervisorType;
public Long getSize() {
return size;
}
public void setSize(Long size) {
this.size = size;
fieldNamesToIncludeInToString.add("size");
}
public Hypervisor.HypervisorType getHypervisorType() {
return hypervisorType;
}
public void setHypervisorType(Hypervisor.HypervisorType hypervisorType) {
this.hypervisorType = hypervisorType;
fieldNamesToIncludeInToString.add("hypervisorType");
}
}

56
server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Template.java Normal file
View File

@ -0,0 +1,56 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.heuristics.presetvariables;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.storage.Storage;
public class Template extends GenericHeuristicPresetVariable {
private Hypervisor.HypervisorType hypervisorType;
private Storage.ImageFormat format;
private Storage.TemplateType templateType;
public Hypervisor.HypervisorType getHypervisorType() {
return hypervisorType;
}
public void setHypervisorType(Hypervisor.HypervisorType hypervisorType) {
this.hypervisorType = hypervisorType;
fieldNamesToIncludeInToString.add("hypervisorType");
}
public Storage.ImageFormat getFormat() {
return format;
}
public void setFormat(Storage.ImageFormat format) {
this.format = format;
fieldNamesToIncludeInToString.add("format");
}
public Storage.TemplateType getTemplateType() {
return templateType;
}
public void setTemplateType(Storage.TemplateType templateType) {
this.templateType = templateType;
fieldNamesToIncludeInToString.add("templateType");
}
}

44
server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Volume.java Normal file
View File

@ -0,0 +1,44 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.heuristics.presetvariables;
import com.cloud.storage.Storage;
public class Volume extends GenericHeuristicPresetVariable {
private Long size;
private Storage.ImageFormat format;
public Long getSize() {
return size;
}
public void setSize(Long size) {
this.size = size;
fieldNamesToIncludeInToString.add("size");
}
public Storage.ImageFormat getFormat() {
return format;
}
public void setFormat(Storage.ImageFormat format) {
this.format = format;
fieldNamesToIncludeInToString.add("format");
}
}

303
server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java
View File

@ -18,21 +18,26 @@
package com.cloud.template;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.event.EventTypes;
import com.cloud.event.UsageEventUtils;
import com.cloud.event.UsageEventVO;
import com.cloud.event.dao.UsageEventDao;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.org.Grouping;
import com.cloud.server.StatsCollector;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.TemplateProfile;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.VMTemplateZoneDao;
import com.cloud.test.TestAppender;
import com.cloud.user.AccountVO;
import com.cloud.user.ResourceLimitService;
import com.cloud.user.dao.AccountDao;
import com.cloud.utils.component.ComponentContext;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
@ -45,9 +50,12 @@ import org.apache.cloudstack.framework.events.Event;
import org.apache.cloudstack.framework.events.EventBus;
import org.apache.cloudstack.framework.events.EventBusException;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -58,6 +66,7 @@ import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import org.mockito.Spy;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
@ -68,9 +77,12 @@ import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.regex.Pattern;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyLong;
@ -119,10 +131,17 @@ public class HypervisorTemplateAdapterTest {
ConfigurationDao _configDao;
@Mock
DataStoreManager storeMgr;
DataStoreManager dataStoreManagerMock;
@Mock
HeuristicRuleHelper heuristicRuleHelperMock;
@Mock
StatsCollector statsCollectorMock;
@Spy
@InjectMocks
HypervisorTemplateAdapter _adapter;
HypervisorTemplateAdapter _adapter = new HypervisorTemplateAdapter();
//UsageEventUtils reflection abuse helpers
private Map<String, Object> oldFields = new HashMap<>();
@ -298,6 +317,282 @@ public class HypervisorTemplateAdapterTest {
cleanupUsageUtils();
}
@Test
public void createTemplateWithinZonesTestZoneIdsNullShouldCallListAllZones() {
TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class);
VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class);
Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(null);
Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class));
_adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock);
Mockito.verify(_dcDao, Mockito.times(1)).listAllZones();
}
@Test
public void createTemplateWithinZonesTestZoneIdsNotNullShouldNotCallListAllZones() {
TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class);
VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class);
List<Long> zoneIds = List.of(1L);
Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds);
Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class));
Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong());
Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class));
_adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock);
Mockito.verify(_dcDao, Mockito.times(0)).listAllZones();
}
@Test
public void createTemplateWithinZonesTestZoneDoesNotHaveActiveHeuristicRulesShouldCallStandardImageStoreAllocation() {
TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class);
VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class);
List<Long> zoneIds = List.of(1L);
Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds);
Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class));
Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong());
Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class));
_adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock);
Mockito.verify(_adapter, Mockito.times(1)).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class));
}
@Test
public void createTemplateWithinZonesTestZoneWithHeuristicRuleShouldCallValidateSecondaryStorageAndCreateTemplate() {
TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class);
VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class);
DataStore dataStoreMock = Mockito.mock(DataStore.class);
List<Long> zoneIds = List.of(1L);
Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds);
Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class));
Mockito.doReturn(dataStoreMock).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong());
Mockito.doNothing().when(_adapter).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class), Mockito.any(VMTemplateVO.class), Mockito.isNull());
_adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock);
Mockito.verify(_adapter, Mockito.times(1)).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class), Mockito.any(VMTemplateVO.class), Mockito.isNull());
}
@Test(expected = CloudRuntimeException.class)
public void getImageStoresThrowsExceptionIfNotFoundTestNullImageStoreShouldThrowCloudRuntimeException() {
TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class);
List<Long> zoneIds = List.of(1L);
Mockito.when(dataStoreManagerMock.getImageStoresByZoneIds(Mockito.anyLong())).thenReturn(null);
_adapter.getImageStoresThrowsExceptionIfNotFound(zoneIds, templateProfileMock);
}
@Test(expected = CloudRuntimeException.class)
public void getImageStoresThrowsExceptionIfNotFoundTestEmptyImageStoreShouldThrowCloudRuntimeException() {
TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class);
List<Long> zoneIds = List.of(1L);
List<DataStore> imageStoresList = new ArrayList<>();
Mockito.when(dataStoreManagerMock.getImageStoresByZoneIds(Mockito.anyLong())).thenReturn(imageStoresList);
_adapter.getImageStoresThrowsExceptionIfNotFound(zoneIds, templateProfileMock);
}
@Test
public void getImageStoresThrowsExceptionIfNotFoundTestNonEmptyImageStoreShouldNotThrowCloudRuntimeException() {
TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class);
List<Long> zoneIds = List.of(1L);
DataStore dataStoreMock = Mockito.mock(DataStore.class);
List<DataStore> imageStoresList = List.of(dataStoreMock);
Mockito.when(dataStoreManagerMock.getImageStoresByZoneIds(Mockito.anyLong())).thenReturn(imageStoresList);
_adapter.getImageStoresThrowsExceptionIfNotFound(zoneIds, templateProfileMock);
}
@Test
public void verifyHeuristicRulesForZoneTestTemplateIsISOFormatShouldCheckForISOHeuristicType() {
VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class);
Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.ISO);
_adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L);
Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.ISO, vmTemplateVOMock);
}
@Test
public void verifyHeuristicRulesForZoneTestTemplateNotISOFormatShouldCheckForTemplateHeuristicType() {
VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class);
Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.QCOW2);
_adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L);
Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.TEMPLATE, vmTemplateVOMock);
}
@Test
public void isZoneAndImageStoreAvailableTestZoneIdIsNullShouldReturnFalse() {
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = null;
Set<Long> zoneSet = null;
boolean isTemplatePrivate = false;
TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
appenderBuilder.addExpectedPattern(Level.WARN, Pattern.quote(String.format("Zone ID is null, cannot allocate ISO/template in image store [%s].", dataStoreMock)));
TestAppender testLogAppender = appenderBuilder.build();
TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender);
boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate);
testLogAppender.assertMessagesLogged();
Assert.assertFalse(result);
}
@Test
public void isZoneAndImageStoreAvailableTestZoneIsNullShouldReturnFalse() {
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = 1L;
Set<Long> zoneSet = null;
boolean isTemplatePrivate = false;
DataCenterVO dataCenterVOMock = null;
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
Mockito.when(dataStoreMock.getId()).thenReturn(2L);
TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
appenderBuilder.addExpectedPattern(Level.WARN, Pattern.quote(String.format("Unable to find zone by id [%s], so skip downloading template to its image store [%s].",
zoneId, dataStoreMock.getId())));
TestAppender testLogAppender = appenderBuilder.build();
TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender);
boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate);
testLogAppender.assertMessagesLogged();
Assert.assertFalse(result);
}
@Test
public void isZoneAndImageStoreAvailableTestZoneIsDisabledShouldReturnFalse() {
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = 1L;
Set<Long> zoneSet = null;
boolean isTemplatePrivate = false;
DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class);
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Disabled);
Mockito.when(dataStoreMock.getId()).thenReturn(2L);
TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Zone [%s] is disabled. Skip downloading template to its image store [%s].", zoneId, dataStoreMock.getId())));
TestAppender testLogAppender = appenderBuilder.build();
TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender);
boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate);
testLogAppender.assertMessagesLogged();
Assert.assertFalse(result);
}
@Test
public void isZoneAndImageStoreAvailableTestImageStoreDoesNotHaveEnoughCapacityShouldReturnFalse() {
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = 1L;
Set<Long> zoneSet = null;
boolean isTemplatePrivate = false;
DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class);
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled);
Mockito.when(dataStoreMock.getId()).thenReturn(2L);
Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(false);
TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Image store doesn't have enough capacity. Skip downloading template to this image store [%s].",
dataStoreMock.getId())));
TestAppender testLogAppender = appenderBuilder.build();
TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender);
boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate);
testLogAppender.assertMessagesLogged();
Assert.assertFalse(result);
}
@Test
public void isZoneAndImageStoreAvailableTestImageStoreHasEnoughCapacityAndZoneSetIsNullShouldReturnTrue() {
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = 1L;
Set<Long> zoneSet = null;
boolean isTemplatePrivate = false;
DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class);
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled);
Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true);
TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Zone set is null; therefore, the ISO/template should be allocated in every secondary storage " +
"of zone [%s].", dataCenterVOMock)));
TestAppender testLogAppender = appenderBuilder.build();
TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender);
boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate);
testLogAppender.assertMessagesLogged();
Assert.assertTrue(result);
}
@Test
public void isZoneAndImageStoreAvailableTestTemplateIsPrivateAndItIsAlreadyAllocatedToTheSameZoneShouldReturnFalse() {
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = 1L;
Set<Long> zoneSet = Set.of(1L);
boolean isTemplatePrivate = true;
DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class);
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled);
Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true);
TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("The template is private and it is already allocated in a secondary storage in zone [%s]; " +
"therefore, image store [%s] will be skipped.", dataCenterVOMock, dataStoreMock)));
TestAppender testLogAppender = appenderBuilder.build();
TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender);
boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate);
testLogAppender.assertMessagesLogged();
Assert.assertFalse(result);
}
@Test
public void isZoneAndImageStoreAvailableTestTemplateIsPrivateAndItIsNotAlreadyAllocatedToTheSameZoneShouldReturnTrue() {
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = 1L;
Set<Long> zoneSet = new HashSet<>();
boolean isTemplatePrivate = true;
DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class);
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled);
Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true);
TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Private template will be allocated in image store [%s] in zone [%s].",
dataStoreMock, dataCenterVOMock)));
TestAppender testLogAppender = appenderBuilder.build();
TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender);
boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate);
testLogAppender.assertMessagesLogged();
Assert.assertTrue(result);
}
@Test
public void testCheckZoneImageStoresDirectDownloadTemplate() {
VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class);
@ -309,7 +604,7 @@ public class HypervisorTemplateAdapterTest {
public void testCheckZoneImageStoresRegularTemplateWithStore() {
VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class);
Mockito.when(templateVO.isDirectDownload()).thenReturn(false);
Mockito.when(storeMgr.getImageStoresByScope(Mockito.any())).thenReturn(List.of(Mockito.mock(DataStore.class)));
Mockito.when(dataStoreManagerMock.getImageStoresByScope(Mockito.any())).thenReturn(List.of(Mockito.mock(DataStore.class)));
_adapter.checkZoneImageStores(templateVO, List.of(1L));
}
@ -317,7 +612,7 @@ public class HypervisorTemplateAdapterTest {
public void testCheckZoneImageStoresRegularTemplateNoStore() {
VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class);
Mockito.when(templateVO.isDirectDownload()).thenReturn(false);
Mockito.when(storeMgr.getImageStoresByScope(Mockito.any())).thenReturn(new ArrayList<>());
Mockito.when(dataStoreManagerMock.getImageStoresByScope(Mockito.any())).thenReturn(new ArrayList<>());
_adapter.checkZoneImageStores(templateVO, List.of(1L));
}
}

209
server/src/test/java/com/cloud/template/TemplateManagerImplTest.java
View File

@ -19,80 +19,6 @@
package com.cloud.template;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import javax.inject.Inject;
import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd;
import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd;
import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd;
import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd;
import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd;
import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd;
import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager;
import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.snapshot.SnapshotHelper;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.template.VnfTemplateManager;
import org.apache.cloudstack.test.utils.SpringUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.core.type.classreading.MetadataReader;
import org.springframework.core.type.classreading.MetadataReaderFactory;
import org.springframework.core.type.filter.TypeFilter;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.support.AnnotationConfigContextLoader;
import com.cloud.agent.AgentManager;
import com.cloud.api.query.dao.UserVmJoinDao;
import com.cloud.configuration.Resource;
@ -106,6 +32,7 @@ import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.hypervisor.HypervisorGuruManager;
import com.cloud.projects.ProjectManager;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.GuestOSVO;
import com.cloud.storage.Snapshot;
import com.cloud.storage.SnapshotVO;
@ -117,6 +44,7 @@ import com.cloud.storage.TemplateProfile;
import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.GuestOSDao;
import com.cloud.storage.dao.LaunchPermissionDao;
import com.cloud.storage.dao.SnapshotDao;
@ -140,6 +68,82 @@ import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.dao.UserVmDao;
import com.cloud.vm.dao.VMInstanceDao;
import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd;
import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd;
import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd;
import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd;
import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd;
import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd;
import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager;
import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
import org.apache.cloudstack.snapshot.SnapshotHelper;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper;
import org.apache.cloudstack.storage.template.VnfTemplateManager;
import org.apache.cloudstack.test.utils.SpringUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.FilterType;
import org.springframework.core.type.classreading.MetadataReader;
import org.springframework.core.type.classreading.MetadataReaderFactory;
import org.springframework.core.type.filter.TypeFilter;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.support.AnnotationConfigContextLoader;
import javax.inject.Inject;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(loader = AnnotationConfigContextLoader.class)
@ -201,6 +205,9 @@ public class TemplateManagerImplTest {
@Inject
VnfTemplateManager vnfTemplateManager;
@Inject
HeuristicRuleHelper heuristicRuleHelperMock;
public class CustomThreadPoolExecutor extends ThreadPoolExecutor {
AtomicInteger ai = new AtomicInteger(0);
public CustomThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
@ -596,6 +603,50 @@ public class TemplateManagerImplTest {
Assert.assertEquals(template, resultTemplate);
}
@Test
public void getImageStoreTestStoreUuidIsNotNullShouldReturnAValidImageStoreIfValidUuid() {
DataStore dataStore = Mockito.mock(DataStore.class);
VolumeVO volumeVO = Mockito.mock(VolumeVO.class);
Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(dataStore);
templateManager.getImageStore("UUID", 1L, volumeVO);
}
@Test(expected = CloudRuntimeException.class)
public void getImageStoreTestStoreUuidIsNotNullShouldThrowCloudRuntimeExceptionIfInvalidUuid() {
VolumeVO volumeVO = Mockito.mock(VolumeVO.class);
Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(null);
templateManager.getImageStore("UUID", 1L, volumeVO);
}
@Test
public void getImageStoreTestStoreUuidIsNullAndThereIsNoActiveHeuristicRulesShouldCallGetImageStoreWithFreeCapacity() {
DataStore dataStore = Mockito.mock(DataStore.class);
VolumeVO volumeVO = Mockito.mock(VolumeVO.class);
Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(null);
Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(VolumeVO.class))).thenReturn(null);
Mockito.when(dataStoreManager.getImageStoreWithFreeCapacity(Mockito.anyLong())).thenReturn(dataStore);
templateManager.getImageStore(null, 1L, volumeVO);
Mockito.verify(dataStoreManager, Mockito.times(1)).getImageStoreWithFreeCapacity(Mockito.anyLong());
}
@Test
public void getImageStoreTestStoreUuidIsNullAndThereIsActiveHeuristicRulesShouldNotCallGetImageStoreWithFreeCapacity() {
DataStore dataStore = Mockito.mock(DataStore.class);
VolumeVO volumeVO = Mockito.mock(VolumeVO.class);
Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(null);
Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(VolumeVO.class))).thenReturn(dataStore);
templateManager.getImageStore(null, 1L, volumeVO);
Mockito.verify(dataStoreManager, Mockito.times(0)).getImageStoreWithFreeCapacity(Mockito.anyLong());
}
@Test
public void testRegisterTemplateWithTemplateType() {
RegisterTemplateCmd cmd = Mockito.mock(RegisterTemplateCmd.class);
@ -915,6 +966,16 @@ public class TemplateManagerImplTest {
return Mockito.mock(SnapshotService.class);
}
@Bean
public SecondaryStorageHeuristicDao secondaryStorageHeuristicDao() {
return Mockito.mock(SecondaryStorageHeuristicDao.class);
}
@Bean
public HeuristicRuleHelper heuristicRuleHelper() {
return Mockito.mock(HeuristicRuleHelper.class);
}
public static class Library implements TypeFilter {
@Override
public boolean match(MetadataReader mdr, MetadataReaderFactory arg1) throws IOException {

205
server/src/test/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelperTest.java Normal file
View File

@ -0,0 +1,205 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.storage.heuristics;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VolumeVO;
import com.cloud.test.TestAppender;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.secstorage.HeuristicVO;
import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao;
import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
import org.apache.cloudstack.storage.heuristics.presetvariables.PresetVariables;
import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter;
import org.apache.log4j.Level;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import java.util.regex.Pattern;
@RunWith(MockitoJUnitRunner.class)
public class HeuristicRuleHelperTest {
@Mock
SecondaryStorageHeuristicDao secondaryStorageHeuristicDaoMock;
@Mock
HeuristicVO heuristicVOMock;
@Mock
VMTemplateVO vmTemplateVOMock;
@Mock
SnapshotInfo snapshotInfoMock;
@Mock
VolumeVO volumeVOMock;
@Mock
DataStoreManager dataStoreManagerMock;
@Mock
DataStore dataStoreMock;
@Spy
@InjectMocks
HeuristicRuleHelper heuristicRuleHelperSpy = new HeuristicRuleHelper();
@Test
public void getImageStoreIfThereIsHeuristicRuleTestZoneDoesNotHaveHeuristicRuleShouldReturnNull() {
Long zoneId = 1L;
Mockito.when(secondaryStorageHeuristicDaoMock.findByZoneIdAndType(Mockito.anyLong(), Mockito.any(HeuristicType.class))).thenReturn(null);
TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
appenderBuilder.addExpectedPattern(Level.DEBUG, Pattern.quote(String.format("No heuristic rules found for zone with ID [%s] and heuristic type [%s]. Returning null.",
zoneId, HeuristicType.TEMPLATE)));
TestAppender testLogAppender = appenderBuilder.build();
TestAppender.safeAddAppender(HeuristicRuleHelper.LOGGER, testLogAppender);
DataStore result = heuristicRuleHelperSpy.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.TEMPLATE, null);
testLogAppender.assertMessagesLogged();
Assert.assertNull(result);
}
@Test
public void getImageStoreIfThereIsHeuristicRuleTestZoneHasHeuristicRuleShouldCallInterpretHeuristicRule() {
Long zoneId = 1L;
Mockito.when(secondaryStorageHeuristicDaoMock.findByZoneIdAndType(Mockito.anyLong(), Mockito.any(HeuristicType.class))).thenReturn(heuristicVOMock);
Mockito.when(heuristicVOMock.getHeuristicRule()).thenReturn("rule");
Mockito.doReturn(null).when(heuristicRuleHelperSpy).interpretHeuristicRule(Mockito.anyString(), Mockito.any(HeuristicType.class), Mockito.isNull(),
Mockito.anyLong());
TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
appenderBuilder.addExpectedPattern(Level.DEBUG, Pattern.quote(String.format("Found the heuristic rule %s to apply for zone with ID [%s].", heuristicVOMock, zoneId)));
TestAppender testLogAppender = appenderBuilder.build();
TestAppender.safeAddAppender(HeuristicRuleHelper.LOGGER, testLogAppender);
DataStore result = heuristicRuleHelperSpy.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.TEMPLATE, null);
testLogAppender.assertMessagesLogged();
Assert.assertNull(result);
}
@Test
public void buildPresetVariablesTestWithTemplateHeuristicTypeShouldSetTemplateAndSecondaryStorageAndAccountPresetVariables() {
Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class));
Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong());
Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong());
heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.TEMPLATE, 1L, vmTemplateVOMock);
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setTemplatePresetVariable(Mockito.any(VMTemplateVO.class));
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong());
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong());
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class));
}
@Test
public void buildPresetVariablesTestWithIsoHeuristicTypeShouldSetTemplateAndSecondaryStorageAndAccountPresetVariables() {
Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class));
Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong());
Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong());
heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.ISO, 1L, vmTemplateVOMock);
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setTemplatePresetVariable(Mockito.any(VMTemplateVO.class));
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong());
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong());
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class));
}
@Test
public void buildPresetVariablesTestWithSnapshotHeuristicTypeShouldSetSnapshotAndSecondaryStorageAndAccountPresetVariables() {
Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class));
Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong());
Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong());
heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.SNAPSHOT, 1L, snapshotInfoMock);
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSnapshotPresetVariable(Mockito.any(SnapshotInfo.class));
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong());
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong());
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class));
}
@Test
public void buildPresetVariablesTestWithSnapshotHeuristicTypeShouldSetVolumeAndSecondaryStorageAndAccountPresetVariables() {
Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class));
Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong());
Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong());
heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.VOLUME, 1L, volumeVOMock);
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setVolumePresetVariable(Mockito.any(VolumeVO.class));
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong());
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong());
Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class));
}
@Test
public void interpretHeuristicRuleTestHeuristicRuleDoesNotReturnAStringShouldThrowCloudRuntimeException() {
String heuristicRule = "1";
Mockito.doNothing().when(heuristicRuleHelperSpy).buildPresetVariables(Mockito.any(JsInterpreter.class), Mockito.any(HeuristicType.class), Mockito.anyLong(),
Mockito.any());
String expectedMessage = String.format("Error while interpreting heuristic rule [%s], the rule did not return a String.", heuristicRule);
CloudRuntimeException assertThrows = Assert.assertThrows(CloudRuntimeException.class,
() -> heuristicRuleHelperSpy.interpretHeuristicRule(heuristicRule, HeuristicType.TEMPLATE, volumeVOMock, 1L));
Assert.assertEquals(expectedMessage, assertThrows.getMessage());
}
@Test
public void interpretHeuristicRuleTestHeuristicRuleReturnAStringWithInvalidUuidShouldThrowCloudRuntimeException() {
String heuristicRule = "'uuid'";
Mockito.doNothing().when(heuristicRuleHelperSpy).buildPresetVariables(Mockito.any(JsInterpreter.class), Mockito.any(HeuristicType.class), Mockito.anyLong(),
Mockito.any());
Mockito.doReturn(null).when(dataStoreManagerMock).getImageStoreByUuid(Mockito.anyString());
String expectedMessage = String.format("Unable to find a secondary storage with the UUID [%s] returned by the heuristic rule [%s]. Check if the rule is " +
"returning a valid UUID.", "uuid", heuristicRule);
CloudRuntimeException assertThrows = Assert.assertThrows(CloudRuntimeException.class,
() -> heuristicRuleHelperSpy.interpretHeuristicRule(heuristicRule, HeuristicType.TEMPLATE, volumeVOMock, 1L));
Assert.assertEquals(expectedMessage, assertThrows.getMessage());
}
@Test
public void interpretHeuristicRuleTestHeuristicRuleReturnAStringWithAValidUuidShouldReturnAImageStore() {
String heuristicRule = "'uuid'";
Mockito.doNothing().when(heuristicRuleHelperSpy).buildPresetVariables(Mockito.any(JsInterpreter.class), Mockito.any(HeuristicType.class), Mockito.anyLong(),
Mockito.any());
Mockito.doReturn(dataStoreMock).when(dataStoreManagerMock).getImageStoreByUuid(Mockito.anyString());
DataStore result = heuristicRuleHelperSpy.interpretHeuristicRule(heuristicRule, HeuristicType.TEMPLATE, volumeVOMock, 1L);
Assert.assertNotNull(result);
}
}