diff --git a/api/src/main/java/com/cloud/server/ResourceTag.java b/api/src/main/java/com/cloud/server/ResourceTag.java index 0bd5d734e30..99eb8603d1c 100644 --- a/api/src/main/java/com/cloud/server/ResourceTag.java +++ b/api/src/main/java/com/cloud/server/ResourceTag.java @@ -58,7 +58,7 @@ public interface ResourceTag extends ControlledEntity, Identity, InternalIdentit AutoScaleVmGroup(false, true), LBStickinessPolicy(false, true), LBHealthCheckPolicy(false, true), - SnapshotPolicy(false, true), + SnapshotPolicy(true, true), GuestOs(false, true), NetworkOffering(false, true), VpcOffering(true, false); diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 7b38a6b1af1..aa6d8a664c8 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -19,6 +19,7 @@ package com.cloud.storage; import java.net.MalformedURLException; +import java.util.Map; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; @@ -93,7 +94,7 @@ public interface VolumeApiService { Volume detachVolumeFromVM(DetachVolumeCmd cmd); - Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup) + Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags) throws ResourceAllocationException; Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException; diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index f24c4f587c4..84de8c9ebac 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -17,14 +17,41 @@ package com.cloud.vm; public interface VmDetailConstants { - public static final String KEYBOARD = "keyboard"; - public static final String NIC_ADAPTER = "nicAdapter"; - public static final String ROOT_DISK_CONTROLLER = "rootDiskController"; - public static final String NESTED_VIRTUALIZATION_FLAG = "nestedVirtualizationFlag"; - public static final String HYPERVISOR_TOOLS_VERSION = "hypervisortoolsversion"; - public static final String DATA_DISK_CONTROLLER = "dataDiskController"; - public static final String SVGA_VRAM_SIZE = "svga.vramSize"; - public static final String CPU_NUMBER = "cpuNumber"; - public static final String CPU_SPEED = "cpuSpeed"; - public static final String MEMORY = "memory"; + String KEYBOARD = "keyboard"; + String CPU_CORE_PER_SOCKET = "cpu.corespersocket"; + String ROOT_DISK_SIZE = "rootdisksize"; + + // VMware specific + String NIC_ADAPTER = "nicAdapter"; + String ROOT_DISK_CONTROLLER = "rootDiskController"; + String DATA_DISK_CONTROLLER = "dataDiskController"; + String SVGA_VRAM_SIZE = "svga.vramSize"; + String NESTED_VIRTUALIZATION_FLAG = "nestedVirtualizationFlag"; + + // XenServer specific (internal) + String HYPERVISOR_TOOLS_VERSION = "hypervisortoolsversion"; + String PLATFORM = "platform"; + String TIME_OFFSET = "timeoffset"; + + // KVM specific (internal) + String KVM_VNC_PORT = "kvm.vnc.port"; + String KVM_VNC_ADDRESS = "kvm.vnc.address"; + + // Mac OSX guest specific (internal) + String SMC_PRESENT = "smc.present"; + String FIRMWARE = "firmware"; + + // VM deployment with custom compute offering params + String CPU_NUMBER = "cpuNumber"; + String CPU_SPEED = "cpuSpeed"; + String MEMORY = "memory"; + + // Misc details for internal usage (not to be set/changed by user or admin) + String CPU_OVER_COMMIT_RATIO = "cpuOvercommitRatio"; + String MEMORY_OVER_COMMIT_RATIO = "memoryOvercommitRatio"; + String MESSAGE_RESERVED_CAPACITY_FREED_FLAG = "Message.ReservedCapacityFreed.Flag"; + String DEPLOY_VM = "deployvm"; + String SSH_PUBLIC_KEY = "SSH.PublicKey"; + String PASSWORD = "password"; + String ENCRYPTED_PASSWORD = "Encrypted.Password"; } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 8cb58d6b1fe..5a40729a58c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -91,6 +91,11 @@ public class ApiConstants { public static final String DIRECT_DOWNLOAD = "directdownload"; public static final String DISK_OFFERING_ID = "diskofferingid"; public static final String NEW_DISK_OFFERING_ID = "newdiskofferingid"; + public static final String DISK_KBS_READ = "diskkbsread"; + public static final String DISK_KBS_WRITE = "diskkbswrite"; + public static final String DISK_IO_READ = "diskioread"; + public static final String DISK_IO_WRITE = "diskiowrite"; + public static final String DISK_IO_PSTOTAL = "diskiopstotal"; public static final String DISK_SIZE = "disksize"; public static final String UTILIZATION = "utilization"; public static final String DRIVER = "driver"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListDetailOptionsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListDetailOptionsCmd.java new file mode 100644 index 00000000000..e53754c099c --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListDetailOptionsCmd.java @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.resource; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.DetailOptionsResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.server.ResourceTag; +import com.google.common.base.Strings; + +@APICommand(name = ListDetailOptionsCmd.APINAME, + description = "Lists all possible details and their options for a resource type such as a VM or a template", + responseObject = DetailOptionsResponse.class, + since = "4.13", + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListDetailOptionsCmd extends BaseCmd { + public final static String APINAME = "listDetailOptions"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, required = true, + description = "the resource type such as UserVm, Template etc.", + validations = {ApiArgValidator.NotNullOrEmpty} + ) + private String resourceType; + + @Parameter(name = ApiConstants.RESOURCE_ID, type = CommandType.STRING, + description = "the UUID of the resource (optional)") + private String resourceId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public ResourceTag.ResourceObjectType getResourceType() { + return _taggedResourceService.getResourceType(resourceType); + } + + public String getResourceId() { + if (!Strings.isNullOrEmpty(resourceId)) { + return _taggedResourceService.getUuid(resourceId, getResourceType()); + } + return null; + } + + ///////////////////////////////////////////////////// + /////////////////// Implementation ////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + @Override + public void execute() { + final DetailOptionsResponse response = _queryService.listDetailOptions(this); + response.setResponseName(getCommandName()); + response.setObjectName("detailoptions"); + setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java index b77b9854590..72437c8d3fe 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java @@ -16,6 +16,10 @@ // under the License. package org.apache.cloudstack.api.command.user.snapshot; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandJobType; import org.apache.cloudstack.api.ApiConstants; @@ -28,6 +32,7 @@ import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.SnapshotPolicyResponse; import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; import com.cloud.event.EventTypes; @@ -83,6 +88,9 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.ASYNC_BACKUP, type = CommandType.BOOLEAN, required = false, description = "asynchronous backup if true") private Boolean asyncBackup; + @Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "Map of tags (key/value pairs)") + private Map tags; + private String syncObjectType = BaseAsyncCmd.snapshotHostSyncObject; // /////////////////////////////////////////////////// @@ -121,6 +129,18 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { } } + public Map getTags() { + Map tagsMap = new HashMap<>(); + if (MapUtils.isNotEmpty(tags)) { + for (Map services : (Collection>)tags.values()) { + String key = services.get("key"); + String value = services.get("value"); + tagsMap.put(key, value); + } + } + return tagsMap; + } + private Long getHostId() { Volume volume = _entityMgr.findById(Volume.class, getVolumeId()); if (volume == null) { @@ -196,7 +216,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { Snapshot snapshot; try { snapshot = - _volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup()); + _volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup(), getTags()); if (snapshot != null) { SnapshotResponse response = _responseGenerator.createSnapshotResponse(snapshot); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java index 4e2e6bd3bbd..898bae5919f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java @@ -16,9 +16,11 @@ // under the License. package org.apache.cloudstack.api.command.user.snapshot; -import org.apache.cloudstack.acl.RoleType; -import org.apache.log4j.Logger; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; @@ -27,6 +29,8 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.SnapshotPolicyResponse; import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.commons.collections.MapUtils; +import org.apache.log4j.Logger; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; @@ -68,6 +72,9 @@ public class CreateSnapshotPolicyCmd extends BaseCmd { @Parameter(name = ApiConstants.FOR_DISPLAY, type = CommandType.BOOLEAN, description = "an optional field, whether to the display the policy to the end user or not", since = "4.4", authorized = {RoleType.Admin}) private Boolean display; + @Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "Map of tags (key/value pairs)") + private Map tags; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -133,6 +140,18 @@ public class CreateSnapshotPolicyCmd extends BaseCmd { return volume.getAccountId(); } + public Map getTags() { + Map tagsMap = new HashMap<>(); + if (MapUtils.isNotEmpty(tags)) { + for (Map services : (Collection>)tags.values()) { + String key = services.get("key"); + String value = services.get("value"); + tagsMap.put(key, value); + } + } + return tagsMap; + } + @Override public void execute() { SnapshotPolicy result = _snapshotService.createPolicy(this, _accountService.getAccount(getEntityOwnerId())); diff --git a/api/src/main/java/org/apache/cloudstack/api/response/DetailOptionsResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/DetailOptionsResponse.java new file mode 100644 index 00000000000..5f6bff39597 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/DetailOptionsResponse.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class DetailOptionsResponse extends BaseResponse { + @SerializedName(ApiConstants.DETAILS) + @Param(description = "Map of all possible details and their possible list of values") + private Map> details; + + public DetailOptionsResponse(Map> details) { + this.details = details; + } + + public void setDetails(Map> details) { + this.details = details; + } + + public Map> getDetails() { + return details; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java index 10710c64803..d1e535ee743 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java @@ -16,18 +16,20 @@ // under the License. package org.apache.cloudstack.api.response; -import com.google.gson.annotations.SerializedName; +import java.util.LinkedHashSet; +import java.util.Set; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.BaseResponseWithTagInformation; import org.apache.cloudstack.api.EntityReference; import com.cloud.serializer.Param; import com.cloud.storage.snapshot.SnapshotPolicy; +import com.google.gson.annotations.SerializedName; @EntityReference(value = SnapshotPolicy.class) -public class SnapshotPolicyResponse extends BaseResponse { +public class SnapshotPolicyResponse extends BaseResponseWithTagInformation { @SerializedName("id") @Param(description = "the ID of the snapshot policy") private String id; @@ -56,6 +58,10 @@ public class SnapshotPolicyResponse extends BaseResponse { @Param(description = "is this policy for display to the regular user", since = "4.4", authorized = {RoleType.Admin}) private Boolean forDisplay; + public SnapshotPolicyResponse() { + tags = new LinkedHashSet(); + } + public String getId() { return id; } @@ -111,4 +117,8 @@ public class SnapshotPolicyResponse extends BaseResponse { public void setForDisplay(Boolean forDisplay) { this.forDisplay = forDisplay; } + + public void setTags(Set tags) { + this.tags = tags; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java index bb2ff7f6d0e..94bb4d14444 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java @@ -16,18 +16,20 @@ // under the License. package org.apache.cloudstack.api.response; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.Set; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponseWithTagInformation; +import org.apache.cloudstack.api.EntityReference; + import com.cloud.serializer.Param; import com.cloud.storage.Snapshot; import com.google.gson.annotations.SerializedName; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseResponse; -import org.apache.cloudstack.api.EntityReference; - -import java.util.Date; -import java.util.List; @EntityReference(value = Snapshot.class) -public class SnapshotResponse extends BaseResponse implements ControlledEntityResponse { +public class SnapshotResponse extends BaseResponseWithTagInformation implements ControlledEntityResponse { @SerializedName(ApiConstants.ID) @Param(description = "ID of the snapshot") private String id; @@ -96,10 +98,6 @@ public class SnapshotResponse extends BaseResponse implements ControlledEntityRe @Param(description = "id of the availability zone") private String zoneId; - @SerializedName(ApiConstants.TAGS) - @Param(description = "the list of resource tags associated with snapshot", responseObject = ResourceTagResponse.class) - private List tags; - @SerializedName(ApiConstants.REVERTABLE) @Param(description = "indicates whether the underlying storage supports reverting the volume to this snapshot") private boolean revertable; @@ -116,6 +114,10 @@ public class SnapshotResponse extends BaseResponse implements ControlledEntityRe @Param(description = "virtual size of backedup snapshot on image store") private long virtualSize; + public SnapshotResponse() { + tags = new LinkedHashSet(); + } + @Override public String getObjectId() { return this.getId(); @@ -206,7 +208,7 @@ public class SnapshotResponse extends BaseResponse implements ControlledEntityRe this.zoneId = zoneId; } - public void setTags(List tags) { + public void setTags(Set tags) { this.tags = tags; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java index bbf6b6cf614..8a2f1a169d6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java @@ -188,11 +188,11 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co @Param(description = "the outgoing network traffic on the host") private Long networkKbsWrite; - @SerializedName("diskkbsread") + @SerializedName(ApiConstants.DISK_KBS_READ) @Param(description = "the read (bytes) of disk on the vm") private Long diskKbsRead; - @SerializedName("diskkbswrite") + @SerializedName(ApiConstants.DISK_KBS_WRITE) @Param(description = "the write (bytes) of disk on the vm") private Long diskKbsWrite; @@ -208,11 +208,11 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co @Param(description = "the target memory in vm") private Long memoryTargetKBs; - @SerializedName("diskioread") + @SerializedName(ApiConstants.DISK_IO_READ) @Param(description = "the read (io) of disk on the vm") private Long diskIORead; - @SerializedName("diskiowrite") + @SerializedName(ApiConstants.DISK_IO_WRITE) @Param(description = "the write (io) of disk on the vm") private Long diskIOWrite; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java index d845e414808..01d2c9b3d8f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java @@ -16,17 +16,18 @@ // under the License. package org.apache.cloudstack.api.response; -import com.cloud.serializer.Param; -import com.cloud.storage.Volume; -import com.google.gson.annotations.SerializedName; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.Set; + import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponseWithTagInformation; import org.apache.cloudstack.api.EntityReference; -import java.util.Date; -import java.util.LinkedHashSet; -import java.util.Set; +import com.cloud.serializer.Param; +import com.cloud.storage.Volume; +import com.google.gson.annotations.SerializedName; @EntityReference(value = Volume.class) @SuppressWarnings("unused") @@ -152,13 +153,29 @@ public class VolumeResponse extends BaseResponseWithTagInformation implements Co private Long bytesWriteRate; @SerializedName("diskIopsReadRate") - @Param(description = "io requests read rate of the disk volume") + @Param(description = "io requests read rate of the disk volume per the disk offering") private Long iopsReadRate; @SerializedName("diskIopsWriteRate") - @Param(description = "io requests write rate of the disk volume") + @Param(description = "io requests write rate of the disk volume per the disk offering") private Long iopsWriteRate; + @SerializedName(ApiConstants.DISK_KBS_READ) + @Param(description = "the read (bytes) of disk on the vm") + private Long diskKbsRead; + + @SerializedName(ApiConstants.DISK_KBS_WRITE) + @Param(description = "the write (bytes) of disk on the vm") + private Long diskKbsWrite; + + @SerializedName(ApiConstants.DISK_IO_READ) + @Param(description = "the read (io) of disk on the vm") + private Long diskIORead; + + @SerializedName(ApiConstants.DISK_IO_WRITE) + @Param(description = "the write (io) of disk on the vm") + private Long diskIOWrite; + @SerializedName(ApiConstants.HYPERVISOR) @Param(description = "Hypervisor the volume belongs to") private String hypervisor; @@ -395,10 +412,42 @@ public class VolumeResponse extends BaseResponseWithTagInformation implements Co this.iopsWriteRate = iopsWriteRate; } + public Long getDiskKbsRead() { + return diskKbsRead; + } + + public void setDiskKbsRead(Long diskKbsRead) { + this.diskKbsRead = diskKbsRead; + } + + public Long getDiskKbsWrite() { + return diskKbsWrite; + } + + public void setDiskKbsWrite(Long diskKbsWrite) { + this.diskKbsWrite = diskKbsWrite; + } + public Long getIopsWriteRate() { return iopsWriteRate; } + public Long getDiskIORead() { + return diskIORead; + } + + public void setDiskIORead(Long diskIORead) { + this.diskIORead = diskIORead; + } + + public Long getDiskIOWrite() { + return diskIOWrite; + } + + public void setDiskIOWrite(Long diskIOWrite) { + this.diskIOWrite = diskIOWrite; + } + public void setHypervisor(String hypervisor) { this.hypervisor = hypervisor; } diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 1f0f933d7d2..618a8f6f8a5 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -20,8 +20,8 @@ import java.util.List; import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.command.admin.domain.ListDomainsCmd; -import org.apache.cloudstack.api.command.admin.host.ListHostsCmd; import org.apache.cloudstack.api.command.admin.host.ListHostTagsCmd; +import org.apache.cloudstack.api.command.admin.host.ListHostsCmd; import org.apache.cloudstack.api.command.admin.internallb.ListInternalLBVMsCmd; import org.apache.cloudstack.api.command.admin.management.ListMgmtsCmd; import org.apache.cloudstack.api.command.admin.router.ListRoutersCmd; @@ -40,6 +40,7 @@ import org.apache.cloudstack.api.command.user.offering.ListDiskOfferingsCmd; import org.apache.cloudstack.api.command.user.offering.ListServiceOfferingsCmd; import org.apache.cloudstack.api.command.user.project.ListProjectInvitationsCmd; import org.apache.cloudstack.api.command.user.project.ListProjectsCmd; +import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; @@ -50,6 +51,7 @@ import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; import org.apache.cloudstack.api.command.user.zone.ListZonesCmd; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.AsyncJobResponse; +import org.apache.cloudstack.api.response.DetailOptionsResponse; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.DomainRouterResponse; @@ -147,6 +149,8 @@ public interface QueryService { ListResponse listIsos(ListIsosCmd cmd); + DetailOptionsResponse listDetailOptions(ListDetailOptionsCmd cmd); + ListResponse searchForAffinityGroups(ListAffinityGroupsCmd cmd); List listResourceDetails(ListResourceDetailsCmd cmd); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java index ceb63ab6e56..4739082cf64 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java @@ -19,9 +19,13 @@ package org.apache.cloudstack.api.command.test; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.isNull; +import java.util.HashMap; +import java.util.Map; + import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd; @@ -32,6 +36,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; import com.cloud.storage.Snapshot; import com.cloud.storage.VolumeApiService; @@ -87,9 +92,8 @@ public class CreateSnapshotCmdTest extends TestCase { VolumeApiService volumeApiService = Mockito.mock(VolumeApiService.class); Snapshot snapshot = Mockito.mock(Snapshot.class); try { - Mockito.when(volumeApiService.takeSnapshot(anyLong(), anyLong(), anyLong(), - any(Account.class), anyBoolean(), isNull(Snapshot.LocationType.class), anyBoolean())).thenReturn(snapshot); + any(Account.class), anyBoolean(), isNull(Snapshot.LocationType.class), anyBoolean(), anyObject())).thenReturn(snapshot); } catch (Exception e) { Assert.fail("Received exception when success expected " + e.getMessage()); @@ -122,7 +126,7 @@ public class CreateSnapshotCmdTest extends TestCase { try { Mockito.when(volumeApiService.takeSnapshot(anyLong(), anyLong(), anyLong(), - any(Account.class), anyBoolean(), isNull(Snapshot.LocationType.class), anyBoolean())).thenReturn(null); + any(Account.class), anyBoolean(), isNull(Snapshot.LocationType.class), anyBoolean(), anyObject())).thenReturn(null); } catch (Exception e) { Assert.fail("Received exception when success expected " + e.getMessage()); } @@ -136,4 +140,24 @@ public class CreateSnapshotCmdTest extends TestCase { Assert.assertEquals("Failed to create snapshot due to an internal error creating snapshot for volume 123", exception.getDescription()); } } + + @Test + public void testParsingTags() { + final CreateSnapshotCmd createSnapshotCmd = new CreateSnapshotCmd(); + final Map tag1 = new HashMap<>(); + tag1.put("key", "key1"); + tag1.put("value", "value1"); + final Map tag2 = new HashMap<>(); + tag2.put("key", "key2"); + tag2.put("value", "value2"); + final Map expectedTags = new HashMap<>(); + expectedTags.put("key1", "value1"); + expectedTags.put("key2", "value2"); + + final Map> tagsParams = new HashMap<>(); + tagsParams.put("0", tag1); + tagsParams.put("1", tag2); + ReflectionTestUtils.setField(createSnapshotCmd, "tags", tagsParams); + Assert.assertEquals(createSnapshotCmd.getTags(), expectedTags); + } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmdTest.java new file mode 100644 index 00000000000..860c2c2c3e7 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmdTest.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.snapshot; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; + +public class CreateSnapshotPolicyCmdTest { + @Test + public void testParsingTags() { + final CreateSnapshotPolicyCmd createSnapshotPolicyCmd = new CreateSnapshotPolicyCmd(); + final Map tag1 = new HashMap<>(); + tag1.put("key", "key1"); + tag1.put("value", "value1"); + final Map tag2 = new HashMap<>(); + tag2.put("key", "key2"); + tag2.put("value", "value2"); + final Map expectedTags = new HashMap<>(); + expectedTags.put("key1", "value1"); + expectedTags.put("key2", "value2"); + + final Map> tagsParams = new HashMap<>(); + tagsParams.put("0", tag1); + tagsParams.put("1", tag2); + ReflectionTestUtils.setField(createSnapshotPolicyCmd, "tags", tagsParams); + Assert.assertEquals(createSnapshotPolicyCmd.getTags(), expectedTags); + } +} \ No newline at end of file diff --git a/core/src/main/java/com/cloud/info/ConsoleProxyInfo.java b/core/src/main/java/com/cloud/info/ConsoleProxyInfo.java index 78cb877e7db..7e1be6a9cad 100644 --- a/core/src/main/java/com/cloud/info/ConsoleProxyInfo.java +++ b/core/src/main/java/com/cloud/info/ConsoleProxyInfo.java @@ -55,18 +55,16 @@ public class ConsoleProxyInfo { private String formatProxyAddress(String consoleProxyUrlDomain, String proxyIpAddress) { StringBuffer sb = new StringBuffer(); - // Domain in format *.example.com, proxy IP is 1.2.3.4 --> 1-2-3-4.example.com - if (consoleProxyUrlDomain.startsWith("*")) { + if (StringUtils.isBlank(consoleProxyUrlDomain)) { + // Blank config, we use the proxy IP + sb.append(proxyIpAddress); + } else if (consoleProxyUrlDomain.startsWith("*")) { + // Domain in format *.example.com, proxy IP is 1.2.3.4 --> 1-2-3-4.example.com sb.append(proxyIpAddress.replaceAll("\\.", "-")); sb.append(consoleProxyUrlDomain.substring(1)); // skip the * - - // Otherwise we assume a valid domain if config not blank - } else if (StringUtils.isNotBlank(consoleProxyUrlDomain)) { - sb.append(consoleProxyUrlDomain); - - // Blank config, we use the proxy IP } else { - sb.append(proxyIpAddress); + // Otherwise we assume a valid domain if config not blank + sb.append(consoleProxyUrlDomain); } return sb.toString(); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 79570534596..1cc925b3ccf 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1071,16 +1071,16 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac long destHostId = dest.getHost().getId(); vm.setPodIdToDeployIn(dest.getPod().getId()); final Long cluster_id = dest.getCluster().getId(); - final ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio"); - final ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, "memoryOvercommitRatio"); - //storing the value of overcommit in the vm_details table for doing a capacity check in case the cluster overcommit ratio is changed. - if (userVmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio") == null && + final ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.CPU_OVER_COMMIT_RATIO); + final ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); + //storing the value of overcommit in the user_vm_details table for doing a capacity check in case the cluster overcommit ratio is changed. + if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) == null && (Float.parseFloat(cluster_detail_cpu.getValue()) > 1f || Float.parseFloat(cluster_detail_ram.getValue()) > 1f)) { - userVmDetailsDao.addDetail(vm.getId(), "cpuOvercommitRatio", cluster_detail_cpu.getValue(), true); - userVmDetailsDao.addDetail(vm.getId(), "memoryOvercommitRatio", cluster_detail_ram.getValue(), true); - } else if (userVmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio") != null) { - userVmDetailsDao.addDetail(vm.getId(), "cpuOvercommitRatio", cluster_detail_cpu.getValue(), true); - userVmDetailsDao.addDetail(vm.getId(), "memoryOvercommitRatio", cluster_detail_ram.getValue(), true); + userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true); + userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true); + } else if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) != null) { + userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true); + userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true); } vmProfile.setCpuOvercommitRatio(Float.parseFloat(cluster_detail_cpu.getValue())); @@ -1160,8 +1160,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac // Remove the information on whether it was a deploy vm request.The deployvm=true information // is set only when the vm is being deployed. When a vm is started from a stop state the // information isn't set, - if (userVmDetailsDao.findDetail(vm.getId(), "deployvm") != null) { - userVmDetailsDao.removeDetail(vm.getId(), "deployvm"); + if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.DEPLOY_VM) != null) { + userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.DEPLOY_VM); } startedVm = vm; @@ -1455,7 +1455,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (platform != null) { final UserVmVO userVm = _userVmDao.findById(vm.getId()); _userVmDao.loadDetails(userVm); - userVm.setDetail("platform", platform); + userVm.setDetail(VmDetailConstants.PLATFORM, platform); _userVmDao.saveDetails(userVm); } } @@ -1731,7 +1731,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (platform != null) { final UserVmVO userVm = _userVmDao.findById(vm.getId()); _userVmDao.loadDetails(userVm); - userVm.setDetail("platform", platform); + userVm.setDetail(VmDetailConstants.PLATFORM, platform); _userVmDao.saveDetails(userVm); } } @@ -3174,16 +3174,16 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private void updateVmMetaData(Long vmId, String platform) { UserVmVO userVm = _userVmDao.findById(vmId); _userVmDao.loadDetails(userVm); - if ( userVm.details.containsKey("timeoffset")) { - userVm.details.remove("timeoffset"); + if ( userVm.details.containsKey(VmDetailConstants.TIME_OFFSET)) { + userVm.details.remove(VmDetailConstants.TIME_OFFSET); } - userVm.setDetail("platform", platform); + userVm.setDetail(VmDetailConstants.PLATFORM, platform); String pvdriver = "xenserver56"; if ( platform.contains("device_id")) { pvdriver = "xenserver61"; } - if (!userVm.details.containsKey("hypervisortoolsversion") || !userVm.details.get("hypervisortoolsversion").equals(pvdriver)) { - userVm.setDetail("hypervisortoolsversion", pvdriver); + if (!userVm.details.containsKey(VmDetailConstants.HYPERVISOR_TOOLS_VERSION) || !userVm.details.get(VmDetailConstants.HYPERVISOR_TOOLS_VERSION).equals(pvdriver)) { + userVm.setDetail(VmDetailConstants.HYPERVISOR_TOOLS_VERSION, pvdriver); } _userVmDao.saveDetails(userVm); } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41200to41300.sql b/engine/schema/src/main/resources/META-INF/db/schema-41200to41300.sql index 9343af2eb77..a582735b80b 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41200to41300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41200to41300.sql @@ -23,6 +23,10 @@ INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '6.7', 128, 0, 13, 32, 1, 1); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'VMware', '6.7', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='VMware' AND hypervisor_version='6.5'; +-- XenServer 7.6 +INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, max_data_volumes_limit, storage_motion_supported) values (UUID(), 'XenServer', '7.6.0', 500, 13, 1); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '7.6.0', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='7.5.0'; + -- DPDK client and server mode support ALTER TABLE `cloud`.`service_offering_details` CHANGE COLUMN `value` `value` TEXT NOT NULL; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index abbe3243c6c..c195712e62a 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -42,68 +42,7 @@ import java.util.TimeZone; import java.util.UUID; import javax.naming.ConfigurationException; - -import org.apache.commons.lang.math.NumberUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; -import org.joda.time.Duration; - -import com.google.gson.Gson; -import com.vmware.vim25.AboutInfo; -import com.vmware.vim25.BoolPolicy; -import com.vmware.vim25.ComputeResourceSummary; -import com.vmware.vim25.CustomFieldStringValue; -import com.vmware.vim25.DVPortConfigInfo; -import com.vmware.vim25.DVPortConfigSpec; -import com.vmware.vim25.DasVmPriority; -import com.vmware.vim25.DatastoreSummary; -import com.vmware.vim25.DistributedVirtualPort; -import com.vmware.vim25.DistributedVirtualSwitchPortConnection; -import com.vmware.vim25.DistributedVirtualSwitchPortCriteria; -import com.vmware.vim25.DynamicProperty; -import com.vmware.vim25.GuestInfo; -import com.vmware.vim25.HostCapability; -import com.vmware.vim25.HostHostBusAdapter; -import com.vmware.vim25.HostInternetScsiHba; -import com.vmware.vim25.ManagedObjectReference; -import com.vmware.vim25.ObjectContent; -import com.vmware.vim25.OptionValue; -import com.vmware.vim25.PerfCounterInfo; -import com.vmware.vim25.PerfEntityMetric; -import com.vmware.vim25.PerfEntityMetricBase; -import com.vmware.vim25.PerfMetricId; -import com.vmware.vim25.PerfMetricIntSeries; -import com.vmware.vim25.PerfMetricSeries; -import com.vmware.vim25.PerfQuerySpec; -import com.vmware.vim25.PerfSampleInfo; -import com.vmware.vim25.RuntimeFaultFaultMsg; -import com.vmware.vim25.ToolsUnavailableFaultMsg; -import com.vmware.vim25.VMwareDVSPortSetting; -import com.vmware.vim25.VimPortType; -import com.vmware.vim25.VirtualDevice; -import com.vmware.vim25.VirtualDeviceBackingInfo; -import com.vmware.vim25.VirtualDeviceConfigSpec; -import com.vmware.vim25.VirtualDeviceConfigSpecOperation; -import com.vmware.vim25.VirtualDisk; -import com.vmware.vim25.VirtualDiskFlatVer2BackingInfo; -import com.vmware.vim25.VirtualEthernetCard; -import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo; -import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo; -import com.vmware.vim25.VirtualEthernetCardOpaqueNetworkBackingInfo; -import com.vmware.vim25.VirtualMachineConfigSpec; -import com.vmware.vim25.VirtualMachineFileInfo; -import com.vmware.vim25.VirtualMachineFileLayoutEx; -import com.vmware.vim25.VirtualMachineFileLayoutExFileInfo; -import com.vmware.vim25.VirtualMachineGuestOsIdentifier; -import com.vmware.vim25.VirtualMachinePowerState; -import com.vmware.vim25.VirtualMachineRelocateSpec; -import com.vmware.vim25.VirtualMachineRelocateSpecDiskLocator; -import com.vmware.vim25.VirtualMachineRuntimeInfo; -import com.vmware.vim25.VirtualMachineToolsStatus; -import com.vmware.vim25.VirtualMachineVideoCard; -import com.vmware.vim25.VirtualUSBController; -import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec; +import javax.xml.datatype.XMLGregorianCalendar; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.storage.command.CopyCommand; @@ -114,6 +53,11 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.math.NumberUtils; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; +import org.joda.time.Duration; import com.cloud.agent.IAgentControl; import com.cloud.agent.api.Answer; @@ -211,6 +155,7 @@ import com.cloud.agent.api.UnregisterVMCommand; import com.cloud.agent.api.UpgradeSnapshotCommand; import com.cloud.agent.api.ValidateSnapshotAnswer; import com.cloud.agent.api.ValidateSnapshotCommand; +import com.cloud.agent.api.VmDiskStatsEntry; import com.cloud.agent.api.VmStatsEntry; import com.cloud.agent.api.VolumeStatsEntry; import com.cloud.agent.api.check.CheckSshAnswer; @@ -310,6 +255,60 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.PowerState; import com.cloud.vm.VirtualMachineName; import com.cloud.vm.VmDetailConstants; +import com.google.gson.Gson; +import com.vmware.vim25.AboutInfo; +import com.vmware.vim25.BoolPolicy; +import com.vmware.vim25.ComputeResourceSummary; +import com.vmware.vim25.CustomFieldStringValue; +import com.vmware.vim25.DVPortConfigInfo; +import com.vmware.vim25.DVPortConfigSpec; +import com.vmware.vim25.DasVmPriority; +import com.vmware.vim25.DatastoreSummary; +import com.vmware.vim25.DistributedVirtualPort; +import com.vmware.vim25.DistributedVirtualSwitchPortConnection; +import com.vmware.vim25.DistributedVirtualSwitchPortCriteria; +import com.vmware.vim25.DynamicProperty; +import com.vmware.vim25.GuestInfo; +import com.vmware.vim25.HostCapability; +import com.vmware.vim25.HostHostBusAdapter; +import com.vmware.vim25.HostInternetScsiHba; +import com.vmware.vim25.ManagedObjectReference; +import com.vmware.vim25.ObjectContent; +import com.vmware.vim25.OptionValue; +import com.vmware.vim25.PerfCounterInfo; +import com.vmware.vim25.PerfEntityMetric; +import com.vmware.vim25.PerfEntityMetricBase; +import com.vmware.vim25.PerfMetricId; +import com.vmware.vim25.PerfMetricIntSeries; +import com.vmware.vim25.PerfMetricSeries; +import com.vmware.vim25.PerfQuerySpec; +import com.vmware.vim25.RuntimeFaultFaultMsg; +import com.vmware.vim25.ToolsUnavailableFaultMsg; +import com.vmware.vim25.VMwareDVSPortSetting; +import com.vmware.vim25.VimPortType; +import com.vmware.vim25.VirtualDevice; +import com.vmware.vim25.VirtualDeviceBackingInfo; +import com.vmware.vim25.VirtualDeviceConfigSpec; +import com.vmware.vim25.VirtualDeviceConfigSpecOperation; +import com.vmware.vim25.VirtualDisk; +import com.vmware.vim25.VirtualDiskFlatVer2BackingInfo; +import com.vmware.vim25.VirtualEthernetCard; +import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo; +import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo; +import com.vmware.vim25.VirtualEthernetCardOpaqueNetworkBackingInfo; +import com.vmware.vim25.VirtualMachineConfigSpec; +import com.vmware.vim25.VirtualMachineFileInfo; +import com.vmware.vim25.VirtualMachineFileLayoutEx; +import com.vmware.vim25.VirtualMachineFileLayoutExFileInfo; +import com.vmware.vim25.VirtualMachineGuestOsIdentifier; +import com.vmware.vim25.VirtualMachinePowerState; +import com.vmware.vim25.VirtualMachineRelocateSpec; +import com.vmware.vim25.VirtualMachineRelocateSpecDiskLocator; +import com.vmware.vim25.VirtualMachineRuntimeInfo; +import com.vmware.vim25.VirtualMachineToolsStatus; +import com.vmware.vim25.VirtualMachineVideoCard; +import com.vmware.vim25.VirtualUSBController; +import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec; public class VmwareResource implements StoragePoolResource, ServerResource, VmwareHostService, VirtualRouterDeployer { private static final Logger s_logger = Logger.getLogger(VmwareResource.class); @@ -1874,7 +1873,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // Check for multi-cores per socket settings int numCoresPerSocket = 1; - String coresPerSocket = vmSpec.getDetails().get("cpu.corespersocket"); + String coresPerSocket = vmSpec.getDetails().get(VmDetailConstants.CPU_CORE_PER_SOCKET); if (coresPerSocket != null) { String apiVersion = HypervisorHostHelper.getVcenterApiVersion(vmMo.getContext()); // Property 'numCoresPerSocket' is supported since vSphere API 5.0 @@ -3469,6 +3468,120 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } protected Answer execute(GetVmDiskStatsCommand cmd) { + try { + final VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); + final ManagedObjectReference perfMgr = getServiceContext().getServiceContent().getPerfManager(); + VimPortType service = getServiceContext().getService(); + + final int intervalSeconds = 300; + final XMLGregorianCalendar startTime = VmwareHelper.getXMLGregorianCalendar(new Date(), intervalSeconds); + final XMLGregorianCalendar endTime = VmwareHelper.getXMLGregorianCalendar(new Date(), 0); + + PerfCounterInfo diskReadIOPerfCounterInfo = null; + PerfCounterInfo diskWriteIOPerfCounterInfo = null; + PerfCounterInfo diskReadKbsPerfCounterInfo = null; + PerfCounterInfo diskWriteKbsPerfCounterInfo = null; + + // https://pubs.vmware.com/vsphere-5-5/topic/com.vmware.wssdk.apiref.doc/virtual_disk_counters.html + List cInfo = getServiceContext().getVimClient().getDynamicProperty(perfMgr, "perfCounter"); + for (PerfCounterInfo info : cInfo) { + if ("virtualdisk".equalsIgnoreCase(info.getGroupInfo().getKey()) && "average".equalsIgnoreCase(info.getRollupType().value())) { + if ("numberReadAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskReadIOPerfCounterInfo = info; + } + if ("numberWriteAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskWriteIOPerfCounterInfo = info; + } + if ("read".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskReadKbsPerfCounterInfo = info; + } + if ("write".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskWriteKbsPerfCounterInfo = info; + } + } + } + + final ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter(); + final DatacenterMO dcMo = new DatacenterMO(getServiceContext(), dcMor); + + final HashMap> vmStatsMap = new HashMap<>(); + for (final String vmName : cmd.getVmNames()) { + final VirtualMachineMO vmMo = dcMo.findVm(vmName); + final List diskStats = new ArrayList<>(); + for (final VirtualDisk disk : vmMo.getAllDiskDevice()) { + final String diskBusName = vmMo.getDeviceBusName(vmMo.getAllDeviceList(), disk); + long readReq = 0; + long readBytes = 0; + long writeReq = 0; + long writeBytes = 0; + + final ArrayList perfMetricsIds = new ArrayList(); + if (diskReadIOPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadIOPerfCounterInfo, diskBusName)); + } + if (diskWriteIOPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteIOPerfCounterInfo, diskBusName)); + } + if (diskReadKbsPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadKbsPerfCounterInfo, diskBusName)); + } + if (diskWriteKbsPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteKbsPerfCounterInfo, diskBusName)); + } + + if (perfMetricsIds.size() > 0) { + final PerfQuerySpec qSpec = new PerfQuerySpec(); + qSpec.setEntity(vmMo.getMor()); + qSpec.setFormat("normal"); + qSpec.setIntervalId(intervalSeconds); + qSpec.setStartTime(startTime); + qSpec.setEndTime(endTime); + qSpec.getMetricId().addAll(perfMetricsIds); + + for (final PerfEntityMetricBase perfValue : service.queryPerf(perfMgr, Collections.singletonList(qSpec))) { + if (!(perfValue instanceof PerfEntityMetric)) { + continue; + } + final List values = ((PerfEntityMetric)perfValue).getValue(); + if (values == null || values.isEmpty()) { + continue; + } + for (final PerfMetricSeries value : values) { + if (!(value instanceof PerfMetricIntSeries) || !value.getId().getInstance().equals(diskBusName)) { + continue; + } + final List perfStats = ((PerfMetricIntSeries)value).getValue(); + if (perfStats.size() > 0) { + long sum = 0; + for (long val : perfStats) { + sum += val; + } + long avg = sum / perfStats.size(); + if (value.getId().getCounterId() == diskReadIOPerfCounterInfo.getKey()) { + readReq = avg; + } else if (value.getId().getCounterId() == diskWriteIOPerfCounterInfo.getKey()) { + writeReq = avg; + } else if (value.getId().getCounterId() == diskReadKbsPerfCounterInfo.getKey()) { + readBytes = avg * 1024; + } else if (value.getId().getCounterId() == diskWriteKbsPerfCounterInfo.getKey()) { + writeBytes = avg * 1024; + } + } + } + } + } + diskStats.add(new VmDiskStatsEntry(vmName, VmwareHelper.getDiskDeviceFileName(disk), writeReq, readReq, writeBytes, readBytes)); + } + if (diskStats.size() > 0) { + vmStatsMap.put(vmName, diskStats); + } + } + if (vmStatsMap.size() > 0) { + return new GetVmDiskStatsAnswer(cmd, "", cmd.getHostName(), vmStatsMap); + } + } catch (Exception e) { + s_logger.error("Unable to execute GetVmDiskStatsCommand due to " + VmwareHelper.getExceptionMessage(e), e); + } return new GetVmDiskStatsAnswer(cmd, null, null, null); } @@ -5857,12 +5970,21 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa HashMap vmResponseMap = new HashMap(); ManagedObjectReference perfMgr = getServiceContext().getServiceContent().getPerfManager(); VimPortType service = getServiceContext().getService(); + PerfCounterInfo rxPerfCounterInfo = null; PerfCounterInfo txPerfCounterInfo = null; + PerfCounterInfo diskReadIOPerfCounterInfo = null; + PerfCounterInfo diskWriteIOPerfCounterInfo = null; + PerfCounterInfo diskReadKbsPerfCounterInfo = null; + PerfCounterInfo diskWriteKbsPerfCounterInfo = null; + + final int intervalSeconds = 300; + final XMLGregorianCalendar startTime = VmwareHelper.getXMLGregorianCalendar(new Date(), intervalSeconds); + final XMLGregorianCalendar endTime = VmwareHelper.getXMLGregorianCalendar(new Date(), 0); List cInfo = getServiceContext().getVimClient().getDynamicProperty(perfMgr, "perfCounter"); for (PerfCounterInfo info : cInfo) { - if ("net".equalsIgnoreCase(info.getGroupInfo().getKey())) { + if ("net".equalsIgnoreCase(info.getGroupInfo().getKey()) && "average".equalsIgnoreCase(info.getRollupType().value())) { if ("transmitted".equalsIgnoreCase(info.getNameInfo().getKey())) { txPerfCounterInfo = info; } @@ -5870,6 +5992,20 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa rxPerfCounterInfo = info; } } + if ("virtualdisk".equalsIgnoreCase(info.getGroupInfo().getKey())) { + if ("numberReadAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskReadIOPerfCounterInfo = info; + } + if ("numberWriteAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskWriteIOPerfCounterInfo = info; + } + if ("read".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskReadKbsPerfCounterInfo = info; + } + if ("write".equalsIgnoreCase(info.getNameInfo().getKey())) { + diskWriteKbsPerfCounterInfo = info; + } + } } int key = ((HostMO)hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME); @@ -5885,8 +6021,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa final String memMbStr = "config.hardware.memoryMB"; final String allocatedCpuStr = "summary.runtime.maxCpuUsage"; - ObjectContent[] ocs = - hyperHost.getVmPropertiesOnHyperHost(new String[] {"name", numCpuStr, cpuUseStr ,guestMemUseStr ,memLimitStr ,memMbStr,allocatedCpuStr ,instanceNameCustomField}); + ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost(new String[] { + "name", numCpuStr, cpuUseStr, guestMemUseStr, memLimitStr, memMbStr,allocatedCpuStr, instanceNameCustomField + }); if (ocs != null && ocs.length > 0) { for (ObjectContent oc : ocs) { @@ -5923,7 +6060,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } maxCpuUsage = (maxCpuUsage/allocatedCpu)*100; - new VirtualMachineMO(hyperHost.getContext(), oc.getObj()); if (vmInternalCSName != null) { name = vmInternalCSName; } else { @@ -5937,60 +6073,86 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa ManagedObjectReference vmMor = hyperHost.findVmOnHyperHost(name).getMor(); assert (vmMor != null); - ArrayList vmNetworkMetrics = new ArrayList(); - // get all the metrics from the available sample period - List perfMetrics = service.queryAvailablePerfMetric(perfMgr, vmMor, null, null, null); - if (perfMetrics != null) { - for (int index = 0; index < perfMetrics.size(); ++index) { - if (((rxPerfCounterInfo != null) && (perfMetrics.get(index).getCounterId() == rxPerfCounterInfo.getKey())) - || ((txPerfCounterInfo != null) && (perfMetrics.get(index).getCounterId() == txPerfCounterInfo.getKey()))) { - vmNetworkMetrics.add(perfMetrics.get(index)); - } - } - } - double networkReadKBs = 0; double networkWriteKBs = 0; - long sampleDuration = 0; + double diskReadIops = 0; + double diskWriteIops = 0; + double diskReadKbs = 0; + double diskWriteKbs = 0; - if (vmNetworkMetrics.size() != 0) { - PerfQuerySpec qSpec = new PerfQuerySpec(); + final ArrayList perfMetricsIds = new ArrayList(); + if (rxPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(rxPerfCounterInfo, "")); + } + if (txPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(txPerfCounterInfo, "")); + } + if (diskReadIOPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadIOPerfCounterInfo, "*")); + } + if (diskWriteIOPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteIOPerfCounterInfo, "*")); + } + if (diskReadKbsPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadKbsPerfCounterInfo, "")); + } + if (diskWriteKbsPerfCounterInfo != null) { + perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteKbsPerfCounterInfo, "")); + } + + if (perfMetricsIds.size() > 0) { + final PerfQuerySpec qSpec = new PerfQuerySpec(); qSpec.setEntity(vmMor); - PerfMetricId[] availableMetricIds = vmNetworkMetrics.toArray(new PerfMetricId[0]); - qSpec.getMetricId().addAll(Arrays.asList(availableMetricIds)); - List qSpecs = new ArrayList(); - qSpecs.add(qSpec); - List values = service.queryPerf(perfMgr, qSpecs); - - for (int i = 0; i < values.size(); ++i) { - List infos = ((PerfEntityMetric)values.get(i)).getSampleInfo(); - if (infos != null && infos.size() > 0) { - int endMs = infos.get(infos.size() - 1).getTimestamp().getSecond() * 1000 + infos.get(infos.size() - 1).getTimestamp().getMillisecond(); - int beginMs = infos.get(0).getTimestamp().getSecond() * 1000 + infos.get(0).getTimestamp().getMillisecond(); - sampleDuration = (endMs - beginMs) / 1000; - List vals = ((PerfEntityMetric)values.get(i)).getValue(); - for (int vi = 0; ((vals != null) && (vi < vals.size())); ++vi) { - if (vals.get(vi) instanceof PerfMetricIntSeries) { - PerfMetricIntSeries val = (PerfMetricIntSeries)vals.get(vi); - List perfValues = val.getValue(); - Long sumRate = 0L; - for (int j = 0; j < infos.size(); j++) { // Size of the array matches the size as the PerfSampleInfo - sumRate += perfValues.get(j); - } - Long averageRate = sumRate / infos.size(); - if (vals.get(vi).getId().getCounterId() == rxPerfCounterInfo.getKey()) { - networkReadKBs = sampleDuration * averageRate; //get the average RX rate multiplied by sampled duration - } - if (vals.get(vi).getId().getCounterId() == txPerfCounterInfo.getKey()) { - networkWriteKBs = sampleDuration * averageRate;//get the average TX rate multiplied by sampled duration - } - } + qSpec.setFormat("normal"); + qSpec.setIntervalId(intervalSeconds); + qSpec.setStartTime(startTime); + qSpec.setEndTime(endTime); + qSpec.getMetricId().addAll(perfMetricsIds); + final List perfValues = service.queryPerf(perfMgr, Collections.singletonList(qSpec)); + for (final PerfEntityMetricBase perfValue : perfValues) { + if (!(perfValue instanceof PerfEntityMetric)) { + continue; + } + final List seriesList = ((PerfEntityMetric) perfValue).getValue(); + for (final PerfMetricSeries series : seriesList) { + if (!(series instanceof PerfMetricIntSeries)) { + continue; + } + final List values = ((PerfMetricIntSeries) series).getValue(); + double sum = 0; + for (final Long value : values) { + sum += value; + } + double avg = sum / values.size(); + if (series.getId().getCounterId() == rxPerfCounterInfo.getKey()) { + networkReadKBs = avg; + } + if (series.getId().getCounterId() == txPerfCounterInfo.getKey()) { + networkWriteKBs = avg; + } + if (series.getId().getCounterId() == diskReadIOPerfCounterInfo.getKey()) { + diskReadIops += avg; + } + if (series.getId().getCounterId() == diskWriteIOPerfCounterInfo.getKey()) { + diskWriteIops += avg; + } + if (series.getId().getCounterId() == diskReadKbsPerfCounterInfo.getKey()) { + diskReadKbs = avg; + } + if (series.getId().getCounterId() == diskWriteKbsPerfCounterInfo.getKey()) { + diskWriteKbs = avg; } } } } - vmResponseMap.put(name, new VmStatsEntry( NumberUtils.toDouble(memkb)*1024,NumberUtils.toDouble(guestMemusage)*1024,NumberUtils.toDouble(memlimit)*1024, - maxCpuUsage, networkReadKBs, networkWriteKBs, NumberUtils.toInt(numberCPUs), "vm")); + + final VmStatsEntry vmStats = new VmStatsEntry( NumberUtils.toDouble(memkb)*1024,NumberUtils.toDouble(guestMemusage)*1024,NumberUtils.toDouble(memlimit)*1024, + maxCpuUsage, networkReadKBs, networkWriteKBs, NumberUtils.toInt(numberCPUs), "vm"); + vmStats.setDiskReadIOs(diskReadIops); + vmStats.setDiskWriteIOs(diskWriteIops); + vmStats.setDiskReadKBs(diskReadKbs); + vmStats.setDiskWriteKBs(diskWriteKbs); + vmResponseMap.put(name, vmStats); } } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index ff7b0a3ab53..79a9fb22972 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -134,6 +134,7 @@ import com.cloud.utils.ssh.SSHCmdHelper; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.PowerState; +import com.cloud.vm.VmDetailConstants; import com.trilead.ssh2.SCPClient; import com.xensource.xenapi.Bond; import com.xensource.xenapi.Connection; @@ -1862,18 +1863,18 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe final Map details = vmSpec.getDetails(); if (details != null) { - final String platformstring = details.get("platform"); + final String platformstring = details.get(VmDetailConstants.PLATFORM); if (platformstring != null && !platformstring.isEmpty()) { final Map platform = StringUtils.stringToMap(platformstring); vm.setPlatform(conn, platform); } else { - final String timeoffset = details.get("timeoffset"); + final String timeoffset = details.get(VmDetailConstants.TIME_OFFSET); if (timeoffset != null) { final Map platform = vm.getPlatform(conn); - platform.put("timeoffset", timeoffset); + platform.put(VmDetailConstants.TIME_OFFSET, timeoffset); vm.setPlatform(conn, platform); } - final String coresPerSocket = details.get("cpu.corespersocket"); + final String coresPerSocket = details.get(VmDetailConstants.CPU_CORE_PER_SOCKET); if (coresPerSocket != null) { final Map platform = vm.getPlatform(conn); platform.put("cores-per-socket", coresPerSocket); @@ -1881,7 +1882,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe } } if (!BootloaderType.CD.equals(vmSpec.getBootloader())) { - final String xenservertoolsversion = details.get("hypervisortoolsversion"); + final String xenservertoolsversion = details.get(VmDetailConstants.HYPERVISOR_TOOLS_VERSION); if ((xenservertoolsversion == null || !xenservertoolsversion.equalsIgnoreCase("xenserver61")) && vmSpec.getGpuDevice() == null) { final Map platform = vm.getPlatform(conn); platform.remove("device_id"); diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixGetVmDiskStatsCommandWrapper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixGetVmDiskStatsCommandWrapper.java index 04090ce7436..d15e0a71727 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixGetVmDiskStatsCommandWrapper.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixGetVmDiskStatsCommandWrapper.java @@ -33,4 +33,4 @@ public final class CitrixGetVmDiskStatsCommandWrapper extends CommandWrapper scaleUpPolicies, List scaleDownPolicies) { diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index d0d87a8eda3..95faaa5f0ad 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -554,7 +554,7 @@ public class ApiResponseHelper implements ResponseGenerator { ResourceTagResponse tagResponse = createResourceTagResponse(tag, true); CollectionUtils.addIgnoreNull(tagResponses, tagResponse); } - snapshotResponse.setTags(tagResponses); + snapshotResponse.setTags(new HashSet<>(tagResponses)); snapshotResponse.setObjectName("snapshot"); return snapshotResponse; @@ -656,6 +656,14 @@ public class ApiResponseHelper implements ResponseGenerator { policyResponse.setForDisplay(policy.isDisplay()); policyResponse.setObjectName("snapshotpolicy"); + List tags = _resourceTagDao.listBy(policy.getId(), ResourceObjectType.SnapshotPolicy); + List tagResponses = new ArrayList(); + for (ResourceTag tag : tags) { + ResourceTagResponse tagResponse = createResourceTagResponse(tag, false); + CollectionUtils.addIgnoreNull(tagResponses, tagResponse); + } + policyResponse.setTags(new HashSet<>(tagResponses)); + return policyResponse; } diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index f116dc6f429..03e9297c663 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -17,12 +17,17 @@ package com.cloud.api.query; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.ListIterator; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.inject.Inject; @@ -62,6 +67,7 @@ import org.apache.cloudstack.api.command.user.offering.ListDiskOfferingsCmd; import org.apache.cloudstack.api.command.user.offering.ListServiceOfferingsCmd; import org.apache.cloudstack.api.command.user.project.ListProjectInvitationsCmd; import org.apache.cloudstack.api.command.user.project.ListProjectsCmd; +import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; @@ -72,6 +78,7 @@ import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; import org.apache.cloudstack.api.command.user.zone.ListZonesCmd; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.AsyncJobResponse; +import org.apache.cloudstack.api.response.DetailOptionsResponse; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.DomainRouterResponse; @@ -215,10 +222,12 @@ import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Func; import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.UserVmVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; @@ -3444,6 +3453,61 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q hypervisorType, true, cmd.listInReadyState(), permittedAccounts, caller, listProjectResourcesCriteria, tags, showRemovedISO, null, null); } + @Override + public DetailOptionsResponse listDetailOptions(final ListDetailOptionsCmd cmd) { + final ResourceObjectType type = cmd.getResourceType(); + final String resourceUuid = cmd.getResourceId(); + final Map> options = new HashMap<>(); + switch (type) { + case Template: + case UserVm: + HypervisorType hypervisorType = HypervisorType.None; + if (!Strings.isNullOrEmpty(resourceUuid) && ResourceObjectType.Template.equals(type)) { + hypervisorType = _templateDao.findByUuid(resourceUuid).getHypervisorType(); + } + if (!Strings.isNullOrEmpty(resourceUuid) && ResourceObjectType.UserVm.equals(type)) { + hypervisorType = _vmInstanceDao.findByUuid(resourceUuid).getHypervisorType(); + } + fillVMOrTemplateDetailOptions(options, hypervisorType); + break; + default: + throw new CloudRuntimeException("Resource type not supported."); + } + if (CallContext.current().getCallingAccount().getType() != Account.ACCOUNT_TYPE_ADMIN) { + final List userBlacklistedSettings = Stream.of(QueryService.UserVMBlacklistedDetails.value().split(",")) + .map(item -> (item).trim()) + .collect(Collectors.toList()); + for (final String detail : userBlacklistedSettings) { + if (options.containsKey(detail)) { + options.remove(detail); + } + } + } + return new DetailOptionsResponse(options); + } + + private void fillVMOrTemplateDetailOptions(final Map> options, final HypervisorType hypervisorType) { + if (options == null) { + throw new CloudRuntimeException("Invalid/null detail-options response object passed"); + } + + options.put(VmDetailConstants.KEYBOARD, Arrays.asList("uk", "us", "jp", "fr")); + options.put(VmDetailConstants.CPU_CORE_PER_SOCKET, Collections.emptyList()); + options.put(VmDetailConstants.ROOT_DISK_SIZE, Collections.emptyList()); + + if (HypervisorType.KVM.equals(hypervisorType)) { + options.put(VmDetailConstants.ROOT_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "virtio")); + } + + if (HypervisorType.VMware.equals(hypervisorType)) { + options.put(VmDetailConstants.NIC_ADAPTER, Arrays.asList("E1000", "PCNet32", "Vmxnet2", "Vmxnet3")); + options.put(VmDetailConstants.ROOT_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "lsilogic", "lsisas1068", "buslogic", "pvscsi")); + options.put(VmDetailConstants.DATA_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "lsilogic", "lsisas1068", "buslogic", "pvscsi")); + options.put(VmDetailConstants.NESTED_VIRTUALIZATION_FLAG, Arrays.asList("true", "false")); + options.put(VmDetailConstants.SVGA_VRAM_SIZE, Collections.emptyList()); + } + } + @Override public ListResponse searchForAffinityGroups(ListAffinityGroupsCmd cmd) { Pair, Integer> result = searchForAffinityGroupsInternal(cmd); @@ -3750,7 +3814,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } response.setResponses(result); return response; - } + } @Override public String getConfigComponentName() { diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java index 61e975211de..c7977356379 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java @@ -36,6 +36,8 @@ import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.Volume; import com.cloud.user.AccountManager; +import com.cloud.user.VmDiskStatisticsVO; +import com.cloud.user.dao.VmDiskStatisticsDao; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -47,6 +49,8 @@ public class VolumeJoinDaoImpl extends GenericDaoBaseWithTagInformation volSearch; @@ -102,6 +106,14 @@ public class VolumeJoinDaoImpl extends GenericDaoBaseWithTagInformation params) throws ConfigurationException { _vmCapacityReleaseInterval = NumbersUtil.parseInt(_configDao.getValue(Config.CapacitySkipcountingHours.key()), 3600); @@ -638,8 +637,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, Float ramOvercommitRatio = 1.0f; long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - vm.getUpdateTime().getTime()) / 1000; if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) { - UserVmDetailVO vmDetailCpu = _userVmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio"); - UserVmDetailVO vmDetailRam = _userVmDetailsDao.findDetail(vm.getId(), "memoryOvercommitRatio"); + UserVmDetailVO vmDetailCpu = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO); + UserVmDetailVO vmDetailRam = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); if (vmDetailCpu != null) { //if vmDetail_cpu is not null it means it is running in a overcommited cluster. cpuOvercommitRatio = Float.parseFloat(vmDetailCpu.getValue()); @@ -669,14 +668,14 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, } else { // signal if not done already, that the VM has been stopped for skip.counting.hours, // hence capacity will not be reserved anymore. - UserVmDetailVO messageSentFlag = _userVmDetailsDao.findDetail(vm.getId(), MESSAGE_RESERVED_CAPACITY_FREED_FLAG); + UserVmDetailVO messageSentFlag = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.MESSAGE_RESERVED_CAPACITY_FREED_FLAG); if (messageSentFlag == null || !Boolean.valueOf(messageSentFlag.getValue())) { _messageBus.publish(_name, "VM_ReservedCapacity_Free", PublishScope.LOCAL, vm); if (vm.getType() == VirtualMachine.Type.User) { UserVmVO userVM = _userVMDao.findById(vm.getId()); _userVMDao.loadDetails(userVM); - userVM.setDetail(MESSAGE_RESERVED_CAPACITY_FREED_FLAG, "true"); + userVM.setDetail(VmDetailConstants.MESSAGE_RESERVED_CAPACITY_FREED_FLAG, "true"); _userVMDao.saveDetails(userVM); } } @@ -903,7 +902,7 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, UserVmVO userVM = _userVMDao.findById(vm.getId()); _userVMDao.loadDetails(userVM); // free the message sent flag if it exists - userVM.setDetail(MESSAGE_RESERVED_CAPACITY_FREED_FLAG, "false"); + userVM.setDetail(VmDetailConstants.MESSAGE_RESERVED_CAPACITY_FREED_FLAG, "false"); _userVMDao.saveDetails(userVM); } diff --git a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java index 4d2452d45ce..76e4fc03ce7 100644 --- a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java +++ b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java @@ -77,6 +77,7 @@ import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; @@ -195,7 +196,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle } private String getSshKey(VirtualMachineProfile profile) { - final UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), "SSH.PublicKey"); + final UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), VmDetailConstants.SSH_PUBLIC_KEY); return (vmDetailSshKey!=null ? vmDetailSshKey.getValue() : null); } @@ -262,7 +263,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle final String password_encrypted = DBEncryptionUtil.encrypt(password); final UserVmVO userVmVO = _userVmDao.findById(vm.getId()); - _userVmDetailsDao.addDetail(vm.getId(), "password", password_encrypted, false); + _userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.PASSWORD, password_encrypted, false); userVmVO.setUpdateParameters(true); _userVmDao.update(userVmVO.getId(), userVmVO); diff --git a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java index b78dcfdbb5d..aefa528bea7 100644 --- a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java +++ b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java @@ -31,6 +31,7 @@ import org.cloud.network.router.deployment.RouterDeploymentDefinitionBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import com.cloud.vm.VmDetailConstants; import com.google.gson.Gson; import org.apache.cloudstack.api.command.admin.router.ConfigureOvsElementCmd; @@ -718,7 +719,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor, RedundantResource, DnsServ final UserVmVO userVmVO = _userVmDao.findById(vm.getId()); _userVmDao.loadDetails(userVmVO); - userVmVO.setDetail("password", password_encrypted); + userVmVO.setDetail(VmDetailConstants.PASSWORD, password_encrypted); _userVmDao.saveDetails(userVmVO); userVmVO.setUpdateParameters(true); diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 27fa42cc4c9..d07a4383d01 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -31,13 +31,6 @@ import java.util.concurrent.ConcurrentHashMap; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.utils.Pair; -import com.cloud.vm.dao.UserVmDetailsDao; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.commons.lang.ObjectUtils; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.cluster.AddClusterCmd; import org.apache.cloudstack.api.command.admin.cluster.DeleteClusterCmd; @@ -49,21 +42,25 @@ import org.apache.cloudstack.api.command.admin.host.ReconnectHostCmd; import org.apache.cloudstack.api.command.admin.host.UpdateHostCmd; import org.apache.cloudstack.api.command.admin.host.UpdateHostPasswordCmd; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.ObjectUtils; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; -import com.cloud.agent.api.GetVncPortCommand; -import com.cloud.agent.api.GetVncPortAnswer; import com.cloud.agent.api.GetGPUStatsAnswer; import com.cloud.agent.api.GetGPUStatsCommand; import com.cloud.agent.api.GetHostStatsAnswer; import com.cloud.agent.api.GetHostStatsCommand; +import com.cloud.agent.api.GetVncPortAnswer; +import com.cloud.agent.api.GetVncPortCommand; import com.cloud.agent.api.MaintainAnswer; import com.cloud.agent.api.MaintainCommand; import com.cloud.agent.api.PropagateResourceEventCommand; @@ -148,6 +145,7 @@ import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; +import com.cloud.utils.Pair; import com.cloud.utils.StringUtils; import com.cloud.utils.UriUtils; import com.cloud.utils.component.Manager; @@ -177,6 +175,8 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VmDetailConstants; +import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; import com.google.gson.Gson; @@ -1314,8 +1314,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, for (VMInstanceVO vm : vms) { GetVncPortAnswer vmVncPortAnswer = (GetVncPortAnswer) _agentMgr.easySend(hostId, new GetVncPortCommand(vm.getId(), vm.getInstanceName())); if (vmVncPortAnswer != null) { - userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.address", vmVncPortAnswer.getAddress(), true); - userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.port", String.valueOf(vmVncPortAnswer.getPort()), true); + userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.KVM_VNC_ADDRESS, vmVncPortAnswer.getAddress(), true); + userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.KVM_VNC_PORT, String.valueOf(vmVncPortAnswer.getPort()), true); } } } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 2c3e71543e3..4beddcc6e02 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -411,6 +411,7 @@ import org.apache.cloudstack.api.command.user.region.ha.gslb.ListGlobalLoadBalan import org.apache.cloudstack.api.command.user.region.ha.gslb.RemoveFromGlobalLoadBalancerRuleCmd; import org.apache.cloudstack.api.command.user.region.ha.gslb.UpdateGlobalLoadBalancerRuleCmd; import org.apache.cloudstack.api.command.user.resource.GetCloudIdentifierCmd; +import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.resource.ListHypervisorsCmd; import org.apache.cloudstack.api.command.user.resource.ListResourceLimitsCmd; import org.apache.cloudstack.api.command.user.resource.UpdateResourceCountCmd; @@ -2900,6 +2901,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(ExtractTemplateCmd.class); cmdList.add(ListTemplatePermissionsCmd.class); cmdList.add(ListTemplatesCmd.class); + cmdList.add(ListDetailOptionsCmd.class); cmdList.add(RegisterTemplateCmd.class); cmdList.add(UpdateTemplateCmd.class); cmdList.add(UpdateTemplatePermissionsCmd.class); diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 7ae7905ce48..b81507ad52b 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -690,14 +690,14 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc s_logger.debug("VmDiskStatsTask is running..."); SearchCriteria sc = createSearchCriteriaForHostTypeRoutingStateUpAndNotInMaintenance(); - sc.addAnd("hypervisorType", SearchCriteria.Op.EQ, HypervisorType.KVM); // support KVM only util 2013.06.25 + sc.addAnd("hypervisorType", SearchCriteria.Op.IN, HypervisorType.KVM, HypervisorType.VMware); List hosts = _hostDao.search(sc, null); for (HostVO host : hosts) { List vms = _userVmDao.listRunningByHostId(host.getId()); List vmIds = new ArrayList(); - for (UserVmVO vm : vms) { + for (UserVmVO vm : vms) { if (vm.getType() == VirtualMachine.Type.User) // user vm vmIds.add(vm.getId()); } diff --git a/server/src/main/java/com/cloud/servlet/ConsoleProxyServlet.java b/server/src/main/java/com/cloud/servlet/ConsoleProxyServlet.java index 8cfaa9fd69b..5a6c84f1479 100644 --- a/server/src/main/java/com/cloud/servlet/ConsoleProxyServlet.java +++ b/server/src/main/java/com/cloud/servlet/ConsoleProxyServlet.java @@ -41,6 +41,7 @@ import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import org.springframework.web.context.support.SpringBeanAutowiringSupport; +import com.cloud.vm.VmDetailConstants; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -421,8 +422,8 @@ public class ConsoleProxyServlet extends HttpServlet { Pair portInfo; if (hostVo.getResourceState().equals(ResourceState.ErrorInMaintenance)) { - UserVmDetailVO detailAddress = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.address"); - UserVmDetailVO detailPort = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.port"); + UserVmDetailVO detailAddress = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.KVM_VNC_ADDRESS); + UserVmDetailVO detailPort = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.KVM_VNC_PORT); portInfo = new Pair<>(detailAddress.getValue(), Integer.valueOf(detailPort.getValue())); } else { portInfo = _ms.getVncPort(vm); @@ -441,7 +442,7 @@ public class ConsoleProxyServlet extends HttpServlet { } String sid = vm.getVncPassword(); - UserVmDetailVO details = _userVmDetailsDao.findDetail(vm.getId(), "keyboard"); + UserVmDetailVO details = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.KEYBOARD); String tag = vm.getUuid(); diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 11ce1d25b5b..5f3ab4c8116 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -80,6 +80,7 @@ import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -114,6 +115,8 @@ import com.cloud.hypervisor.HypervisorCapabilitiesVO; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.org.Grouping; import com.cloud.serializer.GsonHelper; +import com.cloud.server.ResourceTag; +import com.cloud.server.TaggedResourceService; import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.dao.DiskOfferingDao; @@ -256,6 +259,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic private StoragePoolTagsDao storagePoolTagsDao; @Inject private StorageUtil storageUtil; + @Inject + public TaggedResourceService taggedResourceService; protected Gson _gson; @@ -2024,6 +2029,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic throw new InvalidParameterValueException("Volume must be in ready state"); } + if (vol.getPoolId() == storagePoolId) { + throw new InvalidParameterValueException("Volume " + vol + " is already on the destination storage pool"); + } + boolean liveMigrateVolume = false; Long instanceId = vol.getInstanceId(); Long srcClusterId = null; @@ -2335,7 +2344,16 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Override @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_CREATE, eventDescription = "taking snapshot", async = true) - public Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup) + public Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map tags) + throws ResourceAllocationException { + final Snapshot snapshot = takeSnapshotInternal(volumeId, policyId, snapshotId, account, quiescevm, locationType, asyncBackup); + if (snapshot != null && MapUtils.isNotEmpty(tags)) { + taggedResourceService.createTags(Collections.singletonList(snapshot.getUuid()), ResourceTag.ResourceObjectType.Snapshot, tags, null); + } + return snapshot; + } + + private Snapshot takeSnapshotInternal(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup) throws ResourceAllocationException { VolumeInfo volume = volFactory.getVolume(volumeId); if (volume == null) { @@ -3380,4 +3398,4 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic public ConfigKey[] getConfigKeys() { return new ConfigKey[] {ConcurrentMigrationsThresholdPerDatastore}; } -} \ No newline at end of file +} diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index ee65486a127..5d98c503ab9 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -16,8 +16,53 @@ // under the License. package com.cloud.storage.snapshot; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd; +import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd; +import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd; +import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; +import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.MapUtils; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + import com.cloud.agent.api.Answer; -import com.cloud.utils.db.GlobalLock; import com.cloud.agent.api.Command; import com.cloud.agent.api.DeleteSnapshotsDirCommand; import com.cloud.alert.AlertManager; @@ -42,6 +87,7 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.projects.Project.ListProjectResourcesCriteria; import com.cloud.resource.ResourceManager; import com.cloud.server.ResourceTag.ResourceObjectType; +import com.cloud.server.TaggedResourceService; import com.cloud.storage.CreateSnapshotPayload; import com.cloud.storage.DataStoreRole; import com.cloud.storage.ScopeType; @@ -80,6 +126,7 @@ import com.cloud.utils.Ternary; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -92,48 +139,6 @@ import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; -import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd; -import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd; -import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd; -import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; -import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; -import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; @Component public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implements SnapshotManager, SnapshotApiService, Configurable { @@ -192,6 +197,8 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement ResourceManager _resourceMgr; @Inject StorageStrategyFactory _storageStrategyFactory; + @Inject + public TaggedResourceService taggedResourceService; private int _totalRetries; private int _pauseInterval; @@ -902,6 +909,11 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement policy.setDisplay(display); _snapshotPolicyDao.update(policy.getId(), policy); _snapSchedMgr.scheduleOrCancelNextSnapshotJobOnDisplayChange(policy, previousDisplay); + taggedResourceService.deleteTags(Collections.singletonList(policy.getUuid()), ResourceObjectType.SnapshotPolicy, null); + } + final Map tags = cmd.getTags(); + if (MapUtils.isNotEmpty(tags)) { + taggedResourceService.createTags(Collections.singletonList(policy.getUuid()), ResourceObjectType.SnapshotPolicy, tags, null); } } finally { createSnapshotPolicyLock.unlock(); @@ -916,9 +928,10 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement } } - protected boolean deletePolicy(long userId, Long policyId) { + protected boolean deletePolicy(Long policyId) { SnapshotPolicyVO snapshotPolicy = _snapshotPolicyDao.findById(policyId); _snapSchedMgr.removeSchedule(snapshotPolicy.getVolumeId(), snapshotPolicy.getId()); + taggedResourceService.deleteTags(Collections.singletonList(snapshotPolicy.getUuid()), ResourceObjectType.SnapshotPolicy, null); return _snapshotPolicyDao.remove(policyId); } @@ -963,8 +976,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement public void deletePoliciesForVolume(Long volumeId) { List policyInstances = listPoliciesforVolume(volumeId); for (SnapshotPolicyVO policyInstance : policyInstances) { - Long policyId = policyInstance.getId(); - deletePolicy(1L, policyId); + deletePolicy(policyInstance.getId()); } // We also want to delete the manual snapshots scheduled for this volume // We can only delete the schedules in the future, not the ones which are already executing. @@ -1321,7 +1333,6 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement public boolean deleteSnapshotPolicies(DeleteSnapshotPoliciesCmd cmd) { Long policyId = cmd.getId(); List policyIds = cmd.getIds(); - Long userId = getSnapshotUserId(); if ((policyId == null) && (policyIds == null)) { throw new InvalidParameterValueException("No policy id (or list of ids) specified."); @@ -1335,6 +1346,10 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement throw new InvalidParameterValueException("There are no policy ids"); } + if (policyIds.contains(Snapshot.MANUAL_POLICY_ID)) { + throw new InvalidParameterValueException("Invalid Policy id given: " + Snapshot.MANUAL_POLICY_ID); + } + for (Long policy : policyIds) { SnapshotPolicyVO snapshotPolicyVO = _snapshotPolicyDao.findById(policy); if (snapshotPolicyVO == null) { @@ -1348,21 +1363,14 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, volume); } - boolean success = true; - - if (policyIds.contains(Snapshot.MANUAL_POLICY_ID)) { - throw new InvalidParameterValueException("Invalid Policy id given: " + Snapshot.MANUAL_POLICY_ID); - } - for (Long pId : policyIds) { - if (!deletePolicy(userId, pId)) { - success = false; + if (!deletePolicy(pId)) { s_logger.warn("Failed to delete snapshot policy with Id: " + policyId); - return success; + return false; } } - return success; + return true; } @Override diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java index dc71b365353..bccf8c68b23 100644 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java @@ -43,6 +43,8 @@ import com.cloud.api.ApiDispatcher; import com.cloud.api.ApiGsonHelper; import com.cloud.event.ActionEventUtils; import com.cloud.event.EventTypes; +import com.cloud.server.ResourceTag; +import com.cloud.server.TaggedResourceService; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotPolicyVO; import com.cloud.storage.SnapshotScheduleVO; @@ -97,6 +99,8 @@ public class SnapshotSchedulerImpl extends ManagerBase implements SnapshotSchedu protected VMSnapshotDao _vmSnapshotDao; @Inject protected VMSnapshotManager _vmSnaphostManager; + @Inject + public TaggedResourceService taggedResourceService; protected AsyncJobDispatcher _asyncDispatcher; @@ -305,6 +309,15 @@ public class SnapshotSchedulerImpl extends ManagerBase implements SnapshotSchedu params.put("ctxUserId", "1"); params.put("ctxAccountId", "" + volume.getAccountId()); params.put("ctxStartEventId", String.valueOf(eventId)); + List resourceTags = taggedResourceService.listByResourceTypeAndId(ResourceTag.ResourceObjectType.SnapshotPolicy, policyId); + if (resourceTags != null && !resourceTags.isEmpty()) { + int tagNumber = 0; + for (ResourceTag resourceTag : resourceTags) { + params.put("tags[" + tagNumber + "].key", resourceTag.getKey()); + params.put("tags[" + tagNumber + "].value", resourceTag.getValue()); + tagNumber++; + } + } final CreateSnapshotCmd cmd = new CreateSnapshotCmd(); ComponentContext.inject(cmd); diff --git a/server/src/main/java/com/cloud/tags/TaggedResourceManagerImpl.java b/server/src/main/java/com/cloud/tags/TaggedResourceManagerImpl.java index ae7949863b4..44876d0ddf5 100644 --- a/server/src/main/java/com/cloud/tags/TaggedResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/tags/TaggedResourceManagerImpl.java @@ -16,13 +16,31 @@ // under the License. package com.cloud.tags; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import javax.persistence.EntityExistsException; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + import com.cloud.dc.DataCenterVO; import com.cloud.domain.PartOf; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; -import com.cloud.offerings.NetworkOfferingVO; import com.cloud.network.LBHealthCheckPolicyVO; import com.cloud.network.as.AutoScaleVmGroupVO; import com.cloud.network.as.AutoScaleVmProfileVO; @@ -43,6 +61,7 @@ import com.cloud.network.vpc.NetworkACLVO; import com.cloud.network.vpc.StaticRouteVO; import com.cloud.network.vpc.VpcOfferingVO; import com.cloud.network.vpc.VpcVO; +import com.cloud.offerings.NetworkOfferingVO; import com.cloud.projects.ProjectVO; import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; @@ -74,24 +93,6 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.NicVO; import com.cloud.vm.UserVmVO; import com.cloud.vm.snapshot.VMSnapshotVO; -import org.apache.cloudstack.api.Identity; -import org.apache.cloudstack.api.InternalIdentity; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; - -import org.apache.commons.collections.MapUtils; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; -import javax.persistence.EntityExistsException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; public class TaggedResourceManagerImpl extends ManagerBase implements TaggedResourceService { public static final Logger s_logger = Logger.getLogger(TaggedResourceManagerImpl.class); @@ -220,6 +221,10 @@ public class TaggedResourceManagerImpl extends ManagerBase implements TaggedReso accountId = ((ProjectVO)entity).getProjectAccountId(); } + if (resourceType == ResourceObjectType.SnapshotPolicy) { + accountId = _entityMgr.findById(VolumeVO.class, ((SnapshotPolicyVO)entity).getVolumeId()).getAccountId(); + } + if (entity instanceof OwnedBy) { accountId = ((OwnedBy)entity).getAccountId(); } @@ -389,7 +394,7 @@ public class TaggedResourceManagerImpl extends ManagerBase implements TaggedReso } if (tagsToDelete.isEmpty()) { - throw new InvalidParameterValueException("Unable to find any tags which conform to specified delete parameters."); + return false; } //Remove the tags diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 9beeb7bc010..5eb96aac11d 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -39,6 +39,7 @@ import com.cloud.utils.EncryptionUtil; import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; import com.cloud.utils.EnumUtils; +import com.cloud.vm.VmDetailConstants; import com.google.common.base.Joiner; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -1918,7 +1919,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } if (cmd.getDetails() != null) { - details.remove("Encrypted.Password"); // new password will be generated during vm deployment from password enabled template + details.remove(VmDetailConstants.ENCRYPTED_PASSWORD); // new password will be generated during vm deployment from password enabled template details.putAll(cmd.getDetails()); } if (!details.isEmpty()) { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index dbd467d2e75..c34670801d9 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -35,6 +35,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -85,6 +86,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -847,7 +849,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } else { final UserVmVO userVm = _vmDao.findById(vmId); _vmDao.loadDetails(userVm); - userVm.setDetail("SSH.PublicKey", sshPublicKey); + userVm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKey); if (template.isEnablePassword()) { userVm.setPassword(password); //update the encrypted password in vm_details table too @@ -2426,11 +2428,36 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (isDisplayVm != null && isDisplayVm != vmInstance.isDisplay()) { updateDisplayVmFlag(isDisplayVm, id, vmInstance); } + final Account caller = CallContext.current().getCallingAccount(); + final List userBlacklistedSettings = Stream.of(QueryService.UserVMBlacklistedDetails.value().split(",")) + .map(item -> (item).trim()) + .collect(Collectors.toList()); if (cleanupDetails){ - userVmDetailsDao.removeDetails(id); - } - else { + if (caller != null && caller.getType() == Account.ACCOUNT_TYPE_ADMIN) { + userVmDetailsDao.removeDetails(id); + } else { + for (final UserVmDetailVO detail : userVmDetailsDao.listDetails(id)) { + if (detail != null && !userBlacklistedSettings.contains(detail.getName())) { + userVmDetailsDao.removeDetail(id, detail.getName()); + } + } + } + } else { if (MapUtils.isNotEmpty(details)) { + if (caller != null && caller.getType() != Account.ACCOUNT_TYPE_ADMIN) { + // Ensure blacklisted detail is not passed by non-root-admin user + for (final String detailName : details.keySet()) { + if (userBlacklistedSettings.contains(detailName)) { + throw new InvalidParameterValueException("You're not allowed to add or edit the restricted setting: " + detailName); + } + } + // Add any hidden/blacklisted detail + for (final UserVmDetailVO detail : userVmDetailsDao.listDetails(id)) { + if (userBlacklistedSettings.contains(detail.getName())) { + details.put(detail.getName(), detail.getValue()); + } + } + } vmInstance.setDetails(details); _vmDao.saveDetails(vmInstance); } @@ -3379,13 +3406,13 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir boolean isIso = Storage.ImageFormat.ISO == template.getFormat(); long size = 0; // custom root disk size, resizes base template to larger size - if (customParameters.containsKey("rootdisksize")) { + if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { // only KVM, XenServer and VMware supports rootdisksize override if (!(hypervisorType == HypervisorType.KVM || hypervisorType == HypervisorType.XenServer || hypervisorType == HypervisorType.VMware || hypervisorType == HypervisorType.Simulator)) { throw new InvalidParameterValueException("Hypervisor " + hypervisorType + " does not support rootdisksize override"); } - Long rootDiskSize = NumbersUtil.parseLong(customParameters.get("rootdisksize"), -1); + Long rootDiskSize = NumbersUtil.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE), -1); if (rootDiskSize <= 0) { throw new InvalidParameterValueException("Root disk size should be a positive number."); } @@ -3768,7 +3795,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } if (sshPublicKey != null) { - vm.setDetail("SSH.PublicKey", sshPublicKey); + vm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKey); } if (keyboard != null && !keyboard.isEmpty()) { @@ -3780,9 +3807,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } Long rootDiskSize = null; // custom root disk size, resizes base template to larger size - if (customParameters.containsKey("rootdisksize")) { + if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { // already verified for positive number - rootDiskSize = Long.parseLong(customParameters.get("rootdisksize")); + rootDiskSize = Long.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE)); VMTemplateVO templateVO = _templateDao.findById(template.getId()); if (templateVO == null) { @@ -3806,10 +3833,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir // If hypervisor is vSphere and OS is OS X, set special settings. if (hypervisorType.equals(HypervisorType.VMware)) { if (guestOS.getDisplayName().toLowerCase().contains("apple mac os")) { - vm.setDetail("smc.present", "TRUE"); + vm.setDetail(VmDetailConstants.SMC_PRESENT, "TRUE"); vm.setDetail(VmDetailConstants.ROOT_DISK_CONTROLLER, "scsi"); vm.setDetail(VmDetailConstants.DATA_DISK_CONTROLLER, "scsi"); - vm.setDetail("firmware", "efi"); + vm.setDetail(VmDetailConstants.FIRMWARE, "efi"); s_logger.info("guestOS is OSX : overwrite root disk controller to scsi, use smc and efi"); } else { String controllerSetting = _configDao.getValue("vmware.root.disk.controller"); @@ -3838,7 +3865,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir vm.setDetail(key, customParameters.get(key)); } } - vm.setDetail("deployvm", "true"); + vm.setDetail(VmDetailConstants.DEPLOY_VM, "true"); _vmDao.saveDetails(vm); s_logger.debug("Allocating in the DB for vm"); @@ -3888,10 +3915,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir s_logger.error(error); throw new InvalidParameterValueException(error); } else if ((rootDiskSize << 30) > templateVO.getSize()) { - if (hypervisorType == HypervisorType.VMware && (vm.getDetails() == null || vm.getDetails().get("rootDiskController") == null)) { + if (hypervisorType == HypervisorType.VMware && (vm.getDetails() == null || vm.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER) == null)) { s_logger.warn("If Root disk controller parameter is not overridden, then Root disk resize may fail because current Root disk controller value is NULL."); - } else if (hypervisorType == HypervisorType.VMware && !vm.getDetails().get("rootDiskController").toLowerCase().contains("scsi")) { - String error = "Found unsupported root disk controller: " + vm.getDetails().get("rootDiskController"); + } else if (hypervisorType == HypervisorType.VMware && !vm.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER).toLowerCase().contains("scsi")) { + String error = "Found unsupported root disk controller: " + vm.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER); s_logger.error(error); throw new InvalidParameterValueException(error); } else { @@ -3899,7 +3926,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } else { s_logger.debug("Root disk size specified is " + (rootDiskSize << 30) + "B and Template root disk size is " + templateVO.getSize() + "B. Both are equal so no need to override"); - customParameters.remove("rootdisksize"); + customParameters.remove(VmDetailConstants.ROOT_DISK_SIZE); } } @@ -4182,7 +4209,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); List vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getHostName(), vm.getId(), - vm.getUuid(), defaultNic.getIPv4Address(), vm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows); + vm.getUuid(), defaultNic.getIPv4Address(), vm.getDetail(VmDetailConstants.SSH_PUBLIC_KEY), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows); String vmName = vm.getInstanceName(); String configDriveIsoRootFolder = "/tmp"; String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso"; @@ -4574,8 +4601,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir // this value is not being sent to the backend; need only for api // display purposes if (template.isEnablePassword()) { - if (vm.getDetail("password") != null) { - userVmDetailsDao.removeDetail(vm.getId(), "password"); + if (vm.getDetail(VmDetailConstants.PASSWORD) != null) { + userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.PASSWORD); } vm.setUpdateParameters(false); _vmDao.update(vm.getId(), vm); @@ -4643,8 +4670,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Override public void collectVmDiskStatistics(final UserVm userVm) { - // support KVM only util 2013.06.25 - if (!userVm.getHypervisorType().equals(HypervisorType.KVM)) { + // Only supported for KVM and VMware + if (!(userVm.getHypervisorType().equals(HypervisorType.KVM) || userVm.getHypervisorType().equals(HypervisorType.VMware))) { return; } s_logger.debug("Collect vm disk statistics from host before stopping Vm"); @@ -6334,7 +6361,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (needRestart) { try { - if (vm.getDetail("password") != null) { + if (vm.getDetail(VmDetailConstants.PASSWORD) != null) { params = new HashMap(); params.put(VirtualMachineProfile.Param.VmPassword, password); } @@ -6347,8 +6374,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (vm.isUpdateParameters()) { vm.setUpdateParameters(false); _vmDao.loadDetails(vm); - if (vm.getDetail("password") != null) { - userVmDetailsDao.removeDetail(vm.getId(), "password"); + if (vm.getDetail(VmDetailConstants.PASSWORD) != null) { + userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.PASSWORD); } _vmDao.update(vm.getId(), vm); } @@ -6526,7 +6553,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } private void encryptAndStorePassword(UserVmVO vm, String password) { - String sshPublicKey = vm.getDetail("SSH.PublicKey"); + String sshPublicKey = vm.getDetail(VmDetailConstants.SSH_PUBLIC_KEY); if (sshPublicKey != null && !sshPublicKey.equals("") && password != null && !password.equals("saved_password")) { if (!sshPublicKey.startsWith("ssh-rsa")) { s_logger.warn("Only RSA public keys can be used to encrypt a vm password."); @@ -6537,7 +6564,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir throw new CloudRuntimeException("Error encrypting password"); } - vm.setDetail("Encrypted.Password", encryptedPasswd); + vm.setDetail(VmDetailConstants.ENCRYPTED_PASSWORD, encryptedPasswd); _vmDao.saveDetails(vm); } } @@ -6569,7 +6596,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir public String getVmUserData(long vmId) { UserVmVO vm = _vmDao.findById(vmId); if (vm == null) { - throw new InvalidParameterValueException("Unable to find virual machine with id " + vmId); + throw new InvalidParameterValueException("Unable to find virtual machine with id " + vmId); } _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -6657,4 +6684,4 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } } -} \ No newline at end of file +} diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index bb5599f0a99..9d1426ce9f1 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -18,6 +18,7 @@ package com.cloud.storage; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doNothing; @@ -65,6 +66,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; import com.cloud.configuration.Resource; import com.cloud.configuration.Resource.ResourceType; @@ -76,6 +78,7 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.org.Grouping; import com.cloud.serializer.GsonHelper; +import com.cloud.server.TaggedResourceService; import com.cloud.storage.Volume.Type; import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.storage.dao.VolumeDao; @@ -409,7 +412,7 @@ public class VolumeApiServiceImplTest { when(volumeDataFactoryMock.getVolume(anyLong())).thenReturn(volumeInfoMock); when(volumeInfoMock.getState()).thenReturn(Volume.State.Allocated); when(volumeInfoMock.getPoolId()).thenReturn(1L); - volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false); + volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null); } @Test @@ -419,7 +422,10 @@ public class VolumeApiServiceImplTest { when(volumeInfoMock.getInstanceId()).thenReturn(null); when(volumeInfoMock.getPoolId()).thenReturn(1L); when(volumeServiceMock.takeSnapshot(Mockito.any(VolumeInfo.class))).thenReturn(snapshotInfoMock); - volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false); + final TaggedResourceService taggedResourceService = Mockito.mock(TaggedResourceService.class); + Mockito.when(taggedResourceService.createTags(anyObject(), anyObject(), anyObject(), anyObject())).thenReturn(null); + ReflectionTestUtils.setField(volumeApiServiceImpl, "taggedResourceService", taggedResourceService); + volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null); } @Test diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 2744714c67b..965377b2c7b 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -22,6 +22,7 @@ import java.util.HashMap; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; import org.apache.cloudstack.context.CallContext; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -44,6 +45,8 @@ import com.cloud.storage.GuestOSVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.UserVO; import com.cloud.uservm.UserVm; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; @@ -76,6 +79,12 @@ public class UserVmManagerImplTest { @Mock private NetworkModel networkModel; + @Mock + private AccountVO callerAccount; + + @Mock + private UserVO callerUser; + private long vmId = 1l; @Before @@ -83,6 +92,14 @@ public class UserVmManagerImplTest { Mockito.when(updateVmCommand.getId()).thenReturn(vmId); Mockito.when(userVmDao.findById(Mockito.eq(vmId))).thenReturn(userVmVoMock); + + Mockito.when(callerAccount.getType()).thenReturn(Account.ACCOUNT_TYPE_ADMIN); + CallContext.register(callerUser, callerAccount); + } + + @After + public void afterTest() { + CallContext.unregister(); } @Test diff --git a/systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf b/systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf new file mode 100644 index 00000000000..ea06042344f --- /dev/null +++ b/systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf @@ -0,0 +1,18 @@ +# event MPM +# StartServers: initial number of server processes to start +# MinSpareThreads: minimum number of worker threads which are kept spare +# MaxSpareThreads: maximum number of worker threads which are kept spare +# ThreadsPerChild: constant number of worker threads in each server process +# MaxRequestWorkers: maximum number of worker threads +# MaxConnectionsPerChild: maximum number of requests a server process serves + + StartServers 1 + MinSpareThreads 25 + MaxSpareThreads 75 + ThreadLimit 64 + ThreadsPerChild 25 + MaxRequestWorkers 30 + MaxConnectionsPerChild 1000 + + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/systemvm/debian/etc/cron.daily/clear_cache b/systemvm/debian/etc/cron.daily/clear_cache new file mode 100755 index 00000000000..e2662f9380f --- /dev/null +++ b/systemvm/debian/etc/cron.daily/clear_cache @@ -0,0 +1,5 @@ +#!/bin/bash + +# clear memory cache to ultimately reduce swapping + +sync && echo 1 > /proc/sys/vm/drop_caches diff --git a/systemvm/debian/etc/cron.hourly/clear_cache b/systemvm/debian/etc/cron.hourly/clear_cache new file mode 100755 index 00000000000..5daa16744d0 --- /dev/null +++ b/systemvm/debian/etc/cron.hourly/clear_cache @@ -0,0 +1,8 @@ +#!/bin/bash + +# clear memory cache to ultimately reduce swapping + +phymem=$(free|awk '/^Mem:/{print $2}') +if [ $phymem -lt 513000 ]; then + sync && echo 1 > /proc/sys/vm/drop_caches +fi diff --git a/systemvm/debian/etc/sysctl.conf b/systemvm/debian/etc/sysctl.conf index 5b5f818e76d..5dd1ae44bc1 100644 --- a/systemvm/debian/etc/sysctl.conf +++ b/systemvm/debian/etc/sysctl.conf @@ -63,3 +63,9 @@ net.ipv6.conf.all.autoconf = 0 # Minimum swappiness without disabling it vm.swappiness=1 + +# make the kernel more aggressive in reclaiming RAM from the disk and swap caches +vm.vfs_cache_pressure = 200 + +# try to maintain 'free' memory thereby reducing the size of disk cache, hence reducing swapping. +vm.min_free_kbytes = 20480 diff --git a/systemvm/debian/etc/vmware-tools/tools.conf b/systemvm/debian/etc/vmware-tools/tools.conf new file mode 100644 index 00000000000..26f96ffc277 --- /dev/null +++ b/systemvm/debian/etc/vmware-tools/tools.conf @@ -0,0 +1,17 @@ +[logging] +# Turns on logging globally. It can still be disabled for each domain. +# log = true + +# Disables core dumps on fatal errors; they're enabled by default. +enableCoreDump = false + +# Defines the "vmsvc" domain, logging to file +# vmsvc.level = message +vmsvc.handler = file +# Setup file rotation - keep 3 files +vmsvc.maxOldLogFiles = 2 +# Max log file size kept: 1 MB +vmsvc.maxLogSize = 1 + +# Defines the "vmtoolsd" domain, and disable logging for it. +# vmtoolsd.level = none diff --git a/systemvm/debian/etc/systemd/system/baremetal-vr.service b/systemvm/debian/lib/systemd/system/baremetal-vr.service similarity index 100% rename from systemvm/debian/etc/systemd/system/baremetal-vr.service rename to systemvm/debian/lib/systemd/system/baremetal-vr.service diff --git a/systemvm/debian/etc/systemd/system/cloud.service b/systemvm/debian/lib/systemd/system/cloud.service similarity index 70% rename from systemvm/debian/etc/systemd/system/cloud.service rename to systemvm/debian/lib/systemd/system/cloud.service index 22439239caa..e3824bf53c2 100644 --- a/systemvm/debian/etc/systemd/system/cloud.service +++ b/systemvm/debian/lib/systemd/system/cloud.service @@ -1,6 +1,6 @@ [Unit] Description=CloudStack Agent service -After=cloud-early-config.service network.target local-fs.target +After=cloud-early-config.service network.target cloud-postinit.service local-fs.target [Install] WantedBy=multi-user.target diff --git a/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service new file mode 100644 index 00000000000..12a0b63c970 --- /dev/null +++ b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service @@ -0,0 +1,9 @@ +[Unit] +Description=Hyper-V file copy service (FCOPY) daemon +ConditionPathExists=/dev/vmbus/hv_fcopy + +[Service] +ExecStart=/usr/sbin/hv_fcopy_daemon -n + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service new file mode 100644 index 00000000000..534a25a7f24 --- /dev/null +++ b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service @@ -0,0 +1,8 @@ +[Unit] +Description=Hyper-V key-value pair (KVP) daemon + +[Service] +ExecStart=/usr/sbin/hv_kvp_daemon -n + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service new file mode 100644 index 00000000000..d4fd675bdc8 --- /dev/null +++ b/systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service @@ -0,0 +1,8 @@ +[Unit] +Description=Hyper-V volume shadow copy service (VSS) daemon + +[Service] +ExecStart=/usr/sbin/hv_vss_daemon -n + +[Install] +WantedBy=multi-user.target diff --git a/systemvm/debian/lib/systemd/system/open-vm-tools.service b/systemvm/debian/lib/systemd/system/open-vm-tools.service new file mode 100644 index 00000000000..e7cdde7e3ff --- /dev/null +++ b/systemvm/debian/lib/systemd/system/open-vm-tools.service @@ -0,0 +1,12 @@ +[Unit] +Description=Service for virtual machines hosted on VMware +Documentation=http://open-vm-tools.sourceforge.net/about.php +DefaultDependencies=no +Before=cloud-early-config.service + +[Service] +ExecStart=/usr/bin/vmtoolsd +TimeoutStopSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/systemvm/debian/lib/systemd/system/xe-daemon.service b/systemvm/debian/lib/systemd/system/xe-daemon.service new file mode 100644 index 00000000000..6dfef7bfdb2 --- /dev/null +++ b/systemvm/debian/lib/systemd/system/xe-daemon.service @@ -0,0 +1,15 @@ +[Unit] +Description=Xen Guest Monitoring Agent +DefaultDependencies=no +After=local-fs.target +Requires=proc-xen.mount +Before=network.target cloud-early-config.service +ConditionPathExists=/proc/xen/capabilities + +[Service] +ExecStartPre=/usr/sbin/xe-linux-distribution /var/cache/xe-linux-distribution +ExecStart=/usr/sbin/xe-daemon +StandardOutput=journal+console + +[Install] +WantedBy=multi-user.target diff --git a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py index e7abb902046..56b44195f6d 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py @@ -139,7 +139,8 @@ class CsDhcp(CsDataBag): logging.error("Caught error while trying to delete entries from dnsmasq.leases file: %s" % e) def preseed(self): - self.add_host("127.0.0.1", "localhost %s" % CsHelper.get_hostname()) + self.add_host("127.0.0.1", "localhost") + self.add_host("127.0.1.1", "%s" % CsHelper.get_hostname()) self.add_host("::1", "localhost ip6-localhost ip6-loopback") self.add_host("ff02::1", "ip6-allnodes") self.add_host("ff02::2", "ip6-allrouters") diff --git a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh index 0208b36d435..5df59a4438e 100755 --- a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh +++ b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh @@ -19,6 +19,9 @@ PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" CMDLINE=/var/cache/cloud/cmdline +rm -f /var/cache/cloud/enabled_svcs +rm -f /var/cache/cloud/disabled_svcs + . /lib/lsb/init-functions log_it() { @@ -56,25 +59,25 @@ hypervisor() { } config_guest() { - if [ "$HYPERVISOR" == "kvm" ] - then - # Configure hot-plug - modprobe acpiphp || true - modprobe pci_hotplug || true - sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab - sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102" /etc/inittab - fi [ ! -d /proc/xen ] && sed -i 's/^vc/#vc/' /etc/inittab && telinit q - [ -d /proc/xen ] && sed -i 's/^#vc/vc/' /etc/inittab && telinit q -} + [ -d /proc/xen ] && sed -i 's/^#vc/vc/' /etc/inittab && telinit q + + systemctl daemon-reload -get_boot_params() { case $HYPERVISOR in xen-pv|xen-domU) + systemctl stop ntpd + systemctl disable ntpd + systemctl start xe-daemon + cat /proc/cmdline > $CMDLINE sed -i "s/%/ /g" $CMDLINE ;; xen-hvm) + systemctl stop ntpd + systemctl disable ntpd + systemctl start xe-daemon + if [ ! -f /usr/bin/xenstore-read ]; then log_it "ERROR: xentools not installed, cannot found xenstore-read" && exit 5 fi @@ -82,7 +85,13 @@ get_boot_params() { sed -i "s/%/ /g" $CMDLINE ;; kvm) + # Configure hot-plug + modprobe acpiphp || true + modprobe pci_hotplug || true + sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab + sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102" /etc/inittab systemctl enable --now qemu-guest-agent + # Wait for $CMDLINE file to be written by the qemu-guest-agent for i in {1..60}; do if [ -s $CMDLINE ]; then @@ -96,13 +105,16 @@ get_boot_params() { fi ;; vmware) + # system time sync'd with host via vmware tools + systemctl stop ntpd + systemctl disable ntpd + systemctl start open-vm-tools + vmtoolsd --cmd 'machine.id.get' > $CMDLINE ;; virtualpc|hyperv) # Hyper-V is recognized as virtualpc hypervisor type. Boot args are passed using KVP Daemon - #waiting for the hv_kvp_daemon to start up - #sleep need to fix the race condition of hv_kvp_daemon and cloud-early-config - [ -f /usr/sbin/hv_kvp_daemon ] && /usr/sbin/hv_kvp_daemon + systemctl start hyperv-daemons.hv-fcopy-daemon.service hyperv-daemons.hv-kvp-daemon.service hyperv-daemons.hv-vss-daemon.service sleep 5 cp -f /var/opt/hyperv/.kvp_pool_0 $CMDLINE cat /dev/null > /var/opt/hyperv/.kvp_pool_0 @@ -117,13 +129,11 @@ get_boot_params() { fi ;; esac -} -get_systemvm_type() { + # Find and export guest type export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE) } - patch_systemvm() { local patchfile=$1 local backupfolder="/tmp/.conf.backup" @@ -172,19 +182,29 @@ patch() { return 0 } +config_sysctl() { + # When there is more memory reset the cache back pressure to default 100 + physmem=$(free|awk '/^Mem:/{print $2}') + if [ $((physmem)) -lt 409600 ]; then + sed -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 200" /etc/sysctl.conf + else + sed -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 100" /etc/sysctl.conf + fi + + sync + sysctl -p +} + bootstrap() { log_it "Bootstrapping systemvm appliance" export HYPERVISOR=$(hypervisor) - [ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out of early init" && exit 10 - log_it "Detected that we are running inside $HYPERVISOR" + [ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out" && exit 10 + log_it "Starting guest services for $HYPERVISOR" config_guest - get_boot_params - get_systemvm_type patch - sync - sysctl -p + config_sysctl log_it "Configuring systemvm type=$TYPE" if [ -f "/opt/cloud/bin/setup/$TYPE.sh" ]; then @@ -192,6 +212,7 @@ bootstrap() { else /opt/cloud/bin/setup/default.sh fi + log_it "Finished setting up systemvm" exit 0 } diff --git a/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh b/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh index 225dc6f5ad2..3f00f3da43a 100755 --- a/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh +++ b/systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh @@ -18,14 +18,13 @@ . /opt/cloud/bin/setup/common.sh -consoleproxy_svcs() { +setup_console_proxy() { + log_it "Setting up console proxy system vm" + echo "cloud" > /var/cache/cloud/enabled_svcs echo "haproxy dnsmasq apache2 nfs-common portmap" > /var/cache/cloud/disabled_svcs mkdir -p /var/log/cloud -} -setup_console_proxy() { - log_it "Setting up console proxy system vm" setup_common eth0 eth1 eth2 setup_system_rfc1918_internal @@ -51,10 +50,4 @@ setup_console_proxy() { rm -f /etc/logrotate.d/cloud } -consoleproxy_svcs -if [ $? -gt 0 ] -then - log_it "Failed to execute consoleproxy_svcs" - exit 1 -fi setup_console_proxy diff --git a/systemvm/debian/opt/cloud/bin/setup/postinit.sh b/systemvm/debian/opt/cloud/bin/setup/postinit.sh index fc92b7e97cd..5e7e4c01a22 100755 --- a/systemvm/debian/opt/cloud/bin/setup/postinit.sh +++ b/systemvm/debian/opt/cloud/bin/setup/postinit.sh @@ -29,36 +29,39 @@ if [ "$TYPE" == "router" ] || [ "$TYPE" == "vpcrouter" ] || [ "$TYPE" == "dhcpsr then if [ -x /opt/cloud/bin/update_config.py ] then - /opt/cloud/bin/update_config.py cmd_line.json || true + /opt/cloud/bin/update_config.py cmd_line.json || true fi fi [ ! -f /var/cache/cloud/enabled_svcs ] && touch /var/cache/cloud/enabled_svcs for svc in $(cat /var/cache/cloud/enabled_svcs) do - systemctl enable --now --no-block $svc + systemctl enable --now --no-block $svc done [ ! -f /var/cache/cloud/disabled_svcs ] && touch /var/cache/cloud/disabled_svcs for svc in $(cat /var/cache/cloud/disabled_svcs) do - systemctl disable --now --no-block $svc + systemctl disable --now --no-block $svc done # Restore the persistent iptables nat, rules and filters for IPv4 and IPv6 if they exist ipv4="/etc/iptables/rules.v4" if [ -e $ipv4 ] then - iptables-restore < $ipv4 + iptables-restore < $ipv4 fi ipv6="/etc/iptables/rules.v6" if [ -e $ipv6 ] then - ip6tables-restore < $ipv6 + ip6tables-restore < $ipv6 fi -# Enable SSH +# Patch known systemd/sshd memory leak - https://github.com/systemd/systemd/issues/8015#issuecomment-476160981 +echo '@include null' >> /etc/pam.d/systemd-user + +# Enable and Start SSH systemctl enable --now --no-block ssh date > /var/cache/cloud/boot_up_done diff --git a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh index 8b6d4ee5a4b..d3a6d21cf65 100755 --- a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh +++ b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh @@ -18,15 +18,12 @@ . /opt/cloud/bin/setup/common.sh -secstorage_svcs() { - echo "apache2 cloud nfs-common portmap" > /var/cache/cloud/enabled_svcs - echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs - mkdir -p /var/log/cloud -} - setup_secstorage() { log_it "Setting up secondary storage system vm" - sysctl vm.min_free_kbytes=8192 + + echo "cloud apache2 nfs-common portmap" > /var/cache/cloud/enabled_svcs + echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs + mkdir -p /var/log/cloud setup_common eth0 eth1 eth2 setup_storage_network @@ -80,10 +77,4 @@ CORS rm -f /etc/logrotate.d/cloud } -secstorage_svcs -if [ $? -gt 0 ] -then - log_it "Failed to execute secstorage_svcs" - exit 1 -fi setup_secstorage diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index 78d868d8b0f..56406b711f5 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -46,7 +46,7 @@ function install_cloud_scripts() { rsync -av ./cloud_scripts/ / chmod +x /opt/cloud/bin/* /opt/cloud/bin/setup/* \ /root/{clearUsageRules.sh,reconfigLB.sh,monitorServices.py} \ - /etc/profile.d/cloud.sh + /etc/profile.d/cloud.sh /etc/cron.daily/* /etc/cron.hourly/* chmod -x /etc/systemd/system/* @@ -64,6 +64,7 @@ function do_signature() { function configure_issue() { cat > /etc/issue <