Merge branch 'master' into storage-offering-domains-zones

api/src/main/java/com/cloud/server/ResourceTag.java

@@ -58,7 +58,7 @@ public interface ResourceTag extends ControlledEntity, Identity, InternalIdentit
         AutoScaleVmGroup(false, true),
         LBStickinessPolicy(false, true),
         LBHealthCheckPolicy(false, true),
-        SnapshotPolicy(false, true),
+        SnapshotPolicy(true, true),
         GuestOs(false, true),
         NetworkOffering(false, true),
         VpcOffering(true, false);

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -19,6 +19,7 @@
 package com.cloud.storage;
 
 import java.net.MalformedURLException;
+import java.util.Map;
 
 import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd;
 import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd;
@@ -93,7 +94,7 @@ public interface VolumeApiService {
 
     Volume detachVolumeFromVM(DetachVolumeCmd cmd);
 
-    Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup)
+    Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map<String, String> tags)
             throws ResourceAllocationException;
 
     Snapshot allocSnapshot(Long volumeId, Long policyId, String snapshotName, Snapshot.LocationType locationType) throws ResourceAllocationException;

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -17,14 +17,41 @@
 package com.cloud.vm;
 
 public interface VmDetailConstants {
-    public static final String KEYBOARD = "keyboard";
+    String KEYBOARD = "keyboard";
-    public static final String NIC_ADAPTER = "nicAdapter";
+    String CPU_CORE_PER_SOCKET = "cpu.corespersocket";
-    public static final String ROOT_DISK_CONTROLLER = "rootDiskController";
+    String ROOT_DISK_SIZE = "rootdisksize";
-    public static final String NESTED_VIRTUALIZATION_FLAG = "nestedVirtualizationFlag";
+
-    public static final String HYPERVISOR_TOOLS_VERSION = "hypervisortoolsversion";
+    // VMware specific
-    public static final String DATA_DISK_CONTROLLER = "dataDiskController";
+    String NIC_ADAPTER = "nicAdapter";
-    public static final String SVGA_VRAM_SIZE = "svga.vramSize";
+    String ROOT_DISK_CONTROLLER = "rootDiskController";
-    public static final String CPU_NUMBER = "cpuNumber";
+    String DATA_DISK_CONTROLLER = "dataDiskController";
-    public static final String CPU_SPEED = "cpuSpeed";
+    String SVGA_VRAM_SIZE = "svga.vramSize";
-    public static final String MEMORY = "memory";
+    String NESTED_VIRTUALIZATION_FLAG = "nestedVirtualizationFlag";
+
+    // XenServer specific (internal)
+    String HYPERVISOR_TOOLS_VERSION = "hypervisortoolsversion";
+    String PLATFORM = "platform";
+    String TIME_OFFSET = "timeoffset";
+
+    // KVM specific (internal)
+    String KVM_VNC_PORT = "kvm.vnc.port";
+    String KVM_VNC_ADDRESS = "kvm.vnc.address";
+
+    // Mac OSX guest specific (internal)
+    String SMC_PRESENT = "smc.present";
+    String FIRMWARE = "firmware";
+
+    // VM deployment with custom compute offering params
+    String CPU_NUMBER = "cpuNumber";
+    String CPU_SPEED = "cpuSpeed";
+    String MEMORY = "memory";
+
+    // Misc details for internal usage (not to be set/changed by user or admin)
+    String CPU_OVER_COMMIT_RATIO = "cpuOvercommitRatio";
+    String MEMORY_OVER_COMMIT_RATIO = "memoryOvercommitRatio";
+    String MESSAGE_RESERVED_CAPACITY_FREED_FLAG = "Message.ReservedCapacityFreed.Flag";
+    String DEPLOY_VM = "deployvm";
+    String SSH_PUBLIC_KEY = "SSH.PublicKey";
+    String PASSWORD = "password";
+    String ENCRYPTED_PASSWORD = "Encrypted.Password";
 }

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -91,6 +91,11 @@ public class ApiConstants {
     public static final String DIRECT_DOWNLOAD = "directdownload";
     public static final String DISK_OFFERING_ID = "diskofferingid";
     public static final String NEW_DISK_OFFERING_ID = "newdiskofferingid";
+    public static final String DISK_KBS_READ = "diskkbsread";
+    public static final String DISK_KBS_WRITE = "diskkbswrite";
+    public static final String DISK_IO_READ = "diskioread";
+    public static final String DISK_IO_WRITE = "diskiowrite";
+    public static final String DISK_IO_PSTOTAL = "diskiopstotal";
     public static final String DISK_SIZE = "disksize";
     public static final String UTILIZATION = "utilization";
     public static final String DRIVER = "driver";

api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListDetailOptionsCmd.java

@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.resource;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiArgValidator;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.DetailOptionsResponse;
+import org.apache.cloudstack.context.CallContext;
+
+import com.cloud.server.ResourceTag;
+import com.google.common.base.Strings;
+
+@APICommand(name = ListDetailOptionsCmd.APINAME,
+        description = "Lists all possible details and their options for a resource type such as a VM or a template",
+        responseObject = DetailOptionsResponse.class,
+        since = "4.13",
+        requestHasSensitiveInfo = false,
+        responseHasSensitiveInfo = false,
+        authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User})
+public class ListDetailOptionsCmd extends BaseCmd {
+    public final static String APINAME = "listDetailOptions";
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, required = true,
+            description = "the resource type such as UserVm, Template etc.",
+            validations = {ApiArgValidator.NotNullOrEmpty}
+    )
+    private String resourceType;
+
+    @Parameter(name = ApiConstants.RESOURCE_ID, type = CommandType.STRING,
+            description = "the UUID of the resource (optional)")
+    private String resourceId;
+
+    /////////////////////////////////////////////////////
+    /////////////////// Accessors ///////////////////////
+    /////////////////////////////////////////////////////
+
+    public ResourceTag.ResourceObjectType getResourceType() {
+        return _taggedResourceService.getResourceType(resourceType);
+    }
+
+    public String getResourceId() {
+        if (!Strings.isNullOrEmpty(resourceId)) {
+            return _taggedResourceService.getUuid(resourceId, getResourceType());
+        }
+        return null;
+    }
+
+    /////////////////////////////////////////////////////
+    /////////////////// Implementation //////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    public String getCommandName() {
+        return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+
+    @Override
+    public void execute() {
+        final DetailOptionsResponse response = _queryService.listDetailOptions(this);
+        response.setResponseName(getCommandName());
+        response.setObjectName("detailoptions");
+        setResponseObject(response);
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java

@@ -16,6 +16,10 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.snapshot;
 
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandJobType;
 import org.apache.cloudstack.api.ApiConstants;
@@ -28,6 +32,7 @@ import org.apache.cloudstack.api.response.DomainResponse;
 import org.apache.cloudstack.api.response.SnapshotPolicyResponse;
 import org.apache.cloudstack.api.response.SnapshotResponse;
 import org.apache.cloudstack.api.response.VolumeResponse;
+import org.apache.commons.collections.MapUtils;
 import org.apache.log4j.Logger;
 
 import com.cloud.event.EventTypes;
@@ -83,6 +88,9 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd {
     @Parameter(name = ApiConstants.ASYNC_BACKUP, type = CommandType.BOOLEAN, required = false, description = "asynchronous backup if true")
     private Boolean asyncBackup;
 
+    @Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "Map of tags (key/value pairs)")
+    private Map tags;
+
     private String syncObjectType = BaseAsyncCmd.snapshotHostSyncObject;
 
     // ///////////////////////////////////////////////////
@@ -121,6 +129,18 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd {
         }
     }
 
+    public Map<String, String> getTags() {
+        Map<String, String> tagsMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(tags)) {
+            for (Map<String, String> services : (Collection<Map<String, String>>)tags.values()) {
+                String key = services.get("key");
+                String value = services.get("value");
+                tagsMap.put(key, value);
+            }
+        }
+        return tagsMap;
+    }
+
     private Long getHostId() {
         Volume volume = _entityMgr.findById(Volume.class, getVolumeId());
         if (volume == null) {
@@ -196,7 +216,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd {
         Snapshot snapshot;
         try {
             snapshot =
-                _volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup());
+                _volumeService.takeSnapshot(getVolumeId(), getPolicyId(), getEntityId(), _accountService.getAccount(getEntityOwnerId()), getQuiescevm(), getLocationType(), getAsyncBackup(), getTags());
 
             if (snapshot != null) {
                 SnapshotResponse response = _responseGenerator.createSnapshotResponse(snapshot);

api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmd.java

@@ -16,9 +16,11 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.snapshot;
 
-import org.apache.cloudstack.acl.RoleType;
+import java.util.Collection;
-import org.apache.log4j.Logger;
+import java.util.HashMap;
+import java.util.Map;
 
+import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.ApiErrorCode;
@@ -27,6 +29,8 @@ import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.response.SnapshotPolicyResponse;
 import org.apache.cloudstack.api.response.VolumeResponse;
+import org.apache.commons.collections.MapUtils;
+import org.apache.log4j.Logger;
 
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.PermissionDeniedException;
@@ -68,6 +72,9 @@ public class CreateSnapshotPolicyCmd extends BaseCmd {
     @Parameter(name = ApiConstants.FOR_DISPLAY, type = CommandType.BOOLEAN, description = "an optional field, whether to the display the policy to the end user or not", since = "4.4", authorized = {RoleType.Admin})
     private Boolean display;
 
+    @Parameter(name = ApiConstants.TAGS, type = CommandType.MAP, description = "Map of tags (key/value pairs)")
+    private Map tags;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -133,6 +140,18 @@ public class CreateSnapshotPolicyCmd extends BaseCmd {
         return volume.getAccountId();
     }
 
+    public Map<String, String> getTags() {
+        Map<String, String> tagsMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(tags)) {
+            for (Map<String, String> services : (Collection<Map<String, String>>)tags.values()) {
+                String key = services.get("key");
+                String value = services.get("value");
+                tagsMap.put(key, value);
+            }
+        }
+        return tagsMap;
+    }
+
     @Override
     public void execute() {
         SnapshotPolicy result = _snapshotService.createPolicy(this, _accountService.getAccount(getEntityOwnerId()));

api/src/main/java/org/apache/cloudstack/api/response/DetailOptionsResponse.java

@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.response;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseResponse;
+
+import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
+
+public class DetailOptionsResponse extends BaseResponse {
+    @SerializedName(ApiConstants.DETAILS)
+    @Param(description = "Map of all possible details and their possible list of values")
+    private Map<String, List<String>> details;
+
+    public DetailOptionsResponse(Map<String, List<String>> details) {
+        this.details = details;
+    }
+
+    public void setDetails(Map<String, List<String>> details) {
+        this.details = details;
+    }
+
+    public Map<String, List<String>> getDetails() {
+        return details;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java

@@ -16,18 +16,20 @@
 // under the License.
 package org.apache.cloudstack.api.response;
 
-import com.google.gson.annotations.SerializedName;
+import java.util.LinkedHashSet;
+import java.util.Set;
 
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.api.BaseResponse;
+import org.apache.cloudstack.api.BaseResponseWithTagInformation;
 import org.apache.cloudstack.api.EntityReference;
 
 import com.cloud.serializer.Param;
 import com.cloud.storage.snapshot.SnapshotPolicy;
+import com.google.gson.annotations.SerializedName;
 
 @EntityReference(value = SnapshotPolicy.class)
-public class SnapshotPolicyResponse extends BaseResponse {
+public class SnapshotPolicyResponse extends BaseResponseWithTagInformation {
     @SerializedName("id")
     @Param(description = "the ID of the snapshot policy")
     private String id;
@@ -56,6 +58,10 @@ public class SnapshotPolicyResponse extends BaseResponse {
     @Param(description = "is this policy for display to the regular user", since = "4.4", authorized = {RoleType.Admin})
     private Boolean forDisplay;
 
+    public SnapshotPolicyResponse() {
+        tags = new LinkedHashSet<ResourceTagResponse>();
+    }
+
     public String getId() {
         return id;
     }
@@ -111,4 +117,8 @@ public class SnapshotPolicyResponse extends BaseResponse {
     public void setForDisplay(Boolean forDisplay) {
         this.forDisplay = forDisplay;
     }
+
+    public void setTags(Set<ResourceTagResponse> tags) {
+        this.tags = tags;
+    }
 }

api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java

@@ -16,18 +16,20 @@
 // under the License.
 package org.apache.cloudstack.api.response;
 
+import java.util.Date;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseResponseWithTagInformation;
+import org.apache.cloudstack.api.EntityReference;
+
 import com.cloud.serializer.Param;
 import com.cloud.storage.Snapshot;
 import com.google.gson.annotations.SerializedName;
-import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.api.BaseResponse;
-import org.apache.cloudstack.api.EntityReference;
-
-import java.util.Date;
-import java.util.List;
 
 @EntityReference(value = Snapshot.class)
-public class SnapshotResponse extends BaseResponse implements ControlledEntityResponse {
+public class SnapshotResponse extends BaseResponseWithTagInformation implements ControlledEntityResponse {
     @SerializedName(ApiConstants.ID)
     @Param(description = "ID of the snapshot")
     private String id;
@@ -96,10 +98,6 @@ public class SnapshotResponse extends BaseResponse implements ControlledEntityRe
     @Param(description = "id of the availability zone")
     private String zoneId;
 
-    @SerializedName(ApiConstants.TAGS)
-    @Param(description = "the list of resource tags associated with snapshot", responseObject = ResourceTagResponse.class)
-    private List<ResourceTagResponse> tags;
-
     @SerializedName(ApiConstants.REVERTABLE)
     @Param(description = "indicates whether the underlying storage supports reverting the volume to this snapshot")
     private boolean revertable;
@@ -116,6 +114,10 @@ public class SnapshotResponse extends BaseResponse implements ControlledEntityRe
     @Param(description = "virtual size of backedup snapshot on image store")
     private long virtualSize;
 
+    public SnapshotResponse() {
+        tags = new LinkedHashSet<ResourceTagResponse>();
+    }
+
     @Override
     public String getObjectId() {
         return this.getId();
@@ -206,7 +208,7 @@ public class SnapshotResponse extends BaseResponse implements ControlledEntityRe
         this.zoneId = zoneId;
     }
 
-    public void setTags(List<ResourceTagResponse> tags) {
+    public void setTags(Set<ResourceTagResponse> tags) {
         this.tags = tags;
     }
 

api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java

@@ -188,11 +188,11 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co
     @Param(description = "the outgoing network traffic on the host")
     private Long networkKbsWrite;
 
-    @SerializedName("diskkbsread")
+    @SerializedName(ApiConstants.DISK_KBS_READ)
     @Param(description = "the read (bytes) of disk on the vm")
     private Long diskKbsRead;
 
-    @SerializedName("diskkbswrite")
+    @SerializedName(ApiConstants.DISK_KBS_WRITE)
     @Param(description = "the write (bytes) of disk on the vm")
     private Long diskKbsWrite;
 
@@ -208,11 +208,11 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co
     @Param(description = "the target memory in vm")
     private Long memoryTargetKBs;
 
-    @SerializedName("diskioread")
+    @SerializedName(ApiConstants.DISK_IO_READ)
     @Param(description = "the read (io) of disk on the vm")
     private Long diskIORead;
 
-    @SerializedName("diskiowrite")
+    @SerializedName(ApiConstants.DISK_IO_WRITE)
     @Param(description = "the write (io) of disk on the vm")
     private Long diskIOWrite;
 

api/src/main/java/org/apache/cloudstack/api/response/VolumeResponse.java

@@ -16,17 +16,18 @@
 // under the License.
 package org.apache.cloudstack.api.response;
 
-import com.cloud.serializer.Param;
+import java.util.Date;
-import com.cloud.storage.Volume;
+import java.util.LinkedHashSet;
-import com.google.gson.annotations.SerializedName;
+import java.util.Set;
+
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponseWithTagInformation;
 import org.apache.cloudstack.api.EntityReference;
 
-import java.util.Date;
+import com.cloud.serializer.Param;
-import java.util.LinkedHashSet;
+import com.cloud.storage.Volume;
-import java.util.Set;
+import com.google.gson.annotations.SerializedName;
 
 @EntityReference(value = Volume.class)
 @SuppressWarnings("unused")
@@ -152,13 +153,29 @@ public class VolumeResponse extends BaseResponseWithTagInformation implements Co
     private Long bytesWriteRate;
 
     @SerializedName("diskIopsReadRate")
-    @Param(description = "io requests read rate of the disk volume")
+    @Param(description = "io requests read rate of the disk volume per the disk offering")
     private Long iopsReadRate;
 
     @SerializedName("diskIopsWriteRate")
-    @Param(description = "io requests write rate of the disk volume")
+    @Param(description = "io requests write rate of the disk volume per the disk offering")
     private Long iopsWriteRate;
 
+    @SerializedName(ApiConstants.DISK_KBS_READ)
+    @Param(description = "the read (bytes) of disk on the vm")
+    private Long diskKbsRead;
+
+    @SerializedName(ApiConstants.DISK_KBS_WRITE)
+    @Param(description = "the write (bytes) of disk on the vm")
+    private Long diskKbsWrite;
+
+    @SerializedName(ApiConstants.DISK_IO_READ)
+    @Param(description = "the read (io) of disk on the vm")
+    private Long diskIORead;
+
+    @SerializedName(ApiConstants.DISK_IO_WRITE)
+    @Param(description = "the write (io) of disk on the vm")
+    private Long diskIOWrite;
+
     @SerializedName(ApiConstants.HYPERVISOR)
     @Param(description = "Hypervisor the volume belongs to")
     private String hypervisor;
@@ -395,10 +412,42 @@ public class VolumeResponse extends BaseResponseWithTagInformation implements Co
         this.iopsWriteRate = iopsWriteRate;
     }
 
+    public Long getDiskKbsRead() {
+        return diskKbsRead;
+    }
+
+    public void setDiskKbsRead(Long diskKbsRead) {
+        this.diskKbsRead = diskKbsRead;
+    }
+
+    public Long getDiskKbsWrite() {
+        return diskKbsWrite;
+    }
+
+    public void setDiskKbsWrite(Long diskKbsWrite) {
+        this.diskKbsWrite = diskKbsWrite;
+    }
+
     public Long getIopsWriteRate() {
         return iopsWriteRate;
     }
 
+    public Long getDiskIORead() {
+        return diskIORead;
+    }
+
+    public void setDiskIORead(Long diskIORead) {
+        this.diskIORead = diskIORead;
+    }
+
+    public Long getDiskIOWrite() {
+        return diskIOWrite;
+    }
+
+    public void setDiskIOWrite(Long diskIOWrite) {
+        this.diskIOWrite = diskIOWrite;
+    }
+
     public void setHypervisor(String hypervisor) {
         this.hypervisor = hypervisor;
     }

api/src/main/java/org/apache/cloudstack/query/QueryService.java

@@ -20,8 +20,8 @@ import java.util.List;
 
 import org.apache.cloudstack.affinity.AffinityGroupResponse;
 import org.apache.cloudstack.api.command.admin.domain.ListDomainsCmd;
-import org.apache.cloudstack.api.command.admin.host.ListHostsCmd;
 import org.apache.cloudstack.api.command.admin.host.ListHostTagsCmd;
+import org.apache.cloudstack.api.command.admin.host.ListHostsCmd;
 import org.apache.cloudstack.api.command.admin.internallb.ListInternalLBVMsCmd;
 import org.apache.cloudstack.api.command.admin.management.ListMgmtsCmd;
 import org.apache.cloudstack.api.command.admin.router.ListRoutersCmd;
@@ -40,6 +40,7 @@ import org.apache.cloudstack.api.command.user.offering.ListDiskOfferingsCmd;
 import org.apache.cloudstack.api.command.user.offering.ListServiceOfferingsCmd;
 import org.apache.cloudstack.api.command.user.project.ListProjectInvitationsCmd;
 import org.apache.cloudstack.api.command.user.project.ListProjectsCmd;
+import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd;
 import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd;
 import org.apache.cloudstack.api.command.user.tag.ListTagsCmd;
 import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd;
@@ -50,6 +51,7 @@ import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd;
 import org.apache.cloudstack.api.command.user.zone.ListZonesCmd;
 import org.apache.cloudstack.api.response.AccountResponse;
 import org.apache.cloudstack.api.response.AsyncJobResponse;
+import org.apache.cloudstack.api.response.DetailOptionsResponse;
 import org.apache.cloudstack.api.response.DiskOfferingResponse;
 import org.apache.cloudstack.api.response.DomainResponse;
 import org.apache.cloudstack.api.response.DomainRouterResponse;
@@ -147,6 +149,8 @@ public interface QueryService {
 
     ListResponse<TemplateResponse> listIsos(ListIsosCmd cmd);
 
+    DetailOptionsResponse listDetailOptions(ListDetailOptionsCmd cmd);
+
     ListResponse<AffinityGroupResponse> searchForAffinityGroups(ListAffinityGroupsCmd cmd);
 
     List<ResourceDetailResponse> listResourceDetails(ListResourceDetailsCmd cmd);

api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java

@@ -19,9 +19,13 @@ package org.apache.cloudstack.api.command.test;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyObject;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.isNull;
 
+import java.util.HashMap;
+import java.util.Map;
+
 import org.apache.cloudstack.api.ResponseGenerator;
 import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd;
@@ -32,6 +36,7 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.mockito.Mockito;
+import org.springframework.test.util.ReflectionTestUtils;
 
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.VolumeApiService;
@@ -87,9 +92,8 @@ public class CreateSnapshotCmdTest extends TestCase {
         VolumeApiService volumeApiService = Mockito.mock(VolumeApiService.class);
         Snapshot snapshot = Mockito.mock(Snapshot.class);
         try {
-
             Mockito.when(volumeApiService.takeSnapshot(anyLong(), anyLong(), anyLong(),
-                    any(Account.class), anyBoolean(), isNull(Snapshot.LocationType.class), anyBoolean())).thenReturn(snapshot);
+                    any(Account.class), anyBoolean(), isNull(Snapshot.LocationType.class), anyBoolean(), anyObject())).thenReturn(snapshot);
 
         } catch (Exception e) {
             Assert.fail("Received exception when success expected " + e.getMessage());
@@ -122,7 +126,7 @@ public class CreateSnapshotCmdTest extends TestCase {
 
         try {
                 Mockito.when(volumeApiService.takeSnapshot(anyLong(), anyLong(), anyLong(),
-                        any(Account.class), anyBoolean(), isNull(Snapshot.LocationType.class), anyBoolean())).thenReturn(null);
+                        any(Account.class), anyBoolean(), isNull(Snapshot.LocationType.class), anyBoolean(), anyObject())).thenReturn(null);
         } catch (Exception e) {
             Assert.fail("Received exception when success expected " + e.getMessage());
         }
@@ -136,4 +140,24 @@ public class CreateSnapshotCmdTest extends TestCase {
             Assert.assertEquals("Failed to create snapshot due to an internal error creating snapshot for volume 123", exception.getDescription());
         }
     }
+
+    @Test
+    public void testParsingTags() {
+        final CreateSnapshotCmd createSnapshotCmd = new CreateSnapshotCmd();
+        final Map<String, String> tag1 = new HashMap<>();
+        tag1.put("key", "key1");
+        tag1.put("value", "value1");
+        final Map<String, String> tag2 = new HashMap<>();
+        tag2.put("key", "key2");
+        tag2.put("value", "value2");
+        final Map<String, String> expectedTags = new HashMap<>();
+        expectedTags.put("key1", "value1");
+        expectedTags.put("key2", "value2");
+
+        final Map<String, Map<String, String>> tagsParams = new HashMap<>();
+        tagsParams.put("0", tag1);
+        tagsParams.put("1", tag2);
+        ReflectionTestUtils.setField(createSnapshotCmd, "tags", tagsParams);
+        Assert.assertEquals(createSnapshotCmd.getTags(), expectedTags);
+    }
 }

api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotPolicyCmdTest.java

@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.snapshot;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.springframework.test.util.ReflectionTestUtils;
+
+public class CreateSnapshotPolicyCmdTest {
+    @Test
+    public void testParsingTags() {
+        final CreateSnapshotPolicyCmd createSnapshotPolicyCmd = new CreateSnapshotPolicyCmd();
+        final Map<String, String> tag1 = new HashMap<>();
+        tag1.put("key", "key1");
+        tag1.put("value", "value1");
+        final Map<String, String> tag2 = new HashMap<>();
+        tag2.put("key", "key2");
+        tag2.put("value", "value2");
+        final Map<String, String> expectedTags = new HashMap<>();
+        expectedTags.put("key1", "value1");
+        expectedTags.put("key2", "value2");
+
+        final Map<String, Map<String, String>> tagsParams = new HashMap<>();
+        tagsParams.put("0", tag1);
+        tagsParams.put("1", tag2);
+        ReflectionTestUtils.setField(createSnapshotPolicyCmd, "tags", tagsParams);
+        Assert.assertEquals(createSnapshotPolicyCmd.getTags(), expectedTags);
+    }
+}

core/src/main/java/com/cloud/info/ConsoleProxyInfo.java

@@ -55,18 +55,16 @@ public class ConsoleProxyInfo {
 
     private String formatProxyAddress(String consoleProxyUrlDomain, String proxyIpAddress) {
         StringBuffer sb = new StringBuffer();
+        if (StringUtils.isBlank(consoleProxyUrlDomain)) {
+            // Blank config, we use the proxy IP
+            sb.append(proxyIpAddress);
+        } else if (consoleProxyUrlDomain.startsWith("*")) {
             // Domain in format *.example.com, proxy IP is 1.2.3.4 --> 1-2-3-4.example.com
-        if (consoleProxyUrlDomain.startsWith("*")) {
             sb.append(proxyIpAddress.replaceAll("\\.", "-"));
             sb.append(consoleProxyUrlDomain.substring(1)); // skip the *
-
-        // Otherwise we assume a valid domain if config not blank
-        } else if (StringUtils.isNotBlank(consoleProxyUrlDomain)) {
-            sb.append(consoleProxyUrlDomain);
-
-        // Blank config, we use the proxy IP
         } else {
-            sb.append(proxyIpAddress);
+            // Otherwise we assume a valid domain if config not blank
+            sb.append(consoleProxyUrlDomain);
         }
         return sb.toString();
     }

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -1071,16 +1071,16 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                 long destHostId = dest.getHost().getId();
                 vm.setPodIdToDeployIn(dest.getPod().getId());
                 final Long cluster_id = dest.getCluster().getId();
-                final ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio");
+                final ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.CPU_OVER_COMMIT_RATIO);
-                final ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, "memoryOvercommitRatio");
+                final ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
-                //storing the value of overcommit in the vm_details table for doing a capacity check in case the cluster overcommit ratio is changed.
+                //storing the value of overcommit in the user_vm_details table for doing a capacity check in case the cluster overcommit ratio is changed.
-                if (userVmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio") == null &&
+                if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) == null &&
                         (Float.parseFloat(cluster_detail_cpu.getValue()) > 1f || Float.parseFloat(cluster_detail_ram.getValue()) > 1f)) {
-                    userVmDetailsDao.addDetail(vm.getId(), "cpuOvercommitRatio", cluster_detail_cpu.getValue(), true);
+                    userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true);
-                    userVmDetailsDao.addDetail(vm.getId(), "memoryOvercommitRatio", cluster_detail_ram.getValue(), true);
+                    userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true);
-                } else if (userVmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio") != null) {
+                } else if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) != null) {
-                    userVmDetailsDao.addDetail(vm.getId(), "cpuOvercommitRatio", cluster_detail_cpu.getValue(), true);
+                    userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true);
-                    userVmDetailsDao.addDetail(vm.getId(), "memoryOvercommitRatio", cluster_detail_ram.getValue(), true);
+                    userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true);
                 }
 
                 vmProfile.setCpuOvercommitRatio(Float.parseFloat(cluster_detail_cpu.getValue()));
@@ -1160,8 +1160,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                             // Remove the information on whether it was a deploy vm request.The deployvm=true information
                             // is set only when the vm is being deployed. When a vm is started from a stop state the
                             // information isn't set,
-                            if (userVmDetailsDao.findDetail(vm.getId(), "deployvm") != null) {
+                            if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.DEPLOY_VM) != null) {
-                                userVmDetailsDao.removeDetail(vm.getId(), "deployvm");
+                                userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.DEPLOY_VM);
                             }
 
                             startedVm = vm;
@@ -1455,7 +1455,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                     if (platform != null) {
                         final UserVmVO userVm = _userVmDao.findById(vm.getId());
                         _userVmDao.loadDetails(userVm);
-                        userVm.setDetail("platform", platform);
+                        userVm.setDetail(VmDetailConstants.PLATFORM, platform);
                         _userVmDao.saveDetails(userVm);
                     }
                 }
@@ -1731,7 +1731,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                         if (platform != null) {
                             final UserVmVO userVm = _userVmDao.findById(vm.getId());
                             _userVmDao.loadDetails(userVm);
-                            userVm.setDetail("platform", platform);
+                            userVm.setDetail(VmDetailConstants.PLATFORM, platform);
                             _userVmDao.saveDetails(userVm);
                         }
                     }
@@ -3174,16 +3174,16 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     private void updateVmMetaData(Long vmId, String platform) {
         UserVmVO userVm = _userVmDao.findById(vmId);
         _userVmDao.loadDetails(userVm);
-        if ( userVm.details.containsKey("timeoffset")) {
+        if ( userVm.details.containsKey(VmDetailConstants.TIME_OFFSET)) {
-            userVm.details.remove("timeoffset");
+            userVm.details.remove(VmDetailConstants.TIME_OFFSET);
         }
-        userVm.setDetail("platform",  platform);
+        userVm.setDetail(VmDetailConstants.PLATFORM,  platform);
         String pvdriver = "xenserver56";
         if ( platform.contains("device_id")) {
             pvdriver = "xenserver61";
         }
-        if (!userVm.details.containsKey("hypervisortoolsversion") || !userVm.details.get("hypervisortoolsversion").equals(pvdriver)) {
+        if (!userVm.details.containsKey(VmDetailConstants.HYPERVISOR_TOOLS_VERSION) || !userVm.details.get(VmDetailConstants.HYPERVISOR_TOOLS_VERSION).equals(pvdriver)) {
-            userVm.setDetail("hypervisortoolsversion", pvdriver);
+            userVm.setDetail(VmDetailConstants.HYPERVISOR_TOOLS_VERSION, pvdriver);
         }
         _userVmDao.saveDetails(userVm);
     }

engine/schema/src/main/resources/META-INF/db/schema-41200to41300.sql

@@ -23,6 +23,10 @@
 INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '6.7', 128, 0, 13, 32, 1, 1);
 INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'VMware', '6.7', guest_os_name, guest_os_id, utc_timestamp(), 0  FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='VMware' AND hypervisor_version='6.5';
 
+-- XenServer 7.6
+INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, max_data_volumes_limit, storage_motion_supported) values (UUID(), 'XenServer', '7.6.0', 500, 13, 1);
+INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '7.6.0', guest_os_name, guest_os_id, utc_timestamp(), 0  FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='7.5.0';
+
 -- DPDK client and server mode support
 ALTER TABLE `cloud`.`service_offering_details` CHANGE COLUMN `value` `value` TEXT NOT NULL;
 

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java

@@ -42,68 +42,7 @@ import java.util.TimeZone;
 import java.util.UUID;
 
 import javax.naming.ConfigurationException;
-
+import javax.xml.datatype.XMLGregorianCalendar;
-import org.apache.commons.lang.math.NumberUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.Logger;
-import org.apache.log4j.NDC;
-import org.joda.time.Duration;
-
-import com.google.gson.Gson;
-import com.vmware.vim25.AboutInfo;
-import com.vmware.vim25.BoolPolicy;
-import com.vmware.vim25.ComputeResourceSummary;
-import com.vmware.vim25.CustomFieldStringValue;
-import com.vmware.vim25.DVPortConfigInfo;
-import com.vmware.vim25.DVPortConfigSpec;
-import com.vmware.vim25.DasVmPriority;
-import com.vmware.vim25.DatastoreSummary;
-import com.vmware.vim25.DistributedVirtualPort;
-import com.vmware.vim25.DistributedVirtualSwitchPortConnection;
-import com.vmware.vim25.DistributedVirtualSwitchPortCriteria;
-import com.vmware.vim25.DynamicProperty;
-import com.vmware.vim25.GuestInfo;
-import com.vmware.vim25.HostCapability;
-import com.vmware.vim25.HostHostBusAdapter;
-import com.vmware.vim25.HostInternetScsiHba;
-import com.vmware.vim25.ManagedObjectReference;
-import com.vmware.vim25.ObjectContent;
-import com.vmware.vim25.OptionValue;
-import com.vmware.vim25.PerfCounterInfo;
-import com.vmware.vim25.PerfEntityMetric;
-import com.vmware.vim25.PerfEntityMetricBase;
-import com.vmware.vim25.PerfMetricId;
-import com.vmware.vim25.PerfMetricIntSeries;
-import com.vmware.vim25.PerfMetricSeries;
-import com.vmware.vim25.PerfQuerySpec;
-import com.vmware.vim25.PerfSampleInfo;
-import com.vmware.vim25.RuntimeFaultFaultMsg;
-import com.vmware.vim25.ToolsUnavailableFaultMsg;
-import com.vmware.vim25.VMwareDVSPortSetting;
-import com.vmware.vim25.VimPortType;
-import com.vmware.vim25.VirtualDevice;
-import com.vmware.vim25.VirtualDeviceBackingInfo;
-import com.vmware.vim25.VirtualDeviceConfigSpec;
-import com.vmware.vim25.VirtualDeviceConfigSpecOperation;
-import com.vmware.vim25.VirtualDisk;
-import com.vmware.vim25.VirtualDiskFlatVer2BackingInfo;
-import com.vmware.vim25.VirtualEthernetCard;
-import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo;
-import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo;
-import com.vmware.vim25.VirtualEthernetCardOpaqueNetworkBackingInfo;
-import com.vmware.vim25.VirtualMachineConfigSpec;
-import com.vmware.vim25.VirtualMachineFileInfo;
-import com.vmware.vim25.VirtualMachineFileLayoutEx;
-import com.vmware.vim25.VirtualMachineFileLayoutExFileInfo;
-import com.vmware.vim25.VirtualMachineGuestOsIdentifier;
-import com.vmware.vim25.VirtualMachinePowerState;
-import com.vmware.vim25.VirtualMachineRelocateSpec;
-import com.vmware.vim25.VirtualMachineRelocateSpecDiskLocator;
-import com.vmware.vim25.VirtualMachineRuntimeInfo;
-import com.vmware.vim25.VirtualMachineToolsStatus;
-import com.vmware.vim25.VirtualMachineVideoCard;
-import com.vmware.vim25.VirtualUSBController;
-import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec;
 
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.storage.command.CopyCommand;
@@ -114,6 +53,11 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
 import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.log4j.Logger;
+import org.apache.log4j.NDC;
+import org.joda.time.Duration;
 
 import com.cloud.agent.IAgentControl;
 import com.cloud.agent.api.Answer;
@@ -211,6 +155,7 @@ import com.cloud.agent.api.UnregisterVMCommand;
 import com.cloud.agent.api.UpgradeSnapshotCommand;
 import com.cloud.agent.api.ValidateSnapshotAnswer;
 import com.cloud.agent.api.ValidateSnapshotCommand;
+import com.cloud.agent.api.VmDiskStatsEntry;
 import com.cloud.agent.api.VmStatsEntry;
 import com.cloud.agent.api.VolumeStatsEntry;
 import com.cloud.agent.api.check.CheckSshAnswer;
@@ -310,6 +255,60 @@ import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.PowerState;
 import com.cloud.vm.VirtualMachineName;
 import com.cloud.vm.VmDetailConstants;
+import com.google.gson.Gson;
+import com.vmware.vim25.AboutInfo;
+import com.vmware.vim25.BoolPolicy;
+import com.vmware.vim25.ComputeResourceSummary;
+import com.vmware.vim25.CustomFieldStringValue;
+import com.vmware.vim25.DVPortConfigInfo;
+import com.vmware.vim25.DVPortConfigSpec;
+import com.vmware.vim25.DasVmPriority;
+import com.vmware.vim25.DatastoreSummary;
+import com.vmware.vim25.DistributedVirtualPort;
+import com.vmware.vim25.DistributedVirtualSwitchPortConnection;
+import com.vmware.vim25.DistributedVirtualSwitchPortCriteria;
+import com.vmware.vim25.DynamicProperty;
+import com.vmware.vim25.GuestInfo;
+import com.vmware.vim25.HostCapability;
+import com.vmware.vim25.HostHostBusAdapter;
+import com.vmware.vim25.HostInternetScsiHba;
+import com.vmware.vim25.ManagedObjectReference;
+import com.vmware.vim25.ObjectContent;
+import com.vmware.vim25.OptionValue;
+import com.vmware.vim25.PerfCounterInfo;
+import com.vmware.vim25.PerfEntityMetric;
+import com.vmware.vim25.PerfEntityMetricBase;
+import com.vmware.vim25.PerfMetricId;
+import com.vmware.vim25.PerfMetricIntSeries;
+import com.vmware.vim25.PerfMetricSeries;
+import com.vmware.vim25.PerfQuerySpec;
+import com.vmware.vim25.RuntimeFaultFaultMsg;
+import com.vmware.vim25.ToolsUnavailableFaultMsg;
+import com.vmware.vim25.VMwareDVSPortSetting;
+import com.vmware.vim25.VimPortType;
+import com.vmware.vim25.VirtualDevice;
+import com.vmware.vim25.VirtualDeviceBackingInfo;
+import com.vmware.vim25.VirtualDeviceConfigSpec;
+import com.vmware.vim25.VirtualDeviceConfigSpecOperation;
+import com.vmware.vim25.VirtualDisk;
+import com.vmware.vim25.VirtualDiskFlatVer2BackingInfo;
+import com.vmware.vim25.VirtualEthernetCard;
+import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo;
+import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo;
+import com.vmware.vim25.VirtualEthernetCardOpaqueNetworkBackingInfo;
+import com.vmware.vim25.VirtualMachineConfigSpec;
+import com.vmware.vim25.VirtualMachineFileInfo;
+import com.vmware.vim25.VirtualMachineFileLayoutEx;
+import com.vmware.vim25.VirtualMachineFileLayoutExFileInfo;
+import com.vmware.vim25.VirtualMachineGuestOsIdentifier;
+import com.vmware.vim25.VirtualMachinePowerState;
+import com.vmware.vim25.VirtualMachineRelocateSpec;
+import com.vmware.vim25.VirtualMachineRelocateSpecDiskLocator;
+import com.vmware.vim25.VirtualMachineRuntimeInfo;
+import com.vmware.vim25.VirtualMachineToolsStatus;
+import com.vmware.vim25.VirtualMachineVideoCard;
+import com.vmware.vim25.VirtualUSBController;
+import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec;
 
 public class VmwareResource implements StoragePoolResource, ServerResource, VmwareHostService, VirtualRouterDeployer {
     private static final Logger s_logger = Logger.getLogger(VmwareResource.class);
@@ -1874,7 +1873,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
 
             // Check for multi-cores per socket settings
             int numCoresPerSocket = 1;
-            String coresPerSocket = vmSpec.getDetails().get("cpu.corespersocket");
+            String coresPerSocket = vmSpec.getDetails().get(VmDetailConstants.CPU_CORE_PER_SOCKET);
             if (coresPerSocket != null) {
                 String apiVersion = HypervisorHostHelper.getVcenterApiVersion(vmMo.getContext());
                 // Property 'numCoresPerSocket' is supported since vSphere API 5.0
@@ -3469,6 +3468,120 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
     }
 
     protected Answer execute(GetVmDiskStatsCommand cmd) {
+        try {
+            final VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext());
+            final ManagedObjectReference perfMgr = getServiceContext().getServiceContent().getPerfManager();
+            VimPortType service = getServiceContext().getService();
+
+            final int intervalSeconds = 300;
+            final XMLGregorianCalendar startTime = VmwareHelper.getXMLGregorianCalendar(new Date(), intervalSeconds);
+            final XMLGregorianCalendar endTime = VmwareHelper.getXMLGregorianCalendar(new Date(), 0);
+
+            PerfCounterInfo diskReadIOPerfCounterInfo = null;
+            PerfCounterInfo diskWriteIOPerfCounterInfo = null;
+            PerfCounterInfo diskReadKbsPerfCounterInfo = null;
+            PerfCounterInfo diskWriteKbsPerfCounterInfo = null;
+
+            // https://pubs.vmware.com/vsphere-5-5/topic/com.vmware.wssdk.apiref.doc/virtual_disk_counters.html
+            List<PerfCounterInfo> cInfo = getServiceContext().getVimClient().getDynamicProperty(perfMgr, "perfCounter");
+            for (PerfCounterInfo info : cInfo) {
+                if ("virtualdisk".equalsIgnoreCase(info.getGroupInfo().getKey()) && "average".equalsIgnoreCase(info.getRollupType().value())) {
+                    if ("numberReadAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) {
+                        diskReadIOPerfCounterInfo = info;
+                    }
+                    if ("numberWriteAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) {
+                        diskWriteIOPerfCounterInfo = info;
+                    }
+                    if ("read".equalsIgnoreCase(info.getNameInfo().getKey())) {
+                        diskReadKbsPerfCounterInfo = info;
+                    }
+                    if ("write".equalsIgnoreCase(info.getNameInfo().getKey())) {
+                        diskWriteKbsPerfCounterInfo = info;
+                    }
+                }
+            }
+
+            final ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
+            final DatacenterMO dcMo = new DatacenterMO(getServiceContext(), dcMor);
+
+            final HashMap<String, List<VmDiskStatsEntry>> vmStatsMap = new HashMap<>();
+            for (final String vmName : cmd.getVmNames()) {
+                final VirtualMachineMO vmMo = dcMo.findVm(vmName);
+                final List<VmDiskStatsEntry> diskStats = new ArrayList<>();
+                for (final VirtualDisk disk : vmMo.getAllDiskDevice()) {
+                    final String diskBusName = vmMo.getDeviceBusName(vmMo.getAllDeviceList(), disk);
+                    long readReq = 0;
+                    long readBytes = 0;
+                    long writeReq = 0;
+                    long writeBytes = 0;
+
+                    final ArrayList<PerfMetricId> perfMetricsIds = new ArrayList<PerfMetricId>();
+                    if (diskReadIOPerfCounterInfo != null) {
+                        perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadIOPerfCounterInfo, diskBusName));
+                    }
+                    if (diskWriteIOPerfCounterInfo != null) {
+                        perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteIOPerfCounterInfo, diskBusName));
+                    }
+                    if (diskReadKbsPerfCounterInfo != null) {
+                        perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadKbsPerfCounterInfo, diskBusName));
+                    }
+                    if (diskWriteKbsPerfCounterInfo != null) {
+                        perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteKbsPerfCounterInfo, diskBusName));
+                    }
+
+                    if (perfMetricsIds.size() > 0) {
+                        final PerfQuerySpec qSpec = new PerfQuerySpec();
+                        qSpec.setEntity(vmMo.getMor());
+                        qSpec.setFormat("normal");
+                        qSpec.setIntervalId(intervalSeconds);
+                        qSpec.setStartTime(startTime);
+                        qSpec.setEndTime(endTime);
+                        qSpec.getMetricId().addAll(perfMetricsIds);
+
+                        for (final PerfEntityMetricBase perfValue : service.queryPerf(perfMgr, Collections.singletonList(qSpec))) {
+                            if (!(perfValue instanceof PerfEntityMetric)) {
+                                continue;
+                            }
+                            final List<PerfMetricSeries> values = ((PerfEntityMetric)perfValue).getValue();
+                            if (values == null || values.isEmpty()) {
+                                continue;
+                            }
+                            for (final PerfMetricSeries value : values) {
+                                if (!(value instanceof PerfMetricIntSeries) || !value.getId().getInstance().equals(diskBusName)) {
+                                    continue;
+                                }
+                                final List<Long> perfStats = ((PerfMetricIntSeries)value).getValue();
+                                if (perfStats.size() > 0) {
+                                    long sum = 0;
+                                    for (long val : perfStats) {
+                                        sum += val;
+                                    }
+                                    long avg = sum / perfStats.size();
+                                    if (value.getId().getCounterId() == diskReadIOPerfCounterInfo.getKey()) {
+                                        readReq = avg;
+                                    } else if (value.getId().getCounterId() == diskWriteIOPerfCounterInfo.getKey()) {
+                                        writeReq = avg;
+                                    } else if (value.getId().getCounterId() == diskReadKbsPerfCounterInfo.getKey()) {
+                                        readBytes = avg * 1024;
+                                    } else if (value.getId().getCounterId() == diskWriteKbsPerfCounterInfo.getKey()) {
+                                        writeBytes = avg * 1024;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    diskStats.add(new VmDiskStatsEntry(vmName, VmwareHelper.getDiskDeviceFileName(disk), writeReq, readReq, writeBytes, readBytes));
+                }
+                if (diskStats.size() > 0) {
+                    vmStatsMap.put(vmName, diskStats);
+                }
+            }
+            if (vmStatsMap.size() > 0) {
+                return new GetVmDiskStatsAnswer(cmd, "", cmd.getHostName(), vmStatsMap);
+            }
+        } catch (Exception e) {
+            s_logger.error("Unable to execute GetVmDiskStatsCommand due to " + VmwareHelper.getExceptionMessage(e), e);
+        }
         return new GetVmDiskStatsAnswer(cmd, null, null, null);
     }
 
@@ -5857,12 +5970,21 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
         HashMap<String, VmStatsEntry> vmResponseMap = new HashMap<String, VmStatsEntry>();
         ManagedObjectReference perfMgr = getServiceContext().getServiceContent().getPerfManager();
         VimPortType service = getServiceContext().getService();
+
         PerfCounterInfo rxPerfCounterInfo = null;
         PerfCounterInfo txPerfCounterInfo = null;
+        PerfCounterInfo diskReadIOPerfCounterInfo = null;
+        PerfCounterInfo diskWriteIOPerfCounterInfo = null;
+        PerfCounterInfo diskReadKbsPerfCounterInfo = null;
+        PerfCounterInfo diskWriteKbsPerfCounterInfo = null;
+
+        final int intervalSeconds = 300;
+        final XMLGregorianCalendar startTime = VmwareHelper.getXMLGregorianCalendar(new Date(), intervalSeconds);
+        final XMLGregorianCalendar endTime = VmwareHelper.getXMLGregorianCalendar(new Date(), 0);
 
         List<PerfCounterInfo> cInfo = getServiceContext().getVimClient().getDynamicProperty(perfMgr, "perfCounter");
         for (PerfCounterInfo info : cInfo) {
-            if ("net".equalsIgnoreCase(info.getGroupInfo().getKey())) {
+            if ("net".equalsIgnoreCase(info.getGroupInfo().getKey()) && "average".equalsIgnoreCase(info.getRollupType().value())) {
                 if ("transmitted".equalsIgnoreCase(info.getNameInfo().getKey())) {
                     txPerfCounterInfo = info;
                 }
@@ -5870,6 +5992,20 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
                     rxPerfCounterInfo = info;
                 }
             }
+            if ("virtualdisk".equalsIgnoreCase(info.getGroupInfo().getKey())) {
+                if ("numberReadAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) {
+                    diskReadIOPerfCounterInfo = info;
+                }
+                if ("numberWriteAveraged".equalsIgnoreCase(info.getNameInfo().getKey())) {
+                    diskWriteIOPerfCounterInfo = info;
+                }
+                if ("read".equalsIgnoreCase(info.getNameInfo().getKey())) {
+                    diskReadKbsPerfCounterInfo = info;
+                }
+                if ("write".equalsIgnoreCase(info.getNameInfo().getKey())) {
+                    diskWriteKbsPerfCounterInfo = info;
+                }
+            }
         }
 
         int key = ((HostMO)hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME);
@@ -5885,8 +6021,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
         final String memMbStr = "config.hardware.memoryMB";
         final String allocatedCpuStr = "summary.runtime.maxCpuUsage";
 
-        ObjectContent[] ocs =
+        ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost(new String[] {
-                hyperHost.getVmPropertiesOnHyperHost(new String[] {"name", numCpuStr, cpuUseStr ,guestMemUseStr ,memLimitStr ,memMbStr,allocatedCpuStr ,instanceNameCustomField});
+                "name", numCpuStr, cpuUseStr, guestMemUseStr, memLimitStr, memMbStr,allocatedCpuStr, instanceNameCustomField
+        });
 
         if (ocs != null && ocs.length > 0) {
             for (ObjectContent oc : ocs) {
@@ -5923,7 +6060,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
                     }
 
                     maxCpuUsage = (maxCpuUsage/allocatedCpu)*100;
-                    new VirtualMachineMO(hyperHost.getContext(), oc.getObj());
                     if (vmInternalCSName != null) {
                         name = vmInternalCSName;
                     } else {
@@ -5937,60 +6073,86 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
                     ManagedObjectReference vmMor = hyperHost.findVmOnHyperHost(name).getMor();
                     assert (vmMor != null);
 
-                    ArrayList<PerfMetricId> vmNetworkMetrics = new ArrayList<PerfMetricId>();
-                    // get all the metrics from the available sample period
-                    List<PerfMetricId> perfMetrics = service.queryAvailablePerfMetric(perfMgr, vmMor, null, null, null);
-                    if (perfMetrics != null) {
-                        for (int index = 0; index < perfMetrics.size(); ++index) {
-                            if (((rxPerfCounterInfo != null) && (perfMetrics.get(index).getCounterId() == rxPerfCounterInfo.getKey()))
-                                    || ((txPerfCounterInfo != null) && (perfMetrics.get(index).getCounterId() == txPerfCounterInfo.getKey()))) {
-                                vmNetworkMetrics.add(perfMetrics.get(index));
-                            }
-                        }
-                    }
-
                     double networkReadKBs = 0;
                     double networkWriteKBs = 0;
-                    long sampleDuration = 0;
+                    double diskReadIops = 0;
+                    double diskWriteIops = 0;
+                    double diskReadKbs = 0;
+                    double diskWriteKbs = 0;
 
-                    if (vmNetworkMetrics.size() != 0) {
+                    final ArrayList<PerfMetricId> perfMetricsIds = new ArrayList<PerfMetricId>();
-                        PerfQuerySpec qSpec = new PerfQuerySpec();
+                    if (rxPerfCounterInfo != null) {
+                        perfMetricsIds.add(VmwareHelper.createPerfMetricId(rxPerfCounterInfo, ""));
+                    }
+                    if (txPerfCounterInfo != null) {
+                        perfMetricsIds.add(VmwareHelper.createPerfMetricId(txPerfCounterInfo, ""));
+                    }
+                    if (diskReadIOPerfCounterInfo != null) {
+                        perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadIOPerfCounterInfo, "*"));
+                    }
+                    if (diskWriteIOPerfCounterInfo != null) {
+                        perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteIOPerfCounterInfo, "*"));
+                    }
+                    if (diskReadKbsPerfCounterInfo != null) {
+                        perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskReadKbsPerfCounterInfo, ""));
+                    }
+                    if (diskWriteKbsPerfCounterInfo != null) {
+                        perfMetricsIds.add(VmwareHelper.createPerfMetricId(diskWriteKbsPerfCounterInfo, ""));
+                    }
+
+                    if (perfMetricsIds.size() > 0) {
+                        final PerfQuerySpec qSpec = new PerfQuerySpec();
                         qSpec.setEntity(vmMor);
-                        PerfMetricId[] availableMetricIds = vmNetworkMetrics.toArray(new PerfMetricId[0]);
+                        qSpec.setFormat("normal");
-                        qSpec.getMetricId().addAll(Arrays.asList(availableMetricIds));
+                        qSpec.setIntervalId(intervalSeconds);
-                        List<PerfQuerySpec> qSpecs = new ArrayList<PerfQuerySpec>();
+                        qSpec.setStartTime(startTime);
-                        qSpecs.add(qSpec);
+                        qSpec.setEndTime(endTime);
-                        List<PerfEntityMetricBase> values = service.queryPerf(perfMgr, qSpecs);
+                        qSpec.getMetricId().addAll(perfMetricsIds);
+                        final List<PerfEntityMetricBase> perfValues = service.queryPerf(perfMgr, Collections.singletonList(qSpec));
+                        for (final PerfEntityMetricBase perfValue : perfValues) {
+                            if (!(perfValue instanceof PerfEntityMetric)) {
+                                continue;
+                            }
+                            final List<PerfMetricSeries> seriesList = ((PerfEntityMetric) perfValue).getValue();
+                            for (final PerfMetricSeries series : seriesList) {
+                                if (!(series instanceof PerfMetricIntSeries)) {
+                                    continue;
+                                }
+                                final List<Long> values = ((PerfMetricIntSeries) series).getValue();
+                                double sum = 0;
+                                for (final Long value : values) {
+                                    sum += value;
+                                }
+                                double avg = sum / values.size();
+                                if (series.getId().getCounterId() == rxPerfCounterInfo.getKey()) {
+                                    networkReadKBs = avg;
+                                }
+                                if (series.getId().getCounterId() == txPerfCounterInfo.getKey()) {
+                                    networkWriteKBs = avg;
+                                }
+                                if (series.getId().getCounterId() == diskReadIOPerfCounterInfo.getKey()) {
+                                    diskReadIops += avg;
+                                }
+                                if (series.getId().getCounterId() == diskWriteIOPerfCounterInfo.getKey()) {
+                                    diskWriteIops += avg;
+                                }
+                                if (series.getId().getCounterId() == diskReadKbsPerfCounterInfo.getKey()) {
+                                    diskReadKbs = avg;
+                                }
+                                if (series.getId().getCounterId() == diskWriteKbsPerfCounterInfo.getKey()) {
+                                    diskWriteKbs = avg;
+                                }
+                            }
+                        }
+                    }
 
-                        for (int i = 0; i < values.size(); ++i) {
+                    final VmStatsEntry vmStats = new VmStatsEntry( NumberUtils.toDouble(memkb)*1024,NumberUtils.toDouble(guestMemusage)*1024,NumberUtils.toDouble(memlimit)*1024,
-                            List<PerfSampleInfo> infos = ((PerfEntityMetric)values.get(i)).getSampleInfo();
+                            maxCpuUsage, networkReadKBs, networkWriteKBs, NumberUtils.toInt(numberCPUs), "vm");
-                            if (infos != null && infos.size() > 0) {
+                    vmStats.setDiskReadIOs(diskReadIops);
-                                int endMs = infos.get(infos.size() - 1).getTimestamp().getSecond() * 1000 + infos.get(infos.size() - 1).getTimestamp().getMillisecond();
+                    vmStats.setDiskWriteIOs(diskWriteIops);
-                                int beginMs = infos.get(0).getTimestamp().getSecond() * 1000 + infos.get(0).getTimestamp().getMillisecond();
+                    vmStats.setDiskReadKBs(diskReadKbs);
-                                sampleDuration = (endMs - beginMs) / 1000;
+                    vmStats.setDiskWriteKBs(diskWriteKbs);
-                                List<PerfMetricSeries> vals = ((PerfEntityMetric)values.get(i)).getValue();
+                    vmResponseMap.put(name, vmStats);
-                                for (int vi = 0; ((vals != null) && (vi < vals.size())); ++vi) {
-                                    if (vals.get(vi) instanceof PerfMetricIntSeries) {
-                                        PerfMetricIntSeries val = (PerfMetricIntSeries)vals.get(vi);
-                                        List<Long> perfValues = val.getValue();
-                                        Long sumRate = 0L;
-                                        for (int j = 0; j < infos.size(); j++) { // Size of the array matches the size as the PerfSampleInfo
-                                            sumRate += perfValues.get(j);
-                                        }
-                                        Long averageRate = sumRate / infos.size();
-                                        if (vals.get(vi).getId().getCounterId() == rxPerfCounterInfo.getKey()) {
-                                            networkReadKBs = sampleDuration * averageRate; //get the average RX rate multiplied by sampled duration
-                                        }
-                                        if (vals.get(vi).getId().getCounterId() == txPerfCounterInfo.getKey()) {
-                                            networkWriteKBs = sampleDuration * averageRate;//get the average TX rate multiplied by sampled duration
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                    vmResponseMap.put(name, new VmStatsEntry( NumberUtils.toDouble(memkb)*1024,NumberUtils.toDouble(guestMemusage)*1024,NumberUtils.toDouble(memlimit)*1024,
-                            maxCpuUsage, networkReadKBs, networkWriteKBs, NumberUtils.toInt(numberCPUs), "vm"));
 
                 }
             }

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java

@@ -134,6 +134,7 @@ import com.cloud.utils.ssh.SSHCmdHelper;
 import com.cloud.utils.ssh.SshHelper;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.PowerState;
+import com.cloud.vm.VmDetailConstants;
 import com.trilead.ssh2.SCPClient;
 import com.xensource.xenapi.Bond;
 import com.xensource.xenapi.Connection;
@@ -1862,18 +1863,18 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
 
         final Map<String, String> details = vmSpec.getDetails();
         if (details != null) {
-            final String platformstring = details.get("platform");
+            final String platformstring = details.get(VmDetailConstants.PLATFORM);
             if (platformstring != null && !platformstring.isEmpty()) {
                 final Map<String, String> platform = StringUtils.stringToMap(platformstring);
                 vm.setPlatform(conn, platform);
             } else {
-                final String timeoffset = details.get("timeoffset");
+                final String timeoffset = details.get(VmDetailConstants.TIME_OFFSET);
                 if (timeoffset != null) {
                     final Map<String, String> platform = vm.getPlatform(conn);
-                    platform.put("timeoffset", timeoffset);
+                    platform.put(VmDetailConstants.TIME_OFFSET, timeoffset);
                     vm.setPlatform(conn, platform);
                 }
-                final String coresPerSocket = details.get("cpu.corespersocket");
+                final String coresPerSocket = details.get(VmDetailConstants.CPU_CORE_PER_SOCKET);
                 if (coresPerSocket != null) {
                     final Map<String, String> platform = vm.getPlatform(conn);
                     platform.put("cores-per-socket", coresPerSocket);
@@ -1881,7 +1882,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
                 }
             }
             if (!BootloaderType.CD.equals(vmSpec.getBootloader())) {
-                final String xenservertoolsversion = details.get("hypervisortoolsversion");
+                final String xenservertoolsversion = details.get(VmDetailConstants.HYPERVISOR_TOOLS_VERSION);
                 if ((xenservertoolsversion == null || !xenservertoolsversion.equalsIgnoreCase("xenserver61")) && vmSpec.getGpuDevice() == null) {
                     final Map<String, String> platform = vm.getPlatform(conn);
                     platform.remove("device_id");

plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java

@@ -17,6 +17,39 @@
 
 package org.apache.cloudstack.metrics;
 
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.inject.Inject;
+
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.ListClustersMetricsCmd;
+import org.apache.cloudstack.api.ListHostsMetricsCmd;
+import org.apache.cloudstack.api.ListInfrastructureCmd;
+import org.apache.cloudstack.api.ListStoragePoolsMetricsCmd;
+import org.apache.cloudstack.api.ListVMsMetricsCmd;
+import org.apache.cloudstack.api.ListVolumesMetricsCmd;
+import org.apache.cloudstack.api.ListZonesMetricsCmd;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.ClusterResponse;
+import org.apache.cloudstack.api.response.HostResponse;
+import org.apache.cloudstack.api.response.StoragePoolResponse;
+import org.apache.cloudstack.api.response.UserVmResponse;
+import org.apache.cloudstack.api.response.VolumeResponse;
+import org.apache.cloudstack.api.response.ZoneResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.response.ClusterMetricsResponse;
+import org.apache.cloudstack.response.HostMetricsResponse;
+import org.apache.cloudstack.response.InfrastructureResponse;
+import org.apache.cloudstack.response.StoragePoolMetricsResponse;
+import org.apache.cloudstack.response.VmMetricsResponse;
+import org.apache.cloudstack.response.VolumeMetricsResponse;
+import org.apache.cloudstack.response.ZoneMetricsResponse;
+import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.commons.beanutils.BeanUtils;
+
 import com.cloud.alert.AlertManager;
 import com.cloud.api.ApiDBUtils;
 import com.cloud.api.query.dao.HostJoinDao;
@@ -45,37 +78,6 @@ import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.dao.DomainRouterDao;
 import com.cloud.vm.dao.VMInstanceDao;
-import org.apache.cloudstack.api.ApiErrorCode;
-import org.apache.cloudstack.api.ListClustersMetricsCmd;
-import org.apache.cloudstack.api.ListHostsMetricsCmd;
-import org.apache.cloudstack.api.ListInfrastructureCmd;
-import org.apache.cloudstack.api.ListStoragePoolsMetricsCmd;
-import org.apache.cloudstack.api.ListVMsMetricsCmd;
-import org.apache.cloudstack.api.ListVolumesMetricsCmd;
-import org.apache.cloudstack.api.ListZonesMetricsCmd;
-import org.apache.cloudstack.api.ServerApiException;
-import org.apache.cloudstack.api.response.ClusterResponse;
-import org.apache.cloudstack.api.response.HostResponse;
-import org.apache.cloudstack.api.response.StoragePoolResponse;
-import org.apache.cloudstack.api.response.UserVmResponse;
-import org.apache.cloudstack.api.response.VolumeResponse;
-import org.apache.cloudstack.api.response.ZoneResponse;
-import org.apache.cloudstack.context.CallContext;
-import org.apache.cloudstack.response.ClusterMetricsResponse;
-import org.apache.cloudstack.response.HostMetricsResponse;
-import org.apache.cloudstack.response.InfrastructureResponse;
-import org.apache.cloudstack.response.StoragePoolMetricsResponse;
-import org.apache.cloudstack.response.VmMetricsResponse;
-import org.apache.cloudstack.response.VolumeMetricsResponse;
-import org.apache.cloudstack.response.ZoneMetricsResponse;
-import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
-import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
-import org.apache.commons.beanutils.BeanUtils;
-
-import javax.inject.Inject;
-import java.lang.reflect.InvocationTargetException;
-import java.util.ArrayList;
-import java.util.List;
 
 public class MetricsServiceImpl extends ComponentLifecycleBase implements MetricsService {
 
@@ -163,6 +165,7 @@ public class MetricsServiceImpl extends ComponentLifecycleBase implements Metric
             }
 
             metricsResponse.setDiskSizeGB(volumeResponse.getSize());
+            metricsResponse.setDiskIopsTotal(volumeResponse.getDiskIORead(), volumeResponse.getDiskIOWrite());
             Account account = CallContext.current().getCallingAccount();
             if (accountMgr.isAdmin(account.getAccountId())) {
                 metricsResponse.setStorageType(volumeResponse.getStorageType(), volumeResponse.getVolumeType());

plugins/metrics/src/main/java/org/apache/cloudstack/response/VmMetricsResponse.java

@@ -17,13 +17,14 @@
 
 package org.apache.cloudstack.response;
 
-import com.cloud.serializer.Param;
+import java.util.Set;
-import com.google.gson.annotations.SerializedName;
+
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.response.NicResponse;
 import org.apache.cloudstack.api.response.UserVmResponse;
 
-import java.util.Set;
+import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
 
 public class VmMetricsResponse extends UserVmResponse {
     @SerializedName(ApiConstants.IP_ADDRESS)
@@ -54,7 +55,7 @@ public class VmMetricsResponse extends UserVmResponse {
     @Param(description = "disk write in MiB")
     private String diskWrite;
 
-    @SerializedName("diskiopstotal")
+    @SerializedName(ApiConstants.DISK_IO_PSTOTAL)
     @Param(description = "the total disk iops")
     private Long diskIopsTotal;
 

plugins/metrics/src/main/java/org/apache/cloudstack/response/VolumeMetricsResponse.java

@@ -17,16 +17,22 @@
 
 package org.apache.cloudstack.response;
 
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.response.VolumeResponse;
+
 import com.cloud.serializer.Param;
 import com.google.common.base.Strings;
 import com.google.gson.annotations.SerializedName;
-import org.apache.cloudstack.api.response.VolumeResponse;
 
 public class VolumeMetricsResponse extends VolumeResponse {
     @SerializedName("sizegb")
     @Param(description = "disk size in GiB")
     private String diskSizeGB;
 
+    @SerializedName(ApiConstants.DISK_IO_PSTOTAL)
+    @Param(description = "the total disk iops")
+    private Long diskIopsTotal;
+
     public void setStorageType(final String storageType, final String volumeType) {
         if (!Strings.isNullOrEmpty(storageType) && !Strings.isNullOrEmpty(volumeType)) {
             this.setStorageType(String.format("%s (%s)", storageType.substring(0, 1).toUpperCase() + storageType.substring(1), volumeType));
@@ -38,4 +44,10 @@ public class VolumeMetricsResponse extends VolumeResponse {
             this.diskSizeGB = String.format("%.2f GB", size / (1024.0 * 1024.0 * 1024.0));
         }
     }
+
+    public void setDiskIopsTotal(final Long diskIoRead, final Long diskIoWrite) {
+        if (diskIoRead != null && diskIoWrite != null) {
+            this.diskIopsTotal = diskIoRead + diskIoWrite;
+        }
+    }
 }

server/src/main/java/com/cloud/api/ApiDBUtils.java

@@ -312,6 +312,7 @@ import com.cloud.vm.UserVmManager;
 import com.cloud.vm.UserVmVO;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VmDetailConstants;
 import com.cloud.vm.VmStats;
 import com.cloud.vm.dao.ConsoleProxyDao;
 import com.cloud.vm.dao.DomainRouterDao;
@@ -1482,7 +1483,7 @@ public class ApiDBUtils {
     }
 
     public static UserVmDetailVO  findPublicKeyByVmId(long vmId) {
-        return s_userVmDetailsDao.findDetail(vmId, "SSH.PublicKey");
+        return s_userVmDetailsDao.findDetail(vmId, VmDetailConstants.SSH_PUBLIC_KEY);
     }
 
     public static void getAutoScaleVmGroupPolicies(long vmGroupId, List<AutoScalePolicy> scaleUpPolicies, List<AutoScalePolicy> scaleDownPolicies) {

server/src/main/java/com/cloud/api/ApiResponseHelper.java

@@ -554,7 +554,7 @@ public class ApiResponseHelper implements ResponseGenerator {
             ResourceTagResponse tagResponse = createResourceTagResponse(tag, true);
             CollectionUtils.addIgnoreNull(tagResponses, tagResponse);
         }
-        snapshotResponse.setTags(tagResponses);
+        snapshotResponse.setTags(new HashSet<>(tagResponses));
 
         snapshotResponse.setObjectName("snapshot");
         return snapshotResponse;
@@ -656,6 +656,14 @@ public class ApiResponseHelper implements ResponseGenerator {
         policyResponse.setForDisplay(policy.isDisplay());
         policyResponse.setObjectName("snapshotpolicy");
 
+        List<? extends ResourceTag> tags = _resourceTagDao.listBy(policy.getId(), ResourceObjectType.SnapshotPolicy);
+        List<ResourceTagResponse> tagResponses = new ArrayList<ResourceTagResponse>();
+        for (ResourceTag tag : tags) {
+            ResourceTagResponse tagResponse = createResourceTagResponse(tag, false);
+            CollectionUtils.addIgnoreNull(tagResponses, tagResponse);
+        }
+        policyResponse.setTags(new HashSet<>(tagResponses));
+
         return policyResponse;
     }
 

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -17,12 +17,17 @@
 package com.cloud.api.query;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.ListIterator;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import javax.inject.Inject;
 
@@ -62,6 +67,7 @@ import org.apache.cloudstack.api.command.user.offering.ListDiskOfferingsCmd;
 import org.apache.cloudstack.api.command.user.offering.ListServiceOfferingsCmd;
 import org.apache.cloudstack.api.command.user.project.ListProjectInvitationsCmd;
 import org.apache.cloudstack.api.command.user.project.ListProjectsCmd;
+import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd;
 import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd;
 import org.apache.cloudstack.api.command.user.tag.ListTagsCmd;
 import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd;
@@ -72,6 +78,7 @@ import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd;
 import org.apache.cloudstack.api.command.user.zone.ListZonesCmd;
 import org.apache.cloudstack.api.response.AccountResponse;
 import org.apache.cloudstack.api.response.AsyncJobResponse;
+import org.apache.cloudstack.api.response.DetailOptionsResponse;
 import org.apache.cloudstack.api.response.DiskOfferingResponse;
 import org.apache.cloudstack.api.response.DomainResponse;
 import org.apache.cloudstack.api.response.DomainRouterResponse;
@@ -215,10 +222,12 @@ import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.SearchCriteria.Func;
 import com.cloud.utils.db.SearchCriteria.Op;
+import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.DomainRouterVO;
 import com.cloud.vm.UserVmVO;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VmDetailConstants;
 import com.cloud.vm.dao.DomainRouterDao;
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.VMInstanceDao;
@@ -3444,6 +3453,61 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
                 hypervisorType, true, cmd.listInReadyState(), permittedAccounts, caller, listProjectResourcesCriteria, tags, showRemovedISO, null, null);
     }
 
+    @Override
+    public DetailOptionsResponse listDetailOptions(final ListDetailOptionsCmd cmd) {
+        final ResourceObjectType type = cmd.getResourceType();
+        final String resourceUuid = cmd.getResourceId();
+        final Map<String, List<String>> options = new HashMap<>();
+        switch (type) {
+            case Template:
+            case UserVm:
+                HypervisorType hypervisorType = HypervisorType.None;
+                if (!Strings.isNullOrEmpty(resourceUuid) && ResourceObjectType.Template.equals(type)) {
+                    hypervisorType = _templateDao.findByUuid(resourceUuid).getHypervisorType();
+                }
+                if (!Strings.isNullOrEmpty(resourceUuid) && ResourceObjectType.UserVm.equals(type)) {
+                    hypervisorType = _vmInstanceDao.findByUuid(resourceUuid).getHypervisorType();
+                }
+                fillVMOrTemplateDetailOptions(options, hypervisorType);
+                break;
+            default:
+                throw new CloudRuntimeException("Resource type not supported.");
+        }
+        if (CallContext.current().getCallingAccount().getType() != Account.ACCOUNT_TYPE_ADMIN) {
+            final List<String> userBlacklistedSettings = Stream.of(QueryService.UserVMBlacklistedDetails.value().split(","))
+                    .map(item -> (item).trim())
+                    .collect(Collectors.toList());
+            for (final String detail : userBlacklistedSettings) {
+                if (options.containsKey(detail)) {
+                    options.remove(detail);
+                }
+            }
+        }
+        return new DetailOptionsResponse(options);
+    }
+
+    private void fillVMOrTemplateDetailOptions(final Map<String, List<String>> options, final HypervisorType hypervisorType) {
+        if (options == null) {
+            throw new CloudRuntimeException("Invalid/null detail-options response object passed");
+        }
+
+        options.put(VmDetailConstants.KEYBOARD, Arrays.asList("uk", "us", "jp", "fr"));
+        options.put(VmDetailConstants.CPU_CORE_PER_SOCKET, Collections.emptyList());
+        options.put(VmDetailConstants.ROOT_DISK_SIZE, Collections.emptyList());
+
+        if (HypervisorType.KVM.equals(hypervisorType)) {
+            options.put(VmDetailConstants.ROOT_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "virtio"));
+        }
+
+        if (HypervisorType.VMware.equals(hypervisorType)) {
+            options.put(VmDetailConstants.NIC_ADAPTER, Arrays.asList("E1000", "PCNet32", "Vmxnet2", "Vmxnet3"));
+            options.put(VmDetailConstants.ROOT_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "lsilogic", "lsisas1068", "buslogic", "pvscsi"));
+            options.put(VmDetailConstants.DATA_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "lsilogic", "lsisas1068", "buslogic", "pvscsi"));
+            options.put(VmDetailConstants.NESTED_VIRTUALIZATION_FLAG, Arrays.asList("true", "false"));
+            options.put(VmDetailConstants.SVGA_VRAM_SIZE, Collections.emptyList());
+        }
+    }
+
     @Override
     public ListResponse<AffinityGroupResponse> searchForAffinityGroups(ListAffinityGroupsCmd cmd) {
         Pair<List<AffinityGroupJoinVO>, Integer> result = searchForAffinityGroupsInternal(cmd);

server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java

@@ -36,6 +36,8 @@ import com.cloud.storage.VMTemplateHostVO;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.storage.Volume;
 import com.cloud.user.AccountManager;
+import com.cloud.user.VmDiskStatisticsVO;
+import com.cloud.user.dao.VmDiskStatisticsDao;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 
@@ -47,6 +49,8 @@ public class VolumeJoinDaoImpl extends GenericDaoBaseWithTagInformation<VolumeJo
     private ConfigurationDao  _configDao;
     @Inject
     public AccountManager _accountMgr;
+    @Inject
+    private VmDiskStatisticsDao vmDiskStatsDao;
 
     private final SearchBuilder<VolumeJoinVO> volSearch;
 
@@ -102,6 +106,14 @@ public class VolumeJoinDaoImpl extends GenericDaoBaseWithTagInformation<VolumeJo
             } else {
                 volResponse.setVirtualMachineDisplayName(volume.getVmName());
             }
+
+            VmDiskStatisticsVO diskStats = vmDiskStatsDao.findBy(volume.getAccountId(), volume.getDataCenterId(), instanceId, volume.getId());
+            if (diskStats != null) {
+                volResponse.setDiskIORead(diskStats.getCurrentIORead());
+                volResponse.setDiskIOWrite(diskStats.getCurrentIOWrite());
+                volResponse.setDiskKbsRead((long) (diskStats.getCurrentBytesRead() / 1024.0));
+                volResponse.setDiskKbsWrite((long) (diskStats.getCurrentBytesWrite() / 1024.0));
+            }
         }
 
         if (volume.getProvisioningType() != null) {

server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java

@@ -89,6 +89,7 @@ import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.Event;
 import com.cloud.vm.VirtualMachine.State;
+import com.cloud.vm.VmDetailConstants;
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.UserVmDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
@@ -138,8 +139,6 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
     @Inject
     MessageBus _messageBus;
 
-    private static final String MESSAGE_RESERVED_CAPACITY_FREED_FLAG = "Message.ReservedCapacityFreed.Flag";
-
     @Override
     public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
         _vmCapacityReleaseInterval = NumbersUtil.parseInt(_configDao.getValue(Config.CapacitySkipcountingHours.key()), 3600);
@@ -638,8 +637,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
             Float ramOvercommitRatio = 1.0f;
             long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - vm.getUpdateTime().getTime()) / 1000;
             if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
-                UserVmDetailVO vmDetailCpu = _userVmDetailsDao.findDetail(vm.getId(), "cpuOvercommitRatio");
+                UserVmDetailVO vmDetailCpu = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO);
-                UserVmDetailVO vmDetailRam = _userVmDetailsDao.findDetail(vm.getId(), "memoryOvercommitRatio");
+                UserVmDetailVO vmDetailRam = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
                 if (vmDetailCpu != null) {
                     //if vmDetail_cpu is not null it means it is running in a overcommited cluster.
                     cpuOvercommitRatio = Float.parseFloat(vmDetailCpu.getValue());
@@ -669,14 +668,14 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
             } else {
                 // signal if not done already, that the VM has been stopped for skip.counting.hours,
                 // hence capacity will not be reserved anymore.
-                UserVmDetailVO messageSentFlag = _userVmDetailsDao.findDetail(vm.getId(), MESSAGE_RESERVED_CAPACITY_FREED_FLAG);
+                UserVmDetailVO messageSentFlag = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.MESSAGE_RESERVED_CAPACITY_FREED_FLAG);
                 if (messageSentFlag == null || !Boolean.valueOf(messageSentFlag.getValue())) {
                     _messageBus.publish(_name, "VM_ReservedCapacity_Free", PublishScope.LOCAL, vm);
 
                     if (vm.getType() == VirtualMachine.Type.User) {
                         UserVmVO userVM = _userVMDao.findById(vm.getId());
                         _userVMDao.loadDetails(userVM);
-                        userVM.setDetail(MESSAGE_RESERVED_CAPACITY_FREED_FLAG, "true");
+                        userVM.setDetail(VmDetailConstants.MESSAGE_RESERVED_CAPACITY_FREED_FLAG, "true");
                         _userVMDao.saveDetails(userVM);
                     }
                 }
@@ -903,7 +902,7 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
           UserVmVO userVM = _userVMDao.findById(vm.getId());
           _userVMDao.loadDetails(userVM);
           // free the message sent flag if it exists
-          userVM.setDetail(MESSAGE_RESERVED_CAPACITY_FREED_FLAG, "false");
+          userVM.setDetail(VmDetailConstants.MESSAGE_RESERVED_CAPACITY_FREED_FLAG, "false");
           _userVMDao.saveDetails(userVM);
 
         }

server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java

@@ -77,6 +77,7 @@ import com.cloud.vm.UserVmVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineManager;
 import com.cloud.vm.VirtualMachineProfile;
+import com.cloud.vm.VmDetailConstants;
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.UserVmDetailsDao;
 
@@ -195,7 +196,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
     }
 
     private String getSshKey(VirtualMachineProfile profile) {
-        final UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), "SSH.PublicKey");
+        final UserVmDetailVO vmDetailSshKey = _userVmDetailsDao.findDetail(profile.getId(), VmDetailConstants.SSH_PUBLIC_KEY);
         return (vmDetailSshKey!=null ? vmDetailSshKey.getValue() : null);
     }
 
@@ -262,7 +263,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle
         final String password_encrypted = DBEncryptionUtil.encrypt(password);
         final UserVmVO userVmVO = _userVmDao.findById(vm.getId());
 
-        _userVmDetailsDao.addDetail(vm.getId(), "password", password_encrypted, false);
+        _userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.PASSWORD,  password_encrypted, false);
 
         userVmVO.setUpdateParameters(true);
         _userVmDao.update(userVmVO.getId(), userVmVO);

server/src/main/java/com/cloud/network/element/VirtualRouterElement.java

@@ -31,6 +31,7 @@ import org.cloud.network.router.deployment.RouterDeploymentDefinitionBuilder;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 
+import com.cloud.vm.VmDetailConstants;
 import com.google.gson.Gson;
 
 import org.apache.cloudstack.api.command.admin.router.ConfigureOvsElementCmd;
@@ -718,7 +719,7 @@ NetworkMigrationResponder, AggregatedCommandExecutor, RedundantResource, DnsServ
         final UserVmVO userVmVO = _userVmDao.findById(vm.getId());
 
         _userVmDao.loadDetails(userVmVO);
-        userVmVO.setDetail("password", password_encrypted);
+        userVmVO.setDetail(VmDetailConstants.PASSWORD, password_encrypted);
         _userVmDao.saveDetails(userVmVO);
 
         userVmVO.setUpdateParameters(true);

server/src/main/java/com/cloud/resource/ResourceManagerImpl.java

@@ -31,13 +31,6 @@ import java.util.concurrent.ConcurrentHashMap;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import com.cloud.utils.Pair;
-import com.cloud.vm.dao.UserVmDetailsDao;
-import org.apache.cloudstack.framework.config.ConfigKey;
-import org.apache.commons.lang.ObjectUtils;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.command.admin.cluster.AddClusterCmd;
 import org.apache.cloudstack.api.command.admin.cluster.DeleteClusterCmd;
@@ -49,21 +42,25 @@ import org.apache.cloudstack.api.command.admin.host.ReconnectHostCmd;
 import org.apache.cloudstack.api.command.admin.host.UpdateHostCmd;
 import org.apache.cloudstack.api.command.admin.host.UpdateHostPasswordCmd;
 import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.ObjectUtils;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
-import com.cloud.agent.api.GetVncPortCommand;
-import com.cloud.agent.api.GetVncPortAnswer;
 import com.cloud.agent.api.GetGPUStatsAnswer;
 import com.cloud.agent.api.GetGPUStatsCommand;
 import com.cloud.agent.api.GetHostStatsAnswer;
 import com.cloud.agent.api.GetHostStatsCommand;
+import com.cloud.agent.api.GetVncPortAnswer;
+import com.cloud.agent.api.GetVncPortCommand;
 import com.cloud.agent.api.MaintainAnswer;
 import com.cloud.agent.api.MaintainCommand;
 import com.cloud.agent.api.PropagateResourceEventCommand;
@@ -148,6 +145,7 @@ import com.cloud.storage.dao.StoragePoolHostDao;
 import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.user.Account;
 import com.cloud.user.AccountManager;
+import com.cloud.utils.Pair;
 import com.cloud.utils.StringUtils;
 import com.cloud.utils.UriUtils;
 import com.cloud.utils.component.Manager;
@@ -177,6 +175,8 @@ import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.State;
 import com.cloud.vm.VirtualMachineManager;
+import com.cloud.vm.VmDetailConstants;
+import com.cloud.vm.dao.UserVmDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
 import com.google.gson.Gson;
 
@@ -1314,8 +1314,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
         for (VMInstanceVO vm : vms) {
             GetVncPortAnswer vmVncPortAnswer = (GetVncPortAnswer) _agentMgr.easySend(hostId, new GetVncPortCommand(vm.getId(), vm.getInstanceName()));
             if (vmVncPortAnswer != null) {
-                userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.address", vmVncPortAnswer.getAddress(), true);
+                userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.KVM_VNC_ADDRESS, vmVncPortAnswer.getAddress(), true);
-                userVmDetailsDao.addDetail(vm.getId(), "kvm.vnc.port", String.valueOf(vmVncPortAnswer.getPort()), true);
+                userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.KVM_VNC_PORT, String.valueOf(vmVncPortAnswer.getPort()), true);
             }
         }
     }

server/src/main/java/com/cloud/server/ManagementServerImpl.java

@@ -411,6 +411,7 @@ import org.apache.cloudstack.api.command.user.region.ha.gslb.ListGlobalLoadBalan
 import org.apache.cloudstack.api.command.user.region.ha.gslb.RemoveFromGlobalLoadBalancerRuleCmd;
 import org.apache.cloudstack.api.command.user.region.ha.gslb.UpdateGlobalLoadBalancerRuleCmd;
 import org.apache.cloudstack.api.command.user.resource.GetCloudIdentifierCmd;
+import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd;
 import org.apache.cloudstack.api.command.user.resource.ListHypervisorsCmd;
 import org.apache.cloudstack.api.command.user.resource.ListResourceLimitsCmd;
 import org.apache.cloudstack.api.command.user.resource.UpdateResourceCountCmd;
@@ -2900,6 +2901,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         cmdList.add(ExtractTemplateCmd.class);
         cmdList.add(ListTemplatePermissionsCmd.class);
         cmdList.add(ListTemplatesCmd.class);
+        cmdList.add(ListDetailOptionsCmd.class);
         cmdList.add(RegisterTemplateCmd.class);
         cmdList.add(UpdateTemplateCmd.class);
         cmdList.add(UpdateTemplatePermissionsCmd.class);

server/src/main/java/com/cloud/server/StatsCollector.java

@@ -690,7 +690,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
                         s_logger.debug("VmDiskStatsTask is running...");
 
                         SearchCriteria<HostVO> sc = createSearchCriteriaForHostTypeRoutingStateUpAndNotInMaintenance();
-                        sc.addAnd("hypervisorType", SearchCriteria.Op.EQ, HypervisorType.KVM); // support KVM only util 2013.06.25
+                        sc.addAnd("hypervisorType", SearchCriteria.Op.IN, HypervisorType.KVM, HypervisorType.VMware);
                         List<HostVO> hosts = _hostDao.search(sc, null);
 
                         for (HostVO host : hosts) {

server/src/main/java/com/cloud/servlet/ConsoleProxyServlet.java

@@ -41,6 +41,7 @@ import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 import org.springframework.web.context.support.SpringBeanAutowiringSupport;
 
+import com.cloud.vm.VmDetailConstants;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 
@@ -421,8 +422,8 @@ public class ConsoleProxyServlet extends HttpServlet {
 
         Pair<String, Integer> portInfo;
         if (hostVo.getResourceState().equals(ResourceState.ErrorInMaintenance)) {
-            UserVmDetailVO detailAddress = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.address");
+            UserVmDetailVO detailAddress = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.KVM_VNC_ADDRESS);
-            UserVmDetailVO detailPort = _userVmDetailsDao.findDetail(vm.getId(), "kvm.vnc.port");
+            UserVmDetailVO detailPort = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.KVM_VNC_PORT);
             portInfo = new Pair<>(detailAddress.getValue(), Integer.valueOf(detailPort.getValue()));
         } else {
             portInfo = _ms.getVncPort(vm);
@@ -441,7 +442,7 @@ public class ConsoleProxyServlet extends HttpServlet {
         }
 
         String sid = vm.getVncPassword();
-        UserVmDetailVO details = _userVmDetailsDao.findDetail(vm.getId(), "keyboard");
+        UserVmDetailVO details = _userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.KEYBOARD);
 
         String tag = vm.getUuid();
 

server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java

@@ -80,6 +80,7 @@ import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
 import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
 import org.apache.log4j.Logger;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -114,6 +115,8 @@ import com.cloud.hypervisor.HypervisorCapabilitiesVO;
 import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao;
 import com.cloud.org.Grouping;
 import com.cloud.serializer.GsonHelper;
+import com.cloud.server.ResourceTag;
+import com.cloud.server.TaggedResourceService;
 import com.cloud.service.dao.ServiceOfferingDetailsDao;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.dao.DiskOfferingDao;
@@ -256,6 +259,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
     private StoragePoolTagsDao storagePoolTagsDao;
     @Inject
     private StorageUtil storageUtil;
+    @Inject
+    public TaggedResourceService taggedResourceService;
 
     protected Gson _gson;
 
@@ -2024,6 +2029,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
             throw new InvalidParameterValueException("Volume must be in ready state");
         }
 
+        if (vol.getPoolId() == storagePoolId) {
+            throw new InvalidParameterValueException("Volume " + vol + " is already on the destination storage pool");
+        }
+
         boolean liveMigrateVolume = false;
         Long instanceId = vol.getInstanceId();
         Long srcClusterId = null;
@@ -2335,7 +2344,16 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
 
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_SNAPSHOT_CREATE, eventDescription = "taking snapshot", async = true)
-    public Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup)
+    public Snapshot takeSnapshot(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup, Map<String, String> tags)
+            throws ResourceAllocationException {
+        final Snapshot snapshot = takeSnapshotInternal(volumeId, policyId, snapshotId, account, quiescevm, locationType, asyncBackup);
+        if (snapshot != null && MapUtils.isNotEmpty(tags)) {
+            taggedResourceService.createTags(Collections.singletonList(snapshot.getUuid()), ResourceTag.ResourceObjectType.Snapshot, tags, null);
+        }
+        return snapshot;
+    }
+
+    private Snapshot takeSnapshotInternal(Long volumeId, Long policyId, Long snapshotId, Account account, boolean quiescevm, Snapshot.LocationType locationType, boolean asyncBackup)
             throws ResourceAllocationException {
         VolumeInfo volume = volFactory.getVolume(volumeId);
         if (volume == null) {

server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java

@@ -16,8 +16,53 @@
 // under the License.
 package com.cloud.storage.snapshot;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import javax.inject.Inject;
+import javax.naming.ConfigurationException;
+
+import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd;
+import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd;
+import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd;
+import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd;
+import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
+import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation;
+import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.commons.collections.MapUtils;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
 import com.cloud.agent.api.Answer;
-import com.cloud.utils.db.GlobalLock;
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.DeleteSnapshotsDirCommand;
 import com.cloud.alert.AlertManager;
@@ -42,6 +87,7 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.projects.Project.ListProjectResourcesCriteria;
 import com.cloud.resource.ResourceManager;
 import com.cloud.server.ResourceTag.ResourceObjectType;
+import com.cloud.server.TaggedResourceService;
 import com.cloud.storage.CreateSnapshotPayload;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.ScopeType;
@@ -80,6 +126,7 @@ import com.cloud.utils.Ternary;
 import com.cloud.utils.concurrency.NamedThreadFactory;
 import com.cloud.utils.db.DB;
 import com.cloud.utils.db.Filter;
+import com.cloud.utils.db.GlobalLock;
 import com.cloud.utils.db.JoinBuilder;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
@@ -92,48 +139,6 @@ import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.snapshot.VMSnapshot;
 import com.cloud.vm.snapshot.VMSnapshotVO;
 import com.cloud.vm.snapshot.dao.VMSnapshotDao;
-import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd;
-import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd;
-import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd;
-import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd;
-import org.apache.cloudstack.api.command.user.snapshot.UpdateSnapshotPolicyCmd;
-import org.apache.cloudstack.context.CallContext;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
-import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
-import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation;
-import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
-import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
-import org.apache.cloudstack.framework.config.ConfigKey;
-import org.apache.cloudstack.framework.config.Configurable;
-import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
-import org.apache.cloudstack.managed.context.ManagedContextRunnable;
-import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
-import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
-import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
-import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-import javax.naming.ConfigurationException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
 
 @Component
 public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implements SnapshotManager, SnapshotApiService, Configurable {
@@ -192,6 +197,8 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
     ResourceManager _resourceMgr;
     @Inject
     StorageStrategyFactory _storageStrategyFactory;
+    @Inject
+    public TaggedResourceService taggedResourceService;
 
     private int _totalRetries;
     private int _pauseInterval;
@@ -902,6 +909,11 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
                     policy.setDisplay(display);
                     _snapshotPolicyDao.update(policy.getId(), policy);
                     _snapSchedMgr.scheduleOrCancelNextSnapshotJobOnDisplayChange(policy, previousDisplay);
+                    taggedResourceService.deleteTags(Collections.singletonList(policy.getUuid()), ResourceObjectType.SnapshotPolicy, null);
+                }
+                final Map<String, String> tags = cmd.getTags();
+                if (MapUtils.isNotEmpty(tags)) {
+                    taggedResourceService.createTags(Collections.singletonList(policy.getUuid()), ResourceObjectType.SnapshotPolicy, tags, null);
                 }
             } finally {
                 createSnapshotPolicyLock.unlock();
@@ -916,9 +928,10 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         }
     }
 
-    protected boolean deletePolicy(long userId, Long policyId) {
+    protected boolean deletePolicy(Long policyId) {
         SnapshotPolicyVO snapshotPolicy = _snapshotPolicyDao.findById(policyId);
         _snapSchedMgr.removeSchedule(snapshotPolicy.getVolumeId(), snapshotPolicy.getId());
+        taggedResourceService.deleteTags(Collections.singletonList(snapshotPolicy.getUuid()), ResourceObjectType.SnapshotPolicy, null);
         return _snapshotPolicyDao.remove(policyId);
     }
 
@@ -963,8 +976,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
     public void deletePoliciesForVolume(Long volumeId) {
         List<SnapshotPolicyVO> policyInstances = listPoliciesforVolume(volumeId);
         for (SnapshotPolicyVO policyInstance : policyInstances) {
-            Long policyId = policyInstance.getId();
+            deletePolicy(policyInstance.getId());
-            deletePolicy(1L, policyId);
         }
         // We also want to delete the manual snapshots scheduled for this volume
         // We can only delete the schedules in the future, not the ones which are already executing.
@@ -1321,7 +1333,6 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
     public boolean deleteSnapshotPolicies(DeleteSnapshotPoliciesCmd cmd) {
         Long policyId = cmd.getId();
         List<Long> policyIds = cmd.getIds();
-        Long userId = getSnapshotUserId();
 
         if ((policyId == null) && (policyIds == null)) {
             throw new InvalidParameterValueException("No policy id (or list of ids) specified.");
@@ -1335,6 +1346,10 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
             throw new InvalidParameterValueException("There are no policy ids");
         }
 
+        if (policyIds.contains(Snapshot.MANUAL_POLICY_ID)) {
+            throw new InvalidParameterValueException("Invalid Policy id given: " + Snapshot.MANUAL_POLICY_ID);
+        }
+
         for (Long policy : policyIds) {
             SnapshotPolicyVO snapshotPolicyVO = _snapshotPolicyDao.findById(policy);
             if (snapshotPolicyVO == null) {
@@ -1348,21 +1363,14 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
             _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, volume);
         }
 
-        boolean success = true;
-
-        if (policyIds.contains(Snapshot.MANUAL_POLICY_ID)) {
-            throw new InvalidParameterValueException("Invalid Policy id given: " + Snapshot.MANUAL_POLICY_ID);
-        }
-
         for (Long pId : policyIds) {
-            if (!deletePolicy(userId, pId)) {
+            if (!deletePolicy(pId)) {
-                success = false;
                 s_logger.warn("Failed to delete snapshot policy with Id: " + policyId);
-                return success;
+                return false;
             }
         }
 
-        return success;
+        return true;
     }
 
     @Override

server/src/main/java/com/cloud/storage/snapshot/SnapshotSchedulerImpl.java

@@ -43,6 +43,8 @@ import com.cloud.api.ApiDispatcher;
 import com.cloud.api.ApiGsonHelper;
 import com.cloud.event.ActionEventUtils;
 import com.cloud.event.EventTypes;
+import com.cloud.server.ResourceTag;
+import com.cloud.server.TaggedResourceService;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.SnapshotPolicyVO;
 import com.cloud.storage.SnapshotScheduleVO;
@@ -97,6 +99,8 @@ public class SnapshotSchedulerImpl extends ManagerBase implements SnapshotSchedu
     protected VMSnapshotDao _vmSnapshotDao;
     @Inject
     protected VMSnapshotManager _vmSnaphostManager;
+    @Inject
+    public TaggedResourceService taggedResourceService;
 
     protected AsyncJobDispatcher _asyncDispatcher;
 
@@ -305,6 +309,15 @@ public class SnapshotSchedulerImpl extends ManagerBase implements SnapshotSchedu
                 params.put("ctxUserId", "1");
                 params.put("ctxAccountId", "" + volume.getAccountId());
                 params.put("ctxStartEventId", String.valueOf(eventId));
+                List<? extends ResourceTag> resourceTags = taggedResourceService.listByResourceTypeAndId(ResourceTag.ResourceObjectType.SnapshotPolicy, policyId);
+                if (resourceTags != null && !resourceTags.isEmpty()) {
+                    int tagNumber = 0;
+                    for (ResourceTag resourceTag : resourceTags) {
+                        params.put("tags[" + tagNumber + "].key", resourceTag.getKey());
+                        params.put("tags[" + tagNumber + "].value", resourceTag.getValue());
+                        tagNumber++;
+                    }
+                }
 
                 final CreateSnapshotCmd cmd = new CreateSnapshotCmd();
                 ComponentContext.inject(cmd);

server/src/main/java/com/cloud/tags/TaggedResourceManagerImpl.java

@@ -16,13 +16,31 @@
 // under the License.
 package com.cloud.tags;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import javax.inject.Inject;
+import javax.naming.ConfigurationException;
+import javax.persistence.EntityExistsException;
+
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+
 import com.cloud.dc.DataCenterVO;
 import com.cloud.domain.PartOf;
 import com.cloud.event.ActionEvent;
 import com.cloud.event.EventTypes;
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.PermissionDeniedException;
-import com.cloud.offerings.NetworkOfferingVO;
 import com.cloud.network.LBHealthCheckPolicyVO;
 import com.cloud.network.as.AutoScaleVmGroupVO;
 import com.cloud.network.as.AutoScaleVmProfileVO;
@@ -43,6 +61,7 @@ import com.cloud.network.vpc.NetworkACLVO;
 import com.cloud.network.vpc.StaticRouteVO;
 import com.cloud.network.vpc.VpcOfferingVO;
 import com.cloud.network.vpc.VpcVO;
+import com.cloud.offerings.NetworkOfferingVO;
 import com.cloud.projects.ProjectVO;
 import com.cloud.server.ResourceTag;
 import com.cloud.server.ResourceTag.ResourceObjectType;
@@ -74,24 +93,6 @@ import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.NicVO;
 import com.cloud.vm.UserVmVO;
 import com.cloud.vm.snapshot.VMSnapshotVO;
-import org.apache.cloudstack.api.Identity;
-import org.apache.cloudstack.api.InternalIdentity;
-import org.apache.cloudstack.context.CallContext;
-import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
-import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.Logger;
-
-import org.apache.commons.collections.MapUtils;
-
-import javax.inject.Inject;
-import javax.naming.ConfigurationException;
-import javax.persistence.EntityExistsException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
 
 public class TaggedResourceManagerImpl extends ManagerBase implements TaggedResourceService {
     public static final Logger s_logger = Logger.getLogger(TaggedResourceManagerImpl.class);
@@ -220,6 +221,10 @@ public class TaggedResourceManagerImpl extends ManagerBase implements TaggedReso
             accountId = ((ProjectVO)entity).getProjectAccountId();
         }
 
+        if (resourceType == ResourceObjectType.SnapshotPolicy) {
+            accountId = _entityMgr.findById(VolumeVO.class, ((SnapshotPolicyVO)entity).getVolumeId()).getAccountId();
+        }
+
         if (entity instanceof OwnedBy) {
             accountId = ((OwnedBy)entity).getAccountId();
         }
@@ -389,7 +394,7 @@ public class TaggedResourceManagerImpl extends ManagerBase implements TaggedReso
         }
 
         if (tagsToDelete.isEmpty()) {
-            throw new InvalidParameterValueException("Unable to find any tags which conform to specified delete parameters.");
+            return false;
         }
 
         //Remove the tags

server/src/main/java/com/cloud/template/TemplateManagerImpl.java

@@ -39,6 +39,7 @@ import com.cloud.utils.EncryptionUtil;
 import com.cloud.utils.DateUtil;
 import com.cloud.utils.Pair;
 import com.cloud.utils.EnumUtils;
+import com.cloud.vm.VmDetailConstants;
 import com.google.common.base.Joiner;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
@@ -1918,7 +1919,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
                 }
             }
             if (cmd.getDetails() != null) {
-                details.remove("Encrypted.Password"); // new password will be generated during vm deployment from password enabled template
+                details.remove(VmDetailConstants.ENCRYPTED_PASSWORD); // new password will be generated during vm deployment from password enabled template
                 details.putAll(cmd.getDetails());
             }
             if (!details.isEmpty()) {

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -35,6 +35,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
@@ -85,6 +86,7 @@ import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.query.QueryService;
 import org.apache.cloudstack.storage.command.DeleteCommand;
 import org.apache.cloudstack.storage.command.DettachCommand;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
@@ -847,7 +849,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         } else {
             final UserVmVO userVm = _vmDao.findById(vmId);
             _vmDao.loadDetails(userVm);
-            userVm.setDetail("SSH.PublicKey", sshPublicKey);
+            userVm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKey);
             if (template.isEnablePassword()) {
                 userVm.setPassword(password);
                 //update the encrypted password in vm_details table too
@@ -2426,11 +2428,36 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         if (isDisplayVm != null && isDisplayVm != vmInstance.isDisplay()) {
             updateDisplayVmFlag(isDisplayVm, id, vmInstance);
         }
+        final Account caller = CallContext.current().getCallingAccount();
+        final List<String> userBlacklistedSettings = Stream.of(QueryService.UserVMBlacklistedDetails.value().split(","))
+                .map(item -> (item).trim())
+                .collect(Collectors.toList());
         if (cleanupDetails){
+            if (caller != null && caller.getType() == Account.ACCOUNT_TYPE_ADMIN) {
                 userVmDetailsDao.removeDetails(id);
+            } else {
+                for (final UserVmDetailVO detail : userVmDetailsDao.listDetails(id)) {
+                    if (detail != null && !userBlacklistedSettings.contains(detail.getName())) {
+                        userVmDetailsDao.removeDetail(id, detail.getName());
                     }
-        else {
+                }
+            }
+        } else {
             if (MapUtils.isNotEmpty(details)) {
+                if (caller != null && caller.getType() != Account.ACCOUNT_TYPE_ADMIN) {
+                    // Ensure blacklisted detail is not passed by non-root-admin user
+                    for (final String detailName : details.keySet()) {
+                        if (userBlacklistedSettings.contains(detailName)) {
+                            throw new InvalidParameterValueException("You're not allowed to add or edit the restricted setting: " + detailName);
+                        }
+                    }
+                    // Add any hidden/blacklisted detail
+                    for (final UserVmDetailVO detail : userVmDetailsDao.listDetails(id)) {
+                        if (userBlacklistedSettings.contains(detail.getName())) {
+                            details.put(detail.getName(), detail.getValue());
+                        }
+                    }
+                }
                 vmInstance.setDetails(details);
                 _vmDao.saveDetails(vmInstance);
             }
@@ -3379,13 +3406,13 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         boolean isIso = Storage.ImageFormat.ISO == template.getFormat();
         long size = 0;
         // custom root disk size, resizes base template to larger size
-        if (customParameters.containsKey("rootdisksize")) {
+        if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) {
             // only KVM, XenServer and VMware supports rootdisksize override
             if (!(hypervisorType == HypervisorType.KVM || hypervisorType == HypervisorType.XenServer || hypervisorType == HypervisorType.VMware || hypervisorType == HypervisorType.Simulator)) {
                 throw new InvalidParameterValueException("Hypervisor " + hypervisorType + " does not support rootdisksize override");
             }
 
-            Long rootDiskSize = NumbersUtil.parseLong(customParameters.get("rootdisksize"), -1);
+            Long rootDiskSize = NumbersUtil.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE), -1);
             if (rootDiskSize <= 0) {
                 throw new InvalidParameterValueException("Root disk size should be a positive number.");
             }
@@ -3768,7 +3795,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                 }
 
                 if (sshPublicKey != null) {
-                    vm.setDetail("SSH.PublicKey", sshPublicKey);
+                    vm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKey);
                 }
 
                 if (keyboard != null && !keyboard.isEmpty()) {
@@ -3780,9 +3807,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                 }
                 Long rootDiskSize = null;
                 // custom root disk size, resizes base template to larger size
-                if (customParameters.containsKey("rootdisksize")) {
+                if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) {
                     // already verified for positive number
-                    rootDiskSize = Long.parseLong(customParameters.get("rootdisksize"));
+                    rootDiskSize = Long.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE));
 
                     VMTemplateVO templateVO = _templateDao.findById(template.getId());
                     if (templateVO == null) {
@@ -3806,10 +3833,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                 // If hypervisor is vSphere and OS is OS X, set special settings.
                 if (hypervisorType.equals(HypervisorType.VMware)) {
                     if (guestOS.getDisplayName().toLowerCase().contains("apple mac os")) {
-                        vm.setDetail("smc.present", "TRUE");
+                        vm.setDetail(VmDetailConstants.SMC_PRESENT, "TRUE");
                         vm.setDetail(VmDetailConstants.ROOT_DISK_CONTROLLER, "scsi");
                         vm.setDetail(VmDetailConstants.DATA_DISK_CONTROLLER, "scsi");
-                        vm.setDetail("firmware", "efi");
+                        vm.setDetail(VmDetailConstants.FIRMWARE, "efi");
                         s_logger.info("guestOS is OSX : overwrite root disk controller to scsi, use smc and efi");
                     } else {
                         String controllerSetting = _configDao.getValue("vmware.root.disk.controller");
@@ -3838,7 +3865,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                         vm.setDetail(key, customParameters.get(key));
                     }
                 }
-                vm.setDetail("deployvm", "true");
+                vm.setDetail(VmDetailConstants.DEPLOY_VM, "true");
                 _vmDao.saveDetails(vm);
 
                 s_logger.debug("Allocating in the DB for vm");
@@ -3888,10 +3915,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             s_logger.error(error);
             throw new InvalidParameterValueException(error);
         } else if ((rootDiskSize << 30) > templateVO.getSize()) {
-            if (hypervisorType == HypervisorType.VMware && (vm.getDetails() == null || vm.getDetails().get("rootDiskController") == null)) {
+            if (hypervisorType == HypervisorType.VMware && (vm.getDetails() == null || vm.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER) == null)) {
                 s_logger.warn("If Root disk controller parameter is not overridden, then Root disk resize may fail because current Root disk controller value is NULL.");
-            } else if (hypervisorType == HypervisorType.VMware && !vm.getDetails().get("rootDiskController").toLowerCase().contains("scsi")) {
+            } else if (hypervisorType == HypervisorType.VMware && !vm.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER).toLowerCase().contains("scsi")) {
-                String error = "Found unsupported root disk controller: " + vm.getDetails().get("rootDiskController");
+                String error = "Found unsupported root disk controller: " + vm.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER);
                 s_logger.error(error);
                 throw new InvalidParameterValueException(error);
             } else {
@@ -3899,7 +3926,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             }
         } else {
             s_logger.debug("Root disk size specified is " + (rootDiskSize << 30) + "B and Template root disk size is " + templateVO.getSize() + "B. Both are equal so no need to override");
-            customParameters.remove("rootdisksize");
+            customParameters.remove(VmDetailConstants.ROOT_DISK_SIZE);
         }
     }
 
@@ -4182,7 +4209,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                 boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
 
                 List<String[]> vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getHostName(), vm.getId(),
-                        vm.getUuid(), defaultNic.getIPv4Address(), vm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
+                        vm.getUuid(), defaultNic.getIPv4Address(), vm.getDetail(VmDetailConstants.SSH_PUBLIC_KEY), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows);
                 String vmName = vm.getInstanceName();
                 String configDriveIsoRootFolder = "/tmp";
                 String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso";
@@ -4574,8 +4601,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             // this value is not being sent to the backend; need only for api
             // display purposes
             if (template.isEnablePassword()) {
-                if (vm.getDetail("password") != null) {
+                if (vm.getDetail(VmDetailConstants.PASSWORD) != null) {
-                    userVmDetailsDao.removeDetail(vm.getId(), "password");
+                    userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.PASSWORD);
                 }
                 vm.setUpdateParameters(false);
                 _vmDao.update(vm.getId(), vm);
@@ -4643,8 +4670,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
 
     @Override
     public void collectVmDiskStatistics(final UserVm userVm) {
-        // support KVM only util 2013.06.25
+        // Only supported for KVM and VMware
-        if (!userVm.getHypervisorType().equals(HypervisorType.KVM)) {
+        if (!(userVm.getHypervisorType().equals(HypervisorType.KVM) || userVm.getHypervisorType().equals(HypervisorType.VMware))) {
             return;
         }
         s_logger.debug("Collect vm disk statistics from host before stopping Vm");
@@ -6334,7 +6361,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
 
             if (needRestart) {
                 try {
-                    if (vm.getDetail("password") != null) {
+                    if (vm.getDetail(VmDetailConstants.PASSWORD) != null) {
                         params = new HashMap<VirtualMachineProfile.Param, Object>();
                         params.put(VirtualMachineProfile.Param.VmPassword, password);
                     }
@@ -6347,8 +6374,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                         if (vm.isUpdateParameters()) {
                             vm.setUpdateParameters(false);
                             _vmDao.loadDetails(vm);
-                            if (vm.getDetail("password") != null) {
+                            if (vm.getDetail(VmDetailConstants.PASSWORD) != null) {
-                                userVmDetailsDao.removeDetail(vm.getId(), "password");
+                                userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.PASSWORD);
                             }
                             _vmDao.update(vm.getId(), vm);
                         }
@@ -6526,7 +6553,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
     }
 
     private void encryptAndStorePassword(UserVmVO vm, String password) {
-        String sshPublicKey = vm.getDetail("SSH.PublicKey");
+        String sshPublicKey = vm.getDetail(VmDetailConstants.SSH_PUBLIC_KEY);
         if (sshPublicKey != null && !sshPublicKey.equals("") && password != null && !password.equals("saved_password")) {
             if (!sshPublicKey.startsWith("ssh-rsa")) {
                 s_logger.warn("Only RSA public keys can be used to encrypt a vm password.");
@@ -6537,7 +6564,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                 throw new CloudRuntimeException("Error encrypting password");
             }
 
-            vm.setDetail("Encrypted.Password", encryptedPasswd);
+            vm.setDetail(VmDetailConstants.ENCRYPTED_PASSWORD, encryptedPasswd);
             _vmDao.saveDetails(vm);
         }
     }
@@ -6569,7 +6596,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
     public String getVmUserData(long vmId) {
         UserVmVO vm = _vmDao.findById(vmId);
         if (vm == null) {
-            throw new InvalidParameterValueException("Unable to find virual machine with id " + vmId);
+            throw new InvalidParameterValueException("Unable to find virtual machine with id " + vmId);
         }
 
         _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, vm);

server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java

@@ -18,6 +18,7 @@ package com.cloud.storage;
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyObject;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doNothing;
@@ -65,6 +66,7 @@ import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.Spy;
 import org.mockito.runners.MockitoJUnitRunner;
+import org.springframework.test.util.ReflectionTestUtils;
 
 import com.cloud.configuration.Resource;
 import com.cloud.configuration.Resource.ResourceType;
@@ -76,6 +78,7 @@ import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.org.Grouping;
 import com.cloud.serializer.GsonHelper;
+import com.cloud.server.TaggedResourceService;
 import com.cloud.storage.Volume.Type;
 import com.cloud.storage.dao.StoragePoolTagsDao;
 import com.cloud.storage.dao.VolumeDao;
@@ -409,7 +412,7 @@ public class VolumeApiServiceImplTest {
         when(volumeDataFactoryMock.getVolume(anyLong())).thenReturn(volumeInfoMock);
         when(volumeInfoMock.getState()).thenReturn(Volume.State.Allocated);
         when(volumeInfoMock.getPoolId()).thenReturn(1L);
-        volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false);
+        volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null);
     }
 
     @Test
@@ -419,7 +422,10 @@ public class VolumeApiServiceImplTest {
         when(volumeInfoMock.getInstanceId()).thenReturn(null);
         when(volumeInfoMock.getPoolId()).thenReturn(1L);
         when(volumeServiceMock.takeSnapshot(Mockito.any(VolumeInfo.class))).thenReturn(snapshotInfoMock);
-        volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false);
+        final TaggedResourceService taggedResourceService = Mockito.mock(TaggedResourceService.class);
+        Mockito.when(taggedResourceService.createTags(anyObject(), anyObject(), anyObject(), anyObject())).thenReturn(null);
+        ReflectionTestUtils.setField(volumeApiServiceImpl, "taggedResourceService", taggedResourceService);
+        volumeApiServiceImpl.takeSnapshot(5L, Snapshot.MANUAL_POLICY_ID, 3L, null, false, null, false, null);
     }
 
     @Test

server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java

@@ -22,6 +22,7 @@ import java.util.HashMap;
 import org.apache.cloudstack.api.BaseCmd.HTTPMethod;
 import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd;
 import org.apache.cloudstack.context.CallContext;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -44,6 +45,8 @@ import com.cloud.storage.GuestOSVO;
 import com.cloud.storage.dao.GuestOSDao;
 import com.cloud.user.Account;
 import com.cloud.user.AccountManager;
+import com.cloud.user.AccountVO;
+import com.cloud.user.UserVO;
 import com.cloud.uservm.UserVm;
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.UserVmDetailsDao;
@@ -76,6 +79,12 @@ public class UserVmManagerImplTest {
     @Mock
     private NetworkModel networkModel;
 
+    @Mock
+    private AccountVO callerAccount;
+
+    @Mock
+    private UserVO callerUser;
+
     private long vmId = 1l;
 
     @Before
@@ -83,6 +92,14 @@ public class UserVmManagerImplTest {
         Mockito.when(updateVmCommand.getId()).thenReturn(vmId);
 
         Mockito.when(userVmDao.findById(Mockito.eq(vmId))).thenReturn(userVmVoMock);
+
+        Mockito.when(callerAccount.getType()).thenReturn(Account.ACCOUNT_TYPE_ADMIN);
+        CallContext.register(callerUser, callerAccount);
+    }
+
+    @After
+    public void afterTest() {
+        CallContext.unregister();
     }
 
     @Test

systemvm/debian/etc/apache2/conf-enabled/mods-enabled/mpm_event.conf

@@ -0,0 +1,18 @@
+# event MPM
+# StartServers: initial number of server processes to start
+# MinSpareThreads: minimum number of worker threads which are kept spare
+# MaxSpareThreads: maximum number of worker threads which are kept spare
+# ThreadsPerChild: constant number of worker threads in each server process
+# MaxRequestWorkers: maximum number of worker threads
+# MaxConnectionsPerChild: maximum number of requests a server process serves
+<IfModule mpm_event_module>
+	StartServers			1
+	MinSpareThreads			25
+	MaxSpareThreads		 	75
+	ThreadLimit			64
+	ThreadsPerChild			25
+	MaxRequestWorkers		30
+	MaxConnectionsPerChild   	1000
+</IfModule>
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

systemvm/debian/etc/cron.daily/clear_cache

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# clear memory cache to ultimately reduce swapping
+
+sync && echo 1 > /proc/sys/vm/drop_caches

systemvm/debian/etc/cron.hourly/clear_cache

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# clear memory cache to ultimately reduce swapping
+
+phymem=$(free|awk '/^Mem:/{print $2}')
+if [ $phymem -lt 513000 ]; then
+  sync && echo 1 > /proc/sys/vm/drop_caches
+fi

systemvm/debian/etc/sysctl.conf

@@ -63,3 +63,9 @@ net.ipv6.conf.all.autoconf = 0
 
 # Minimum swappiness without disabling it
 vm.swappiness=1
+
+# make the kernel more aggressive in reclaiming RAM from the disk and swap caches
+vm.vfs_cache_pressure = 200
+
+# try to maintain 'free' memory thereby reducing the size of disk cache, hence reducing swapping.
+vm.min_free_kbytes = 20480

systemvm/debian/etc/vmware-tools/tools.conf

@@ -0,0 +1,17 @@
+[logging]
+# Turns on logging globally. It can still be disabled for each domain.
+# log = true
+
+# Disables core dumps on fatal errors; they're enabled by default.
+enableCoreDump = false
+
+# Defines the "vmsvc" domain, logging to file
+# vmsvc.level = message
+vmsvc.handler = file
+# Setup file rotation - keep 3 files
+vmsvc.maxOldLogFiles = 2
+# Max log file size kept: 1 MB
+vmsvc.maxLogSize = 1
+
+# Defines the "vmtoolsd" domain, and disable logging for it.
+# vmtoolsd.level = none

systemvm/debian/lib/systemd/system/cloud.service

@@ -1,6 +1,6 @@
 [Unit]
 Description=CloudStack Agent service
-After=cloud-early-config.service network.target local-fs.target
+After=cloud-early-config.service network.target cloud-postinit.service local-fs.target
 
 [Install]
 WantedBy=multi-user.target

systemvm/debian/lib/systemd/system/hyperv-daemons.hv-fcopy-daemon.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Hyper-V file copy service (FCOPY) daemon
+ConditionPathExists=/dev/vmbus/hv_fcopy
+
+[Service]
+ExecStart=/usr/sbin/hv_fcopy_daemon -n
+
+[Install]
+WantedBy=multi-user.target

systemvm/debian/lib/systemd/system/hyperv-daemons.hv-kvp-daemon.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=Hyper-V key-value pair (KVP) daemon
+
+[Service]
+ExecStart=/usr/sbin/hv_kvp_daemon -n
+
+[Install]
+WantedBy=multi-user.target

systemvm/debian/lib/systemd/system/hyperv-daemons.hv-vss-daemon.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=Hyper-V volume shadow copy service (VSS) daemon
+
+[Service]
+ExecStart=/usr/sbin/hv_vss_daemon -n
+
+[Install]
+WantedBy=multi-user.target

systemvm/debian/lib/systemd/system/open-vm-tools.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Service for virtual machines hosted on VMware
+Documentation=http://open-vm-tools.sourceforge.net/about.php
+DefaultDependencies=no
+Before=cloud-early-config.service
+
+[Service]
+ExecStart=/usr/bin/vmtoolsd
+TimeoutStopSec=5
+
+[Install]
+WantedBy=multi-user.target

systemvm/debian/lib/systemd/system/xe-daemon.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Xen Guest Monitoring Agent
+DefaultDependencies=no
+After=local-fs.target
+Requires=proc-xen.mount
+Before=network.target cloud-early-config.service
+ConditionPathExists=/proc/xen/capabilities
+
+[Service]
+ExecStartPre=/usr/sbin/xe-linux-distribution /var/cache/xe-linux-distribution
+ExecStart=/usr/sbin/xe-daemon
+StandardOutput=journal+console
+
+[Install]
+WantedBy=multi-user.target

systemvm/debian/opt/cloud/bin/cs/CsDhcp.py

@@ -139,7 +139,8 @@ class CsDhcp(CsDataBag):
             logging.error("Caught error while trying to delete entries from dnsmasq.leases file: %s" % e)
 
     def preseed(self):
-        self.add_host("127.0.0.1", "localhost %s" % CsHelper.get_hostname())
+        self.add_host("127.0.0.1", "localhost")
+        self.add_host("127.0.1.1", "%s" % CsHelper.get_hostname())
         self.add_host("::1", "localhost ip6-localhost ip6-loopback")
         self.add_host("ff02::1", "ip6-allnodes")
         self.add_host("ff02::2", "ip6-allrouters")

systemvm/debian/opt/cloud/bin/setup/bootstrap.sh

@@ -19,6 +19,9 @@
 PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin"
 CMDLINE=/var/cache/cloud/cmdline
 
+rm -f /var/cache/cloud/enabled_svcs
+rm -f /var/cache/cloud/disabled_svcs
+
 . /lib/lsb/init-functions
 
 log_it() {
@@ -56,25 +59,25 @@ hypervisor() {
 }
 
 config_guest() {
-  if [ "$HYPERVISOR" == "kvm" ]
-  then
-    # Configure hot-plug
-    modprobe acpiphp || true
-    modprobe pci_hotplug || true
-    sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab
-    sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102" /etc/inittab
-  fi
   [ ! -d /proc/xen ] && sed -i 's/^vc/#vc/' /etc/inittab && telinit q
   [ -d /proc/xen ] && sed -i 's/^#vc/vc/' /etc/inittab && telinit q
-}
 
-get_boot_params() {
+  systemctl daemon-reload
+
   case $HYPERVISOR in
      xen-pv|xen-domU)
+          systemctl stop ntpd
+          systemctl disable ntpd
+          systemctl start xe-daemon
+
           cat /proc/cmdline > $CMDLINE
           sed -i "s/%/ /g" $CMDLINE
           ;;
      xen-hvm)
+          systemctl stop ntpd
+          systemctl disable ntpd
+          systemctl start xe-daemon
+
           if [ ! -f /usr/bin/xenstore-read ]; then
             log_it "ERROR: xentools not installed, cannot found xenstore-read" && exit 5
           fi
@@ -82,7 +85,13 @@ get_boot_params() {
           sed -i "s/%/ /g" $CMDLINE
           ;;
      kvm)
+          # Configure hot-plug
+          modprobe acpiphp || true
+          modprobe pci_hotplug || true
+          sed -i -e "/^s0:2345:respawn.*/d" /etc/inittab
+          sed -i -e "/6:23:respawn/a\s0:2345:respawn:/sbin/getty -L 115200 ttyS0 vt102" /etc/inittab
           systemctl enable --now qemu-guest-agent
+
           # Wait for $CMDLINE file to be written by the qemu-guest-agent
           for i in {1..60}; do
             if [ -s $CMDLINE ]; then
@@ -96,13 +105,16 @@ get_boot_params() {
           fi
           ;;
      vmware)
+          # system time sync'd with host via vmware tools
+          systemctl stop ntpd
+          systemctl disable ntpd
+          systemctl start open-vm-tools
+
           vmtoolsd --cmd 'machine.id.get' > $CMDLINE
           ;;
      virtualpc|hyperv)
           # Hyper-V is recognized as virtualpc hypervisor type. Boot args are passed using KVP Daemon
-          #waiting for the hv_kvp_daemon to start up
+          systemctl start hyperv-daemons.hv-fcopy-daemon.service hyperv-daemons.hv-kvp-daemon.service hyperv-daemons.hv-vss-daemon.service
-          #sleep  need to fix the race condition of hv_kvp_daemon and cloud-early-config
-          [ -f /usr/sbin/hv_kvp_daemon ] && /usr/sbin/hv_kvp_daemon
           sleep 5
           cp -f /var/opt/hyperv/.kvp_pool_0 $CMDLINE
           cat /dev/null > /var/opt/hyperv/.kvp_pool_0
@@ -117,13 +129,11 @@ get_boot_params() {
           fi
           ;;
   esac
-}
 
-get_systemvm_type() {
+  # Find and export guest type
   export TYPE=$(grep -Po 'type=\K[a-zA-Z]*' $CMDLINE)
 }
 
-
 patch_systemvm() {
   local patchfile=$1
   local backupfolder="/tmp/.conf.backup"
@@ -172,19 +182,29 @@ patch() {
   return 0
 }
 
+config_sysctl() {
+  # When there is more memory reset the cache back pressure to default 100
+  physmem=$(free|awk '/^Mem:/{print $2}')
+  if [ $((physmem)) -lt 409600 ]; then
+      sed  -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 200" /etc/sysctl.conf
+  else
+      sed  -i "/^vm.vfs_cache_pressure/ c\vm.vfs_cache_pressure = 100" /etc/sysctl.conf
+  fi
+
+  sync
+  sysctl -p
+}
+
 bootstrap() {
   log_it "Bootstrapping systemvm appliance"
 
   export HYPERVISOR=$(hypervisor)
-  [ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out of early init" && exit 10
+  [ $? -ne 0 ] && log_it "Failed to detect hypervisor type, bailing out" && exit 10
-  log_it "Detected that we are running inside $HYPERVISOR"
+  log_it "Starting guest services for $HYPERVISOR"
 
   config_guest
-  get_boot_params
-  get_systemvm_type
   patch
-  sync
+  config_sysctl
-  sysctl -p
 
   log_it "Configuring systemvm type=$TYPE"
   if [ -f "/opt/cloud/bin/setup/$TYPE.sh" ]; then
@@ -192,6 +212,7 @@ bootstrap() {
   else
       /opt/cloud/bin/setup/default.sh
   fi
+
   log_it "Finished setting up systemvm"
   exit 0
 }

systemvm/debian/opt/cloud/bin/setup/consoleproxy.sh

@@ -18,14 +18,13 @@
 
 . /opt/cloud/bin/setup/common.sh
 
-consoleproxy_svcs() {
+setup_console_proxy() {
+  log_it "Setting up console proxy system vm"
+
   echo "cloud" > /var/cache/cloud/enabled_svcs
   echo "haproxy dnsmasq apache2 nfs-common portmap" > /var/cache/cloud/disabled_svcs
   mkdir -p /var/log/cloud
-}
 
-setup_console_proxy() {
-  log_it "Setting up console proxy system vm"
   setup_common eth0 eth1 eth2
   setup_system_rfc1918_internal
 
@@ -51,10 +50,4 @@ setup_console_proxy() {
   rm -f /etc/logrotate.d/cloud
 }
 
-consoleproxy_svcs
-if [ $? -gt 0 ]
-then
-  log_it "Failed to execute consoleproxy_svcs"
-  exit 1
-fi
 setup_console_proxy

systemvm/debian/opt/cloud/bin/setup/postinit.sh

@@ -58,7 +58,10 @@ then
   ip6tables-restore < $ipv6
 fi
 
-# Enable SSH
+# Patch known systemd/sshd memory leak - https://github.com/systemd/systemd/issues/8015#issuecomment-476160981
+echo '@include null' >> /etc/pam.d/systemd-user
+
+# Enable and Start SSH
 systemctl enable --now --no-block ssh
 
 date > /var/cache/cloud/boot_up_done

systemvm/debian/opt/cloud/bin/setup/secstorage.sh

@@ -18,15 +18,12 @@
 
 . /opt/cloud/bin/setup/common.sh
 
-secstorage_svcs() {
-  echo "apache2 cloud nfs-common portmap" > /var/cache/cloud/enabled_svcs
-  echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs
-  mkdir -p /var/log/cloud
-}
-
 setup_secstorage() {
   log_it "Setting up secondary storage system vm"
-  sysctl vm.min_free_kbytes=8192
+
+  echo "cloud apache2 nfs-common portmap" > /var/cache/cloud/enabled_svcs
+  echo "conntrackd keepalived haproxy dnsmasq" > /var/cache/cloud/disabled_svcs
+  mkdir -p /var/log/cloud
 
   setup_common eth0 eth1 eth2
   setup_storage_network
@@ -80,10 +77,4 @@ CORS
   rm -f /etc/logrotate.d/cloud
 }
 
-secstorage_svcs
-if [ $? -gt 0 ]
-then
-  log_it "Failed to execute secstorage_svcs"
-  exit 1
-fi
 setup_secstorage

tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh

@@ -46,7 +46,7 @@ function install_cloud_scripts() {
   rsync -av ./cloud_scripts/ /
   chmod +x /opt/cloud/bin/* /opt/cloud/bin/setup/* \
     /root/{clearUsageRules.sh,reconfigLB.sh,monitorServices.py} \
-    /etc/profile.d/cloud.sh
+    /etc/profile.d/cloud.sh /etc/cron.daily/* /etc/cron.hourly/*
 
   chmod -x /etc/systemd/system/*
 
@@ -64,6 +64,7 @@ function do_signature() {
 
 function configure_issue() {
   cat > /etc/issue <<EOF
+ESC [ 2J
    __?.o/  Apache CloudStack SystemVM $CLOUDSTACK_RELEASE
   (  )#    https://cloudstack.apache.org
  (___(_)   Debian GNU/Linux 9 \n \l
@@ -108,6 +109,18 @@ function configure_services() {
   systemctl disable strongswan
   systemctl disable x11-common
   systemctl disable xl2tpd
+  systemctl disable vgauth
+  systemctl disable sshd
+  systemctl disable nfs-common
+  systemctl disable portmap
+
+  # Disable guest services which will selectively be started based on hypervisor
+  systemctl disable open-vm-tools
+  systemctl disable xe-daemon
+  systemctl disable hyperv-daemons.hv-fcopy-daemon.service
+  systemctl disable hyperv-daemons.hv-kvp-daemon.service
+  systemctl disable hyperv-daemons.hv-vss-daemon.service
+  systemctl disable qemu-guest-agent
 
   configure_apache2
   configure_strongswan

ui/config.js

@@ -29,11 +29,7 @@ cloudStackOptions = {
         "jp": "label.japanese.keyboard",
         "sc": "label.simplified.chinese.keyboard"
     },
-    hiddenFields: {
+    hiddenFields: { // Fields to be hidden only for users in the tables below
-        "metrics.zones":[], // Options - "name", "state", "clusters", "cpuused", "cpuallocated", "memused", "memallocated"
-        "metrics.clusters": [], // Options - "name", "state", "hosts", "cpuused", "cpuallocated", "memused", "memallocated"
-        "metrics.hosts": [], // Options - "name", "state", "powerstate", "instances", "cpuused", "memused", "network"
-        "metrics.storagepool": [], // Options - "name", "property", "disk",
         "metrics.instances": [], // Options - "name", "state", "ipaddress", "zonename", "cpuused", "memused", "network", "disk"
         "metrics.volumes": [] // Options - "name", "state", "vmname", "sizegb", "physicalsize", "utilization", "storagetype", "storage"
     }

ui/css/cloudstack3.css

@@ -4027,7 +4027,7 @@ textarea {
   font-size: 14px;
 }
 
-.ui-dialog div.form-container div.value label {
+#label_delete_volumes label {
   display: block;
   width: 119px;
   margin-top: 2px;
@@ -8447,6 +8447,10 @@ div#details-tab-aclRules td.cidrlist span {
   display: inline-block;
 }
 
+.recurring-snapshots .schedule .forms .formContainer {
+  min-height: 250px;
+}
+
 .recurring-snapshots .schedule .add-snapshot-actions {
   float: left;
   clear: both;
@@ -8510,6 +8514,11 @@ div#details-tab-aclRules td.cidrlist span {
   left: 5px;
 }
 
+.recurring-snapshots .schedule .forms .tagger form div.value label {
+  width: 25px;
+  top: 10px;
+}
+
 .recurring-snapshots .schedule .forms form label.error {
   float: left;
   width: 100%;
@@ -8522,6 +8531,10 @@ div#details-tab-aclRules td.cidrlist span {
   margin: 8px 0 0;
 }
 
+.recurring-snapshots .schedule .forms .tagger form .field {
+  margin: 0;
+}
+
 .recurring-snapshots .schedule .forms form .name {
   float: left;
   width: 72px;
@@ -8660,6 +8673,11 @@ div#details-tab-aclRules td.cidrlist span {
   padding: 0;
 }
 
+.recurring-snapshots .ui-tabs .tagger ul {
+    margin: 16px auto auto;
+    padding-bottom: 10px;
+}
+
 .recurring-snapshots .ui-tabs ul li a {
   width: 76px;
   background: url("../images/sprites.png") no-repeat -521px -533px;
@@ -10692,6 +10710,11 @@ div#details-tab-aclRules td.cidrlist span {
   display: none;
 }
 
+.ui-dialog .tagger .tag-info.inside-form {
+  display: block;
+  text-align: left;
+}
+
 .ui-dialog.editTags .ui-button {
   float: right;
 }
@@ -13023,6 +13046,29 @@ div.gpugroups div.list-view {
   background: transparent url("../images/icons.png") no-repeat -626px -209px;
 }
 
+ul.ui-autocomplete.ui-menu {
+  width: 250px;
+  max-height: 100px;
+  background: #eee;
+  padding: 5px;
+  text-align: left;
+  overflow-y: auto;
+  overflow-x: hidden;
+  z-index: 100;
+}
+
+.ui-menu .ui-menu-item {
+  cursor: pointer;
+}
+
+.ui-menu .ui-menu-item .ui-state-active {
+  background: #CBDDF3;
+}
+
+.ui-helper-hidden-accessible {
+  display: none;
+}
+
 .copy-template-destination-list div.text-search {
   right: 5px;
 }

ui/css/custom.css

@@ -0,0 +1,23 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*
+* Use custom.css to override the default CloudStack styles
+*/
+
+/*# sourceMappingURL=src/sourcemaps/custom.css.map */

ui/css/src/scss/cloudstack3.scss

@@ -88,5 +88,3 @@
 @import 'components/button-export';
 @import 'components/jquery-ui';
 @import 'components/token-input-facebook';
-
-@import 'custom/custom';

ui/css/src/scss/custom.scss

@@ -0,0 +1,21 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*
+* Use custom.css to override the default CloudStack styles
+*/

ui/css/src/scss/custom/custom.scss

@@ -1,21 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//* Use custom.scss to override the default CloudStack styles
-//* Use it as your global @import file for many scss files.
-//* Use lines like @import "database.scss"; inside here only.
-//* Custom scss is imported on bottom of cloudstack3.scss

ui/index.html

@@ -26,6 +26,7 @@
         <link type="text/css" rel="stylesheet" href="lib/jquery-ui/css/jquery-ui.css" />
         <link type="text/css" rel="stylesheet" href="lib/reset.css"/>
         <link type="text/css" rel="stylesheet" href="css/cloudstack3.css" />
+        <link type="text/css" rel="stylesheet" href="css/custom.css" />
     </head>
     <body>
         <!-- CloudStack widget content -->
@@ -1553,7 +1554,7 @@
                         </ul>
 
                         <!-- Hourly -->
-                        <div id="recurring-snapshots-hourly">
+                        <div id="recurring-snapshots-hourly" class="formContainer">
                             <form>
                                 <input type="hidden" name="snapshot-type" value="hourly" />
 
@@ -1583,11 +1584,14 @@
                                         <label for="maxsnaps"><translate key="label.snapshots" /></label>
                                     </div>
                                 </div>
+
+                                <!-- Tags -->
+                                <div class="field taggerContainer"></div>
                             </form>
                         </div>
 
                         <!-- Daily -->
-                        <div id="recurring-snapshots-daily">
+                        <div id="recurring-snapshots-daily" class="formContainer">
                             <form>
                                 <input type="hidden" name="snapshot-type" value="daily" />
 
@@ -1617,11 +1621,14 @@
                                         <label for="maxsnaps"><translate key="label.snapshots" /></label>
                                     </div>
                                 </div>
+
+                                <!-- Tags -->
+                                <div class="field taggerContainer"></div>
                             </form>
                         </div>
 
                         <!-- Weekly -->
-                        <div id="recurring-snapshots-weekly">
+                        <div id="recurring-snapshots-weekly" class="formContainer">
                             <form>
                                 <input type="hidden" name="snapshot-type" value="weekly" />
 
@@ -1659,11 +1666,14 @@
                                         <label for="maxsnaps"><translate key="label.snapshots" /></label>
                                     </div>
                                 </div>
+
+                                <!-- Tags -->
+                                <div class="field taggerContainer"></div>
                             </form>
                         </div>
 
                         <!-- Monthly -->
-                        <div id="recurring-snapshots-monthly">
+                        <div id="recurring-snapshots-monthly" class="formContainer">
                             <form>
                                 <input type="hidden" name="snapshot-type" value="monthly" />
 
@@ -1701,6 +1711,9 @@
                                         <label for="maxsnaps"><translate key="label.snapshots" /></label>
                                     </div>
                                 </div>
+
+                                <!-- Tags -->
+                                <div class="field taggerContainer"></div>
                             </form>
                         </div>
                     </div>

ui/scripts/globalSettings.js

@@ -168,7 +168,6 @@
                                 }],
                                 dataProvider: function(args) {
                                     var items = [];
-                                    console.log(args);
                                     $.ajax({
                                         url: createURL("listLdapConfigurations&hostname=" + args.context.ldapConfiguration[0].hostname),
                                         dataType: "json",

ui/scripts/instances.js

@@ -3329,6 +3329,7 @@
 					settings: {
 						title: 'label.settings',
 						custom: cloudStack.uiCustom.granularDetails({
+                        resourceType: 'UserVm',
 							dataProvider: function(args) {
 								$.ajax({
 									url: createURL('listVirtualMachines&id=' + args.context.instances[0].id),
@@ -3388,7 +3389,6 @@
                                         // It could happen that a stale web page has been opened up when VM was stopped but
                                         // vm was turned on through another route - UI or API. so we should check again.
                                         var existingDetails = virtualMachine.details;
-                                        console.log(existingDetails);
                                         var newDetails = {};
                                         for (d in existingDetails) {
                                             if (d != data.name) {

ui/scripts/metrics.js

@@ -607,6 +607,21 @@
                 },
                 storage: {
                     label: 'label.metrics.storagepool'
+                },
+                disk: {
+                    label: 'label.metrics.disk.usage',
+                    collapsible: true,
+                    columns: {
+                        diskioread: {
+                            label: 'label.metrics.disk.read'
+                        },
+                        diskiowrite: {
+                            label: 'label.metrics.disk.write'
+                        },
+                        diskiopstotal: {
+                            label: 'label.metrics.disk.iops.total'
+                        }
+                    }
                 }
             },
             dataProvider: function(args) {

ui/scripts/roles.js

@@ -327,31 +327,24 @@
                                             });
                                         }
                                     });
-                                    var setupAutocompletion = function() {
-                                        var $target = $($.find('input[name="rule"]'));
-                                        if ($target.hasClass('ui-autocomplete')) {
-                                            $target.autocomplete('destroy');
-                                        }
-                                        $($.find('input[name="rule"]')).autocomplete({
-                                            source: apiList,
-                                            autoFocus:true
-                                        });
-                                    };
-                                    if (apiList.length == 0) {
                                     $.ajax({
                                         url: createURL("listApis"),
                                         dataType: "json",
                                         success: function(json) {
+                                            var apis = [];
                                             var response = json.listapisresponse.api;
                                             $.each(response, function(idx, api) {
-                                                    apiList.push(api.name);
+                                                apis.push(api.name);
+                                            });
+                                            $($.find('input[name="rule"]')).autocomplete({
+                                                minLength: 0,
+                                                delay: 0,
+                                                source: apis.sort()
+                                            }).focus(function() {
+                                                $(this).data("uiAutocomplete").search($(this).val());
                                             });
-                                                setupAutocompletion();
                                         }
                                     });
-                                    } else {
-                                        setupAutocompletion();
-                                    }
                                 }
                             });
                         }

ui/scripts/storage.js

@@ -937,6 +937,10 @@
                                         asyncBackup: {
                                             label: 'label.async.backup',
                                             isBoolean: true
+                                        },
+                                        tags: {
+                                            label: 'label.tags',
+                                            tagger: true
                                         }
                                     }
                                 },
@@ -951,6 +955,15 @@
                                             name: args.data.name
                                         });
                                     }
+                                    if (!$.isEmptyObject(args.data.tags)) {
+                                        $(args.data.tags).each(function(idx, tagData) {
+                                            var formattedTagData = {};
+                                            formattedTagData["tags[" + _s(idx) + "].key"] = _s(tagData.key);
+                                            formattedTagData["tags[" + _s(idx) + "].value"] = _s(tagData.value);
+                                            $.extend(data, formattedTagData);
+                                        });
+                                    }
+
                                     $.ajax({
                                         url: createURL("createSnapshot"),
                                         data: data,
@@ -1016,7 +1029,9 @@
                                                 var snap = args.snapshot;
 
                                                 var data = {
-                                                    keep: snap.maxsnaps,
+                                                    volumeid: args.context.volumes[0].id,
+                                                    intervaltype: snap['snapshot-type'],
+                                                    maxsnaps: snap.maxsnaps,
                                                     timezone: snap.timezone
                                                 };
 
@@ -1069,15 +1084,18 @@
                                                         break;
                                                 }
 
+                                                if (!$.isEmptyObject(snap.tags)) {
+                                                    $(snap.tags).each(function(idx, tagData) {
+                                                        var formattedTagData = {};
+                                                        formattedTagData["tags[" + _s(idx) + "].key"] = _s(tagData.key);
+                                                        formattedTagData["tags[" + _s(idx) + "].value"] = _s(tagData.value);
+                                                        $.extend(data, formattedTagData);
+                                                    });
+                                                }
+
                                                 $.ajax({
                                                     url: createURL('createSnapshotPolicy'),
-                                                    data: {
+                                                    data: data,
-                                                        volumeid: args.context.volumes[0].id,
-                                                        intervaltype: snap['snapshot-type'],
-                                                        maxsnaps: snap.maxsnaps,
-                                                        schedule: data.schedule,
-                                                        timezone: snap.timezone
-                                                    },
                                                     dataType: 'json',
                                                     async: true,
                                                     success: function(successData) {

ui/scripts/templates.js

@@ -2170,6 +2170,7 @@
 							settings: {
 								title: 'label.settings',
 								custom: cloudStack.uiCustom.granularDetails({
+                                    resourceType: 'Template',
 									dataProvider: function(args) {
 										$.ajax({
 											url: createURL('listTemplates'),

ui/scripts/ui-custom/granularSettings.js

@@ -57,6 +57,7 @@
     cloudStack.uiCustom.granularDetails = function(args) {
         var dataProvider = args.dataProvider;
         var actions = args.actions;
+        var resourceType = args.resourceType;
 
         return function(args) {
             var context = args.context;
@@ -109,6 +110,52 @@
                         },
                         createForm: {
                             title: 'Add New Setting',
+                            preFilter: function(args) {
+                                var data = {
+                                    resourcetype: resourceType
+                                };
+                                if (resourceType === 'UserVm') {
+                                    data.resourceid = args.context.instances[0].id;
+                                }
+                                if (resourceType === 'Template') {
+                                    data.resourceid = args.context.templates[0].id;
+                                }
+
+                                $.ajax({
+                                    url: createURL("listDetailOptions"),
+                                    data: data,
+                                    dataType: "json",
+                                    success: function(json) {
+                                        var details = json.listdetailoptionsresponse.detailoptions.details;
+                                        var keys = [];
+                                        Object.keys(details).forEach(function(key,index) {
+                                            keys.push(key);
+                                        });
+                                        $(args.$form.find('input[name="name"]')).blur();
+                                        $(args.$form.find('input[name="name"]')).autocomplete({
+                                            minLength: 0,
+                                            delay: 0,
+                                            source: keys.sort(),
+                                            select: function(event, ui) {
+                                                const key = ui.item.value;
+                                                const options = details[key];
+                                                $(args.$form.find('input[name="value"]')).autocomplete({
+                                                    minLength: 0,
+                                                    delay: 0,
+                                                    autoFocus: true,
+                                                    source: options.sort()
+                                                }).focus(function() {
+                                                    $(this).data("uiAutocomplete").search($(this).val());
+                                                });
+                                            }
+                                        }).focus(function() {
+                                            $(this).data("uiAutocomplete").search($(this).val());
+                                        });
+                                    }
+                                });
+
+                                return true;
+                            },
                             fields: {
                                 name: {
                                     label: 'label.name',
@@ -127,7 +174,45 @@
                         action: actions.add
                     }
                 },
-                dataProvider: dataProvider
+                dataProvider: function(args) {
+                    var data = {
+                        resourcetype: resourceType
+                    };
+                    if (resourceType === 'UserVm') {
+                        data.resourceid = args.context.instances[0].id;
+                    }
+                    if (resourceType === 'Template') {
+                        data.resourceid = args.context.templates[0].id;
+                    }
+
+                    $.ajax({
+                        url: createURL("listDetailOptions"),
+                        data: data,
+                        dataType: "json",
+                        success: function(json) {
+                            var details = json.listdetailoptionsresponse.detailoptions.details;
+                            var tbody = $.find('#details-tab-settings .data-table tbody');
+                            $(tbody).on('DOMNodeInserted', "tr", function() {
+                                $.each($.find('#details-tab-settings .data-table tbody tr'), function(idx, row) {
+                                    const key = $(row).find('td.name').attr('title');
+                                    const options = details[key];
+                                    if (options) {
+                                        $($(row).find('input.edit')).autocomplete({
+                                            minLength: 0,
+                                            delay: 0,
+                                            autoFocus: true,
+                                            source: options.sort()
+                                        }).focus(function() {
+                                            $(this).data("uiAutocomplete").search("");
+                                        });
+                                        $(row).find('input.edit').blur();
+                                    }
+                                });
+                            });
+                            dataProvider(args);
+                        }
+                    });
+                }
             };
 
             var $listView = $('<div>').listView({

ui/scripts/ui-custom/recurringSnapshots.js

@@ -61,6 +61,10 @@
                 }
             });
 
+            $($snapshots.find('.taggerContainer')).each(function() {
+                $('<div>').taggerInForm().appendTo(this);
+            });
+
             // Form validation
             $snapshots.find('form').validate();
 
@@ -70,7 +74,7 @@
 
                 if (!$form.valid()) return false;
 
-                var formData = cloudStack.serializeForm($form);
+                var formData = $.extend(cloudStack.serializeForm($form), {'tags' : cloudStack.getTagsFromForm($form)});
 
                 actions.add({
                     context: context,

ui/scripts/ui/dialog.js

@@ -149,7 +149,6 @@
 
             var isAsync = false;
             var isNoDialog = args.noDialog ? args.noDialog : false;
-
             $(fields).each(function(idx, element) {
                 var key = this;
                 var field = args.form.fields[key];
@@ -190,7 +189,6 @@
                  closeOnEscape: false
                  }); */
                 // Label field
-
                 var $name = $('<div>').addClass('name')
                         .appendTo($formItem)
                         .append(
@@ -702,6 +700,10 @@
 
                         this.oldUnit = newUnit;
                     })
+                } else if (field.tagger) {
+                    $name.hide();
+                    $value.hide();
+                    $input = $('<div>').taggerInForm().appendTo($formItem);
                 } else { //text field
                     $input = $('<input>').attr({
                         name: key,
@@ -717,10 +719,12 @@
                     $input.addClass("disallowSpecialCharacters");
                 }
 
+                if (!field.tagger) { // Tagger has it's own validation
                     if (field.validation != null)
                         $input.data('validation-rules', field.validation);
                     else
                         $input.data('validation-rules', {});
+                }
 
                 var fieldLabel = field.label;
 
@@ -775,6 +779,13 @@
             var complete = function($formContainer) {
                 var $form = $formContainer.find('form');
                 var data = cloudStack.serializeForm($form);
+                if (!data.tags) {
+                    // Some APIs consume tags as a string (such as disk offering creation).
+                    // The UI of those use a tagger that is not a custom cloudStack.tagger
+                    // but rather a string. That case is handled by usual serialization. We
+                    // only need to check extract tags when the string tags are not present.
+                    $.extend(data, {'tags' : cloudStack.getTagsFromForm($form)});
+                }
 
                 if (!$formContainer.find('form').valid()) {
                     // Ignore hidden field validation

ui/scripts/ui/widgets/tagger.js

@@ -46,10 +46,7 @@
 
             $keyField.append($keyLabel, $key);
             $valueField.append($valueLabel, $value);
-            $form.append(
+            $form.append($keyField, $valueField, $submit);
-                $keyField, $valueField,
-                $submit
-            );
 
             $form.validate();
 
@@ -80,9 +77,11 @@
                         }
                     });
 
+                    if (args.isAsyncSubmit) {
                         // Prevent input during submission
                         $key.attr('disabled', 'disabled');
                         $value.attr('disabled', 'disabled');
+                    }
 
                     return false;
                 } :
@@ -102,6 +101,8 @@
 
             $label.append($key, '<span>=</span>', $value);
             $label.attr('title', title);
+            $label.attr('cloudstack_tag_key', _s(data.key));
+            $label.attr('cloudstack_tag_value', _s(data.value));
             $remove.click(function() {
                 if (onRemove) onRemove($li, data);
             });
@@ -173,6 +174,7 @@
             };
 
             var $inputArea = elems.inputArea({
+                isAsyncSubmit: true,
                 onSubmit: function(args) {
                     var data = args.data;
                     var success = args.response.success;
@@ -252,4 +254,53 @@
             });
         }
     });
+
+    $.widget('cloudStack.taggerInForm', {
+        _init: function(args) {
+            var $container = this.element.addClass('tagger');
+            var $tagArea = $('<ul>').addClass('tags');
+            var $title = elems.info(_l('label.tags')).addClass('title inside-form');
+            var $loading = $('<div>').addClass('loading-overlay');
+            var $tags = {};
+
+            var onRemoveItem = function($item, data) {
+                $item.remove();
+                if ($tags[data.key]) delete $tags[data.key];
+                else {
+                    cloudStack.dialog.notice({
+                        message: "Unexpected error occured in attempting deletion"
+                    });
+                }
+            };
+
+            var $inputArea = elems.inputArea({
+                isAsyncSubmit: false,
+                onSubmit: function(args) {
+                    var data = args.data;
+                    if ($tags[data.key]) {
+                        cloudStack.dialog.notice({
+                            message: "Key already present. Please delete previous and add again."
+                        });
+                    } else {
+                        var success = args.response.success;
+                        var title = data.key + ' = ' + data.value;
+                        elems.tagItem(title, onRemoveItem, data).appendTo($tagArea);
+                        success();
+                        $tags[data.key] = data.value;
+                    }
+                }
+            });
+
+            $container.append($title, $inputArea, $tagArea);
+        }
+    });
+
+    cloudStack.getTagsFromForm = function($form) {
+        var tagLabels = $($form).find('.tagger .tags .label');
+        var tags = [];
+        $(tagLabels).each(function() {
+            tags.push({'key' : $(this).attr('cloudstack_tag_key'), 'value' : $(this).attr('cloudstack_tag_value')});
+        });
+        return tags;
+    };
 }(jQuery, cloudStack));

utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java

@@ -30,9 +30,11 @@ public class ImageStoreUtil {
 
         //if ssvm url domain is present, use it to construct hostname in the format 1-2-3-4.domain
         // if the domain name is not present, ssl validation fails and has to be ignored
-        if(StringUtils.isNotBlank(ssvmUrlDomain)) {
+        if(StringUtils.isNotBlank(ssvmUrlDomain) && ssvmUrlDomain.startsWith("*")) {
             hostname = ipAddress.replace(".", "-");
             hostname = hostname + ssvmUrlDomain.substring(1);
+        } else if (StringUtils.isNotBlank(ssvmUrlDomain)) {
+            hostname = ssvmUrlDomain;
         }
 
         //only https works with postupload and url format is fixed

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java

@@ -2709,7 +2709,7 @@ public class VirtualMachineMO extends BaseMO {
         return pathList;
     }
 
-    private String getDeviceBusName(List<VirtualDevice> allDevices, VirtualDevice theDevice) throws Exception {
+    public String getDeviceBusName(List<VirtualDevice> allDevices, VirtualDevice theDevice) throws Exception {
         for (VirtualDevice device : allDevices) {
             if (device.getKey() == theDevice.getControllerKey().intValue()) {
                 if (device instanceof VirtualIDEController) {

vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java

@@ -25,11 +25,16 @@ import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
+import java.util.Date;
+import java.util.GregorianCalendar;
 import java.util.List;
 import java.util.Random;
 import java.util.UUID;
 
 import javax.annotation.Nonnull;
+import javax.xml.datatype.DatatypeConfigurationException;
+import javax.xml.datatype.DatatypeFactory;
+import javax.xml.datatype.XMLGregorianCalendar;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
@@ -41,6 +46,8 @@ import com.vmware.vim25.ManagedObjectReference;
 import com.vmware.vim25.MethodFault;
 import com.vmware.vim25.ObjectContent;
 import com.vmware.vim25.OptionValue;
+import com.vmware.vim25.PerfCounterInfo;
+import com.vmware.vim25.PerfMetricId;
 import com.vmware.vim25.ResourceAllocationInfo;
 import com.vmware.vim25.VirtualCdrom;
 import com.vmware.vim25.VirtualCdromIsoBackingInfo;
@@ -637,6 +644,25 @@ public class VmwareHelper {
         return usbController;
     }
 
+    public static PerfMetricId createPerfMetricId(PerfCounterInfo counterInfo, String instance) {
+        PerfMetricId metricId = new PerfMetricId();
+        metricId.setCounterId(counterInfo.getKey());
+        metricId.setInstance(instance);
+        return metricId;
+    }
+
+    public static String getDiskDeviceFileName(VirtualDisk diskDevice) {
+        VirtualDeviceBackingInfo backingInfo = diskDevice.getBacking();
+        if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) {
+            final String vmdkName = ((VirtualDiskFlatVer2BackingInfo)backingInfo).getFileName().replace(".vmdk", "");
+            if (vmdkName.contains("/")) {
+                return vmdkName.split("/", 2)[1];
+            }
+            return vmdkName;
+        }
+        return null;
+    }
+
     public static ManagedObjectReference getDiskDeviceDatastore(VirtualDisk diskDevice) throws Exception {
         VirtualDeviceBackingInfo backingInfo = diskDevice.getBacking();
         assert (backingInfo instanceof VirtualDiskFlatVer2BackingInfo);
@@ -773,4 +799,13 @@ public class VmwareHelper {
         return DiskControllerType.getType(dataDiskController) == DiskControllerType.osdefault;
     }
 
+    public static XMLGregorianCalendar getXMLGregorianCalendar(final Date date, final int offsetSeconds) throws DatatypeConfigurationException {
+        if (offsetSeconds > 0) {
+            date.setTime(date.getTime() - offsetSeconds * 1000);
+        }
+        final GregorianCalendar gregorianCalendar = new GregorianCalendar();
+        gregorianCalendar.setTime(date);
+        return DatatypeFactory.newInstance().newXMLGregorianCalendar(gregorianCalendar);
+    }
+
 }