Merge remote-tracking branch 'apache/4.20'

api/src/main/java/com/cloud/bgp/BGPService.java

@@ -21,6 +21,7 @@ import com.cloud.network.Network;
 import com.cloud.network.vpc.Vpc;
 import com.cloud.utils.Pair;
 import org.apache.cloudstack.api.command.user.bgp.ListASNumbersCmd;
+import org.apache.cloudstack.network.BgpPeer;
 
 import java.util.List;
 
@@ -36,4 +37,8 @@ public interface BGPService {
     boolean applyBgpPeers(Network network, boolean continueOnError) throws ResourceUnavailableException;
 
     boolean applyBgpPeers(Vpc vpc, boolean continueOnError) throws ResourceUnavailableException;
+
+    List<? extends BgpPeer> getBgpPeersForNetwork(Network network);
+
+    List<? extends BgpPeer> getBgpPeersForVpc(Vpc vpc);
 }

api/src/main/java/com/cloud/event/EventTypes.java

@@ -292,6 +292,7 @@ public class EventTypes {
 
     //register for user API and secret keys
     public static final String EVENT_REGISTER_FOR_SECRET_API_KEY = "REGISTER.USER.KEY";
+    public static final String API_KEY_ACCESS_UPDATE = "API.KEY.ACCESS.UPDATE";
 
     // Template Events
     public static final String EVENT_TEMPLATE_CREATE = "TEMPLATE.CREATE";

api/src/main/java/com/cloud/hypervisor/Hypervisor.java

@@ -20,37 +20,57 @@ import com.cloud.storage.Storage.ImageFormat;
 import org.apache.commons.lang3.StringUtils;
 
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+import java.util.EnumSet;
+import java.util.stream.Collectors;
+
+import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.DirectDownloadTemplate;
+import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.RootDiskSizeOverride;
+import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.VmStorageMigration;
 
 public class Hypervisor {
     public static class HypervisorType {
+        public enum Functionality {
+            DirectDownloadTemplate,
+            RootDiskSizeOverride,
+            VmStorageMigration
+        }
+
         private static final Map<String, HypervisorType> hypervisorTypeMap = new LinkedHashMap<>();
         public static final HypervisorType None = new HypervisorType("None"); //for storage hosts
-        public static final HypervisorType XenServer = new HypervisorType("XenServer", ImageFormat.VHD);
-        public static final HypervisorType KVM = new HypervisorType("KVM", ImageFormat.QCOW2);
-        public static final HypervisorType VMware = new HypervisorType("VMware", ImageFormat.OVA);
+        public static final HypervisorType XenServer = new HypervisorType("XenServer", ImageFormat.VHD, EnumSet.of(RootDiskSizeOverride, VmStorageMigration));
+        public static final HypervisorType KVM = new HypervisorType("KVM", ImageFormat.QCOW2, EnumSet.of(DirectDownloadTemplate, RootDiskSizeOverride, VmStorageMigration));
+        public static final HypervisorType VMware = new HypervisorType("VMware", ImageFormat.OVA, EnumSet.of(RootDiskSizeOverride, VmStorageMigration));
         public static final HypervisorType Hyperv = new HypervisorType("Hyperv");
         public static final HypervisorType VirtualBox = new HypervisorType("VirtualBox");
         public static final HypervisorType Parralels = new HypervisorType("Parralels");
         public static final HypervisorType BareMetal = new HypervisorType("BareMetal");
-        public static final HypervisorType Simulator = new HypervisorType("Simulator");
+        public static final HypervisorType Simulator = new HypervisorType("Simulator", null, EnumSet.of(RootDiskSizeOverride, VmStorageMigration));
         public static final HypervisorType Ovm = new HypervisorType("Ovm", ImageFormat.RAW);
         public static final HypervisorType Ovm3 = new HypervisorType("Ovm3", ImageFormat.RAW);
         public static final HypervisorType LXC = new HypervisorType("LXC");
-        public static final HypervisorType Custom = new HypervisorType("Custom");
+        public static final HypervisorType Custom = new HypervisorType("Custom", null, EnumSet.of(RootDiskSizeOverride));
         public static final HypervisorType Any = new HypervisorType("Any"); /*If you don't care about the hypervisor type*/
         private final String name;
         private final ImageFormat imageFormat;
+        private final Set<Functionality> supportedFunctionalities;
 
         public HypervisorType(String name) {
-            this(name, null);
+            this(name, null, EnumSet.noneOf(Functionality.class));
         }
 
         public HypervisorType(String name, ImageFormat imageFormat) {
+            this(name, imageFormat, EnumSet.noneOf(Functionality.class));
+        }
+
+        public HypervisorType(String name, ImageFormat imageFormat, Set<Functionality> supportedFunctionalities) {
             this.name = name;
             this.imageFormat = imageFormat;
+            this.supportedFunctionalities = supportedFunctionalities;
             if (name.equals("Parralels")){ // typo in the original code
                 hypervisorTypeMap.put("parallels", this);
             } else {
@@ -81,6 +101,12 @@ public class Hypervisor {
             return hypervisorType;
         }
 
+        public static List<HypervisorType> getListOfHypervisorsSupportingFunctionality(Functionality functionality) {
+            return hypervisorTypeMap.values().stream()
+                    .filter(hypervisor -> hypervisor.supportedFunctionalities.contains(functionality))
+                    .collect(Collectors.toList());
+        }
+
         /**
          * Returns the display name of a hypervisor type in case the custom hypervisor is used,
          * using the 'hypervisor.custom.display.name' setting. Otherwise, returns hypervisor name
@@ -102,6 +128,15 @@ public class Hypervisor {
             return name;
         }
 
+        /**
+         * Make this method to be part of the properties of the hypervisor type itself.
+         *
+         * @return true if the hypervisor plugin support the specified functionality
+         */
+        public boolean isFunctionalitySupported(Functionality functionality) {
+            return supportedFunctionalities.contains(functionality);
+        }
+
         @Override
         public int hashCode() {
             return Objects.hash(name);

api/src/main/java/com/cloud/server/ManagementServerHostStats.java

@@ -32,6 +32,8 @@ public interface ManagementServerHostStats {
 
     String getManagementServerHostUuid();
 
+    long getManagementServerRunId();
+
     long getSessions();
 
     double getCpuUtilization();

api/src/main/java/com/cloud/user/Account.java

@@ -93,4 +93,8 @@ public interface Account extends ControlledEntity, InternalIdentity, Identity {
 
     boolean isDefault();
 
+    public void setApiKeyAccess(Boolean apiKeyAccess);
+
+    public Boolean getApiKeyAccess();
+
 }

api/src/main/java/com/cloud/user/AccountService.java

@@ -19,6 +19,7 @@ package com.cloud.user;
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.utils.Pair;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
@@ -127,9 +128,9 @@ public interface AccountService {
      */
     UserAccount getUserAccountById(Long userId);
 
-    public Map<String, String> getKeys(GetUserKeysCmd cmd);
+    public Pair<Boolean, Map<String, String>> getKeys(GetUserKeysCmd cmd);
 
-    public Map<String, String> getKeys(Long userId);
+    public Pair<Boolean, Map<String, String>> getKeys(Long userId);
 
     /**
      * Lists user two-factor authentication provider plugins

api/src/main/java/com/cloud/user/User.java

@@ -94,4 +94,9 @@ public interface User extends OwnedBy, InternalIdentity {
     public boolean isUser2faEnabled();
 
     public String getKeyFor2fa();
+
+    public void setApiKeyAccess(Boolean apiKeyAccess);
+
+    public Boolean getApiKeyAccess();
+
 }

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -35,6 +35,7 @@ public class ApiConstants {
     public static final String ALLOW_USER_FORCE_STOP_VM = "allowuserforcestopvm";
     public static final String ANNOTATION = "annotation";
     public static final String API_KEY = "apikey";
+    public static final String API_KEY_ACCESS = "apikeyaccess";
     public static final String ARCHIVED = "archived";
     public static final String ARCH = "arch";
     public static final String AS_NUMBER = "asnumber";
@@ -381,6 +382,14 @@ public class ApiConstants {
     public static final String PATH = "path";
     public static final String PAYLOAD = "payload";
     public static final String PAYLOAD_URL = "payloadurl";
+    public static final String PEERS = "peers";
+    public static final String PEER_ID = "peerid";
+    public static final String PEER_NAME = "peername";
+    public static final String PEER_MSID = "peermsid";
+    public static final String PEER_RUNID = "peerrunid";
+    public static final String PEER_SERVICE_IP = "peerserviceip";
+    public static final String PEER_SERVICE_PORT = "peerserviceport";
+    public static final String PEER_STATE = "peerstate";
     public static final String POD_ID = "podid";
     public static final String POD_NAME = "podname";
     public static final String POD_IDS = "podids";
@@ -986,6 +995,7 @@ public class ApiConstants {
     public static final String ACL_NAME = "aclname";
     public static final String NUMBER = "number";
     public static final String IS_DYNAMICALLY_SCALABLE = "isdynamicallyscalable";
+    public static final String ROUTED_MODE_ENABLED = "routedmodeenabled";
     public static final String ROUTING = "isrouting";
     public static final String ROUTING_MODE = "routingmode";
     public static final String MAX_CONNECTIONS = "maxconnections";
@@ -1238,4 +1248,30 @@ public class ApiConstants {
     public enum DomainDetails {
         all, resource, min;
     }
+
+    public enum ApiKeyAccess {
+        DISABLED(false),
+        ENABLED(true),
+        INHERIT(null);
+
+        Boolean apiKeyAccess;
+
+        ApiKeyAccess(Boolean keyAccess) {
+            apiKeyAccess = keyAccess;
+        }
+
+        public Boolean toBoolean() {
+            return apiKeyAccess;
+        }
+
+        public static ApiKeyAccess fromBoolean(Boolean value) {
+            if (value == null) {
+                return INHERIT;
+            } else if (value) {
+                return ENABLED;
+            } else {
+                return DISABLED;
+            }
+        }
+    }
 }

api/src/main/java/org/apache/cloudstack/api/command/admin/account/UpdateAccountCmd.java

@@ -21,7 +21,9 @@ import java.util.Map;
 
 import javax.inject.Inject;
 
+import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.ApiCommandResourceType;
+import org.apache.cloudstack.api.command.user.UserCmd;
 import org.apache.cloudstack.api.response.RoleResponse;
 
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
@@ -40,8 +42,8 @@ import org.apache.cloudstack.region.RegionService;
 import com.cloud.user.Account;
 
 @APICommand(name = "updateAccount", description = "Updates account information for the authenticated user", responseObject = AccountResponse.class, entityType = {Account.class},
-        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
-public class UpdateAccountCmd extends BaseCmd {
+        responseView = ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
+public class UpdateAccountCmd extends BaseCmd implements UserCmd {
 
     /////////////////////////////////////////////////////
     //////////////// API parameters /////////////////////
@@ -70,6 +72,9 @@ public class UpdateAccountCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ACCOUNT_DETAILS, type = CommandType.MAP, description = "Details for the account used to store specific parameters")
     private Map details;
 
+    @Parameter(name = ApiConstants.API_KEY_ACCESS, type = CommandType.STRING, description = "Determines if Api key access for this user is enabled, disabled or inherits the value from its parent, the domain level setting api.key.access", since = "4.20.1.0", authorized = {RoleType.Admin})
+    private String apiKeyAccess;
+
     @Inject
     RegionService _regionService;
 
@@ -109,6 +114,10 @@ public class UpdateAccountCmd extends BaseCmd {
         return params;
     }
 
+    public String getApiKeyAccess() {
+        return apiKeyAccess;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -131,7 +140,7 @@ public class UpdateAccountCmd extends BaseCmd {
     public void execute() {
         Account result = _regionService.updateAccount(this);
         if (result != null){
-            AccountResponse response = _responseGenerator.createAccountResponse(ResponseView.Full, result);
+            AccountResponse response = _responseGenerator.createAccountResponse(getResponseView(), result);
             response.setResponseName(getCommandName());
             setResponseObject(response);
         } else {

api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java

@@ -23,6 +23,7 @@ import org.apache.cloudstack.api.BaseListCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.ManagementServerResponse;
+import org.apache.commons.lang3.BooleanUtils;
 
 @APICommand(name = "listManagementServers", description = "Lists management servers.", responseObject = ManagementServerResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
@@ -39,6 +40,11 @@ public class ListMgmtsCmd extends BaseListCmd {
     @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name of the management server")
     private String hostName;
 
+    @Parameter(name = ApiConstants.PEERS, type = CommandType.BOOLEAN,
+            description = "Whether to return the management server peers or not. By default, the management server peers will not be returned.",
+            since = "4.20.0.0")
+    private Boolean peers;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -51,6 +57,10 @@ public class ListMgmtsCmd extends BaseListCmd {
         return hostName;
     }
 
+    public Boolean getPeers() {
+        return BooleanUtils.toBooleanDefaultIfNull(peers, false);
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserKeysCmd.java

@@ -20,6 +20,7 @@ package org.apache.cloudstack.api.command.admin.user;
 
 import com.cloud.user.Account;
 import com.cloud.user.User;
+import com.cloud.utils.Pair;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
@@ -54,11 +55,13 @@ public class GetUserKeysCmd extends BaseCmd{
         else return Account.ACCOUNT_ID_SYSTEM;
     }
     public void execute(){
-        Map<String, String> keys = _accountService.getKeys(this);
+        Pair<Boolean, Map<String, String>> keys = _accountService.getKeys(this);
+
         RegisterResponse response = new RegisterResponse();
         if(keys != null){
-            response.setApiKey(keys.get("apikey"));
-            response.setSecretKey(keys.get("secretkey"));
+            response.setApiKeyAccess(keys.first());
+            response.setApiKey(keys.second().get("apikey"));
+            response.setSecretKey(keys.second().get("secretkey"));
         }
 
         response.setObjectName("userkeys");

api/src/main/java/org/apache/cloudstack/api/command/admin/user/ListUsersCmd.java

@@ -19,20 +19,23 @@ package org.apache.cloudstack.api.command.admin.user;
 import com.cloud.server.ResourceIcon;
 import com.cloud.server.ResourceTag;
 import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.command.user.UserCmd;
 import org.apache.cloudstack.api.response.ResourceIconResponse;
 
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseListAccountResourcesCmd;
 import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ResponseObject.ResponseView;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.UserResponse;
 
 import java.util.List;
 
 @APICommand(name = "listUsers", description = "Lists user accounts", responseObject = UserResponse.class,
-        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
-public class ListUsersCmd extends BaseListAccountResourcesCmd {
+        responseView = ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
+public class ListUsersCmd extends BaseListAccountResourcesCmd implements UserCmd {
 
 
     /////////////////////////////////////////////////////
@@ -53,6 +56,9 @@ public class ListUsersCmd extends BaseListAccountResourcesCmd {
     @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, description = "List user by the username")
     private String username;
 
+    @Parameter(name = ApiConstants.API_KEY_ACCESS, type = CommandType.STRING, description = "List users by the Api key access value", since = "4.20.1.0", authorized = {RoleType.Admin})
+    private String apiKeyAccess;
+
     @Parameter(name = ApiConstants.SHOW_RESOURCE_ICON, type = CommandType.BOOLEAN,
             description = "flag to display the resource icon for users")
     private Boolean showIcon;
@@ -77,6 +83,10 @@ public class ListUsersCmd extends BaseListAccountResourcesCmd {
         return username;
     }
 
+    public String getApiKeyAccess() {
+        return apiKeyAccess;
+    }
+
     public Boolean getShowIcon() {
         return showIcon != null ? showIcon : false;
     }
@@ -87,7 +97,7 @@ public class ListUsersCmd extends BaseListAccountResourcesCmd {
 
     @Override
     public void execute() {
-        ListResponse<UserResponse> response = _queryService.searchForUsers(this);
+        ListResponse<UserResponse> response = _queryService.searchForUsers(getResponseView(), this);
         response.setResponseName(getCommandName());
         this.setResponseObject(response);
         if (response != null && response.getCount() > 0 && getShowIcon()) {

api/src/main/java/org/apache/cloudstack/api/command/admin/user/UpdateUserCmd.java

@@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.admin.user;
 
 import javax.inject.Inject;
 
+import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.api.ApiConstants;
@@ -69,6 +70,9 @@ public class UpdateUserCmd extends BaseCmd {
     @Parameter(name = ApiConstants.USER_SECRET_KEY, type = CommandType.STRING, description = "The secret key for the user. Must be specified with userApiKey")
     private String secretKey;
 
+    @Parameter(name = ApiConstants.API_KEY_ACCESS, type = CommandType.STRING, description = "Determines if Api key access for this user is enabled, disabled or inherits the value from its parent, the owning account", since = "4.20.1.0", authorized = {RoleType.Admin})
+    private String apiKeyAccess;
+
     @Parameter(name = ApiConstants.TIMEZONE,
             type = CommandType.STRING,
             description = "Specifies a timezone for this command. For more information on the timezone parameter, see Time Zone Format.")
@@ -120,6 +124,10 @@ public class UpdateUserCmd extends BaseCmd {
         return secretKey;
     }
 
+    public String getApiKeyAccess() {
+        return apiKeyAccess;
+    }
+
     public String getTimezone() {
         return timezone;
     }

api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java

@@ -20,6 +20,7 @@ import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.List;
 
+import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.api.ApiConstants;
@@ -70,6 +71,9 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd implements UserC
                description = "comma separated list of account details requested, value can be a list of [ all, resource, min]")
     private List<String> viewDetails;
 
+    @Parameter(name = ApiConstants.API_KEY_ACCESS, type = CommandType.STRING, description = "List accounts by the Api key access value", since = "4.20.1.0", authorized = {RoleType.Admin})
+    private String apiKeyAccess;
+
     @Parameter(name = ApiConstants.SHOW_RESOURCE_ICON, type = CommandType.BOOLEAN,
             description = "flag to display the resource icon for accounts")
     private Boolean showIcon;
@@ -120,6 +124,10 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd implements UserC
         return dv;
     }
 
+    public String getApiKeyAccess() {
+        return apiKeyAccess;
+    }
+
     public boolean getShowIcon() {
         return showIcon != null ? showIcon : false;
     }

api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java

@@ -362,7 +362,9 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd {
                     "Parameter zoneids cannot combine all zones (-1) option with other zones");
 
         String customHypervisor = HypervisorGuru.HypervisorCustomDisplayName.value();
-        if (isDirectDownload() && !(getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.KVM.toString())
+        if (isDirectDownload() &&
+                !(Hypervisor.HypervisorType.getType(getHypervisor())
+                        .isFunctionalitySupported(Hypervisor.HypervisorType.Functionality.DirectDownloadTemplate)
                 || getHypervisor().equalsIgnoreCase(customHypervisor))) {
             throw new ServerApiException(ApiErrorCode.PARAM_ERROR, String.format("Parameter directdownload " +
                     "is only allowed for KVM or %s templates", customHypervisor));

api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java

@@ -73,6 +73,10 @@ public class ResizeVolumeCmd extends BaseAsyncCmd implements UserCmd {
                description = "new disk offering id")
     private Long newDiskOfferingId;
 
+    @Parameter(name = ApiConstants.AUTO_MIGRATE, type = CommandType.BOOLEAN, required = false,
+            description = "Flag to allow automatic migration of the volume to another suitable storage pool that accommodates the new size", since = "4.20.1")
+    private Boolean autoMigrate;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -129,6 +133,10 @@ public class ResizeVolumeCmd extends BaseAsyncCmd implements UserCmd {
         return newDiskOfferingId;
     }
 
+    public boolean getAutoMigrate() {
+        return autoMigrate == null ? false : autoMigrate;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java

@@ -271,6 +271,10 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou
     @Param(description = "The tagged resource limit and count for the account", since = "4.20.0")
     List<TaggedResourceLimitAndCountResponse> taggedResources;
 
+    @SerializedName(ApiConstants.API_KEY_ACCESS)
+    @Param(description = "whether api key access is Enabled, Disabled or set to Inherit (it inherits the value from the parent)", since = "4.20.1.0")
+    ApiConstants.ApiKeyAccess apiKeyAccess;
+
     @Override
     public String getObjectId() {
         return id;
@@ -554,4 +558,8 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou
     public void setTaggedResourceLimitsAndCounts(List<TaggedResourceLimitAndCountResponse> taggedResourceLimitsAndCounts) {
         this.taggedResources = taggedResourceLimitsAndCounts;
     }
+
+    public void setApiKeyAccess(Boolean apiKeyAccess) {
+        this.apiKeyAccess = ApiConstants.ApiKeyAccess.fromBoolean(apiKeyAccess);
+    }
 }

api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java

@@ -24,7 +24,9 @@ import org.apache.cloudstack.api.BaseResponse;
 import org.apache.cloudstack.api.EntityReference;
 import org.apache.cloudstack.management.ManagementServerHost.State;
 
+import java.util.ArrayList;
 import java.util.Date;
+import java.util.List;
 
 @EntityReference(value = ManagementServerHost.class)
 public class ManagementServerResponse extends BaseResponse {
@@ -76,6 +78,10 @@ public class ManagementServerResponse extends BaseResponse {
     @Param(description = "the IP Address for this Management Server")
     private String serviceIp;
 
+    @SerializedName(ApiConstants.PEERS)
+    @Param(description = "the Management Server Peers")
+    private List<PeerManagementServerNodeResponse> peers;
+
     public String getId() {
         return this.id;
     }
@@ -171,4 +177,19 @@ public class ManagementServerResponse extends BaseResponse {
     public String getKernelVersion() {
         return kernelVersion;
     }
+
+    public List<PeerManagementServerNodeResponse> getPeers() {
+        return peers;
+    }
+
+    public void setPeers(List<PeerManagementServerNodeResponse> peers) {
+        this.peers = peers;
+    }
+
+    public void addPeer(PeerManagementServerNodeResponse peer) {
+        if (peers == null) {
+            peers = new ArrayList<>();
+        }
+        peers.add(peer);
+    }
 }

api/src/main/java/org/apache/cloudstack/api/response/PeerManagementServerNodeResponse.java

@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.response;
+
+import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseResponse;
+import org.apache.cloudstack.management.ManagementServerHost.State;
+
+import java.util.Date;
+
+public class PeerManagementServerNodeResponse extends BaseResponse {
+
+    @SerializedName(ApiConstants.STATE)
+    @Param(description = "the state of the management server peer")
+    private State state;
+
+    @SerializedName(ApiConstants.LAST_UPDATED)
+    @Param(description = "the last updated time of the management server peer state")
+    private Date lastUpdated;
+
+    @SerializedName(ApiConstants.PEER_ID)
+    @Param(description = "the ID of the peer management server")
+    private String peerId;
+
+    @SerializedName(ApiConstants.PEER_NAME)
+    @Param(description = "the name of the peer management server")
+    private String peerName;
+
+    @SerializedName(ApiConstants.PEER_MSID)
+    @Param(description = "the management ID of the peer management server")
+    private String peerMsId;
+
+    @SerializedName(ApiConstants.PEER_RUNID)
+    @Param(description = "the run ID of the peer management server")
+    private String peerRunId;
+
+    @SerializedName(ApiConstants.PEER_STATE)
+    @Param(description = "the state of the peer management server")
+    private String peerState;
+
+    @SerializedName(ApiConstants.PEER_SERVICE_IP)
+    @Param(description = "the IP Address for the peer Management Server")
+    private String peerServiceIp;
+
+    @SerializedName(ApiConstants.PEER_SERVICE_PORT)
+    @Param(description = "the service port for the peer Management Server")
+    private String peerServicePort;
+
+    public void setState(State state) {
+        this.state = state;
+    }
+
+    public void setLastUpdated(Date lastUpdated) {
+        this.lastUpdated = lastUpdated;
+    }
+
+    public void setPeerId(String peerId) {
+        this.peerId = peerId;
+    }
+
+    public void setPeerName(String peerName) {
+        this.peerName = peerName;
+    }
+
+    public void setPeerMsId(String peerMsId) {
+        this.peerMsId = peerMsId;
+    }
+
+    public void setPeerRunId(String peerRunId) {
+        this.peerRunId = peerRunId;
+    }
+
+    public void setPeerState(String peerState) {
+        this.peerState = peerState;
+    }
+
+    public void setPeerServiceIp(String peerServiceIp) {
+        this.peerServiceIp = peerServiceIp;
+    }
+
+    public void setPeerServicePort(String peerServicePort) {
+        this.peerServicePort = peerServicePort;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/response/RegisterResponse.java

@@ -18,19 +18,24 @@ package org.apache.cloudstack.api.response;
 
 import com.google.gson.annotations.SerializedName;
 
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponse;
 
 import com.cloud.serializer.Param;
 
 public class RegisterResponse extends BaseResponse {
-    @SerializedName("apikey")
+    @SerializedName(ApiConstants.API_KEY)
     @Param(description = "the api key of the registered user", isSensitive = true)
     private String apiKey;
 
-    @SerializedName("secretkey")
+    @SerializedName(ApiConstants.SECRET_KEY)
     @Param(description = "the secret key of the registered user", isSensitive = true)
     private String secretKey;
 
+    @SerializedName(ApiConstants.API_KEY_ACCESS)
+    @Param(description = "whether api key access is allowed or not", isSensitive = true)
+    private Boolean apiKeyAccess;
+
     public String getApiKey() {
         return apiKey;
     }
@@ -46,4 +51,8 @@ public class RegisterResponse extends BaseResponse {
     public void setSecretKey(String secretKey) {
         this.secretKey = secretKey;
     }
+
+    public void setApiKeyAccess(Boolean apiKeyAccess) {
+        this.apiKeyAccess = apiKeyAccess;
+    }
 }

api/src/main/java/org/apache/cloudstack/api/response/UserResponse.java

@@ -128,6 +128,10 @@ public class UserResponse extends BaseResponse implements SetResourceIconRespons
     @Param(description = "true if user has two factor authentication is mandated", since = "4.18.0.0")
     private Boolean is2FAmandated;
 
+    @SerializedName(ApiConstants.API_KEY_ACCESS)
+    @Param(description = "whether api key access is Enabled, Disabled or set to Inherit (it inherits the value from the parent)", since = "4.20.1.0")
+    ApiConstants.ApiKeyAccess apiKeyAccess;
+
     @Override
     public String getObjectId() {
         return this.getId();
@@ -309,4 +313,8 @@ public class UserResponse extends BaseResponse implements SetResourceIconRespons
     public void set2FAmandated(Boolean is2FAmandated) {
         this.is2FAmandated = is2FAmandated;
     }
+
+    public void setApiKeyAccess(Boolean apiKeyAccess) {
+        this.apiKeyAccess = ApiConstants.ApiKeyAccess.fromBoolean(apiKeyAccess);
+    }
 }

api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java

@@ -157,6 +157,11 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso
     @Param(description = "AS Number Range")
     private String asnRange;
 
+    @SerializedName(ApiConstants.ROUTED_MODE_ENABLED)
+    @Param(description = "true, if routed network/vpc is enabled", since = "4.20.1")
+    private boolean routedModeEnabled = false;
+
+
     public ZoneResponse() {
         tags = new LinkedHashSet<ResourceTagResponse>();
     }
@@ -412,4 +417,12 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso
     public String getAsnRange() {
         return asnRange;
     }
+
+    public boolean isRoutedModeEnabled() {
+        return routedModeEnabled;
+    }
+
+    public void setRoutedModeEnabled(boolean routedModeEnabled) {
+        this.routedModeEnabled = routedModeEnabled;
+    }
 }

api/src/main/java/org/apache/cloudstack/network/RoutedIpv4Manager.java

@@ -57,6 +57,13 @@ import java.util.List;
 
 public interface RoutedIpv4Manager extends PluggableService, Configurable {
 
+    ConfigKey<Boolean> RoutedNetworkVpcEnabled = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK, Boolean.class,
+            "routed.network.vpc.enabled",
+            "true",
+            "If true, the Routed network and VPC are enabled in the zone.",
+            true,
+            ConfigKey.Scope.Zone);
+
     ConfigKey<Integer> RoutedNetworkIPv4MaxCidrSize = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK, Integer.class,
             "routed.network.ipv4.max.cidr.size", "30", "The maximum value of the cidr size for isolated networks in ROUTED mode",
             true, ConfigKey.Scope.Account);
@@ -196,4 +203,6 @@ public interface RoutedIpv4Manager extends PluggableService, Configurable {
     void removeBgpPeersByAccountId(long accountId);
 
     void removeBgpPeersByDomainId(long domainId);
+
+    Boolean isRoutedNetworkVpcEnabled(long zoneId);
 }

api/src/main/java/org/apache/cloudstack/query/QueryService.java

@@ -19,6 +19,7 @@ package org.apache.cloudstack.query;
 import java.util.List;
 
 import org.apache.cloudstack.affinity.AffinityGroupResponse;
+import org.apache.cloudstack.api.ResponseObject;
 import org.apache.cloudstack.api.command.admin.domain.ListDomainsCmd;
 import org.apache.cloudstack.api.command.admin.host.ListHostTagsCmd;
 import org.apache.cloudstack.api.command.admin.host.ListHostsCmd;
@@ -130,7 +131,7 @@ public interface QueryService {
     ConfigKey<Boolean> ReturnVmStatsOnVmList = new ConfigKey<>("Advanced", Boolean.class, "list.vm.default.details.stats", "true",
             "Determines whether VM stats should be returned when details are not explicitly specified in listVirtualMachines API request. When false, details default to [group, nics, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp]. When true, all details are returned including 'stats'.", true, ConfigKey.Scope.Global);
 
-    ListResponse<UserResponse> searchForUsers(ListUsersCmd cmd) throws PermissionDeniedException;
+    ListResponse<UserResponse> searchForUsers(ResponseObject.ResponseView responseView, ListUsersCmd cmd) throws PermissionDeniedException;
 
     ListResponse<UserResponse> searchForUsers(Long domainId, boolean recursive) throws PermissionDeniedException;
 

engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java

@@ -40,6 +40,7 @@ public interface CapacityManager {
     static final String StorageCapacityDisableThresholdCK = "pool.storage.capacity.disablethreshold";
     static final String StorageOverprovisioningFactorCK = "storage.overprovisioning.factor";
     static final String StorageAllocatedCapacityDisableThresholdCK = "pool.storage.allocated.capacity.disablethreshold";
+    static final String StorageAllocatedCapacityDisableThresholdForVolumeResizeCK = "pool.storage.allocated.resize.capacity.disablethreshold";
 
     static final ConfigKey<Float> CpuOverprovisioningFactor =
             new ConfigKey<>(
@@ -118,6 +119,17 @@ public interface CapacityManager {
                     "Percentage (as a value between 0 and 1) of secondary storage capacity threshold.",
                     true);
 
+    static final ConfigKey<Double> StorageAllocatedCapacityDisableThresholdForVolumeSize =
+            new ConfigKey<>(
+                    ConfigKey.CATEGORY_ALERT,
+                    Double.class,
+                    StorageAllocatedCapacityDisableThresholdForVolumeResizeCK,
+                    "0.90",
+                    "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for volume resize. " +
+                            "This is applicable only when volume.resize.allowed.beyond.allocation is set to true.",
+                    true,
+                    ConfigKey.Scope.Zone);
+
     public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId);
 
     void allocateVmCapacity(VirtualMachine vm, boolean fromLastHost);

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -209,6 +209,11 @@ public interface StorageManager extends StorageService {
     ConfigKey<Long> HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000",
             "The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true);
 
+    ConfigKey<Boolean> AllowVolumeReSizeBeyondAllocation = new ConfigKey<Boolean>("Advanced", Boolean.class, "volume.resize.allowed.beyond.allocation", "false",
+            "Determines whether volume size can exceed the pool capacity allocation disable threshold (pool.storage.allocated.capacity.disablethreshold) " +
+                    "when resize a volume upto resize capacity disable threshold (pool.storage.allocated.resize.capacity.disablethreshold)",
+            true, ConfigKey.Scope.Zone);
+
     /**
      * should we execute in sequence not involving any storages?
      * @return tru if commands should execute in sequence

engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java

@@ -31,7 +31,7 @@ public interface VMSnapshotManager extends VMSnapshotService, Manager {
     static final ConfigKey<Integer> VMSnapshotExpireInterval = new ConfigKey<Integer>("Advanced", Integer.class, "vmsnapshot.expire.interval", "-1",
             "VM Snapshot expire interval in hours", true, ConfigKey.Scope.Account);
 
-    public static final int VMSNAPSHOTMAX = 10;
+    ConfigKey<Integer> VMSnapshotMax = new ConfigKey<Integer>("Advanced", Integer.class, "vmsnapshot.max", "10", "Maximum vm snapshots for a single vm", true, ConfigKey.Scope.Global);
 
     /**
      * Delete all VM snapshots belonging to one VM

engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java

@@ -88,6 +88,7 @@ import com.cloud.upgrade.dao.Upgrade41800to41810;
 import com.cloud.upgrade.dao.Upgrade41810to41900;
 import com.cloud.upgrade.dao.Upgrade41900to41910;
 import com.cloud.upgrade.dao.Upgrade41910to42000;
+import com.cloud.upgrade.dao.Upgrade42000to42010;
 import com.cloud.upgrade.dao.Upgrade420to421;
 import com.cloud.upgrade.dao.Upgrade421to430;
 import com.cloud.upgrade.dao.Upgrade430to440;
@@ -230,6 +231,7 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
                 .next("4.18.1.0", new Upgrade41810to41900())
                 .next("4.19.0.0", new Upgrade41900to41910())
                 .next("4.19.1.0", new Upgrade41910to42000())
+                .next("4.20.0.0", new Upgrade42000to42010())
                 .build();
     }
 

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42000to42010.java

@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.upgrade.dao;
+
+import java.io.InputStream;
+import java.sql.Connection;
+
+import com.cloud.upgrade.SystemVmTemplateRegistration;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class Upgrade42000to42010 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate {
+    private SystemVmTemplateRegistration systemVmTemplateRegistration;
+
+    @Override
+    public String[] getUpgradableVersionRange() {
+        return new String[] {"4.20.0.0", "4.20.1.0"};
+    }
+
+    @Override
+    public String getUpgradedVersion() {
+        return "4.20.1.0";
+    }
+
+    @Override
+    public boolean supportsRollingUpgrade() {
+        return false;
+    }
+
+    @Override
+    public InputStream[] getPrepareScripts() {
+        final String scriptFile = "META-INF/db/schema-42000to42010.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[] {script};
+    }
+
+    @Override
+    public void performDataMigration(Connection conn) {
+    }
+
+    @Override
+    public InputStream[] getCleanupScripts() {
+        final String scriptFile = "META-INF/db/schema-42000to42010-cleanup.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[] {script};
+    }
+
+    private void initSystemVmTemplateRegistration() {
+        systemVmTemplateRegistration = new SystemVmTemplateRegistration("");
+    }
+
+    @Override
+    public void updateSystemVmTemplates(Connection conn) {
+        logger.debug("Updating System Vm template IDs");
+        initSystemVmTemplateRegistration();
+        try {
+            systemVmTemplateRegistration.updateSystemVmTemplates(conn);
+        } catch (Exception e) {
+            throw new CloudRuntimeException("Failed to find / register SystemVM template(s)");
+        }
+    }
+}

engine/schema/src/main/java/com/cloud/user/AccountVO.java

@@ -77,6 +77,9 @@ public class AccountVO implements Account {
     @Column(name = "default")
     boolean isDefault;
 
+    @Column(name = "api_key_access")
+    private Boolean apiKeyAccess;
+
     public AccountVO() {
         uuid = UUID.randomUUID().toString();
     }
@@ -229,4 +232,14 @@ public class AccountVO implements Account {
     public String reflectionToString() {
         return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "uuid", "accountName", "domainId");
     }
+
+    @Override
+    public void setApiKeyAccess(Boolean apiKeyAccess) {
+        this.apiKeyAccess = apiKeyAccess;
+    }
+
+    @Override
+    public Boolean getApiKeyAccess() {
+        return apiKeyAccess;
+    }
 }

engine/schema/src/main/java/com/cloud/user/UserVO.java

@@ -115,6 +115,9 @@ public class UserVO implements User, Identity, InternalIdentity {
     @Column(name = "key_for_2fa")
     private String keyFor2fa;
 
+    @Column(name = "api_key_access")
+    private Boolean apiKeyAccess;
+
     public UserVO() {
         this.uuid = UUID.randomUUID().toString();
     }
@@ -350,4 +353,13 @@ public class UserVO implements User, Identity, InternalIdentity {
         this.user2faProvider = user2faProvider;
     }
 
+    @Override
+    public void setApiKeyAccess(Boolean apiKeyAccess) {
+        this.apiKeyAccess = apiKeyAccess;
+    }
+
+    @Override
+    public Boolean getApiKeyAccess() {
+        return apiKeyAccess;
+    }
 }

engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java

@@ -41,8 +41,8 @@ import java.util.List;
 
 @Component
 public class AccountDaoImpl extends GenericDaoBase<AccountVO, Long> implements AccountDao {
-    private static final String FIND_USER_ACCOUNT_BY_API_KEY = "SELECT u.id, u.username, u.account_id, u.secret_key, u.state, "
-        + "a.id, a.account_name, a.type, a.role_id, a.domain_id, a.state " + "FROM `cloud`.`user` u, `cloud`.`account` a "
+    private static final String FIND_USER_ACCOUNT_BY_API_KEY = "SELECT u.id, u.username, u.account_id, u.secret_key, u.state, u.api_key_access, "
+        + "a.id, a.account_name, a.type, a.role_id, a.domain_id, a.state, a.api_key_access " + "FROM `cloud`.`user` u, `cloud`.`account` a "
         + "WHERE u.account_id = a.id AND u.api_key = ? and u.removed IS NULL";
 
     protected final SearchBuilder<AccountVO> AllFieldsSearch;
@@ -148,13 +148,25 @@ public class AccountDaoImpl extends GenericDaoBase<AccountVO, Long> implements A
                 u.setAccountId(rs.getLong(3));
                 u.setSecretKey(DBEncryptionUtil.decrypt(rs.getString(4)));
                 u.setState(State.getValueOf(rs.getString(5)));
+                boolean apiKeyAccess = rs.getBoolean(6);
+                if (rs.wasNull()) {
+                    u.setApiKeyAccess(null);
+                } else {
+                    u.setApiKeyAccess(apiKeyAccess);
+                }
 
-                AccountVO a = new AccountVO(rs.getLong(6));
-                a.setAccountName(rs.getString(7));
-                a.setType(Account.Type.getFromValue(rs.getInt(8)));
-                a.setRoleId(rs.getLong(9));
-                a.setDomainId(rs.getLong(10));
-                a.setState(State.getValueOf(rs.getString(11)));
+                AccountVO a = new AccountVO(rs.getLong(7));
+                a.setAccountName(rs.getString(8));
+                a.setType(Account.Type.getFromValue(rs.getInt(9)));
+                a.setRoleId(rs.getLong(10));
+                a.setDomainId(rs.getLong(11));
+                a.setState(State.getValueOf(rs.getString(12)));
+                apiKeyAccess = rs.getBoolean(13);
+                if (rs.wasNull()) {
+                    a.setApiKeyAccess(null);
+                } else {
+                    a.setApiKeyAccess(apiKeyAccess);
+                }
 
                 userAcctPair = new Pair<User, Account>(u, a);
             }

engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml

@@ -117,6 +117,7 @@
   <bean id="loadBalancerCertMapDaoImpl" class="com.cloud.network.dao.LoadBalancerCertMapDaoImpl" />
   <bean id="managementServerHostDaoImpl" class="com.cloud.cluster.dao.ManagementServerHostDaoImpl" />
   <bean id="managementServerHostPeerDaoImpl" class="com.cloud.cluster.dao.ManagementServerHostPeerDaoImpl" />
+  <bean id="managementServerHostPeerJoinDaoImpl" class="com.cloud.cluster.dao.ManagementServerHostPeerJoinDaoImpl" />
   <bean id="managementServerStatusDaoImpl" class="com.cloud.cluster.dao.ManagementServerStatusDaoImpl" />
   <bean id="networkAccountDaoImpl" class="com.cloud.network.dao.NetworkAccountDaoImpl" />
   <bean id="networkACLDaoImpl" class="com.cloud.network.vpc.dao.NetworkACLDaoImpl" />

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_foreign_key.sql

@@ -0,0 +1,28 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`;
+
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY` (
+	IN in_table_name VARCHAR(200)
+    , IN in_key_name VARCHAR(200)
+    , IN in_foreign_key VARCHAR(200)
+    , IN in_references VARCHAR(1000)
+)
+BEGIN
+
+    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT_WS(' ', 'ALTER TABLE ', in_table_name, ' ADD CONSTRAINT ', in_key_name, ' FOREIGN KEY ', in_foreign_key, ' REFERENCES ', in_references, ' ON DELETE CASCADE'); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;

engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql

@@ -425,3 +425,10 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervi
 
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_instance', 'delete_protection', 'boolean DEFAULT FALSE COMMENT "delete protection for vm" ');
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'delete_protection', 'boolean DEFAULT FALSE COMMENT "delete protection for volumes" ');
+
+-- Modify index for mshost_peer
+DELETE FROM `cloud`.`mshost_peer`;
+CALL `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`('cloud.mshost_peer','fk_mshost_peer__owner_mshost');
+CALL `cloud`.`IDEMPOTENT_DROP_INDEX`('i_mshost_peer__owner_peer_runid','mshost_peer');
+CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.mshost_peer', 'i_mshost_peer__owner_peer', '(owner_mshost, peer_mshost)');
+CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.mshost_peer', 'fk_mshost_peer__owner_mshost', '(owner_mshost)', '`mshost`(`id`)');

engine/schema/src/main/resources/META-INF/db/schema-42000to42010-cleanup.sql

@@ -0,0 +1,20 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade cleanup from 4.20.0.0 to 4.20.1.0
+--;

engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql

@@ -0,0 +1,24 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade from 4.20.0.0 to 4.20.1.0
+--;
+
+-- Add column api_key_access to user and account tables
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the user" AFTER `secret_key`');
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" ');

engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql

@@ -31,6 +31,7 @@ select
     `account`.`cleanup_needed` AS `cleanup_needed`,
     `account`.`network_domain` AS `network_domain` ,
     `account`.`default` AS `default`,
+    `account`.`api_key_access` AS `api_key_access`,
     `domain`.`id` AS `domain_id`,
     `domain`.`uuid` AS `domain_uuid`,
     `domain`.`name` AS `domain_name`,

engine/schema/src/main/resources/META-INF/db/views/cloud.mshost_peer_view.sql

@@ -0,0 +1,44 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+
+DROP VIEW IF EXISTS `cloud`.`mshost_peer_view`;
+
+CREATE VIEW `cloud`.`mshost_peer_view` AS
+SELECT
+    `mshost_peer`.`id` AS `id`,
+    `mshost_peer`.`peer_state` AS `peer_state`,
+    `mshost_peer`.`last_update` AS `last_update`,
+    `owner_mshost`.`id` AS `owner_mshost_id`,
+    `owner_mshost`.`msid` AS `owner_mshost_msid`,
+    `owner_mshost`.`runid` AS `owner_mshost_runid`,
+    `owner_mshost`.`name` AS `owner_mshost_name`,
+    `owner_mshost`.`uuid` AS `owner_mshost_uuid`,
+    `owner_mshost`.`state` AS `owner_mshost_state`,
+    `owner_mshost`.`service_ip` AS `owner_mshost_service_ip`,
+    `owner_mshost`.`service_port` AS `owner_mshost_service_port`,
+    `peer_mshost`.`id` AS `peer_mshost_id`,
+    `peer_mshost`.`msid` AS `peer_mshost_msid`,
+    `peer_mshost`.`runid` AS `peer_mshost_runid`,
+    `peer_mshost`.`name` AS `peer_mshost_name`,
+    `peer_mshost`.`uuid` AS `peer_mshost_uuid`,
+    `peer_mshost`.`state` AS `peer_mshost_state`,
+    `peer_mshost`.`service_ip` AS `peer_mshost_service_ip`,
+    `peer_mshost`.`service_port` AS `peer_mshost_service_port`
+FROM `cloud`.`mshost_peer`
+LEFT JOIN `cloud`.`mshost` AS owner_mshost on `mshost_peer`.`owner_mshost` = `owner_mshost`.`id`
+LEFT JOIN `cloud`.`mshost` AS peer_mshost on `mshost_peer`.`peer_mshost` = `peer_mshost`.`id`;

engine/schema/src/main/resources/META-INF/db/views/cloud.user_view.sql

@@ -39,6 +39,7 @@ select
     user.incorrect_login_attempts,
     user.source,
     user.default,
+    user.api_key_access,
     account.id account_id,
     account.uuid account_uuid,
     account.account_name account_name,

engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java

@@ -419,7 +419,7 @@ public class DefaultVMSnapshotStrategy extends ManagerBase implements VMSnapshot
                 if (answer != null && answer.getDetails() != null)
                     errMsg = errMsg + " due to " + answer.getDetails();
                 logger.error(errMsg);
-                throw new CloudRuntimeException(errMsg);
+                throw new CloudRuntimeException(String.format("Unable to revert VM %s to snapshot %s.", userVm.getInstanceName(), vmSnapshotVO.getName()));
             }
         } catch (OperationTimedoutException e) {
             logger.debug("Failed to revert vm snapshot", e);

framework/cluster/src/main/java/com/cloud/cluster/ClusterManager.java

@@ -27,9 +27,9 @@ import com.cloud.utils.component.Manager;
 public interface ClusterManager extends Manager {
     static final String ALERT_SUBJECT = "cluster-alert";
     final ConfigKey<Integer> HeartbeatInterval = new ConfigKey<Integer>(Integer.class, "cluster.heartbeat.interval", "management-server", "1500",
-        "Interval to check for the heart beat between management server nodes", false);
+        "Interval (in milliseconds) to check for the heart beat between management server nodes", false);
     final ConfigKey<Integer> HeartbeatThreshold = new ConfigKey<Integer>(Integer.class, "cluster.heartbeat.threshold", "management-server", "150000",
-        "Threshold before self-fence the management server", true);
+        "Threshold (in milliseconds) before self-fence the management server. The threshold should be larger than management.server.stats.interval", true);
 
     /**
      * Adds a new packet to the incoming queue.

framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java

@@ -758,21 +758,16 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C
         }
 
         switch (msg.getMessageType()) {
-        case nodeAdded: {
-            final List<ManagementServerHostVO> l = msg.getNodes();
-            if (l != null && l.size() > 0) {
-                for (final ManagementServerHostVO mshost : l) {
-                    _mshostPeerDao.updatePeerInfo(_mshostId, mshost.getId(), mshost.getRunid(), ManagementServerHost.State.Up);
-                }
-            }
-        }
+        case nodeAdded:
         break;
 
         case nodeRemoved: {
             final List<ManagementServerHostVO> l = msg.getNodes();
             if (l != null && l.size() > 0) {
                 for (final ManagementServerHostVO mshost : l) {
-                    _mshostPeerDao.updatePeerInfo(_mshostId, mshost.getId(), mshost.getRunid(), ManagementServerHost.State.Down);
+                    if (mshost.getId() != _mshostId) {
+                        _mshostPeerDao.updatePeerInfo(_mshostId, mshost.getId(), mshost.getRunid(), ManagementServerHost.State.Down);
+                    }
                 }
             }
         }
@@ -823,8 +818,9 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C
 
             final List<ManagementServerHostVO> downHostList = new ArrayList<ManagementServerHostVO>();
             for (final ManagementServerHostVO host : inactiveList) {
-                if (!pingManagementNode(host)) {
-                    logger.warn("Management node " + host.getId() + " is detected inactive by timestamp and also not pingable");
+                // Check if peer state is Up in the period
+                if (!_mshostPeerDao.isPeerUpState(_mshostId, host.getId(), new Date(cutTime.getTime() - HeartbeatThreshold.value()))) {
+                    logger.warn("Management node " + host.getId() + " is detected inactive by timestamp and did not send node status to this node");
                     downHostList.add(host);
                 }
             }
@@ -898,6 +894,44 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C
 
         final Profiler profilerInvalidatedNodeList = new Profiler();
         profilerInvalidatedNodeList.start();
+        processInvalidatedNodes(invalidatedNodeList);
+        profilerInvalidatedNodeList.stop();
+
+        final Profiler profilerRemovedList = new Profiler();
+        profilerRemovedList.start();
+        processRemovedNodes(cutTime, removedNodeList);
+        profilerRemovedList.stop();
+
+        final Profiler profilerNewList = new Profiler();
+        profilerNewList.start();
+        processNewNodes(cutTime, currentList);
+        profilerNewList.stop();
+
+        final Profiler profilerInactiveList = new Profiler();
+        profilerInactiveList.start();
+        processInactiveNodes(cutTime);
+        profilerInactiveList.stop();
+
+        profiler.stop();
+
+        logger.debug(String.format("Peer scan is finished. profiler: %s , profilerQueryActiveList: %s, " +
+                        ", profilerSyncClusterInfo: %s, profilerInvalidatedNodeList: %s, profilerRemovedList: %s," +
+                        ", profilerNewList: %s, profilerInactiveList: %s",
+                profiler, profilerQueryActiveList, profilerSyncClusterInfo, profilerInvalidatedNodeList, profilerRemovedList,
+                profilerNewList, profilerInactiveList));
+
+        if (profiler.getDurationInMillis() >= HeartbeatInterval.value()) {
+            if (logger.isDebugEnabled()) {
+                logger.debug(String.format("Peer scan takes too long to finish. profiler: %s , profilerQueryActiveList: %s, " +
+                        ", profilerSyncClusterInfo: %s, profilerInvalidatedNodeList: %s, profilerRemovedList: %s," +
+                        ", profilerNewList: %s, profilerInactiveList: %s",
+                        profiler, profilerQueryActiveList, profilerSyncClusterInfo, profilerInvalidatedNodeList, profilerRemovedList,
+                        profilerNewList, profilerInactiveList));
+            }
+        }
+    }
+
+    private void processInvalidatedNodes(List<ManagementServerHostVO> invalidatedNodeList) {
         // process invalidated node list
         if (invalidatedNodeList.size() > 0) {
             for (final ManagementServerHostVO mshost : invalidatedNodeList) {
@@ -911,16 +945,16 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C
 
             queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeRemoved, invalidatedNodeList));
         }
-        profilerInvalidatedNodeList.stop();
+    }
 
-        final Profiler profilerRemovedList = new Profiler();
-        profilerRemovedList.start();
+    private void processRemovedNodes(Date cutTime, List<ManagementServerHostVO> removedNodeList) {
         // process removed node list
         final Iterator<ManagementServerHostVO> it = removedNodeList.iterator();
         while (it.hasNext()) {
             final ManagementServerHostVO mshost = it.next();
-            if (!pingManagementNode(mshost)) {
-                logger.warn("Management node " + mshost.getId() + " is detected inactive by timestamp and also not pingable");
+            // Check if peer state is Up in the period
+            if (!_mshostPeerDao.isPeerUpState(_mshostId, mshost.getId(), new Date(cutTime.getTime() - HeartbeatThreshold.value()))) {
+                logger.warn("Management node " + mshost.getId() + " is detected inactive by timestamp and did not send node status to this node");
                 _activePeers.remove(mshost.getId());
                 try {
                     JmxUtil.unregisterMBean("ClusterManager", "Node " + mshost.getId());
@@ -928,7 +962,7 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C
                     logger.warn("Unable to deregiester cluster node from JMX monitoring due to exception " + e.toString());
                 }
             } else {
-                logger.info("Management node " + mshost.getId() + " is detected inactive by timestamp but is pingable");
+                logger.info("Management node " + mshost.getId() + " is detected inactive by timestamp but sent node status to this node");
                 it.remove();
             }
         }
@@ -936,8 +970,9 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C
         if (removedNodeList.size() > 0) {
             queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeRemoved, removedNodeList));
         }
-        profilerRemovedList.stop();
+    }
 
+    private void processNewNodes(Date cutTime, List<ManagementServerHostVO> currentList) {
         final List<ManagementServerHostVO> newNodeList = new ArrayList<ManagementServerHostVO>();
         for (final ManagementServerHostVO mshost : currentList) {
             if (!_activePeers.containsKey(mshost.getId())) {
@@ -959,18 +994,31 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C
         if (newNodeList.size() > 0) {
             queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeAdded, newNodeList));
         }
+    }
 
-        profiler.stop();
-
-        if (profiler.getDurationInMillis() >= HeartbeatInterval.value()) {
-            if (logger.isDebugEnabled()) {
-                logger.debug("Peer scan takes too long to finish. profiler: " + profiler.toString() + ", profilerQueryActiveList: " +
-                        profilerQueryActiveList.toString() + ", profilerSyncClusterInfo: " + profilerSyncClusterInfo.toString() + ", profilerInvalidatedNodeList: " +
-                        profilerInvalidatedNodeList.toString() + ", profilerRemovedList: " + profilerRemovedList.toString());
+    private void processInactiveNodes(Date cutTime) {
+        final List<ManagementServerHostVO> inactiveList = _mshostDao.getInactiveList(new Date(cutTime.getTime() - HeartbeatThreshold.value()));
+        if (inactiveList.size() > 0) {
+            if (logger.isInfoEnabled()) {
+                logger.info(String.format("Found %s inactive management server node based on timestamp", inactiveList.size()));
             }
+            for (final ManagementServerHostVO host : inactiveList) {
+                logger.info(String.format("management server node msid: %s, name: %s, service ip: %s, version: %s",
+                        host.getMsid(),  host.getName(), host.getServiceIP(), host.getVersion()));
+                // Check if any peer state is Up in the period
+                if (ManagementServerHost.State.Up.equals(host.getState()) &&
+                        !_mshostPeerDao.isPeerUpState(host.getId(), new Date(cutTime.getTime() - HeartbeatThreshold.value()))) {
+                    logger.warn("Management node " + host.getId() + " is detected inactive by timestamp and did not send node status to all other nodes");
+                    host.setState(ManagementServerHost.State.Down);
+                    _mshostDao.update(host.getId(), host);
+                }
+            }
+        } else {
+            logger.info("No inactive management server node found");
         }
     }
 
+
     private static ManagementServerHostVO getInListById(final Long id, final List<ManagementServerHostVO> l) {
         for (final ManagementServerHostVO mshost : l) {
             if (mshost.getId() == id) {

framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostPeerJoinVO.java

@@ -0,0 +1,177 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.cluster;
+
+import java.util.Date;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+
+import org.apache.cloudstack.management.ManagementServerHost;
+
+@Entity
+@Table(name = "mshost_peer_view")
+public class ManagementServerHostPeerJoinVO {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Column(name = "id")
+    private long id;
+
+    @Column(name = "peer_state")
+    @Enumerated(value = EnumType.STRING)
+    private ManagementServerHost.State peerState;
+
+    @Temporal(TemporalType.TIMESTAMP)
+    @Column(name = "last_update")
+    private Date lastUpdateTime;
+
+    @Column(name = "owner_mshost_id")
+    private long ownerMshostId;
+
+    @Column(name = "owner_mshost_msid")
+    private long ownerMshostMsId;
+
+    @Column(name = "owner_mshost_runid")
+    private long ownerMshostRunId;
+
+    @Column(name = "owner_mshost_name")
+    private String ownerMshostName;
+
+    @Column(name = "owner_mshost_uuid")
+    private String ownerMshostUuid;
+
+    @Column(name = "owner_mshost_state")
+    private String ownerMshostState;
+
+    @Column(name = "owner_mshost_service_ip")
+    private String ownerMshostServiceIp;
+
+    @Column(name = "owner_mshost_service_port")
+    private Integer ownerMshostServicePort;
+
+    @Column(name = "peer_mshost_id")
+    private long peerMshostId;
+
+    @Column(name = "peer_mshost_msid")
+    private long peerMshostMsId;
+
+    @Column(name = "peer_mshost_runid")
+    private long peerMshostRunId;
+
+    @Column(name = "peer_mshost_name")
+    private String peerMshostName;
+
+    @Column(name = "peer_mshost_uuid")
+    private String peerMshostUuid;
+
+    @Column(name = "peer_mshost_state")
+    private String peerMshostState;
+
+    @Column(name = "peer_mshost_service_ip")
+    private String peerMshostServiceIp;
+
+    @Column(name = "peer_mshost_service_port")
+    private Integer peerMshostServicePort;
+
+    public ManagementServerHostPeerJoinVO() {
+    }
+
+    public long getId() {
+        return id;
+    }
+
+    public ManagementServerHost.State getPeerState() {
+        return peerState;
+    }
+
+    public Date getLastUpdateTime() {
+        return lastUpdateTime;
+    }
+
+    public long getOwnerMshostId() {
+        return ownerMshostId;
+    }
+
+    public long getOwnerMshostMsId() {
+        return ownerMshostMsId;
+    }
+
+    public long getOwnerMshostRunId() {
+        return ownerMshostRunId;
+    }
+
+    public String getOwnerMshostName() {
+        return ownerMshostName;
+    }
+
+    public String getOwnerMshostUuid() {
+        return ownerMshostUuid;
+    }
+
+    public String getOwnerMshostState() {
+        return ownerMshostState;
+    }
+
+    public String getOwnerMshostServiceIp() {
+        return ownerMshostServiceIp;
+    }
+
+    public Integer getOwnerMshostServicePort() {
+        return ownerMshostServicePort;
+    }
+
+    public long getPeerMshostId() {
+        return peerMshostId;
+    }
+
+    public long getPeerMshostMsId() {
+        return peerMshostMsId;
+    }
+
+    public long getPeerMshostRunId() {
+        return peerMshostRunId;
+    }
+
+    public String getPeerMshostName() {
+        return peerMshostName;
+    }
+
+    public String getPeerMshostUuid() {
+        return peerMshostUuid;
+    }
+
+    public String getPeerMshostState() {
+        return peerMshostState;
+    }
+
+    public String getPeerMshostServiceIp() {
+        return peerMshostServiceIp;
+    }
+
+    public Integer getPeerMshostServicePort() {
+        return peerMshostServicePort;
+    }
+}

framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java

@@ -130,7 +130,7 @@ public class ManagementServerHostDaoImpl extends GenericDaoBase<ManagementServer
         try {
             txn.start();
 
-            pstmt = txn.prepareAutoCloseStatement("update mshost set last_update=?, removed=null, alert_count=0 where id=? and runid=?");
+            pstmt = txn.prepareAutoCloseStatement("update mshost set last_update=?, removed=null, alert_count=0, state='Up' where id=? and runid=?");
             pstmt.setString(1, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), lastUpdate));
             pstmt.setLong(2, id);
             pstmt.setLong(3, runid);

framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerDao.java

@@ -20,10 +20,17 @@ import org.apache.cloudstack.management.ManagementServerHost;
 import com.cloud.cluster.ManagementServerHostPeerVO;
 import com.cloud.utils.db.GenericDao;
 
+import java.util.Date;
+
 public interface ManagementServerHostPeerDao extends GenericDao<ManagementServerHostPeerVO, Long> {
     void clearPeerInfo(long ownerMshost);
 
     void updatePeerInfo(long ownerMshost, long peerMshost, long peerRunid, ManagementServerHost.State peerState);
 
-    int countStateSeenInPeers(long mshost, long runid, ManagementServerHost.State state);
+    int countStateSeenInPeers(long peerMshost, long runid, ManagementServerHost.State state);
+
+    boolean isPeerUpState(long peerMshost, Date cutTime);
+
+    boolean isPeerUpState(long ownerMshost, long peerMshost, Date cutTime);
+
 }

framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerDaoImpl.java

@@ -16,10 +16,10 @@
 // under the License.
 package com.cloud.cluster.dao;
 
+import java.util.Date;
 import java.util.List;
 
 
-
 import org.apache.cloudstack.management.ManagementServerHost;
 import com.cloud.cluster.ManagementServerHostPeerVO;
 import com.cloud.utils.db.DB;
@@ -33,10 +33,12 @@ public class ManagementServerHostPeerDaoImpl extends GenericDaoBase<ManagementSe
     private final SearchBuilder<ManagementServerHostPeerVO> ClearPeerSearch;
     private final SearchBuilder<ManagementServerHostPeerVO> FindForUpdateSearch;
     private final SearchBuilder<ManagementServerHostPeerVO> CountSearch;
+    private final SearchBuilder<ManagementServerHostPeerVO> ActiveSearch;
 
     public ManagementServerHostPeerDaoImpl() {
         ClearPeerSearch = createSearchBuilder();
         ClearPeerSearch.and("ownerMshost", ClearPeerSearch.entity().getOwnerMshost(), SearchCriteria.Op.EQ);
+        ClearPeerSearch.or("peerMshost", ClearPeerSearch.entity().getPeerMshost(), SearchCriteria.Op.EQ);
         ClearPeerSearch.done();
 
         FindForUpdateSearch = createSearchBuilder();
@@ -50,6 +52,13 @@ public class ManagementServerHostPeerDaoImpl extends GenericDaoBase<ManagementSe
         CountSearch.and("peerRunid", CountSearch.entity().getPeerRunid(), SearchCriteria.Op.EQ);
         CountSearch.and("peerState", CountSearch.entity().getPeerState(), SearchCriteria.Op.EQ);
         CountSearch.done();
+
+        ActiveSearch = createSearchBuilder();
+        ActiveSearch.and("ownerMshost", ActiveSearch.entity().getOwnerMshost(), SearchCriteria.Op.EQ);
+        ActiveSearch.and("peerMshost", ActiveSearch.entity().getPeerMshost(), SearchCriteria.Op.EQ);
+        ActiveSearch.and("peerState", ActiveSearch.entity().getPeerState(), SearchCriteria.Op.EQ);
+        ActiveSearch.and("lastUpdateTime", ActiveSearch.entity().getLastUpdateTime(), SearchCriteria.Op.GT);
+        ActiveSearch.done();
     }
 
     @Override
@@ -57,6 +66,7 @@ public class ManagementServerHostPeerDaoImpl extends GenericDaoBase<ManagementSe
     public void clearPeerInfo(long ownerMshost) {
         SearchCriteria<ManagementServerHostPeerVO> sc = ClearPeerSearch.create();
         sc.setParameters("ownerMshost", ownerMshost);
+        sc.setParameters("peerMshost", ownerMshost);
 
         expunge(sc);
     }
@@ -71,11 +81,12 @@ public class ManagementServerHostPeerDaoImpl extends GenericDaoBase<ManagementSe
             SearchCriteria<ManagementServerHostPeerVO> sc = FindForUpdateSearch.create();
             sc.setParameters("ownerMshost", ownerMshost);
             sc.setParameters("peerMshost", peerMshost);
-            sc.setParameters("peerRunid", peerRunid);
             List<ManagementServerHostPeerVO> l = listBy(sc);
             if (l.size() == 1) {
                 ManagementServerHostPeerVO peer = l.get(0);
+                peer.setPeerRunid(peerRunid);
                 peer.setPeerState(peerState);
+                peer.setLastUpdateTime(new Date());
                 update(peer.getId(), peer);
             } else {
                 ManagementServerHostPeerVO peer = new ManagementServerHostPeerVO(ownerMshost, peerMshost, peerRunid, peerState);
@@ -90,13 +101,36 @@ public class ManagementServerHostPeerDaoImpl extends GenericDaoBase<ManagementSe
 
     @Override
     @DB
-    public int countStateSeenInPeers(long mshost, long runid, ManagementServerHost.State state) {
+    public int countStateSeenInPeers(long peerMshost, long runid, ManagementServerHost.State state) {
         SearchCriteria<ManagementServerHostPeerVO> sc = CountSearch.create();
-        sc.setParameters("peerMshost", mshost);
+        sc.setParameters("peerMshost", peerMshost);
         sc.setParameters("peerRunid", runid);
         sc.setParameters("peerState", state);
 
         List<ManagementServerHostPeerVO> l = listBy(sc);
         return l.size();
     }
+
+    @Override
+    @DB
+    public boolean isPeerUpState(long peerMshost, Date cutTime) {
+        SearchCriteria<ManagementServerHostPeerVO> sc = ActiveSearch.create();
+        sc.setParameters("peerMshost", peerMshost);
+        sc.setParameters("peerState", ManagementServerHost.State.Up);
+        sc.setParameters("lastUpdateTime", cutTime);
+
+        return listBy(sc).size() > 0;
+    }
+
+    @Override
+    @DB
+    public boolean isPeerUpState(long ownerMshost, long peerMshost, Date cutTime) {
+        SearchCriteria<ManagementServerHostPeerVO> sc = ActiveSearch.create();
+        sc.setParameters("ownerMshost", ownerMshost);
+        sc.setParameters("peerMshost", peerMshost);
+        sc.setParameters("peerState", ManagementServerHost.State.Up);
+        sc.setParameters("lastUpdateTime", cutTime);
+
+        return listBy(sc).size() > 0;
+    }
 }

framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerJoinDao.java

@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.cluster.dao;
+
+import com.cloud.cluster.ManagementServerHostPeerJoinVO;
+import com.cloud.utils.db.GenericDao;
+
+import java.util.List;
+
+public interface ManagementServerHostPeerJoinDao extends GenericDao<ManagementServerHostPeerJoinVO, Long> {
+
+    List<ManagementServerHostPeerJoinVO> listByOwnerMshostId(long ownerMshostId);
+}

framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerJoinDaoImpl.java

@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.cluster.dao;
+
+import java.util.List;
+
+import com.cloud.cluster.ManagementServerHostPeerJoinVO;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+
+public class ManagementServerHostPeerJoinDaoImpl extends GenericDaoBase<ManagementServerHostPeerJoinVO, Long> implements ManagementServerHostPeerJoinDao {
+
+    private final SearchBuilder<ManagementServerHostPeerJoinVO> AllFieldSearch;
+
+    public ManagementServerHostPeerJoinDaoImpl() {
+        AllFieldSearch = createSearchBuilder();
+        AllFieldSearch.and("ownerMshostId", AllFieldSearch.entity().getOwnerMshostId(), SearchCriteria.Op.EQ);
+        AllFieldSearch.done();
+    }
+
+    @Override
+    public List<ManagementServerHostPeerJoinVO> listByOwnerMshostId(long ownerMshostId) {
+        SearchCriteria<ManagementServerHostPeerJoinVO> sc = AllFieldSearch.create();
+        sc.setParameters("ownerMshostId", ownerMshostId);
+        return listBy(sc);
+    }
+}

framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java

@@ -34,6 +34,7 @@ public class ConfigKey<T> {
     public static final String CATEGORY_ADVANCED = "Advanced";
     public static final String CATEGORY_ALERT = "Alert";
     public static final String CATEGORY_NETWORK = "Network";
+    public static final String CATEGORY_SYSTEM = "System";
 
     public enum Scope {
         Global, Zone, Cluster, StoragePool, Account, ManagementServer, ImageStore, Domain

framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java

@@ -157,7 +157,7 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager {
                 .map(quotaUsageVO -> new Pair<>(quotaUsageVO.getStartDate(), quotaUsageVO.getEndDate()))
                 .collect(Collectors.toCollection(LinkedHashSet::new));
 
-        logger.info(String.format("Processing quota balance for account[{}] between [{}] and [{}].", accountToString, startDate, lastQuotaUsageEndDate));
+        logger.info("Processing quota balance for account [{}] between [{}] and [{}].", accountToString, startDate, lastQuotaUsageEndDate);
 
         long accountId = accountVo.getAccountId();
         long domainId = accountVo.getDomainId();

plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java

@@ -486,12 +486,12 @@ public class MockAccountManager extends ManagerBase implements AccountManager {
     }
 
     @Override
-    public Map<String, String> getKeys(GetUserKeysCmd cmd){
+    public Pair<Boolean, Map<String, String>> getKeys(GetUserKeysCmd cmd){
         return null;
     }
 
     @Override
-    public Map<String, String> getKeys(Long userId) {
+    public Pair<Boolean, Map<String, String>> getKeys(Long userId) {
         return null;
     }
 

pom.xml

@@ -50,7 +50,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <project.systemvm.template.location>https://download.cloudstack.org/systemvm</project.systemvm.template.location>
-        <project.systemvm.template.version>4.21.0.0-SNAPSHOT</project.systemvm.template.version>
+        <project.systemvm.template.version>4.20.0.0</project.systemvm.template.version>
         <sonar.organization>apache</sonar.organization>
         <sonar.host.url>https://sonarcloud.io</sonar.host.url>
 

server/src/main/java/com/cloud/api/ApiDBUtils.java

@@ -1945,11 +1945,11 @@ public class ApiDBUtils {
     }
 
     public static UserResponse newUserResponse(UserAccountJoinVO usr) {
-        return newUserResponse(usr, null);
+        return newUserResponse(ResponseView.Restricted, null, usr);
     }
 
-    public static UserResponse newUserResponse(UserAccountJoinVO usr, Long domainId) {
-        UserResponse response = s_userAccountJoinDao.newUserResponse(usr);
+    public static UserResponse newUserResponse(ResponseView view, Long domainId, UserAccountJoinVO usr) {
+        UserResponse response = s_userAccountJoinDao.newUserResponse(view, usr);
         if(!AccountManager.UseSecretKeyInResponse.value()){
             response.setSecretKey(null);
         }

server/src/main/java/com/cloud/api/ApiServer.java

@@ -188,6 +188,7 @@ import com.cloud.utils.exception.ExceptionProxyObject;
 import com.cloud.utils.net.NetUtils;
 import com.google.gson.reflect.TypeToken;
 
+import static com.cloud.user.AccountManagerImpl.apiKeyAccess;
 import static org.apache.cloudstack.user.UserPasswordResetManager.UserPasswordResetEnabled;
 
 @Component
@@ -896,6 +897,34 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
         }
     }
 
+    protected boolean verifyApiKeyAccessAllowed(User user, Account account) {
+        Boolean apiKeyAccessEnabled = user.getApiKeyAccess();
+        if (apiKeyAccessEnabled != null) {
+            if (Boolean.TRUE.equals(apiKeyAccessEnabled)) {
+                return true;
+            } else {
+                logger.info("Api-Key access is disabled for the User " + user.toString());
+                return false;
+            }
+        }
+        apiKeyAccessEnabled = account.getApiKeyAccess();
+        if (apiKeyAccessEnabled != null) {
+            if (Boolean.TRUE.equals(apiKeyAccessEnabled)) {
+                return true;
+            } else {
+                logger.info("Api-Key access is disabled for the Account " + account.toString());
+                return false;
+            }
+        }
+        apiKeyAccessEnabled = apiKeyAccess.valueIn(account.getDomainId());
+        if (Boolean.TRUE.equals(apiKeyAccessEnabled)) {
+                return true;
+        } else {
+            logger.info("Api-Key access is disabled by the Domain level setting api.key.access");
+        }
+        return false;
+    }
+
     @Override
     public boolean verifyRequest(final Map<String, Object[]> requestParameters, final Long userId, InetAddress remoteAddress) throws ServerApiException {
         try {
@@ -1012,6 +1041,10 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
                 return false;
             }
 
+            if (!verifyApiKeyAccessAllowed(user, account)) {
+                return false;
+            }
+
             if (!commandAvailable(remoteAddress, commandName, user)) {
                 return false;
             }

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -36,7 +36,6 @@ import java.util.stream.Stream;
 
 import javax.inject.Inject;
 
-import com.cloud.cpu.CPU;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.acl.ControlledEntity.ACLType;
 import org.apache.cloudstack.acl.SecurityChecker;
@@ -114,6 +113,7 @@ import org.apache.cloudstack.api.response.IpQuarantineResponse;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.ManagementServerResponse;
 import org.apache.cloudstack.api.response.ObjectStoreResponse;
+import org.apache.cloudstack.api.response.PeerManagementServerNodeResponse;
 import org.apache.cloudstack.api.response.ProjectAccountResponse;
 import org.apache.cloudstack.api.response.ProjectInvitationResponse;
 import org.apache.cloudstack.api.response.ProjectResponse;
@@ -214,8 +214,11 @@ import com.cloud.api.query.vo.TemplateJoinVO;
 import com.cloud.api.query.vo.UserAccountJoinVO;
 import com.cloud.api.query.vo.UserVmJoinVO;
 import com.cloud.api.query.vo.VolumeJoinVO;
+import com.cloud.cluster.ManagementServerHostPeerJoinVO;
 import com.cloud.cluster.ManagementServerHostVO;
 import com.cloud.cluster.dao.ManagementServerHostDao;
+import com.cloud.cluster.dao.ManagementServerHostPeerJoinDao;
+import com.cloud.cpu.CPU;
 import com.cloud.dc.ClusterVO;
 import com.cloud.dc.DataCenter;
 import com.cloud.dc.DedicatedResourceVO;
@@ -607,6 +610,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
     @Inject
     private ClusterDao clusterDao;
 
+    @Inject
+    private ManagementServerHostPeerJoinDao mshostPeerJoinDao;
+
 
     private SearchCriteria<ServiceOfferingJoinVO> getMinimumCpuServiceOfferingJoinSearchCriteria(int cpu) {
         SearchCriteria<ServiceOfferingJoinVO> sc = _srvOfferingJoinDao.createSearchCriteria();
@@ -655,10 +661,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
      * .api.command.admin.user.ListUsersCmd)
      */
     @Override
-    public ListResponse<UserResponse> searchForUsers(ListUsersCmd cmd) throws PermissionDeniedException {
+    public ListResponse<UserResponse> searchForUsers(ResponseView responseView, ListUsersCmd cmd) throws PermissionDeniedException {
         Pair<List<UserAccountJoinVO>, Integer> result = searchForUsersInternal(cmd);
         ListResponse<UserResponse> response = new ListResponse<UserResponse>();
-        List<UserResponse> userResponses = ViewResponseHelper.createUserResponse(CallContext.current().getCallingAccount().getDomainId(),
+        if (CallContext.current().getCallingAccount().getType() == Account.Type.ADMIN) {
+            responseView = ResponseView.Full;
+        }
+        List<UserResponse> userResponses = ViewResponseHelper.createUserResponse(responseView, CallContext.current().getCallingAccount().getDomainId(),
                 result.first().toArray(new UserAccountJoinVO[result.first().size()]));
         response.setResponses(userResponses, result.second());
         return response;
@@ -685,10 +694,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         Object state = null;
         String keyword = null;
 
-        Pair<List<UserAccountJoinVO>, Integer> result =  getUserListInternal(caller, permittedAccounts, listAll, id, username, type, accountName, state, keyword, domainId, recursive,
-                null);
+        Pair<List<UserAccountJoinVO>, Integer> result =  getUserListInternal(caller, permittedAccounts, listAll, id,
+                username, type, accountName, state, keyword, null, domainId, recursive, null);
         ListResponse<UserResponse> response = new ListResponse<UserResponse>();
-        List<UserResponse> userResponses = ViewResponseHelper.createUserResponse(CallContext.current().getCallingAccount().getDomainId(),
+        List<UserResponse> userResponses = ViewResponseHelper.createUserResponse(ResponseView.Restricted, CallContext.current().getCallingAccount().getDomainId(),
                 result.first().toArray(new UserAccountJoinVO[result.first().size()]));
         response.setResponses(userResponses, result.second());
         return response;
@@ -713,6 +722,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         String accountName = cmd.getAccountName();
         Object state = cmd.getState();
         String keyword = cmd.getKeyword();
+        String apiKeyAccess = cmd.getApiKeyAccess();
 
         Long domainId = cmd.getDomainId();
         boolean recursive = cmd.isRecursive();
@@ -721,11 +731,11 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
 
         Filter searchFilter = new Filter(UserAccountJoinVO.class, "id", true, startIndex, pageSizeVal);
 
-        return getUserListInternal(caller, permittedAccounts, listAll, id, username, type, accountName, state, keyword, domainId, recursive, searchFilter);
+        return getUserListInternal(caller, permittedAccounts, listAll, id, username, type, accountName, state, keyword, apiKeyAccess, domainId, recursive, searchFilter);
     }
 
     private Pair<List<UserAccountJoinVO>, Integer> getUserListInternal(Account caller, List<Long> permittedAccounts, boolean listAll, Long id, Object username, Object type,
-            String accountName, Object state, String keyword, Long domainId, boolean recursive, Filter searchFilter) {
+            String accountName, Object state, String keyword, String apiKeyAccess, Long domainId, boolean recursive, Filter searchFilter) {
         Ternary<Long, Boolean, ListProjectResourcesCriteria> domainIdRecursiveListProject = new Ternary<Long, Boolean, ListProjectResourcesCriteria>(domainId, recursive, null);
         accountMgr.buildACLSearchParameters(caller, id, accountName, null, permittedAccounts, domainIdRecursiveListProject, listAll, false);
         domainId = domainIdRecursiveListProject.first();
@@ -751,6 +761,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         sb.and("domainId", sb.entity().getDomainId(), Op.EQ);
         sb.and("accountName", sb.entity().getAccountName(), Op.EQ);
         sb.and("state", sb.entity().getState(), Op.EQ);
+        if (apiKeyAccess != null) {
+            sb.and("apiKeyAccess", sb.entity().getApiKeyAccess(), Op.EQ);
+        }
 
         if ((accountName == null) && (domainId != null)) {
             sb.and("domainPath", sb.entity().getDomainPath(), Op.LIKE);
@@ -805,6 +818,15 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
             sc.setParameters("state", state);
         }
 
+        if (apiKeyAccess != null) {
+            try {
+                ApiConstants.ApiKeyAccess access = ApiConstants.ApiKeyAccess.valueOf(apiKeyAccess.toUpperCase());
+                sc.setParameters("apiKeyAccess", access.toBoolean());
+            } catch (IllegalArgumentException ex) {
+                throw new InvalidParameterValueException("ApiKeyAccess value can only be Enabled/Disabled/Inherit");
+            }
+        }
+
         return _userAccountJoinDao.searchAndCount(sc, searchFilter);
     }
 
@@ -2891,6 +2913,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         Object state = cmd.getState();
         Object isCleanupRequired = cmd.isCleanupRequired();
         Object keyword = cmd.getKeyword();
+        String apiKeyAccess = cmd.getApiKeyAccess();
 
         SearchBuilder<AccountVO> accountSearchBuilder = _accountDao.createSearchBuilder();
         accountSearchBuilder.select(null, Func.DISTINCT, accountSearchBuilder.entity().getId()); // select distinct
@@ -2903,6 +2926,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         accountSearchBuilder.and("typeNEQ", accountSearchBuilder.entity().getType(), SearchCriteria.Op.NEQ);
         accountSearchBuilder.and("idNEQ", accountSearchBuilder.entity().getId(), SearchCriteria.Op.NEQ);
         accountSearchBuilder.and("type2NEQ", accountSearchBuilder.entity().getType(), SearchCriteria.Op.NEQ);
+        if (apiKeyAccess != null) {
+            accountSearchBuilder.and("apiKeyAccess", accountSearchBuilder.entity().getApiKeyAccess(), Op.EQ);
+        }
 
         if (domainId != null && isRecursive) {
             SearchBuilder<DomainVO> domainSearch = _domainDao.createSearchBuilder();
@@ -2966,6 +2992,15 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
             }
         }
 
+        if (apiKeyAccess != null) {
+            try {
+                ApiConstants.ApiKeyAccess access = ApiConstants.ApiKeyAccess.valueOf(apiKeyAccess.toUpperCase());
+                sc.setParameters("apiKeyAccess", access.toBoolean());
+            } catch (IllegalArgumentException ex) {
+                throw new InvalidParameterValueException("ApiKeyAccess value can only be Enabled/Disabled/Inherit");
+            }
+        }
+
         Pair<List<AccountVO>, Integer> uniqueAccountPair = _accountDao.searchAndCount(sc, searchFilter);
         Integer count = uniqueAccountPair.second();
         List<Long> accountIds = uniqueAccountPair.first().stream().map(AccountVO::getId).collect(Collectors.toList());
@@ -5342,7 +5377,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         List<ManagementServerResponse> hostResponses = new ArrayList<>();
 
         for (ManagementServerJoinVO host : result.first()) {
-            ManagementServerResponse hostResponse = createManagementServerResponse(host);
+            ManagementServerResponse hostResponse = createManagementServerResponse(host, cmd.getPeers());
             hostResponses.add(hostResponse);
         }
 
@@ -5365,7 +5400,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         return managementServerJoinDao.searchAndCount(sc, null);
     }
 
-    protected ManagementServerResponse createManagementServerResponse(ManagementServerJoinVO mgmt) {
+    protected ManagementServerResponse createManagementServerResponse(ManagementServerJoinVO mgmt, boolean listPeers) {
         ManagementServerResponse mgmtResponse = new ManagementServerResponse();
         mgmtResponse.setId(mgmt.getUuid());
         mgmtResponse.setName(mgmt.getName());
@@ -5378,10 +5413,34 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         mgmtResponse.setLastServerStop(mgmt.getLastJvmStop());
         mgmtResponse.setLastBoot(mgmt.getLastSystemBoot());
         mgmtResponse.setServiceIp(mgmt.getServiceIP());
+        if (listPeers) {
+            List<ManagementServerHostPeerJoinVO> peers = mshostPeerJoinDao.listByOwnerMshostId(mgmt.getId());
+            for (ManagementServerHostPeerJoinVO peer: peers) {
+                mgmtResponse.addPeer(createPeerManagementServerNodeResponse(peer));
+            }
+        }
         mgmtResponse.setObjectName("managementserver");
         return mgmtResponse;
     }
 
+    private PeerManagementServerNodeResponse createPeerManagementServerNodeResponse(ManagementServerHostPeerJoinVO peer) {
+        PeerManagementServerNodeResponse response = new PeerManagementServerNodeResponse();
+
+        response.setState(peer.getPeerState());
+        response.setLastUpdated(peer.getLastUpdateTime());
+
+        response.setPeerId(peer.getPeerMshostUuid());
+        response.setPeerName(peer.getPeerMshostName());
+        response.setPeerMsId(String.valueOf(peer.getPeerMshostMsId()));
+        response.setPeerRunId(String.valueOf(peer.getPeerMshostRunId()));
+        response.setPeerState(peer.getPeerMshostState());
+        response.setPeerServiceIp(peer.getPeerMshostServiceIp());
+        response.setPeerServicePort(String.valueOf(peer.getPeerMshostServicePort()));
+
+        response.setObjectName("peermanagementserver");
+        return response;
+    }
+
     @Override
     public List<RouterHealthCheckResultResponse> listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd) {
         logger.info("Executing health check command " + cmd);

server/src/main/java/com/cloud/api/query/ViewResponseHelper.java

@@ -105,13 +105,13 @@ public class ViewResponseHelper {
     protected Logger logger = LogManager.getLogger(getClass());
 
     public static List<UserResponse> createUserResponse(UserAccountJoinVO... users) {
-        return createUserResponse(null, users);
+        return createUserResponse(ResponseView.Restricted, null, users);
     }
 
-    public static List<UserResponse> createUserResponse(Long domainId, UserAccountJoinVO... users) {
+    public static List<UserResponse> createUserResponse(ResponseView responseView, Long domainId, UserAccountJoinVO... users) {
         List<UserResponse> respList = new ArrayList<UserResponse>();
         for (UserAccountJoinVO vt : users) {
-            respList.add(ApiDBUtils.newUserResponse(vt, domainId));
+            respList.add(ApiDBUtils.newUserResponse(responseView, domainId, vt));
         }
         return respList;
     }

server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java

@@ -82,6 +82,9 @@ public class AccountJoinDaoImpl extends GenericDaoBase<AccountJoinVO, Long> impl
         accountResponse.setNetworkDomain(account.getNetworkDomain());
         accountResponse.setDefaultZone(account.getDataCenterUuid());
         accountResponse.setIsDefault(account.isDefault());
+        if (view == ResponseView.Full) {
+            accountResponse.setApiKeyAccess(account.getApiKeyAccess());
+        }
 
         // get network stat
         accountResponse.setBytesReceived(account.getBytesReceived());

server/src/main/java/com/cloud/api/query/dao/DataCenterJoinDaoImpl.java

@@ -34,6 +34,7 @@ import org.apache.cloudstack.api.response.ResourceIconResponse;
 import org.apache.cloudstack.api.response.ResourceTagResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.network.RoutedIpv4Manager;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.springframework.stereotype.Component;
@@ -141,6 +142,8 @@ public class DataCenterJoinDaoImpl extends GenericDaoBase<DataCenterJoinVO, Long
         String asRange = asNumberRange.stream().map(range -> range.getStartASNumber() + "-" + range.getEndASNumber()).collect(Collectors.joining(", "));
         zoneResponse.setAsnRange(asRange);
 
+        zoneResponse.setRoutedModeEnabled(RoutedIpv4Manager.RoutedNetworkVpcEnabled.valueIn(dataCenter.getId()));
+
         zoneResponse.setResourceDetails(ApiDBUtils.getResourceDetails(dataCenter.getId(), ResourceObjectType.Zone));
         zoneResponse.setHasAnnotation(annotationDao.hasAnnotations(dataCenter.getUuid(), AnnotationService.EntityType.ZONE.name(),
                 _accountMgr.isRootAdmin(CallContext.current().getCallingAccount().getId())));

server/src/main/java/com/cloud/api/query/dao/UserAccountJoinDao.java

@@ -18,6 +18,7 @@ package com.cloud.api.query.dao;
 
 import java.util.List;
 
+import org.apache.cloudstack.api.ResponseObject;
 import org.apache.cloudstack.api.response.UserResponse;
 
 import com.cloud.api.query.vo.UserAccountJoinVO;
@@ -27,7 +28,7 @@ import com.cloud.utils.db.GenericDao;
 
 public interface UserAccountJoinDao extends GenericDao<UserAccountJoinVO, Long> {
 
-    UserResponse newUserResponse(UserAccountJoinVO usr);
+    UserResponse newUserResponse(ResponseObject.ResponseView responseView, UserAccountJoinVO usr);
 
     UserAccountJoinVO newUserView(User usr);
 

server/src/main/java/com/cloud/api/query/dao/UserAccountJoinDaoImpl.java

@@ -20,6 +20,7 @@ import java.util.List;
 
 
 import com.cloud.user.AccountManagerImpl;
+import org.apache.cloudstack.api.ResponseObject.ResponseView;
 import org.springframework.stereotype.Component;
 
 import org.apache.cloudstack.api.response.UserResponse;
@@ -52,7 +53,7 @@ public class UserAccountJoinDaoImpl extends GenericDaoBase<UserAccountJoinVO, Lo
     }
 
     @Override
-    public UserResponse newUserResponse(UserAccountJoinVO usr) {
+    public UserResponse newUserResponse(ResponseView view, UserAccountJoinVO usr) {
         UserResponse userResponse = new UserResponse();
         userResponse.setAccountId(usr.getAccountUuid());
         userResponse.setAccountName(usr.getAccountName());
@@ -75,6 +76,9 @@ public class UserAccountJoinDaoImpl extends GenericDaoBase<UserAccountJoinVO, Lo
         long domainId = usr.getDomainId();
         boolean is2FAmandated = Boolean.TRUE.equals(AccountManagerImpl.enableUserTwoFactorAuthentication.valueIn(domainId)) && Boolean.TRUE.equals(AccountManagerImpl.mandateUserTwoFactorAuthentication.valueIn(domainId));
         userResponse.set2FAmandated(is2FAmandated);
+        if (view == ResponseView.Full) {
+            userResponse.setApiKeyAccess(usr.getApiKeyAccess());
+        }
 
         // set async job
         if (usr.getJobId() != null) {

server/src/main/java/com/cloud/api/query/vo/AccountJoinVO.java

@@ -189,6 +189,9 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident
     @Column(name = "default")
     boolean isDefault;
 
+    @Column(name = "api_key_access")
+    Boolean apiKeyAccess;
+
     public AccountJoinVO() {
     }
 
@@ -393,4 +396,8 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident
     public boolean isDefault() {
         return isDefault;
     }
+
+    public Boolean getApiKeyAccess() {
+        return apiKeyAccess;
+    }
 }

server/src/main/java/com/cloud/api/query/vo/UserAccountJoinVO.java

@@ -133,6 +133,9 @@ public class UserAccountJoinVO extends BaseViewVO implements InternalIdentity, I
     @Column(name = "is_user_2fa_enabled")
     boolean user2faEnabled;
 
+    @Column(name = "api_key_access")
+    Boolean apiKeyAccess;
+
     public UserAccountJoinVO() {
     }
 
@@ -281,4 +284,8 @@ public class UserAccountJoinVO extends BaseViewVO implements InternalIdentity, I
     public boolean isUser2faEnabled() {
         return user2faEnabled;
     }
+
+    public Boolean getApiKeyAccess() {
+        return apiKeyAccess;
+    }
 }

server/src/main/java/com/cloud/bgp/BGPServiceImpl.java

@@ -24,6 +24,7 @@ import com.cloud.dc.dao.ASNumberRangeDao;
 import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.domain.Domain;
 import com.cloud.domain.dao.DomainDao;
+import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.network.Network;
 import com.cloud.network.NetworkModel;
@@ -54,6 +55,7 @@ import com.cloud.utils.db.TransactionStatus;
 import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.cloudstack.api.command.user.bgp.ListASNumbersCmd;
 import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.network.BgpPeer;
 import org.apache.cloudstack.network.BgpPeerVO;
 import org.apache.cloudstack.network.RoutedIpv4Manager;
 import org.apache.cloudstack.network.dao.BgpPeerDao;
@@ -116,6 +118,9 @@ public class BGPServiceImpl implements BGPService {
             LOGGER.error(msg);
             throw new InvalidParameterException(msg);
         }
+        if (!routedIpv4Manager.isRoutedNetworkVpcEnabled(zoneId)) {
+            throw new InvalidParameterValueException("Cannot create ASN range as Routed networks and VPCs are not enabled for the zone.");
+        }
         if (startASNumber > endASNumber) {
             String msg = "Please specify a valid AS Number range";
             LOGGER.error(msg);
@@ -391,19 +396,7 @@ public class BGPServiceImpl implements BGPService {
         if (gatewayProviderStr != null) {
             NetworkElement provider = networkModel.getElementImplementingProvider(gatewayProviderStr);
             if (provider != null && provider instanceof BgpServiceProvider) {
-                List<BgpPeerVO> bgpPeers;
-                if (network.getVpcId() != null) {
-                    bgpPeers = bgpPeerDao.listNonRevokeByVpcId(network.getVpcId());
-                } else {
-                    bgpPeers = bgpPeerDao.listNonRevokeByNetworkId(network.getId());
-                }
-                if (CollectionUtils.isEmpty(bgpPeers)) {
-                    Account owner = accountDao.findByIdIncludingRemoved(network.getAccountId());
-                    List<Long> bgpPeerIds = routedIpv4Manager.getBgpPeerIdsForAccount(owner, network.getDataCenterId());
-                    bgpPeers = bgpPeerIds.stream()
-                            .map(bgpPeerId -> bgpPeerDao.findById(bgpPeerId))
-                            .collect(Collectors.toList());
-                }
+                List<? extends BgpPeer> bgpPeers = getBgpPeersForNetwork(network);
                 LOGGER.debug(String.format("Applying BPG Peers for network [%s]: [%s]", network, bgpPeers));
                 return ((BgpServiceProvider) provider).applyBgpPeers(null, network, bgpPeers);
             }
@@ -420,14 +413,7 @@ public class BGPServiceImpl implements BGPService {
         if (gatewayProviderStr != null) {
             NetworkElement provider = networkModel.getElementImplementingProvider(gatewayProviderStr);
             if (provider != null && provider instanceof BgpServiceProvider) {
-                List<BgpPeerVO> bgpPeers = bgpPeerDao.listNonRevokeByVpcId(vpc.getId());
-                if (CollectionUtils.isEmpty(bgpPeers)) {
-                    Account owner = accountDao.findByIdIncludingRemoved(vpc.getAccountId());
-                    List<Long> bgpPeerIds = routedIpv4Manager.getBgpPeerIdsForAccount(owner, vpc.getZoneId());
-                    bgpPeers = bgpPeerIds.stream()
-                            .map(bgpPeerId -> bgpPeerDao.findById(bgpPeerId))
-                            .collect(Collectors.toList());
-                }
+                List<? extends BgpPeer> bgpPeers = getBgpPeersForVpc(vpc);
                 LOGGER.debug(String.format("Applying BPG Peers for VPC [%s]: [%s]", vpc, bgpPeers));
                 return ((BgpServiceProvider) provider).applyBgpPeers(vpc, null, bgpPeers);
 
@@ -435,4 +421,35 @@ public class BGPServiceImpl implements BGPService {
         }
         return true;
     }
+
+    @Override
+    public List<? extends BgpPeer> getBgpPeersForNetwork(Network network) {
+        List<BgpPeerVO> bgpPeers;
+        if (network.getVpcId() != null) {
+            bgpPeers = bgpPeerDao.listNonRevokeByVpcId(network.getVpcId());
+        } else {
+            bgpPeers = bgpPeerDao.listNonRevokeByNetworkId(network.getId());
+        }
+        if (CollectionUtils.isEmpty(bgpPeers)) {
+            Account owner = accountDao.findByIdIncludingRemoved(network.getAccountId());
+            List<Long> bgpPeerIds = routedIpv4Manager.getBgpPeerIdsForAccount(owner, network.getDataCenterId());
+            bgpPeers = bgpPeerIds.stream()
+                    .map(bgpPeerId -> bgpPeerDao.findById(bgpPeerId))
+                    .collect(Collectors.toList());
+        }
+        return bgpPeers;
+    }
+
+    @Override
+    public List<? extends BgpPeer> getBgpPeersForVpc(Vpc vpc) {
+        List<BgpPeerVO> bgpPeers = bgpPeerDao.listNonRevokeByVpcId(vpc.getId());
+        if (CollectionUtils.isEmpty(bgpPeers)) {
+            Account owner = accountDao.findByIdIncludingRemoved(vpc.getAccountId());
+            List<Long> bgpPeerIds = routedIpv4Manager.getBgpPeerIdsForAccount(owner, vpc.getZoneId());
+            bgpPeers = bgpPeerIds.stream()
+                    .map(bgpPeerId -> bgpPeerDao.findById(bgpPeerId))
+                    .collect(Collectors.toList());
+        }
+        return bgpPeers;
+    }
 }

server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java

@@ -1254,6 +1254,7 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {CpuOverprovisioningFactor, MemOverprovisioningFactor, StorageCapacityDisableThreshold, StorageOverprovisioningFactor,
-            StorageAllocatedCapacityDisableThreshold, StorageOperationsExcludeCluster, ImageStoreNFSVersion, SecondaryStorageCapacityThreshold};
+                StorageAllocatedCapacityDisableThreshold, StorageOperationsExcludeCluster, ImageStoreNFSVersion, SecondaryStorageCapacityThreshold,
+                StorageAllocatedCapacityDisableThresholdForVolumeSize };
     }
 }

server/src/main/java/com/cloud/configuration/Config.java

@@ -1740,7 +1740,6 @@ DefaultMaxAccountProjects(
                     null),
 
     // VMSnapshots
-    VMSnapshotMax("Advanced", VMSnapshotManager.class, Integer.class, "vmsnapshot.max", "10", "Maximum vm snapshots for a vm", null),
     VMSnapshotCreateWait("Advanced", VMSnapshotManager.class, Integer.class, "vmsnapshot.create.wait", "1800", "In second, timeout for create vm snapshot", null),
 
     CloudDnsName("Advanced", ManagementServer.class, String.class, "cloud.dns.name", null, "DNS name of the cloud for the GSLB service", null),

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -53,6 +53,7 @@ import org.apache.cloudstack.agent.lb.IndirectAgentLB;
 import org.apache.cloudstack.agent.lb.IndirectAgentLBServiceImpl;
 import org.apache.cloudstack.annotation.AnnotationService;
 import org.apache.cloudstack.annotation.dao.AnnotationDao;
+import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.command.admin.config.ResetCfgCmd;
 import org.apache.cloudstack.api.command.admin.config.UpdateCfgCmd;
@@ -106,6 +107,7 @@ import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
 import org.apache.cloudstack.framework.messagebus.MessageBus;
 import org.apache.cloudstack.framework.messagebus.MessageSubscriber;
 import org.apache.cloudstack.framework.messagebus.PublishScope;
+import org.apache.cloudstack.network.RoutedIpv4Manager;
 import org.apache.cloudstack.query.QueryService;
 import org.apache.cloudstack.region.PortableIp;
 import org.apache.cloudstack.region.PortableIpDao;
@@ -309,6 +311,7 @@ import com.googlecode.ipv6.IPv6Network;
 import static com.cloud.configuration.Config.SecStorageAllowedInternalDownloadSites;
 import static com.cloud.offering.NetworkOffering.RoutingMode.Dynamic;
 import static com.cloud.offering.NetworkOffering.RoutingMode.Static;
+import static org.apache.cloudstack.framework.config.ConfigKey.CATEGORY_SYSTEM;
 
 public class ConfigurationManagerImpl extends ManagerBase implements ConfigurationManager, ConfigurationService, Configurable {
     public static final String PERACCOUNT = "peraccount";
@@ -590,6 +593,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         weightBasedParametersForValidation.add(Config.LocalStorageCapacityThreshold.key());
         weightBasedParametersForValidation.add(CapacityManager.StorageAllocatedCapacityDisableThreshold.key());
         weightBasedParametersForValidation.add(CapacityManager.StorageCapacityDisableThreshold.key());
+        weightBasedParametersForValidation.add(CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.key());
         weightBasedParametersForValidation.add(DeploymentClusterPlanner.ClusterCPUCapacityDisableThreshold.key());
         weightBasedParametersForValidation.add(DeploymentClusterPlanner.ClusterMemoryCapacityDisableThreshold.key());
         weightBasedParametersForValidation.add(Config.AgentLoadThreshold.key());
@@ -706,6 +710,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                 value = DBEncryptionUtil.encrypt(value);
             }
 
+            ApiCommandResourceType resourceType;
             ConfigKey.Scope scopeVal = ConfigKey.Scope.valueOf(scope);
             switch (scopeVal) {
             case Zone:
@@ -713,6 +718,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                 if (zone == null) {
                     throw new InvalidParameterValueException("unable to find zone by id " + resourceId);
                 }
+                resourceType = ApiCommandResourceType.Zone;
                 _dcDetailsDao.addDetail(resourceId, name, value, true);
                 break;
             case Cluster:
@@ -720,6 +726,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                 if (cluster == null) {
                     throw new InvalidParameterValueException("unable to find cluster by id " + resourceId);
                 }
+                resourceType = ApiCommandResourceType.Cluster;
                 String newName = name;
                 if (name.equalsIgnoreCase("cpu.overprovisioning.factor")) {
                     newName = "cpuOvercommitRatio";
@@ -742,6 +749,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                 if (pool == null) {
                     throw new InvalidParameterValueException("unable to find storage pool by id " + resourceId);
                 }
+                resourceType = ApiCommandResourceType.StoragePool;
                 if(name.equals(CapacityManager.StorageOverprovisioningFactor.key())) {
                     if(!pool.getPoolType().supportsOverProvisioning() ) {
                         throw new InvalidParameterValueException("Unable to update storage pool with id " + resourceId + ". Overprovision not supported for " + pool.getPoolType());
@@ -763,6 +771,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                 if (account == null) {
                     throw new InvalidParameterValueException("unable to find account by id " + resourceId);
                 }
+                resourceType = ApiCommandResourceType.Account;
                 AccountDetailVO accountDetailVO = _accountDetailsDao.findDetail(resourceId, name);
                 if (accountDetailVO == null) {
                     accountDetailVO = new AccountDetailVO(resourceId, name, value);
@@ -776,6 +785,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
             case ImageStore:
                 final ImageStoreVO imgStore = _imageStoreDao.findById(resourceId);
                 Preconditions.checkState(imgStore != null);
+                resourceType = ApiCommandResourceType.ImageStore;
                 _imageStoreDetailsDao.addDetail(resourceId, name, value, true);
                 break;
 
@@ -784,6 +794,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                 if (domain == null) {
                     throw new InvalidParameterValueException("unable to find domain by id " + resourceId);
                 }
+                resourceType = ApiCommandResourceType.Domain;
                 DomainDetailVO domainDetailVO = _domainDetailsDao.findDetail(resourceId, name);
                 if (domainDetailVO == null) {
                     domainDetailVO = new DomainDetailVO(resourceId, name, value);
@@ -798,6 +809,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                 throw new InvalidParameterValueException("Scope provided is invalid");
             }
 
+            CallContext.current().setEventResourceType(resourceType);
+            CallContext.current().setEventResourceId(resourceId);
+            CallContext.current().setEventDetails(String.format(" Name: %s, New Value: %s, Scope: %s", name, value, scope));
+
             _configDepot.invalidateConfigCache(name, scopeVal, resourceId);
             return valueEncrypted ? DBEncryptionUtil.decrypt(value) : value;
         }
@@ -955,6 +970,11 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
             category = config.getCategory();
         }
 
+        if (CATEGORY_SYSTEM.equals(category) && !_accountMgr.isRootAdmin(caller.getId())) {
+            logger.warn("Only Root Admin is allowed to edit the configuration " + name);
+            throw new CloudRuntimeException("Only Root Admin is allowed to edit this configuration.");
+        }
+
         if (value == null) {
             return _configDao.findByName(name);
         }
@@ -1006,7 +1026,6 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
         if (value.isEmpty() || value.equals("null")) {
             value = (id == null) ? null : "";
         }
-
         final String updatedValue = updateConfiguration(userId, name, category, value, scope, id);
         if (value == null && updatedValue == null || updatedValue.equalsIgnoreCase(value)) {
             return _configDao.findByName(name);
@@ -6673,6 +6692,16 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
                 throw new InvalidParameterValueException("networkMode should be set only for Isolated network offerings");
             }
             if (NetworkOffering.NetworkMode.ROUTED.equals(networkMode)) {
+                if (!RoutedIpv4Manager.RoutedNetworkVpcEnabled.value()) {
+                    throw new InvalidParameterValueException(String.format("Configuration %s needs to be enabled for Routed networks", RoutedIpv4Manager.RoutedNetworkVpcEnabled.key()));
+                }
+                if (zoneIds != null) {
+                    for (Long zoneId: zoneIds) {
+                        if (!RoutedIpv4Manager.RoutedNetworkVpcEnabled.valueIn(zoneId)) {
+                            throw new InvalidParameterValueException(String.format("Configuration %s needs to be enabled for Routed networks in zone (ID: %s)", RoutedIpv4Manager.RoutedNetworkVpcEnabled.key(), zoneId));
+                        }
+                    }
+                }
                 boolean useVirtualRouterOnly = true;
                 for (Service service : serviceProviderMap.keySet()) {
                     Set<Provider> providers = serviceProviderMap.get(service);

server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java

@@ -1795,7 +1795,7 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
                 useLocalStorage = diskOffering.isUseLocalStorage();
             }
             diskProfile.setUseLocalStorage(useLocalStorage);
-            logger.debug(String.format("Calling StoragePoolAllocators to find suitable pools to allocate volume [{}] necessary to deploy VM [{}].", toBeCreated.getUuid(), vmProfile.getUuid()));
+            logger.debug("Calling StoragePoolAllocators to find suitable pools to allocate volume [{}] necessary to deploy VM [{}].", toBeCreated.getUuid(), vmProfile.getUuid());
             boolean foundPotentialPools = tryToFindPotentialPoolsToAlocateVolume(vmProfile, plan, avoid, returnUpTo, suitableVolumeStoragePools, toBeCreated, diskProfile);
 
             if (avoid.getPoolsToAvoid() != null) {

server/src/main/java/com/cloud/network/NetworkServiceImpl.java

@@ -1384,7 +1384,7 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
         }
     }
 
-    void validateNetworkCidrSize(Account caller, Integer cidrSize, String cidr, NetworkOffering networkOffering, long accountId) {
+    void validateNetworkCidrSize(Account caller, Integer cidrSize, String cidr, NetworkOffering networkOffering, long accountId, long zoneId) {
         if (!GuestType.Isolated.equals(networkOffering.getGuestType())) {
             if (cidrSize != null) {
                 throw new InvalidParameterValueException("network cidr size is only applicable on Isolated networks");
@@ -1405,11 +1405,11 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
             if (cidrSize == null) {
                 throw new InvalidParameterValueException("network cidr or cidr size is required for Isolated networks with ROUTED mode");
             }
-            Integer maxCidrSize = routedIpv4Manager.RoutedNetworkIPv4MaxCidrSize.valueIn(accountId);
+            Integer maxCidrSize = RoutedIpv4Manager.RoutedNetworkIPv4MaxCidrSize.valueIn(accountId);
             if (cidrSize > maxCidrSize) {
                 throw new InvalidParameterValueException("network cidr size cannot be bigger than maximum cidr size " + maxCidrSize);
             }
-            Integer minCidrSize = routedIpv4Manager.RoutedNetworkIPv4MinCidrSize.valueIn(accountId);
+            Integer minCidrSize = RoutedIpv4Manager.RoutedNetworkIPv4MinCidrSize.valueIn(accountId);
             if (cidrSize < minCidrSize) {
                 throw new InvalidParameterValueException("network cidr size cannot be smaller than minimum cidr size " + minCidrSize);
             }
@@ -1656,11 +1656,16 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
             }
         }
 
+        if (NetworkOffering.NetworkMode.ROUTED.equals(ntwkOff.getNetworkMode())
+                && !routedIpv4Manager.isRoutedNetworkVpcEnabled(zone.getId())) {
+            throw new InvalidParameterValueException("Routed network is not enabled in this zone");
+        }
+
         if (isNonVpcNetworkSupportingDynamicRouting(ntwkOff) && ntwkOff.isSpecifyAsNumber() && asNumber == null) {
             throw new InvalidParameterValueException("AS number is required for the network but not passed.");
         }
 
-        validateNetworkCidrSize(caller, networkCidrSize, cidr, ntwkOff, owner.getAccountId());
+        validateNetworkCidrSize(caller, networkCidrSize, cidr, ntwkOff, owner.getAccountId(), zone.getId());
 
         validateSharedNetworkRouterIPs(gateway, startIP, endIP, netmask, routerIPv4, routerIPv6, startIPv6, endIPv6, ip6Cidr, ntwkOff);
 

server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -65,10 +65,8 @@ import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
 import org.apache.cloudstack.lb.ApplicationLoadBalancerRuleVO;
 import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
-import org.apache.cloudstack.network.BgpPeerVO;
+import org.apache.cloudstack.network.BgpPeer;
 import org.apache.cloudstack.network.RoutedIpv4Manager;
-import org.apache.cloudstack.network.dao.BgpPeerDao;
-import org.apache.cloudstack.network.dao.BgpPeerNetworkMapDao;
 import org.apache.cloudstack.network.topology.NetworkTopology;
 import org.apache.cloudstack.network.topology.NetworkTopologyContext;
 import org.apache.cloudstack.utils.CloudStackVersion;
@@ -114,6 +112,7 @@ import com.cloud.api.query.dao.DomainRouterJoinDao;
 import com.cloud.api.query.dao.UserVmJoinDao;
 import com.cloud.api.query.vo.DomainRouterJoinVO;
 import com.cloud.api.query.vo.UserVmJoinVO;
+import com.cloud.bgp.BGPService;
 import com.cloud.cluster.ManagementServerHostVO;
 import com.cloud.cluster.dao.ManagementServerHostDao;
 import com.cloud.configuration.Config;
@@ -348,9 +347,7 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
     @Inject
     RoutedIpv4Manager routedIpv4Manager;
     @Inject
-    BgpPeerDao bgpPeerDao;
-    @Inject
-    BgpPeerNetworkMapDao bgpPeerNetworkMapDao;
+    BGPService bgpService;
 
     private int _routerRamSize;
     private int _routerCpuMHz;
@@ -2508,12 +2505,12 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
         if (guestNetwork.getVpcId() != null) {
             final Vpc vpc = _vpcDao.findById(guestNetwork.getVpcId());
             if (routedIpv4Manager.isDynamicRoutedVpc(vpc)) {
-                final List<BgpPeerVO> bgpPeers = bgpPeerDao.listNonRevokeByVpcId(guestNetwork.getVpcId());
+                List<? extends BgpPeer> bgpPeers = bgpService.getBgpPeersForVpc(vpc);
                 _commandSetupHelper.createBgpPeersCommands(bgpPeers, router, cmds, guestNetwork);
             }
         } else {
             if (routedIpv4Manager.isDynamicRoutedNetwork(guestNetwork)) {
-                final List<BgpPeerVO> bgpPeers = bgpPeerDao.listNonRevokeByNetworkId(guestNetworkId);
+                List<? extends BgpPeer> bgpPeers = bgpService.getBgpPeersForNetwork(guestNetwork);
                 _commandSetupHelper.createBgpPeersCommands(bgpPeers, router, cmds, guestNetwork);
             }
         }

server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java

@@ -502,6 +502,18 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
             }
             networkMode = NetworkOffering.NetworkMode.valueOf(networkModeStr);
         }
+        if (NetworkOffering.NetworkMode.ROUTED.equals(networkMode)) {
+            if (!RoutedIpv4Manager.RoutedNetworkVpcEnabled.value()) {
+                throw new InvalidParameterValueException(String.format("Configuration %s needs to be enabled for Routed VPCs", RoutedIpv4Manager.RoutedNetworkVpcEnabled.key()));
+            }
+            if (zoneIds != null) {
+                for (Long zoneId: zoneIds) {
+                    if (!RoutedIpv4Manager.RoutedNetworkVpcEnabled.valueIn(zoneId)) {
+                        throw new InvalidParameterValueException(String.format("Configuration %s needs to be enabled for Routed VPCs in zone (ID: %s)", RoutedIpv4Manager.RoutedNetworkVpcEnabled.key(), zoneId));
+                    }
+                }
+            }
+        }
         boolean specifyAsNumber = cmd.getSpecifyAsNumber();
         String routingModeString = cmd.getRoutingMode();
 
@@ -1163,12 +1175,17 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
             throw ex;
         }
 
+        if (NetworkOffering.NetworkMode.ROUTED.equals(vpcOff.getNetworkMode())
+                && !routedIpv4Manager.RoutedNetworkVpcEnabled.valueIn(zoneId)) {
+            throw new InvalidParameterValueException("Routed VPC is not enabled in this zone");
+        }
+
         if (NetworkOffering.RoutingMode.Dynamic.equals(vpcOff.getRoutingMode()) && vpcOff.isSpecifyAsNumber() && asNumber == null) {
             throw new InvalidParameterValueException("AS number is required for the VPC but not passed.");
         }
 
         // Validate VPC cidr/cidrsize
-        validateVpcCidrSize(caller, owner.getAccountId(), vpcOff, cidr, cidrSize);
+        validateVpcCidrSize(caller, owner.getAccountId(), vpcOff, cidr, cidrSize, zoneId);
 
         // Validate BGP peers
         if (CollectionUtils.isNotEmpty(bgpPeerIds)) {
@@ -1253,7 +1270,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
         return newVpc;
     }
 
-    private void validateVpcCidrSize(Account caller, long accountId, VpcOffering vpcOffering, String cidr, Integer cidrSize) {
+    private void validateVpcCidrSize(Account caller, long accountId, VpcOffering vpcOffering, String cidr, Integer cidrSize, long zoneId) {
         if (ObjectUtils.allNull(cidr, cidrSize)) {
             throw new InvalidParameterValueException("VPC cidr or cidr size must be specified");
         }

server/src/main/java/com/cloud/server/ManagementServerHostStatsEntry.java

@@ -24,6 +24,7 @@ public class ManagementServerHostStatsEntry implements ManagementServerHostStats
 
     private long managementServerHostId;
     private String managementServerHostUuid;
+    private long managementServerRunId;
 
     private Date collectionTime;
     private long sessions;
@@ -94,6 +95,15 @@ public class ManagementServerHostStatsEntry implements ManagementServerHostStats
         this.managementServerHostUuid = managementServerHostUuid;
     }
 
+    @Override
+    public long getManagementServerRunId() {
+        return managementServerRunId;
+    }
+
+    public void setManagementServerRunId(long managementServerRunId) {
+        this.managementServerRunId = managementServerRunId;
+    }
+
     @Override
     public Date getCollectionTime(){
         return collectionTime;

server/src/main/java/com/cloud/server/ManagementServerImpl.java

@@ -68,11 +68,6 @@ import org.apache.cloudstack.api.command.admin.affinitygroup.UpdateVMAffinityGro
 import org.apache.cloudstack.api.command.admin.alert.GenerateAlertCmd;
 import org.apache.cloudstack.api.command.admin.autoscale.CreateCounterCmd;
 import org.apache.cloudstack.api.command.admin.autoscale.DeleteCounterCmd;
-import org.apache.cloudstack.api.command.admin.bgp.CreateASNRangeCmd;
-import org.apache.cloudstack.api.command.admin.bgp.DeleteASNRangeCmd;
-import org.apache.cloudstack.api.command.admin.bgp.ListASNRangesCmd;
-import org.apache.cloudstack.api.command.user.bgp.ListASNumbersCmd;
-import org.apache.cloudstack.api.command.admin.bgp.ReleaseASNumberCmd;
 import org.apache.cloudstack.api.command.admin.cluster.AddClusterCmd;
 import org.apache.cloudstack.api.command.admin.cluster.DeleteClusterCmd;
 import org.apache.cloudstack.api.command.admin.cluster.ListClustersCmd;
@@ -4017,12 +4012,6 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         cmdList.add(RemoveSecondaryStorageSelectorCmd.class);
         cmdList.add(ListAffectedVmsForStorageScopeChangeCmd.class);
 
-        cmdList.add(CreateASNRangeCmd.class);
-        cmdList.add(ListASNRangesCmd.class);
-        cmdList.add(DeleteASNRangeCmd.class);
-        cmdList.add(ListASNumbersCmd.class);
-        cmdList.add(ReleaseASNumberCmd.class);
-
         // Out-of-band management APIs for admins
         cmdList.add(EnableOutOfBandManagementForHostCmd.class);
         cmdList.add(DisableOutOfBandManagementForHostCmd.class);

server/src/main/java/com/cloud/server/StatsCollector.java

@@ -95,6 +95,7 @@ import com.cloud.cluster.ClusterServicePdu;
 import com.cloud.cluster.ManagementServerHostVO;
 import com.cloud.cluster.ManagementServerStatusVO;
 import com.cloud.cluster.dao.ManagementServerHostDao;
+import com.cloud.cluster.dao.ManagementServerHostPeerDao;
 import com.cloud.cluster.dao.ManagementServerStatusDao;
 import com.cloud.dc.Vlan.VlanType;
 import com.cloud.dc.VlanVO;
@@ -346,6 +347,8 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
     @Inject
     private ManagementServerStatusDao managementServerStatusDao;
     @Inject
+    private ManagementServerHostPeerDao managementServerHostPeerDao;
+    @Inject
     VirtualMachineManager virtualMachineManager;
 
     private final ConcurrentHashMap<String, ManagementServerHostStats> managementServerHostStats = new ConcurrentHashMap<>();
@@ -796,6 +799,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
             logger.trace("Metrics collection start...");
             newEntry.setManagementServerHostId(mshost.getId());
             newEntry.setManagementServerHostUuid(mshost.getUuid());
+            newEntry.setManagementServerRunId(mshost.getRunid());
             newEntry.setDbLocal(isDbLocal());
             newEntry.setUsageLocal(isUsageLocal());
             retrieveSession(newEntry);
@@ -1153,6 +1157,9 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
             try {
                 hostStatsEntry = gson.fromJson(pdu.getJsonPackage(),new TypeToken<ManagementServerHostStatsEntry>(){}.getType());
                 managementServerHostStats.put(hostStatsEntry.getManagementServerHostUuid(), hostStatsEntry);
+
+                // Update peer state to Up in mshost_peer
+                updatePeerInfo(hostStatsEntry);
             } catch (JsonParseException e) {
                 logger.error("Exception in decoding of other MS hosts status from : " + pdu.getSourcePeer());
                 if (logger.isDebugEnabled()) {
@@ -1162,6 +1169,23 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
             return null;
         }
 
+        private void updatePeerInfo(ManagementServerHostStatsEntry hostStatsEntry) {
+            // Update msId to id of the management server if msId is same as managementServerNodeId
+            if (msId == managementServerNodeId) {
+                ManagementServerHostVO mgmtServerVo = managementServerHostDao.findByMsid(managementServerNodeId);
+                if (mgmtServerVo != null) {
+                    msId = mgmtServerVo.getId();
+                } else {
+                    logger.warn(String.format("Cannot find management server with msid [%s]. Therefore, do not update peer info.", managementServerNodeId));
+                    return;
+                }
+            }
+            // Update peer state to Up in mshost_peer
+            if (msId != hostStatsEntry.getManagementServerHostId()) {
+                managementServerHostPeerDao.updatePeerInfo(msId, hostStatsEntry.getManagementServerHostId(), hostStatsEntry.getManagementServerRunId(), ManagementServerHost.State.Up);
+            }
+        }
+
         @Override
         public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
             // do nothing, but wait for the status to come through

server/src/main/java/com/cloud/storage/StorageManagerImpl.java

@@ -3101,7 +3101,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         } else {
             final StoragePoolVO poolVO = _storagePoolDao.findById(pool.getId());
             final long allocatedSizeWithTemplate = _capacityMgr.getAllocatedPoolCapacity(poolVO, null);
-            return checkPoolforSpace(pool, allocatedSizeWithTemplate, totalAskingSize);
+            return checkPoolforSpace(pool, allocatedSizeWithTemplate, totalAskingSize, true);
         }
     }
 
@@ -3164,6 +3164,10 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
     }
 
     protected boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemplate, long totalAskingSize) {
+        return checkPoolforSpace(pool, allocatedSizeWithTemplate, totalAskingSize, false);
+    }
+
+    protected boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemplate, long totalAskingSize, boolean forVolumeResize) {
         // allocated space includes templates
         StoragePoolVO poolVO = _storagePoolDao.findById(pool.getId());
 
@@ -3196,10 +3200,22 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
         if (usedPercentage > storageAllocatedThreshold) {
             if (logger.isDebugEnabled()) {
                 logger.debug("Insufficient un-allocated capacity on: " + pool.getId() + " for storage allocation since its allocated percentage: " + usedPercentage
-                        + " has crossed the allocated pool.storage.allocated.capacity.disablethreshold: " + storageAllocatedThreshold + ", skipping this pool");
+                        + " has crossed the allocated pool.storage.allocated.capacity.disablethreshold: " + storageAllocatedThreshold);
+            }
+            if (!forVolumeResize) {
+                return false;
+            }
+            if (!AllowVolumeReSizeBeyondAllocation.valueIn(pool.getDataCenterId())) {
+                logger.debug(String.format("Skipping the pool %s as %s is false", pool, AllowVolumeReSizeBeyondAllocation.key()));
+                return false;
             }
 
-            return false;
+            double storageAllocatedThresholdForResize = CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.valueIn(pool.getDataCenterId());
+            if (usedPercentage > storageAllocatedThresholdForResize) {
+                logger.debug(String.format("Skipping the pool %s since its allocated percentage: %s has crossed the allocated %s: %s",
+                        pool, usedPercentage, CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.key(), storageAllocatedThresholdForResize));
+                return false;
+            }
         }
 
         if (totalOverProvCapacity < (allocatedSizeWithTemplate + totalAskingSize)) {
@@ -4050,7 +4066,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
                 MountDisabledStoragePool,
                 VmwareCreateCloneFull,
                 VmwareAllowParallelExecution,
-                DataStoreDownloadFollowRedirects
+                DataStoreDownloadFollowRedirects,
+                AllowVolumeReSizeBeyondAllocation
         };
     }
 

server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java

@@ -1093,6 +1093,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
         Long newMaxIops = cmd.getMaxIops();
         Integer newHypervisorSnapshotReserve = null;
         boolean shrinkOk = cmd.isShrinkOk();
+        boolean autoMigrateVolume = cmd.getAutoMigrate();
 
         VolumeVO volume = _volsDao.findById(cmd.getEntityId());
         if (volume == null) {
@@ -1154,8 +1155,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
                 newSize = volume.getSize();
             }
 
-            newMinIops = cmd.getMinIops();
-
             if (newMinIops != null) {
                 if (!volume.getVolumeType().equals(Volume.Type.ROOT) && (diskOffering.isCustomizedIops() == null || !diskOffering.isCustomizedIops())) {
                     throw new InvalidParameterValueException("The current disk offering does not support customization of the 'Min IOPS' parameter.");
@@ -1165,8 +1164,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
                 newMinIops = volume.getMinIops();
             }
 
-            newMaxIops = cmd.getMaxIops();
-
             if (newMaxIops != null) {
                 if (!volume.getVolumeType().equals(Volume.Type.ROOT) && (diskOffering.isCustomizedIops() == null || !diskOffering.isCustomizedIops())) {
                     throw new InvalidParameterValueException("The current disk offering does not support customization of the 'Max IOPS' parameter.");
@@ -1288,6 +1285,54 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
             return volume;
         }
 
+        Long newDiskOfferingId = newDiskOffering != null ? newDiskOffering.getId() : diskOffering.getId();
+
+        boolean volumeMigrateRequired = false;
+        List<? extends StoragePool> suitableStoragePoolsWithEnoughSpace = null;
+        StoragePoolVO storagePool = _storagePoolDao.findById(volume.getPoolId());
+        if (!storageMgr.storagePoolHasEnoughSpaceForResize(storagePool, currentSize, newSize)) {
+            if (!autoMigrateVolume) {
+                throw new CloudRuntimeException(String.format("Failed to resize volume %s since the storage pool does not have enough space to accommodate new size for the volume %s, try with automigrate set to true in order to check in the other suitable pools for the new size and then migrate & resize volume there.", volume.getUuid(), volume.getName()));
+            }
+            Pair<List<? extends StoragePool>, List<? extends StoragePool>> poolsPair = managementService.listStoragePoolsForSystemMigrationOfVolume(volume.getId(), newDiskOfferingId, currentSize, newMinIops, newMaxIops, true, false);
+            List<? extends StoragePool> suitableStoragePools = poolsPair.second();
+            if (CollectionUtils.isEmpty(poolsPair.first()) && CollectionUtils.isEmpty(poolsPair.second())) {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Volume resize failed for volume ID: %s as no suitable pool(s) found for migrating to support new disk offering or new size", volume.getUuid()));
+            }
+            final Long newSizeFinal = newSize;
+            suitableStoragePoolsWithEnoughSpace = suitableStoragePools.stream().filter(pool -> storageMgr.storagePoolHasEnoughSpaceForResize(pool, 0L, newSizeFinal)).collect(Collectors.toList());
+            if (CollectionUtils.isEmpty(suitableStoragePoolsWithEnoughSpace)) {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Volume resize failed for volume ID: %s as no suitable pool(s) with enough space found.", volume.getUuid()));
+            }
+            Collections.shuffle(suitableStoragePoolsWithEnoughSpace);
+            volumeMigrateRequired = true;
+        }
+
+        boolean volumeResizeRequired = false;
+        if (currentSize != newSize || !compareEqualsIncludingNullOrZero(newMaxIops, volume.getMaxIops()) || !compareEqualsIncludingNullOrZero(newMinIops, volume.getMinIops())) {
+            volumeResizeRequired = true;
+        }
+        if (!volumeMigrateRequired && !volumeResizeRequired && newDiskOffering != null) {
+            _volsDao.updateDiskOffering(volume.getId(), newDiskOffering.getId());
+            volume = _volsDao.findById(volume.getId());
+            updateStorageWithTheNewDiskOffering(volume, newDiskOffering);
+
+            return volume;
+        }
+
+        if (volumeMigrateRequired) {
+            MigrateVolumeCmd migrateVolumeCmd = new MigrateVolumeCmd(volume.getId(), suitableStoragePoolsWithEnoughSpace.get(0).getId(), newDiskOfferingId, true);
+            try {
+                Volume result = migrateVolume(migrateVolumeCmd);
+                volume = (result != null) ? _volsDao.findById(result.getId()) : null;
+                if (volume == null) {
+                    throw new CloudRuntimeException(String.format("Volume resize operation failed for volume ID: %s as migration failed to storage pool %s accommodating new size", volume.getUuid(), suitableStoragePoolsWithEnoughSpace.get(0).getId()));
+                }
+            } catch (Exception e) {
+                throw new CloudRuntimeException(String.format("Volume resize operation failed for volume ID: %s as migration failed to storage pool %s accommodating new size", volume.getUuid(), suitableStoragePoolsWithEnoughSpace.get(0).getId()));
+            }
+        }
+
         UserVmVO userVm = _userVmDao.findById(volume.getInstanceId());
 
         if (userVm != null) {
@@ -1973,6 +2018,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
     public Volume changeDiskOfferingForVolume(ChangeOfferingForVolumeCmd cmd) throws ResourceAllocationException {
         Long newSize = cmd.getSize();
         Long newMinIops = cmd.getMinIops();
+
         Long newMaxIops = cmd.getMaxIops();
         Long newDiskOfferingId = cmd.getNewDiskOfferingId();
         boolean shrinkOk = cmd.isShrinkOk();
@@ -2055,7 +2101,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
 
         StoragePoolVO existingStoragePool = _storagePoolDao.findById(volume.getPoolId());
 
-        Pair<List<? extends StoragePool>, List<? extends StoragePool>> poolsPair = managementService.listStoragePoolsForSystemMigrationOfVolume(volume.getId(), newDiskOffering.getId(), newSize, newMinIops, newMaxIops, true, false);
+        Pair<List<? extends StoragePool>, List<? extends StoragePool>> poolsPair = managementService.listStoragePoolsForSystemMigrationOfVolume(volume.getId(), newDiskOffering.getId(), currentSize, newMinIops, newMaxIops, true, false);
         List<? extends StoragePool> suitableStoragePools = poolsPair.second();
 
         if (!suitableStoragePools.stream().anyMatch(p -> (p.getId() == existingStoragePool.getId()))) {
@@ -2077,10 +2123,16 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
             if (CollectionUtils.isEmpty(poolsPair.first()) && CollectionUtils.isEmpty(poolsPair.second())) {
                 throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Volume change offering operation failed for volume ID: %s as no suitable pool(s) found for migrating to support new disk offering", volume.getUuid()));
             }
-            Collections.shuffle(suitableStoragePools);
-            MigrateVolumeCmd migrateVolumeCmd = new MigrateVolumeCmd(volume.getId(), suitableStoragePools.get(0).getId(), newDiskOffering.getId(), true);
+            final Long newSizeFinal = newSize;
+            List<? extends StoragePool> suitableStoragePoolsWithEnoughSpace = suitableStoragePools.stream().filter(pool -> storageMgr.storagePoolHasEnoughSpaceForResize(pool, 0L, newSizeFinal)).collect(Collectors.toList());
+            if (CollectionUtils.isEmpty(suitableStoragePoolsWithEnoughSpace)) {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Volume change offering operation failed for volume ID: %s as no suitable pool(s) with enough space found for volume migration.", volume.getUuid()));
+            }
+            Collections.shuffle(suitableStoragePoolsWithEnoughSpace);
+            MigrateVolumeCmd migrateVolumeCmd = new MigrateVolumeCmd(volume.getId(), suitableStoragePoolsWithEnoughSpace.get(0).getId(), newDiskOffering.getId(), true);
             try {
-                volume = (VolumeVO) migrateVolume(migrateVolumeCmd);
+                Volume result = migrateVolume(migrateVolumeCmd);
+                volume = (result != null) ? _volsDao.findById(result.getId()) : null;
                 if (volume == null) {
                     throw new CloudRuntimeException(String.format("Volume change offering operation failed for volume ID: %s migration failed to storage pool %s", volume.getUuid(), suitableStoragePools.get(0).getId()));
                 }

server/src/main/java/com/cloud/user/AccountManagerImpl.java

@@ -373,6 +373,13 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
             "totp",
             "The default user two factor authentication provider. Eg. totp, staticpin", true, ConfigKey.Scope.Domain);
 
+    public static final ConfigKey<Boolean> apiKeyAccess = new ConfigKey<>(ConfigKey.CATEGORY_SYSTEM, Boolean.class,
+            "api.key.access",
+            "true",
+            "Determines whether API (api-key/secret-key) access is allowed or not. Editable only by Root Admin.",
+            true,
+            ConfigKey.Scope.Domain);
+
     protected AccountManagerImpl() {
         super();
     }
@@ -1463,6 +1470,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
         logger.debug("Updating user with Id: " + user.getUuid());
 
         validateAndUpdateApiAndSecretKeyIfNeeded(updateUserCmd, user);
+        validateAndUpdateUserApiKeyAccess(updateUserCmd, user);
         Account account = retrieveAndValidateAccount(user);
 
         validateAndUpdateFirstNameIfNeeded(updateUserCmd, user);
@@ -1682,6 +1690,38 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
         user.setSecretKey(secretKey);
     }
 
+    protected void validateAndUpdateUserApiKeyAccess(UpdateUserCmd updateUserCmd, UserVO user) {
+        if (updateUserCmd.getApiKeyAccess() != null) {
+            try {
+                ApiConstants.ApiKeyAccess access = ApiConstants.ApiKeyAccess.valueOf(updateUserCmd.getApiKeyAccess().toUpperCase());
+                user.setApiKeyAccess(access.toBoolean());
+                Long callingUserId = CallContext.current().getCallingUserId();
+                Account callingAccount = CallContext.current().getCallingAccount();
+                ActionEventUtils.onActionEvent(callingUserId, callingAccount.getAccountId(), callingAccount.getDomainId(),
+                        EventTypes.API_KEY_ACCESS_UPDATE, "Api key access was changed for the User to " + access.toString(),
+                        user.getId(), ApiCommandResourceType.User.toString());
+            } catch (IllegalArgumentException ex) {
+                throw new InvalidParameterValueException("ApiKeyAccess value can only be Enabled/Disabled/Inherit");
+            }
+        }
+    }
+
+    protected void validateAndUpdateAccountApiKeyAccess(UpdateAccountCmd updateAccountCmd, AccountVO account) {
+        if (updateAccountCmd.getApiKeyAccess() != null) {
+            try {
+                ApiConstants.ApiKeyAccess access = ApiConstants.ApiKeyAccess.valueOf(updateAccountCmd.getApiKeyAccess().toUpperCase());
+                account.setApiKeyAccess(access.toBoolean());
+                Long callingUserId = CallContext.current().getCallingUserId();
+                Account callingAccount = CallContext.current().getCallingAccount();
+                ActionEventUtils.onActionEvent(callingUserId, callingAccount.getAccountId(), callingAccount.getDomainId(),
+                        EventTypes.API_KEY_ACCESS_UPDATE, "Api key access was changed for the Account to " + access.toString(),
+                        account.getId(), ApiCommandResourceType.Account.toString());
+            } catch (IllegalArgumentException ex) {
+                throw new InvalidParameterValueException("ApiKeyAccess value can only be Enabled/Disabled/Inherit");
+            }
+        }
+    }
+
     /**
      * Searches for a user with the given userId. If no user is found we throw an {@link InvalidParameterValueException}.
      */
@@ -2048,6 +2088,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
         Account caller = getCurrentCallingAccount();
         checkAccess(caller, _domainMgr.getDomain(account.getDomainId()));
 
+        validateAndUpdateAccountApiKeyAccess(cmd, acctForUpdate);
+
         if(newAccountName != null) {
 
             if (newAccountName.isEmpty()) {
@@ -2794,18 +2836,18 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
     }
 
     @Override
-    public Map<String, String> getKeys(GetUserKeysCmd cmd) {
+    public Pair<Boolean, Map<String, String>> getKeys(GetUserKeysCmd cmd) {
         final long userId = cmd.getID();
         return getKeys(userId);
     }
 
     @Override
-    public Map<String, String> getKeys(Long userId) {
+    public Pair<Boolean, Map<String, String>> getKeys(Long userId) {
         User user = getActiveUser(userId);
         if (user == null) {
             throw new InvalidParameterValueException("Unable to find user by id");
         }
-        final ControlledEntity account = getAccount(getUserAccountById(userId).getAccountId()); //Extracting the Account from the userID of the requested user.
+        final Account account = getAccount(getUserAccountById(userId).getAccountId()); //Extracting the Account from the userID of the requested user.
         User caller = CallContext.current().getCallingUser();
         preventRootDomainAdminAccessToRootAdminKeys(caller, account);
         checkAccess(caller, account);
@@ -2814,7 +2856,15 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
         keys.put("apikey", user.getApiKey());
         keys.put("secretkey", user.getSecretKey());
 
-        return keys;
+        Boolean apiKeyAccess = user.getApiKeyAccess();
+        if (apiKeyAccess == null) {
+            apiKeyAccess = account.getApiKeyAccess();
+            if (apiKeyAccess == null) {
+                apiKeyAccess = AccountManagerImpl.apiKeyAccess.valueIn(account.getDomainId());
+            }
+        }
+
+        return new Pair<Boolean, Map<String, String>>(apiKeyAccess, keys);
     }
 
     protected void preventRootDomainAdminAccessToRootAdminKeys(User caller, ControlledEntity account) {
@@ -3320,7 +3370,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {UseSecretKeyInResponse, enableUserTwoFactorAuthentication,
-                userTwoFactorAuthenticationDefaultProvider, mandateUserTwoFactorAuthentication, userTwoFactorAuthenticationIssuer};
+                userTwoFactorAuthenticationDefaultProvider, mandateUserTwoFactorAuthentication, userTwoFactorAuthenticationIssuer, apiKeyAccess};
     }
 
     public List<UserTwoFactorAuthenticator> getUserTwoFactorAuthenticationProviders() {

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.vm;
 
+import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality;
 import static com.cloud.storage.Volume.IOPS_LIMIT;
 import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
 import static org.apache.cloudstack.api.ApiConstants.MAX_IOPS;
@@ -679,23 +680,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
     private static final ConfigKey<Boolean> VmDestroyForcestop = new ConfigKey<Boolean>("Advanced", Boolean.class, "vm.destroy.forcestop", "false",
             "On destroy, force-stop takes this value ", true);
 
-    public static final List<HypervisorType> VM_STORAGE_MIGRATION_SUPPORTING_HYPERVISORS = new ArrayList<>(Arrays.asList(
-            HypervisorType.KVM,
-            HypervisorType.VMware,
-            HypervisorType.XenServer,
-            HypervisorType.Simulator
-    ));
-
-    protected static final List<HypervisorType> ROOT_DISK_SIZE_OVERRIDE_SUPPORTING_HYPERVISORS = Arrays.asList(
-            HypervisorType.KVM,
-            HypervisorType.XenServer,
-            HypervisorType.VMware,
-            HypervisorType.Simulator,
-            HypervisorType.Custom
-    );
-
-    private static final List<HypervisorType> HYPERVISORS_THAT_CAN_DO_STORAGE_MIGRATION_ON_NON_USER_VMS = Arrays.asList(HypervisorType.KVM, HypervisorType.VMware);
-
     @Override
     public UserVmVO getVirtualMachine(long vmId) {
         return _vmDao.findById(vmId);
@@ -4563,7 +4547,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
      * @throws InvalidParameterValueException if the hypervisor does not support rootdisksize override
      */
     protected void verifyIfHypervisorSupportsRootdiskSizeOverride(HypervisorType hypervisorType) {
-        if (!ROOT_DISK_SIZE_OVERRIDE_SUPPORTING_HYPERVISORS.contains(hypervisorType)) {
+        if (!hypervisorType.isFunctionalitySupported(Functionality.RootDiskSizeOverride)) {
             throw new InvalidParameterValueException("Hypervisor " + hypervisorType + " does not support rootdisksize override");
         }
     }
@@ -6606,9 +6590,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         }
 
         HypervisorType hypervisorType = vm.getHypervisorType();
-        if (vm.getType() != VirtualMachine.Type.User && !HYPERVISORS_THAT_CAN_DO_STORAGE_MIGRATION_ON_NON_USER_VMS.contains(hypervisorType)) {
-            throw new InvalidParameterValueException(String.format("Unable to migrate storage of non-user VMs for hypervisor [%s]. Operation only supported for the following"
-                    + " hypervisors: [%s].", hypervisorType, HYPERVISORS_THAT_CAN_DO_STORAGE_MIGRATION_ON_NON_USER_VMS));
+        List<HypervisorType> supportedHypervisorsForNonUserVMStorageMigration = HypervisorType.getListOfHypervisorsSupportingFunctionality(Functionality.VmStorageMigration)
+                .stream().filter(hypervisor -> !hypervisor.equals(HypervisorType.XenServer)).collect(Collectors.toList());
+        if (vm.getType() != VirtualMachine.Type.User && !supportedHypervisorsForNonUserVMStorageMigration.contains(hypervisorType)) {
+            throw new InvalidParameterValueException(String.format(
+                    "Unable to migrate storage of non-user VMs for hypervisor [%s]. Operation only supported for the following hypervisors: [%s].",
+                    hypervisorType, supportedHypervisorsForNonUserVMStorageMigration));
         }
 
         List<VolumeVO> vols = _volsDao.findByInstance(vm.getId());
@@ -7318,8 +7305,11 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             throw new InvalidParameterValueException("Live Migration of GPU enabled VM is not supported");
         }
 
-        if (!VM_STORAGE_MIGRATION_SUPPORTING_HYPERVISORS.contains(vm.getHypervisorType())) {
-            throw new InvalidParameterValueException(String.format("Unsupported hypervisor: %s for VM migration, we support XenServer/VMware/KVM only", vm.getHypervisorType()));
+        if (!vm.getHypervisorType().isFunctionalitySupported(Functionality.VmStorageMigration)) {
+            throw new InvalidParameterValueException(
+                    String.format("Unsupported hypervisor: %s for VM migration, we support [%s] only",
+                            vm.getHypervisorType(),
+                            HypervisorType.getListOfHypervisorsSupportingFunctionality(Functionality.VmStorageMigration)));
         }
 
         if (_vmSnapshotDao.findByVm(vmId).size() > 0) {

server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java

@@ -174,7 +174,6 @@ public class VMSnapshotManagerImpl extends MutualExclusiveIdsManagerBase impleme
 
     VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this);
 
-    int _vmSnapshotMax;
     int _wait;
 
     static final ConfigKey<Long> VmJobCheckInterval = new ConfigKey<Long>("Advanced",
@@ -188,8 +187,6 @@ public class VMSnapshotManagerImpl extends MutualExclusiveIdsManagerBase impleme
             throw new ConfigurationException("Unable to get the configuration dao.");
         }
 
-        _vmSnapshotMax = NumbersUtil.parseInt(_configDao.getValue("vmsnapshot.max"), VMSNAPSHOTMAX);
-
         String value = _configDao.getValue("vmsnapshot.create.wait");
         _wait = NumbersUtil.parseInt(value, 1800);
 
@@ -398,8 +395,10 @@ public class VMSnapshotManagerImpl extends MutualExclusiveIdsManagerBase impleme
         _accountMgr.checkAccess(caller, null, true, userVmVo);
 
         // check max snapshot limit for per VM
-        if (_vmSnapshotDao.findByVm(vmId).size() >= _vmSnapshotMax) {
-            throw new CloudRuntimeException("Creating vm snapshot failed due to a VM can just have : " + _vmSnapshotMax + " VM snapshots. Please delete old ones");
+        int vmSnapshotMax = VMSnapshotManager.VMSnapshotMax.value();
+
+        if (_vmSnapshotDao.findByVm(vmId).size() >= vmSnapshotMax) {
+            throw new CloudRuntimeException("Creating vm snapshot failed due to a VM can just have : " + vmSnapshotMax + " VM snapshots. Please delete old ones");
         }
 
         // check if there are active volume snapshots tasks
@@ -1391,6 +1390,6 @@ public class VMSnapshotManagerImpl extends MutualExclusiveIdsManagerBase impleme
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] {VMSnapshotExpireInterval};
+        return new ConfigKey<?>[] {VMSnapshotExpireInterval, VMSnapshotMax};
     }
 }

server/src/main/java/org/apache/cloudstack/network/RoutedIpv4ManagerImpl.java

@@ -57,6 +57,10 @@ import com.cloud.utils.db.TransactionStatus;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.net.NetUtils;
 
+import org.apache.cloudstack.api.command.admin.bgp.CreateASNRangeCmd;
+import org.apache.cloudstack.api.command.admin.bgp.DeleteASNRangeCmd;
+import org.apache.cloudstack.api.command.admin.bgp.ListASNRangesCmd;
+import org.apache.cloudstack.api.command.admin.bgp.ReleaseASNumberCmd;
 import org.apache.cloudstack.api.command.admin.network.CreateIpv4SubnetForZoneCmd;
 import org.apache.cloudstack.api.command.admin.network.CreateIpv4SubnetForGuestNetworkCmd;
 import org.apache.cloudstack.api.command.admin.network.DedicateIpv4SubnetForZoneCmd;
@@ -74,6 +78,7 @@ import org.apache.cloudstack.api.command.admin.network.bgp.DeleteBgpPeerCmd;
 import org.apache.cloudstack.api.command.admin.network.bgp.ListBgpPeersCmd;
 import org.apache.cloudstack.api.command.admin.network.bgp.ReleaseDedicatedBgpPeerCmd;
 import org.apache.cloudstack.api.command.admin.network.bgp.UpdateBgpPeerCmd;
+import org.apache.cloudstack.api.command.user.bgp.ListASNumbersCmd;
 import org.apache.cloudstack.api.command.user.network.routing.CreateRoutingFirewallRuleCmd;
 import org.apache.cloudstack.api.command.user.network.routing.DeleteRoutingFirewallRuleCmd;
 import org.apache.cloudstack.api.command.user.network.routing.ListRoutingFirewallRulesCmd;
@@ -154,6 +159,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey[] {
+                RoutedNetworkVpcEnabled,
                 RoutedNetworkIPv4MaxCidrSize, RoutedNetworkIPv4MinCidrSize, RoutedIPv4NetworkCidrAutoAllocationEnabled,
                 RoutedVpcIPv4MaxCidrSize, RoutedVpcIPv4MinCidrSize, UseSystemBgpPeers
         };
@@ -162,19 +168,25 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
     @Override
     public List<Class<?>> getCommands() {
         final List<Class<?>> cmdList = new ArrayList<Class<?>>();
+        if (!RoutedNetworkVpcEnabled.value()) {
+            return cmdList;
+        }
         cmdList.add(CreateIpv4SubnetForZoneCmd.class);
         cmdList.add(DeleteIpv4SubnetForZoneCmd.class);
         cmdList.add(ListIpv4SubnetsForZoneCmd.class);
         cmdList.add(UpdateIpv4SubnetForZoneCmd.class);
         cmdList.add(DedicateIpv4SubnetForZoneCmd.class);
         cmdList.add(ReleaseDedicatedIpv4SubnetForZoneCmd.class);
+
         cmdList.add(CreateIpv4SubnetForGuestNetworkCmd.class);
         cmdList.add(ListIpv4SubnetsForGuestNetworkCmd.class);
         cmdList.add(DeleteIpv4SubnetForGuestNetworkCmd.class);
+
         cmdList.add(CreateRoutingFirewallRuleCmd.class);
         cmdList.add(ListRoutingFirewallRulesCmd.class);
         cmdList.add(UpdateRoutingFirewallRuleCmd.class);
         cmdList.add(DeleteRoutingFirewallRuleCmd.class);
+
         cmdList.add(CreateBgpPeerCmd.class);
         cmdList.add(DeleteBgpPeerCmd.class);
         cmdList.add(ListBgpPeersCmd.class);
@@ -183,6 +195,13 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         cmdList.add(ReleaseDedicatedBgpPeerCmd.class);
         cmdList.add(ChangeBgpPeersForNetworkCmd.class);
         cmdList.add(ChangeBgpPeersForVpcCmd.class);
+
+        cmdList.add(CreateASNRangeCmd.class);
+        cmdList.add(ListASNRangesCmd.class);
+        cmdList.add(DeleteASNRangeCmd.class);
+        cmdList.add(ListASNumbersCmd.class);
+        cmdList.add(ReleaseASNumberCmd.class);
+
         return cmdList;
     }
 
@@ -198,6 +217,8 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
             throw new InvalidParameterValueException("Invalid IPv4 subnet: " + subnet);
         }
 
+        checkIfRoutedNetworkVpcEnabled(zoneId);
+
         // check conflicts
         List<DataCenterIpv4GuestSubnetVO> existingSubnets = dataCenterIpv4GuestSubnetDao.listByDataCenterId(zoneId);
         checkConflicts(existingSubnets, subnet, null);
@@ -319,7 +340,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         return dataCenterIpv4GuestSubnetDao.findById(subnetId);
     }
 
-    private void checkConflicts(List<DataCenterIpv4GuestSubnetVO> existingSubnets, String newSubnet, Long ignoreSubnetId) {
+    protected void checkConflicts(List<DataCenterIpv4GuestSubnetVO> existingSubnets, String newSubnet, Long ignoreSubnetId) {
         for (DataCenterIpv4GuestSubnetVO existing : existingSubnets) {
             if ((ignoreSubnetId == null || existing.getId() != ignoreSubnetId) && NetUtils.isNetworksOverlap(existing.getSubnet(), newSubnet)) {
                 throw new InvalidParameterValueException(String.format("Existing zone subnet %s has overlap with: %s", existing.getSubnet(), newSubnet));
@@ -553,11 +574,6 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
                 response.setParentSubnet(parent.getSubnet());
                 zoneId = parent.getDataCenterId();
             }
-        } else if (subnet.getNetworkId() != null) {
-            Network network = ApiDBUtils.findNetworkById(subnet.getNetworkId());
-            if (network != null) {
-                zoneId = network.getDataCenterId();
-            }
         }
         if (zoneId != null) {
             DataCenter zone = ApiDBUtils.findZoneById(zoneId);
@@ -590,7 +606,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         return getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(vpcCidrSize, vpc.getDomainId(), vpc.getAccountId(), vpc.getZoneId());
     }
 
-    private Ipv4GuestSubnetNetworkMap getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, Long ownerDomainId, Long ownerAccountId, Long zoneId) {
+    protected Ipv4GuestSubnetNetworkMap getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, Long ownerDomainId, Long ownerAccountId, Long zoneId) {
         validateNetworkCidrSize(ownerAccountId, cidrSize);
         List<DataCenterIpv4GuestSubnetVO> subnets = getZoneSubnetsForAccount(ownerDomainId, ownerAccountId, zoneId);
         for (DataCenterIpv4GuestSubnetVO subnet : subnets) {
@@ -602,7 +618,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         return null;
     }
 
-    private Ipv4GuestSubnetNetworkMap getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, DataCenterIpv4GuestSubnetVO subnet) {
+    protected Ipv4GuestSubnetNetworkMap getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(Integer cidrSize, DataCenterIpv4GuestSubnetVO subnet) {
         Ipv4GuestSubnetNetworkMap map = ipv4GuestSubnetNetworkMapDao.findFirstAvailable(subnet.getId(), cidrSize);
         if (map != null) {
             return map;
@@ -615,7 +631,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         return null;
     }
 
-    private void getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(String networkCidr, Long ownerDomainId, Long ownerAccountId, Long zoneId) {
+    protected void getOrCreateIpv4SubnetForGuestNetworkOrVpcInternal(String networkCidr, Long ownerDomainId, Long ownerAccountId, Long zoneId) {
         Ipv4GuestSubnetNetworkMapVO subnetMap = ipv4GuestSubnetNetworkMapDao.findBySubnet(networkCidr);
         if (subnetMap != null) {
             // check if the subnet is in use
@@ -650,7 +666,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         }
     }
 
-    private DataCenterIpv4GuestSubnet getParentOfNetworkCidr(Long zoneId, String networkCidr) {
+    protected DataCenterIpv4GuestSubnet getParentOfNetworkCidr(Long zoneId, String networkCidr) {
         List<DataCenterIpv4GuestSubnetVO> existingSubnets = dataCenterIpv4GuestSubnetDao.listByDataCenterId(zoneId);
         for (DataCenterIpv4GuestSubnetVO existing : existingSubnets) {
             if (NetUtils.isNetworkAWithinNetworkB(networkCidr, existing.getSubnet())) {
@@ -689,14 +705,22 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
 
     private List<DataCenterIpv4GuestSubnetVO> getZoneSubnetsForAccount(long domainId, long accountId, long zoneId) {
         // Get dedicated guest subnets for the account
-        List<DataCenterIpv4GuestSubnetVO> subnets = dataCenterIpv4GuestSubnetDao.listByDataCenterIdAndAccountId(zoneId, accountId);
+        List<DataCenterIpv4GuestSubnetVO> subnets = new ArrayList<>();
+        subnets.addAll(dataCenterIpv4GuestSubnetDao.listByDataCenterIdAndAccountId(zoneId, accountId));
         subnets.addAll(dataCenterIpv4GuestSubnetDao.listByDataCenterIdAndDomainId(zoneId, domainId));
         // Get non-dedicated zone guest subnets for the account
         subnets.addAll(dataCenterIpv4GuestSubnetDao.listNonDedicatedByDataCenterId(zoneId));
         return subnets;
     }
 
-    private Ipv4GuestSubnetNetworkMap createIpv4SubnetFromParentSubnet(DataCenterIpv4GuestSubnet parent, Integer networkCidrSize) {
+    protected Ipv4GuestSubnetNetworkMap createIpv4SubnetFromParentSubnet(DataCenterIpv4GuestSubnet parent, Integer networkCidrSize) {
+        String networkCidr = createIpv4SubnetStringFromParentSubnet(parent, networkCidrSize);
+        // create DB record
+        Ipv4GuestSubnetNetworkMapVO subnetMap = new Ipv4GuestSubnetNetworkMapVO(parent.getId(), NetUtils.transformCidr(networkCidr), null, State.Free);
+        return ipv4GuestSubnetNetworkMapDao.persist(subnetMap);
+    }
+
+    protected String createIpv4SubnetStringFromParentSubnet(DataCenterIpv4GuestSubnet parent, Integer networkCidrSize) {
         DataCenterIpv4GuestSubnetVO subnetVO = dataCenterIpv4GuestSubnetDao.findById(parent.getId());
         if (subnetVO == null) {
             throw new InvalidParameterValueException(String.format("Invalid subnet ID: %s", parent.getId()));
@@ -733,9 +757,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         if (networkCidr == null) {
             throw new CloudRuntimeException("Failed to automatically allocate a subnet with specified cidrsize");
         }
-        // create DB record
-        Ipv4GuestSubnetNetworkMapVO subnetMap = new Ipv4GuestSubnetNetworkMapVO(parent.getId(), NetUtils.transformCidr(networkCidr), null, State.Free);
-        return ipv4GuestSubnetNetworkMapDao.persist(subnetMap);
+        return networkCidr;
     }
 
     private String getFreeNetworkCidr(List<Pair<Long, Integer>> subnetsInFreeIpRanges, int networkCidrSize) {
@@ -752,7 +774,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         return null;
     }
 
-    private Ipv4GuestSubnetNetworkMap createIpv4SubnetFromParentSubnet(DataCenterIpv4GuestSubnet parent, String networkCidr) {
+    protected Ipv4GuestSubnetNetworkMap createIpv4SubnetFromParentSubnet(DataCenterIpv4GuestSubnet parent, String networkCidr) {
         // Validate the network cidr
         if (!NetUtils.isNetworkAWithinNetworkB(networkCidr, parent.getSubnet())) {
             throw new InvalidParameterValueException(String.format("networkCidr %s is not within parent cidr: %s", networkCidr, parent.getSubnet()));
@@ -941,9 +963,11 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         }
         if (!FirewallRule.Purpose.Firewall.equals(rule.getPurpose())) {
             logger.error(String.format("Cannot apply routing firewall rule with ID: %d as purpose %s is not %s", id, rule.getPurpose(), FirewallRule.Purpose.Firewall));
+            return false;
         }
         logger.debug(String.format("Applying routing firewall rules for rule with ID: %s", rule.getUuid()));
-        List<FirewallRuleVO> rules = firewallDao.listByNetworkPurposeTrafficType(rule.getNetworkId(), rule.getPurpose(), FirewallRule.TrafficType.Egress);
+        List<FirewallRuleVO> rules = new ArrayList<>();
+        rules.addAll(firewallDao.listByNetworkPurposeTrafficType(rule.getNetworkId(), rule.getPurpose(), FirewallRule.TrafficType.Egress));
         rules.addAll(firewallDao.listByNetworkPurposeTrafficType(rule.getNetworkId(), rule.getPurpose(), FirewallRule.TrafficType.Ingress));
         return firewallManager.applyFirewallRules(rules, false, CallContext.current().getCallingAccount());
     }
@@ -1014,6 +1038,8 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         String password = createBgpPeerCmd.getPassword();
         Map<String, String> detailsStr = createBgpPeerCmd.getDetails();
 
+        checkIfRoutedNetworkVpcEnabled(zoneId);
+
         if (ObjectUtils.allNull(ip4Address, ip6Address)) {
             throw new InvalidParameterValueException("At least one of IPv4 and IPv6 address must be specified.");
         }
@@ -1398,7 +1424,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         return changeBgpPeersForNetworkInternal(network, null);
     }
 
-    private Network changeBgpPeersForNetworkInternal(Network network, List<Long> bgpPeerIds) {
+    protected Network changeBgpPeersForNetworkInternal(Network network, List<Long> bgpPeerIds) {
         final List<Long> bgpPeerIdsToBeAdded;
         if (CollectionUtils.isNotEmpty(bgpPeerIds)) {
             bgpPeerIdsToBeAdded = new ArrayList<>(bgpPeerIds);
@@ -1529,7 +1555,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
         return bgpPeerDao.listAvailableBgpPeerIdsForAccount(zoneId, owner.getDomainId(), owner.getId(), UseSystemBgpPeers.valueIn(owner.getId()));
     }
 
-    private Vpc changeBgpPeersForVpcInternal(Vpc vpc, List<Long> bgpPeerIds) {
+    protected Vpc changeBgpPeersForVpcInternal(Vpc vpc, List<Long> bgpPeerIds) {
         final List<Long> bgpPeerIdsToBeAdded;
         if (CollectionUtils.isNotEmpty(bgpPeerIds)) {
             bgpPeerIdsToBeAdded = new ArrayList<>(bgpPeerIds);
@@ -1618,4 +1644,15 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou
     public void removeBgpPeersByDomainId(long domainId) {
         bgpPeerDao.removeByDomainId(domainId);
     }
+
+    @Override
+    public Boolean isRoutedNetworkVpcEnabled(long zoneId) {
+        return RoutedNetworkVpcEnabled.valueIn(zoneId);
+    }
+
+    private void checkIfRoutedNetworkVpcEnabled(long zoneId) {
+        if (!isRoutedNetworkVpcEnabled(zoneId)) {
+            throw new InvalidParameterValueException("Routed networks and VPCs are not enabled for the zone.");
+        }
+    }
 }

server/src/test/java/com/cloud/api/ApiServerTest.java

@@ -17,6 +17,8 @@
 package com.cloud.api;
 
 import com.cloud.domain.Domain;
+import com.cloud.user.Account;
+import com.cloud.user.User;
 import com.cloud.user.UserAccount;
 import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.cloudstack.framework.config.ConfigKey;
@@ -147,4 +149,31 @@ public class ApiServerTest {
         Mockito.when(domain.getState()).thenReturn(Domain.State.Inactive);
         apiServer.forgotPassword(userAccount, domain);
     }
+
+    @Test
+    public void testVerifyApiKeyAccessAllowed() {
+        Long domainId = 1L;
+        User user = Mockito.mock(User.class);
+        Account account = Mockito.mock(Account.class);
+
+        Mockito.when(user.getApiKeyAccess()).thenReturn(true);
+        Assert.assertEquals(true, apiServer.verifyApiKeyAccessAllowed(user, account));
+        Mockito.verify(account, Mockito.never()).getApiKeyAccess();
+
+        Mockito.when(user.getApiKeyAccess()).thenReturn(false);
+        Assert.assertEquals(false, apiServer.verifyApiKeyAccessAllowed(user, account));
+        Mockito.verify(account, Mockito.never()).getApiKeyAccess();
+
+        Mockito.when(user.getApiKeyAccess()).thenReturn(null);
+        Mockito.when(account.getApiKeyAccess()).thenReturn(true);
+        Assert.assertEquals(true, apiServer.verifyApiKeyAccessAllowed(user, account));
+
+        Mockito.when(user.getApiKeyAccess()).thenReturn(null);
+        Mockito.when(account.getApiKeyAccess()).thenReturn(false);
+        Assert.assertEquals(false, apiServer.verifyApiKeyAccessAllowed(user, account));
+
+        Mockito.when(user.getApiKeyAccess()).thenReturn(null);
+        Mockito.when(account.getApiKeyAccess()).thenReturn(null);
+        Assert.assertEquals(true, apiServer.verifyApiKeyAccessAllowed(user, account));
+    }
 }

server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java

@@ -17,13 +17,18 @@
 
 package com.cloud.api.query;
 
+import com.cloud.api.ApiDBUtils;
 import com.cloud.api.query.dao.TemplateJoinDao;
+import com.cloud.api.query.dao.UserAccountJoinDao;
 import com.cloud.api.query.dao.UserVmJoinDao;
 import com.cloud.api.query.vo.EventJoinVO;
 import com.cloud.api.query.vo.TemplateJoinVO;
+import com.cloud.api.query.vo.UserAccountJoinVO;
 import com.cloud.api.query.vo.UserVmJoinVO;
 import com.cloud.dc.ClusterVO;
 import com.cloud.dc.dao.ClusterDao;
+import com.cloud.domain.DomainVO;
+import com.cloud.domain.dao.DomainDao;
 import com.cloud.event.EventVO;
 import com.cloud.event.dao.EventDao;
 import com.cloud.event.dao.EventJoinDao;
@@ -45,6 +50,7 @@ import com.cloud.user.AccountManager;
 import com.cloud.user.AccountVO;
 import com.cloud.user.User;
 import com.cloud.user.UserVO;
+import com.cloud.user.dao.AccountDao;
 import com.cloud.utils.Pair;
 import com.cloud.utils.db.EntityManager;
 import com.cloud.utils.db.Filter;
@@ -56,8 +62,11 @@ import com.cloud.vm.dao.VMInstanceDao;
 
 import org.apache.cloudstack.acl.SecurityChecker;
 import org.apache.cloudstack.api.ApiCommandResourceType;
+import org.apache.cloudstack.api.ResponseObject;
 import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd;
+import org.apache.cloudstack.api.command.admin.user.ListUsersCmd;
 import org.apache.cloudstack.api.command.admin.vm.ListAffectedVmsForStorageScopeChangeCmd;
+import org.apache.cloudstack.api.command.user.account.ListAccountsCmd;
 import org.apache.cloudstack.api.command.user.bucket.ListBucketsCmd;
 import org.apache.cloudstack.api.command.user.event.ListEventsCmd;
 import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd;
@@ -65,6 +74,7 @@ import org.apache.cloudstack.api.response.DetailOptionsResponse;
 import org.apache.cloudstack.api.response.EventResponse;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.ObjectStoreResponse;
+import org.apache.cloudstack.api.response.UserResponse;
 import org.apache.cloudstack.api.response.VirtualMachineResponse;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao;
@@ -150,6 +160,15 @@ public class QueryManagerImplTest {
     @Mock
     UserVmJoinDao userVmJoinDao;
 
+    @Mock
+    UserAccountJoinDao userAccountJoinDao;
+
+    @Mock
+    DomainDao domainDao;
+
+    @Mock
+    AccountDao accountDao;
+
     private AccountVO account;
     private UserVO user;
 
@@ -477,4 +496,79 @@ public class QueryManagerImplTest {
         Assert.assertEquals(response.getResponses().get(0).getId(), instanceUuid);
         Assert.assertEquals(response.getResponses().get(0).getName(), vmName);
     }
+
+    @Test
+    public void testSearchForUsers() {
+        ListUsersCmd cmd = Mockito.mock(ListUsersCmd.class);
+        String username = "Admin";
+        String accountName = "Admin";
+        Account.Type accountType = Account.Type.ADMIN;
+        Long domainId = 1L;
+        String apiKeyAccess = "Disabled";
+        Mockito.when(cmd.getUsername()).thenReturn(username);
+        Mockito.when(cmd.getAccountName()).thenReturn(accountName);
+        Mockito.when(cmd.getAccountType()).thenReturn(accountType);
+        Mockito.when(cmd.getDomainId()).thenReturn(domainId);
+        Mockito.when(cmd.getApiKeyAccess()).thenReturn(apiKeyAccess);
+
+        UserAccountJoinVO user = new UserAccountJoinVO();
+        DomainVO domain = Mockito.mock(DomainVO.class);
+        SearchBuilder<UserAccountJoinVO> sb = Mockito.mock(SearchBuilder.class);
+        SearchCriteria<UserAccountJoinVO> sc = Mockito.mock(SearchCriteria.class);
+        List<UserAccountJoinVO> users = new ArrayList<>();
+        Pair<List<UserAccountJoinVO>, Integer> result = new Pair<>(users, 0);
+        UserResponse response = Mockito.mock(UserResponse.class);
+
+        Mockito.when(userAccountJoinDao.createSearchBuilder()).thenReturn(sb);
+        Mockito.when(sb.entity()).thenReturn(user);
+        Mockito.when(sb.create()).thenReturn(sc);
+        Mockito.when(userAccountJoinDao.searchAndCount(any(SearchCriteria.class), any(Filter.class))).thenReturn(result);
+
+        queryManager.searchForUsers(ResponseObject.ResponseView.Restricted, cmd);
+
+        Mockito.verify(sc).setParameters("username", username);
+        Mockito.verify(sc).setParameters("accountName", accountName);
+        Mockito.verify(sc).setParameters("type", accountType);
+        Mockito.verify(sc).setParameters("domainId", domainId);
+        Mockito.verify(sc).setParameters("apiKeyAccess", false);
+        Mockito.verify(userAccountJoinDao, Mockito.times(1)).searchAndCount(
+                any(SearchCriteria.class), any(Filter.class));
+    }
+
+    @Test
+    public void testSearchForAccounts() {
+        ListAccountsCmd cmd = Mockito.mock(ListAccountsCmd.class);
+        Long domainId = 1L;
+        String accountName = "Admin";
+        Account.Type accountType = Account.Type.ADMIN;
+        String apiKeyAccess = "Enabled";
+        Mockito.when(cmd.getId()).thenReturn(null);
+        Mockito.when(cmd.getDomainId()).thenReturn(domainId);
+        Mockito.when(cmd.getSearchName()).thenReturn(accountName);
+        Mockito.when(cmd.getAccountType()).thenReturn(accountType);
+        Mockito.when(cmd.getApiKeyAccess()).thenReturn(apiKeyAccess);
+
+        DomainVO domain = Mockito.mock(DomainVO.class);
+        SearchBuilder<AccountVO> sb = Mockito.mock(SearchBuilder.class);
+        SearchCriteria<AccountVO> sc = Mockito.mock(SearchCriteria.class);
+        Pair<List<AccountVO>, Integer> uniqueAccountPair = new Pair<>(new ArrayList<>(), 0);
+        Mockito.when(domainDao.findById(domainId)).thenReturn(domain);
+        Mockito.doNothing().when(accountManager).checkAccess(account, domain);
+
+        Mockito.when(accountDao.createSearchBuilder()).thenReturn(sb);
+        Mockito.when(sb.entity()).thenReturn(account);
+        Mockito.when(sb.create()).thenReturn(sc);
+        Mockito.when(accountDao.searchAndCount(any(SearchCriteria.class), any(Filter.class))).thenReturn(uniqueAccountPair);
+
+        try (MockedStatic<ApiDBUtils> apiDBUtilsMocked = Mockito.mockStatic(ApiDBUtils.class)) {
+            queryManager.searchForAccounts(cmd);
+        }
+
+        Mockito.verify(sc).setParameters("domainId", domainId);
+        Mockito.verify(sc).setParameters("accountName", accountName);
+        Mockito.verify(sc).setParameters("type", accountType);
+        Mockito.verify(sc).setParameters("apiKeyAccess", true);
+        Mockito.verify(accountDao, Mockito.times(1)).searchAndCount(
+                any(SearchCriteria.class), any(Filter.class));
+    }
 }

server/src/test/java/com/cloud/bgp/BGPServiceImplTest.java

@@ -18,14 +18,39 @@
 
 package com.cloud.bgp;
 
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.network.Network;
+import com.cloud.network.NetworkModel;
+import com.cloud.network.dao.NetworkServiceMapDao;
+import com.cloud.network.element.VirtualRouterElement;
+import com.cloud.network.element.VpcVirtualRouterElement;
+import com.cloud.network.vpc.Vpc;
+import com.cloud.network.vpc.dao.VpcServiceMapDao;
+import com.cloud.user.AccountVO;
+import com.cloud.user.dao.AccountDao;
+import org.apache.cloudstack.network.BgpPeerVO;
+import org.apache.cloudstack.network.RoutedIpv4Manager;
+import org.apache.cloudstack.network.dao.BgpPeerDao;
+
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.InjectMocks;
-
+import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
 @RunWith(MockitoJUnitRunner.class)
 public class BGPServiceImplTest {
 
@@ -33,6 +58,24 @@ public class BGPServiceImplTest {
     @InjectMocks
     BGPServiceImpl bGPServiceImplSpy = new BGPServiceImpl();
 
+    @Mock
+    RoutedIpv4Manager routedIpv4Manager;
+
+    @Mock
+    NetworkServiceMapDao ntwkSrvcDao;
+
+    @Mock
+    NetworkModel networkModel;
+
+    @Mock
+    BgpPeerDao bgpPeerDao;
+
+    @Mock
+    AccountDao accountDao;
+
+    @Mock
+    VpcServiceMapDao vpcServiceMapDao;
+
     @Test
     public void testASNumbersOverlap() {
         Assert.assertEquals(bGPServiceImplSpy.isASNumbersOverlap(1,2,3,4), false);
@@ -42,4 +85,92 @@ public class BGPServiceImplTest {
         Assert.assertEquals(bGPServiceImplSpy.isASNumbersOverlap(1,4,2,3), true);
         Assert.assertEquals(bGPServiceImplSpy.isASNumbersOverlap(3,4,1,2), false);
     }
+
+    @Test
+    public void testApplyBgpPeersForIsolatedNetwork() throws ResourceUnavailableException {
+        Long networkId = 11L;
+        Network network = Mockito.mock(Network.class);
+        when(network.getId()).thenReturn(networkId);
+        when(network.getVpcId()).thenReturn(null);
+
+        when(routedIpv4Manager.isDynamicRoutedNetwork(network)).thenReturn(true);
+        when(ntwkSrvcDao.getProviderForServiceInNetwork(networkId, Network.Service.Gateway)).thenReturn("VirtualRouter");
+        VirtualRouterElement virtualRouterElement = Mockito.mock(VirtualRouterElement.class);
+        when(networkModel.getElementImplementingProvider("VirtualRouter")).thenReturn(virtualRouterElement);
+
+        BgpPeerVO bgpPeer1 = Mockito.mock(BgpPeerVO.class);
+        List<BgpPeerVO> bgpPeers = Arrays.asList(bgpPeer1);
+        when(bgpPeerDao.listNonRevokeByNetworkId(networkId)).thenReturn(bgpPeers);
+        doReturn(true).when(virtualRouterElement).applyBgpPeers(null, network, bgpPeers);
+
+        bGPServiceImplSpy.applyBgpPeers(network, true);
+
+        verify(virtualRouterElement).applyBgpPeers(null, network, bgpPeers);
+    }
+
+    @Test
+    public void testApplyBgpPeersForVpcTier() throws ResourceUnavailableException {
+        Long networkId = 11L;
+        Long accountId = 12L;
+        Long vpcId = 13L;
+        Long zoneId = 1L;
+        Network network = Mockito.mock(Network.class);
+        when(network.getId()).thenReturn(networkId);
+        when(network.getVpcId()).thenReturn(vpcId);
+        when(network.getAccountId()).thenReturn(accountId);
+        when(network.getDataCenterId()).thenReturn(zoneId);
+
+        when(routedIpv4Manager.isDynamicRoutedNetwork(network)).thenReturn(true);
+        when(ntwkSrvcDao.getProviderForServiceInNetwork(networkId, Network.Service.Gateway)).thenReturn("VirtualRouter");
+        VirtualRouterElement virtualRouterElement = Mockito.mock(VirtualRouterElement.class);
+        when(networkModel.getElementImplementingProvider("VirtualRouter")).thenReturn(virtualRouterElement);
+
+        when(bgpPeerDao.listNonRevokeByVpcId(vpcId)).thenReturn(new ArrayList<>());
+
+        AccountVO owner = Mockito.mock(AccountVO.class);
+        when(accountDao.findByIdIncludingRemoved(accountId)).thenReturn(owner);
+
+        Long bgpPeerId1 = 14L;
+        BgpPeerVO bgpPeer1 = Mockito.mock(BgpPeerVO.class);
+        when(bgpPeerDao.findById(bgpPeerId1)).thenReturn(bgpPeer1);
+        when(routedIpv4Manager.getBgpPeerIdsForAccount(owner, zoneId)).thenReturn(Arrays.asList(bgpPeerId1));
+
+        doReturn(true).when(virtualRouterElement).applyBgpPeers(eq(null), eq(network), any());
+
+        bGPServiceImplSpy.applyBgpPeers(network, true);
+
+        verify(virtualRouterElement).applyBgpPeers(eq(null), eq(network), any());
+    }
+
+    @Test
+    public void testApplyBgpPeersForVpcWithBgpPeers() throws ResourceUnavailableException {
+        Long accountId = 12L;
+        Long vpcId = 13L;
+        Long zoneId = 1L;
+        Vpc vpc = Mockito.mock(Vpc.class);
+        when(vpc.getId()).thenReturn(vpcId);
+        when(vpc.getAccountId()).thenReturn(accountId);
+        when(vpc.getZoneId()).thenReturn(zoneId);
+
+        when(routedIpv4Manager.isDynamicRoutedVpc(vpc)).thenReturn(true);
+        when(vpcServiceMapDao.getProviderForServiceInVpc(vpcId, Network.Service.Gateway)).thenReturn("VPCVirtualRouter");
+        VpcVirtualRouterElement vpcVirtualRouterElement = Mockito.mock(VpcVirtualRouterElement.class);
+        when(networkModel.getElementImplementingProvider("VPCVirtualRouter")).thenReturn(vpcVirtualRouterElement);
+
+        when(bgpPeerDao.listNonRevokeByVpcId(vpcId)).thenReturn(new ArrayList<>());
+
+        AccountVO owner = Mockito.mock(AccountVO.class);
+        when(accountDao.findByIdIncludingRemoved(accountId)).thenReturn(owner);
+
+        Long bgpPeerId1 = 14L;
+        BgpPeerVO bgpPeer1 = Mockito.mock(BgpPeerVO.class);
+        when(bgpPeerDao.findById(bgpPeerId1)).thenReturn(bgpPeer1);
+        when(routedIpv4Manager.getBgpPeerIdsForAccount(owner, zoneId)).thenReturn(Arrays.asList(bgpPeerId1));
+
+        doReturn(true).when(vpcVirtualRouterElement).applyBgpPeers(eq(vpc), eq(null), any());
+
+        bGPServiceImplSpy.applyBgpPeers(vpc, true);
+
+        verify(vpcVirtualRouterElement).applyBgpPeers(eq(vpc), eq(null), any());
+    }
 }

server/src/test/java/com/cloud/network/NetworkServiceImplTest.java

@@ -22,6 +22,7 @@ import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.ArgumentMatchers.nullable;
 import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.when;
@@ -29,6 +30,7 @@ import static org.mockito.Mockito.doReturn;
 
 import java.lang.reflect.Field;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
@@ -46,6 +48,7 @@ import com.cloud.user.dao.AccountDao;
 import com.cloud.utils.net.Ip;
 import com.cloud.exception.InsufficientAddressCapacityException;
 import org.apache.cloudstack.alert.AlertService;
+import org.apache.cloudstack.api.command.admin.network.CreateNetworkCmdByAdmin;
 import org.apache.cloudstack.api.command.user.address.UpdateQuarantinedIpCmd;
 import org.apache.cloudstack.api.command.user.network.CreateNetworkCmd;
 import org.apache.cloudstack.api.command.user.network.UpdateNetworkCmd;
@@ -753,13 +756,14 @@ public class NetworkServiceImplTest {
     }
 
     @Test
-    public void testCreateIpv4RoutedNetwork() throws InsufficientCapacityException, ResourceAllocationException {
+    public void testCreateIpv4RoutedNetwork() {
         registerCallContext();
         CreateNetworkCmd cmd = Mockito.mock(CreateNetworkCmd.class);
         Mockito.when(cmd.getCidrSize()).thenReturn(24);
         prepareCreateNetworkDnsMocks(cmd, Network.GuestType.Isolated, false, false, true);
         when(networkOfferingVO.getNetworkMode()).thenReturn(NetworkOffering.NetworkMode.ROUTED);
         when(networkOfferingVO.getRoutingMode()).thenReturn(NetworkOffering.RoutingMode.Static);
+        when(routedIpv4Manager.isRoutedNetworkVpcEnabled(nullable(Long.class))).thenReturn(true);
         when(routedIpv4Manager.isVirtualRouterGateway(networkOfferingVO)).thenReturn(true);
         doNothing().when(routedIpv4Manager).assignIpv4SubnetToNetwork(nullable(Network.class));
 
@@ -812,6 +816,91 @@ public class NetworkServiceImplTest {
                 null, null, null, null, null, null, null, new Pair<>(0, privateMtu), null);
     }
 
+    public void testCreateIpv4RoutedNetworkWithBgpPeersFailure1() {
+        registerCallContext();
+        CreateNetworkCmdByAdmin cmd = Mockito.mock(CreateNetworkCmdByAdmin.class);
+        Mockito.when(cmd.getCidrSize()).thenReturn(24);
+        List<Long> bgpPeerIds = Arrays.asList(11L, 12L);
+        Mockito.when(cmd.getBgpPeerIds()).thenReturn(bgpPeerIds);
+
+        prepareCreateNetworkDnsMocks(cmd, Network.GuestType.Isolated, false, true, true);
+        when(networkOfferingVO.getNetworkMode()).thenReturn(NetworkOffering.NetworkMode.ROUTED);
+        when(networkOfferingVO.getRoutingMode()).thenReturn(NetworkOffering.RoutingMode.Static);
+        when(routedIpv4Manager.isRoutedNetworkVpcEnabled(nullable(Long.class))).thenReturn(true);
+        when(routedIpv4Manager.isVirtualRouterGateway(networkOfferingVO)).thenReturn(true);
+
+        DataCenterVO zone = Mockito.mock(DataCenterVO.class);
+        when(cmd.getZoneId()).thenReturn(zoneId);
+        when(dcDao.findById(zoneId)).thenReturn(zone);
+        when(zone.getId()).thenReturn(zoneId);
+
+        try {
+            service.createGuestNetwork(cmd);
+        } catch (InsufficientCapacityException | ResourceAllocationException e) {
+            Assert.fail(String.format("failure with exception: %s", e.getMessage()));
+        } catch (InvalidParameterValueException ex) {
+            Assert.assertEquals("The BGP peers of VPC tiers will inherit from the VPC, do not add separately.", ex.getMessage());
+        }
+    }
+
+    @Test
+    public void testCreateIpv4RoutedNetworkWithBgpPeersFailure2() {
+        registerCallContext();
+        CreateNetworkCmdByAdmin cmd = Mockito.mock(CreateNetworkCmdByAdmin.class);
+        Mockito.when(cmd.getCidrSize()).thenReturn(24);
+        List<Long> bgpPeerIds = Arrays.asList(11L, 12L);
+        Mockito.when(cmd.getBgpPeerIds()).thenReturn(bgpPeerIds);
+
+        prepareCreateNetworkDnsMocks(cmd, Network.GuestType.Isolated, false, false, true);
+        when(networkOfferingVO.getNetworkMode()).thenReturn(NetworkOffering.NetworkMode.ROUTED);
+        when(networkOfferingVO.getRoutingMode()).thenReturn(NetworkOffering.RoutingMode.Static);
+        when(routedIpv4Manager.isRoutedNetworkVpcEnabled(nullable(Long.class))).thenReturn(true);
+        when(routedIpv4Manager.isVirtualRouterGateway(networkOfferingVO)).thenReturn(true);
+
+        DataCenterVO zone = Mockito.mock(DataCenterVO.class);
+        when(cmd.getZoneId()).thenReturn(zoneId);
+        when(dcDao.findById(zoneId)).thenReturn(zone);
+        when(zone.getId()).thenReturn(zoneId);
+
+        try {
+            service.createGuestNetwork(cmd);
+        } catch (InsufficientCapacityException | ResourceAllocationException e) {
+            Assert.fail(String.format("failure with exception: %s", e.getMessage()));
+        } catch (InvalidParameterValueException ex) {
+            Assert.assertEquals("The network offering does not support Dynamic routing", ex.getMessage());
+        }
+    }
+
+    @Test
+    public void testCreateIpv4RoutedNetworkWithBgpPeersFailure3() {
+        registerCallContext();
+        CreateNetworkCmdByAdmin cmd = Mockito.mock(CreateNetworkCmdByAdmin.class);
+        Mockito.when(cmd.getCidrSize()).thenReturn(24);
+        List<Long> bgpPeerIds = Arrays.asList(11L, 12L);
+        Mockito.when(cmd.getBgpPeerIds()).thenReturn(bgpPeerIds);
+
+        prepareCreateNetworkDnsMocks(cmd, Network.GuestType.Isolated, false, false, true);
+        when(networkOfferingVO.getNetworkMode()).thenReturn(NetworkOffering.NetworkMode.ROUTED);
+        when(networkOfferingVO.getRoutingMode()).thenReturn(NetworkOffering.RoutingMode.Static);
+        when(routedIpv4Manager.isRoutedNetworkVpcEnabled(nullable(Long.class))).thenReturn(true);
+        when(routedIpv4Manager.isVirtualRouterGateway(networkOfferingVO)).thenReturn(true);
+        when(routedIpv4Manager.isDynamicRoutedNetwork(networkOfferingVO)).thenReturn(true);
+        doThrow(new InvalidParameterValueException("validation error")).when(routedIpv4Manager).validateBgpPeers(nullable(Account.class), nullable(Long.class), any(List.class));
+
+        DataCenterVO zone = Mockito.mock(DataCenterVO.class);
+        when(cmd.getZoneId()).thenReturn(zoneId);
+        when(dcDao.findById(zoneId)).thenReturn(zone);
+        when(zone.getId()).thenReturn(zoneId);
+
+        try {
+            service.createGuestNetwork(cmd);
+        } catch (InsufficientCapacityException | ResourceAllocationException e) {
+            Assert.fail(String.format("failure with exception: %s", e.getMessage()));
+        } catch (InvalidParameterValueException ex) {
+            Assert.assertEquals("validation error", ex.getMessage());
+        }
+    }
+
     @Test
     public void testCheckAndUpdateNetworkResetSuccess() {
         NetworkVO networkVO = new NetworkVO();

server/src/test/java/com/cloud/network/element/VirtualRouterElementTest.java

@@ -65,6 +65,7 @@ import com.cloud.network.router.VirtualRouter;
 import com.cloud.network.router.VirtualRouter.RedundantState;
 import com.cloud.network.router.VpcVirtualNetworkApplianceManagerImpl;
 import com.cloud.network.rules.dao.PortForwardingRulesDao;
+import com.cloud.network.vpc.VpcVO;
 import com.cloud.offering.NetworkOffering;
 import com.cloud.offerings.NetworkOfferingVO;
 import com.cloud.offerings.dao.NetworkOfferingDao;
@@ -98,7 +99,11 @@ import com.cloud.vm.dao.UserVmDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.network.BgpPeerVO;
 import org.apache.cloudstack.network.router.deployment.RouterDeploymentDefinitionBuilder;
+import org.apache.cloudstack.network.topology.NetworkTopology;
+import org.apache.cloudstack.network.topology.NetworkTopologyContext;
+import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -111,6 +116,7 @@ import org.mockito.junit.MockitoJUnitRunner;
 import org.mockito.stubbing.Answer;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -122,6 +128,7 @@ import static org.mockito.ArgumentMatchers.anyList;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
@@ -178,6 +185,7 @@ public class VirtualRouterElementTest {
     @Mock private ResourceManager _resourceMgr;
     @Mock private UserVmManager _userVmMgr;
     @Mock private VirtualMachineManager _itMgr;
+    @Mock private NetworkTopologyContext networkTopologyContext;
 
     @InjectMocks
     private RouterDeploymentDefinitionBuilder routerDeploymentDefinitionBuilder;
@@ -517,4 +525,52 @@ public class VirtualRouterElementTest {
         assertTrue(counterNames.contains(AutoScaleCounterType.Memory.getName()));
         assertTrue(counterNames.contains(AutoScaleCounterType.VirtualRouter.getName()));
     }
+
+    @Test
+    public void testApplyBgpPeersForVpc() throws ResourceUnavailableException {
+        List<BgpPeerVO> bgpPeers = Mockito.mock(List.class);
+        VpcVO vpc = Mockito.mock(VpcVO.class);
+
+        DomainRouterVO router = Mockito.mock(DomainRouterVO.class);
+        when(router.getState()).thenReturn(VirtualMachine.State.Running);
+        long zoneId = 10L;
+        long vpcId = 11L;
+        when(vpc.getId()).thenReturn(vpcId);
+        when(vpc.getZoneId()).thenReturn(zoneId);
+        when(_routerDao.listByVpcId(vpcId)).thenReturn(Arrays.asList(router));
+        DataCenterVO dc = Mockito.mock(DataCenterVO.class);
+        when(_dcDao.findById(zoneId)).thenReturn(dc);
+        NetworkTopology networkTopology = Mockito.mock(NetworkTopology.class);
+        when(networkTopologyContext.retrieveNetworkTopology(dc)).thenReturn(networkTopology);
+        doReturn(true).when(networkTopology).applyBgpPeers(any(), any(), any());
+
+        boolean result = virtualRouterElement.applyBgpPeers(vpc, null, bgpPeers);
+
+        Assert.assertTrue(result);
+        verify(networkTopology).applyBgpPeers(any(), any(), any());
+    }
+
+    @Test
+    public void testApplyBgpPeersForNetwork() throws ResourceUnavailableException {
+        List<BgpPeerVO> bgpPeers = Mockito.mock(List.class);
+        NetworkVO network = Mockito.mock(NetworkVO.class);
+
+        DomainRouterVO router = Mockito.mock(DomainRouterVO.class);
+        when(router.getState()).thenReturn(VirtualMachine.State.Running);
+        long zoneId = 10L;
+        long networkId = 11L;
+        when(network.getId()).thenReturn(networkId);
+        when(network.getDataCenterId()).thenReturn(zoneId);
+        when(_routerDao.listByNetworkAndRole(networkId, VirtualRouter.Role.VIRTUAL_ROUTER)).thenReturn(Arrays.asList(router));
+        DataCenterVO dc = Mockito.mock(DataCenterVO.class);
+        when(_dcDao.findById(zoneId)).thenReturn(dc);
+        NetworkTopology networkTopology = Mockito.mock(NetworkTopology.class);
+        when(networkTopologyContext.retrieveNetworkTopology(dc)).thenReturn(networkTopology);
+        doReturn(true).when(networkTopology).applyBgpPeers(any(), any(), any());
+
+        boolean result = virtualRouterElement.applyBgpPeers(null, network, bgpPeers);
+
+        Assert.assertTrue(result);
+        verify(networkTopology).applyBgpPeers(any(), any(), any());
+    }
 }

server/src/test/java/com/cloud/network/router/CommandSetupHelperTest.java

@@ -17,12 +17,15 @@
 package com.cloud.network.router;
 
 import com.cloud.agent.api.Command;
+import com.cloud.agent.api.routing.SetBgpPeersCommand;
 import com.cloud.agent.api.routing.VmDataCommand;
 import com.cloud.agent.manager.Commands;
 import com.cloud.configuration.ConfigurationManager;
+import com.cloud.dc.ASNumberVO;
 import com.cloud.dc.DataCenter;
 import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.VlanVO;
+import com.cloud.dc.dao.ASNumberDao;
 import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.dc.dao.VlanDao;
 import com.cloud.network.NetworkModel;
@@ -43,6 +46,8 @@ import com.cloud.utils.net.Ip;
 import com.cloud.vm.NicVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.dao.NicDao;
+import org.apache.cloudstack.network.BgpPeerVO;
+import org.apache.cloudstack.network.dao.BgpPeerDetailsDao;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -55,10 +60,13 @@ import org.mockito.junit.MockitoJUnitRunner;
 import org.springframework.test.util.ReflectionTestUtils;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.mockito.Mockito.when;
+
 @RunWith(MockitoJUnitRunner.class)
 public class CommandSetupHelperTest {
 
@@ -88,6 +96,10 @@ public class CommandSetupHelperTest {
     RouterControlHelper routerControlHelper;
     @Mock
     DataCenterDao dcDao;
+    @Mock
+    ASNumberDao asNumberDao;
+    @Mock
+    BgpPeerDetailsDao bgpPeerDetailsDao;
 
     @Before
     public void setUp() {
@@ -173,24 +185,90 @@ public class CommandSetupHelperTest {
         VpcVO vpc = new VpcVO();
         DataCenterVO dc = new DataCenterVO(1L, null, null, null, null, null, null, null, null, null, DataCenter.NetworkType.Advanced, null, null);
 
-        Mockito.when(router.getId()).thenReturn(14L);
-        Mockito.when(router.getDataCenterId()).thenReturn(4L);
-        Mockito.when(nicDao.listByVmId(ArgumentMatchers.anyLong())).thenReturn(List.of(nicVO));
-        Mockito.when(networkDao.findById(ArgumentMatchers.anyLong())).thenReturn(networkVO);
-        Mockito.when(ipAddressDao.listByAssociatedVpc(ArgumentMatchers.anyLong(), ArgumentMatchers.nullable(Boolean.class))).thenReturn(userIps);
-        Mockito.when(vlanDao.findById(ArgumentMatchers.anyLong())).thenReturn(vlanVO);
-        Mockito.when(networkModel.getNetworkRate(ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong())).thenReturn(1200);
-        Mockito.when(networkModel.getNetwork(ArgumentMatchers.anyLong())).thenReturn(networkVO);
-        Mockito.when(networkOfferingDao.findById(ArgumentMatchers.anyLong())).thenReturn(networkOfferingVO);
-        Mockito.when(configurationManager.getNetworkOfferingNetworkRate(ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong())).thenReturn(1200);
-        Mockito.when(networkModel.isSecurityGroupSupportedInNetwork(networkVO)).thenReturn(false);
-        Mockito.when(networkOfferingDetailsDao.getNtwkOffDetails(ArgumentMatchers.anyLong())).thenReturn(details);
-        Mockito.when(networkDetailsDao.findDetail(ArgumentMatchers.anyLong(), ArgumentMatchers.anyString())).thenReturn(null);
-        Mockito.when(vpcDao.findById(ArgumentMatchers.anyLong())).thenReturn(vpc);
-        Mockito.when(routerControlHelper.getRouterControlIp(ArgumentMatchers.anyLong())).thenReturn("10.1.11.101");
-        Mockito.when(dcDao.findById(ArgumentMatchers.anyLong())).thenReturn(dc);
+        when(router.getId()).thenReturn(14L);
+        when(router.getDataCenterId()).thenReturn(4L);
+        when(nicDao.listByVmId(ArgumentMatchers.anyLong())).thenReturn(List.of(nicVO));
+        when(networkDao.findById(ArgumentMatchers.anyLong())).thenReturn(networkVO);
+        when(ipAddressDao.listByAssociatedVpc(ArgumentMatchers.anyLong(), ArgumentMatchers.nullable(Boolean.class))).thenReturn(userIps);
+        when(vlanDao.findById(ArgumentMatchers.anyLong())).thenReturn(vlanVO);
+        when(networkModel.getNetworkRate(ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong())).thenReturn(1200);
+        when(networkModel.getNetwork(ArgumentMatchers.anyLong())).thenReturn(networkVO);
+        when(networkOfferingDao.findById(ArgumentMatchers.anyLong())).thenReturn(networkOfferingVO);
+        when(configurationManager.getNetworkOfferingNetworkRate(ArgumentMatchers.anyLong(), ArgumentMatchers.anyLong())).thenReturn(1200);
+        when(networkModel.isSecurityGroupSupportedInNetwork(networkVO)).thenReturn(false);
+        when(networkOfferingDetailsDao.getNtwkOffDetails(ArgumentMatchers.anyLong())).thenReturn(details);
+        when(networkDetailsDao.findDetail(ArgumentMatchers.anyLong(), ArgumentMatchers.anyString())).thenReturn(null);
+        when(vpcDao.findById(ArgumentMatchers.anyLong())).thenReturn(vpc);
+        when(routerControlHelper.getRouterControlIp(ArgumentMatchers.anyLong())).thenReturn("10.1.11.101");
+        when(dcDao.findById(ArgumentMatchers.anyLong())).thenReturn(dc);
 
         commandSetupHelper.createVpcAssociatePublicIPCommands(router, pubIpList, commands, vlanMacAddress);
         Assert.assertEquals(2, commands.size());
     }
+
+    @Test
+    public void testCreateBgpPeersCommandsForNetwork() {
+        BgpPeerVO bgpPeer1 = Mockito.mock(BgpPeerVO.class);
+        BgpPeerVO bgpPeer2 = Mockito.mock(BgpPeerVO.class);
+        List<BgpPeerVO> bgpPeers = Arrays.asList(bgpPeer1, bgpPeer2);
+        Commands cmds = new Commands(Command.OnError.Stop);
+        VirtualRouter router = Mockito.mock(VirtualRouter.class);
+        NetworkVO network = Mockito.mock(NetworkVO.class);
+
+        long zoneId = 10L;
+        long networkId = 11L;
+        when(router.getDataCenterId()).thenReturn(zoneId);
+        when(router.getVpcId()).thenReturn(null);
+        when(network.getId()).thenReturn(networkId);
+        ASNumberVO asNumberVO = Mockito.mock(ASNumberVO.class);
+        when(asNumberDao.findByZoneAndNetworkId(zoneId, networkId)).thenReturn(asNumberVO);
+        DataCenterVO dc = Mockito.mock(DataCenterVO.class);
+        when(dcDao.findById(zoneId)).thenReturn(dc);
+        when(dc.getNetworkType()).thenReturn(DataCenter.NetworkType.Advanced);
+
+        commandSetupHelper.createBgpPeersCommands(bgpPeers, router, cmds, network);
+
+        Assert.assertEquals(1, cmds.size());
+        Command cmd = cmds.toCommands()[0];
+        Assert.assertTrue(cmd instanceof SetBgpPeersCommand);
+        Assert.assertEquals(2, ((SetBgpPeersCommand) cmd).getBpgPeers().length);
+    }
+
+    @Test
+    public void testCreateBgpPeersCommandsForVpc() {
+        BgpPeerVO bgpPeer1 = Mockito.mock(BgpPeerVO.class);
+        BgpPeerVO bgpPeer2 = Mockito.mock(BgpPeerVO.class);
+        List<BgpPeerVO> bgpPeers = Arrays.asList(bgpPeer1, bgpPeer2);
+        Commands cmds = new Commands(Command.OnError.Stop);
+        VirtualRouter router = Mockito.mock(VirtualRouter.class);
+        NetworkVO network = Mockito.mock(NetworkVO.class);
+
+        long zoneId = 10L;
+        long vpcId = 11L;
+        when(router.getDataCenterId()).thenReturn(zoneId);
+        when(router.getVpcId()).thenReturn(vpcId);
+        ASNumberVO asNumberVO = Mockito.mock(ASNumberVO.class);
+        when(asNumberDao.findByZoneAndVpcId(zoneId, vpcId)).thenReturn(asNumberVO);
+
+        long networkOfferingId = 12L;
+        NetworkOfferingVO offering = Mockito.mock(NetworkOfferingVO.class);
+        when(networkOfferingDao.findByIdIncludingRemoved(networkOfferingId)).thenReturn(offering);
+        when(offering.getRoutingMode()).thenReturn(NetworkOffering.RoutingMode.Dynamic);
+        NetworkVO network1 = Mockito.mock(NetworkVO.class);
+        when(network1.getNetworkOfferingId()).thenReturn(networkOfferingId);
+        NetworkVO network2 = Mockito.mock(NetworkVO.class);
+        when(network2.getNetworkOfferingId()).thenReturn(networkOfferingId);
+        when(networkDao.listByVpc(vpcId)).thenReturn(Arrays.asList(network1, network2));
+
+        DataCenterVO dc = Mockito.mock(DataCenterVO.class);
+        when(dcDao.findById(zoneId)).thenReturn(dc);
+        when(dc.getNetworkType()).thenReturn(DataCenter.NetworkType.Advanced);
+
+        commandSetupHelper.createBgpPeersCommands(bgpPeers, router, cmds, network);
+
+        Assert.assertEquals(1, cmds.size());
+        Command cmd = cmds.toCommands()[0];
+        Assert.assertTrue(cmd instanceof SetBgpPeersCommand);
+        Assert.assertEquals(4, ((SetBgpPeersCommand) cmd).getBpgPeers().length);
+    }
 }

server/src/test/java/com/cloud/network/router/VirtualNetworkApplianceManagerImplTest.java

@@ -19,7 +19,10 @@ package com.cloud.network.router;
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.CheckS2SVpnConnectionsAnswer;
 import com.cloud.agent.api.CheckS2SVpnConnectionsCommand;
+import com.cloud.agent.api.Command;
+import com.cloud.agent.manager.Commands;
 import com.cloud.alert.AlertManager;
+import com.cloud.bgp.BGPService;
 import com.cloud.cluster.dao.ManagementServerHostDao;
 import com.cloud.dc.dao.ClusterDao;
 import com.cloud.dc.dao.DataCenterDao;
@@ -30,6 +33,8 @@ import com.cloud.host.HostVO;
 import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor;
+import com.cloud.network.Network;
+import com.cloud.network.NetworkModel;
 import com.cloud.network.Site2SiteVpnConnection;
 import com.cloud.network.dao.FirewallRulesDao;
 import com.cloud.network.dao.IPAddressDao;
@@ -37,6 +42,7 @@ import com.cloud.network.dao.LoadBalancerDao;
 import com.cloud.network.dao.LoadBalancerVMMapDao;
 import com.cloud.network.dao.MonitoringServiceDao;
 import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.dao.NetworkVO;
 import com.cloud.network.dao.OpRouterMonitorServiceDao;
 import com.cloud.network.dao.PhysicalNetworkServiceProviderDao;
 import com.cloud.network.dao.RemoteAccessVpnDao;
@@ -49,6 +55,8 @@ import com.cloud.network.dao.UserIpv6AddressDao;
 import com.cloud.network.dao.VirtualRouterProviderDao;
 import com.cloud.network.dao.VpnUserDao;
 import com.cloud.network.rules.dao.PortForwardingRulesDao;
+import com.cloud.network.vpc.VpcVO;
+import com.cloud.network.vpc.dao.VpcDao;
 import com.cloud.network.vpn.Site2SiteVpnManager;
 import com.cloud.offerings.dao.NetworkOfferingDao;
 import com.cloud.service.dao.ServiceOfferingDao;
@@ -69,12 +77,15 @@ import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.UserVmDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.network.BgpPeer;
+import org.apache.cloudstack.network.RoutedIpv4Manager;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.junit.MockitoJUnitRunner;
 
 import java.util.ArrayList;
@@ -233,6 +244,20 @@ public class VirtualNetworkApplianceManagerImplTest {
     @InjectMocks
     private VirtualNetworkApplianceManagerImpl virtualNetworkApplianceManagerImpl;
 
+    @Mock
+    private NetworkModel _networkModel;
+
+    @Mock
+    private RoutedIpv4Manager routedIpv4Manager;
+
+    @Mock
+    private CommandSetupHelper _commandSetupHelper;
+
+    @Mock
+    private VpcDao _vpcDao;
+
+    @Mock
+    private BGPService bgpService;
 
     //    @InjectMocks
     //    private VirtualNetworkApplianceManagerImpl virtualNetworkApplianceManagerImpl;
@@ -328,4 +353,42 @@ public class VirtualNetworkApplianceManagerImplTest {
         result = virtualNetworkApplianceManagerImpl.checkLogrotateTimerPattern(foo);
         Assert.assertTrue(result);
     }
+
+    @Test
+    public void testFinalizeNetworkRulesForNetwork() {
+        Long guestNetworkId = 10L;
+        Commands cmds = new Commands(Command.OnError.Stop);
+        DomainRouterVO router = Mockito.mock(DomainRouterVO.class);
+
+        NetworkVO network = Mockito.mock(NetworkVO.class);
+        when(_networkDao.findById(guestNetworkId)).thenReturn(network);
+        when(network.getVpcId()).thenReturn(null);
+        when(routedIpv4Manager.isDynamicRoutedNetwork(network)).thenReturn(true);
+        List<? extends BgpPeer> bgpPeers = Mockito.mock(List.class);
+        doReturn(bgpPeers).when(bgpService).getBgpPeersForNetwork(network);
+        virtualNetworkApplianceManagerImpl.finalizeNetworkRulesForNetwork(cmds, router, Network.Provider.VirtualRouter, guestNetworkId);
+
+        Mockito.verify(_commandSetupHelper).createBgpPeersCommands(bgpPeers, router, cmds, network);
+    }
+
+    @Test
+    public void testFinalizeNetworkRulesForVpcNetwork() {
+        Long guestNetworkId = 10L;
+        Long vpcId = 11L;
+        Commands cmds = new Commands(Command.OnError.Stop);
+        DomainRouterVO router = Mockito.mock(DomainRouterVO.class);
+
+        NetworkVO network = Mockito.mock(NetworkVO.class);
+        when(_networkDao.findById(guestNetworkId)).thenReturn(network);
+        when(network.getVpcId()).thenReturn(vpcId);
+        VpcVO vpc = Mockito.mock(VpcVO.class);
+        when(_vpcDao.findById(vpcId)).thenReturn(vpc);
+        when(routedIpv4Manager.isDynamicRoutedVpc(vpc)).thenReturn(true);
+        List<? extends BgpPeer> bgpPeers = Mockito.mock(List.class);
+        doReturn(bgpPeers).when(bgpService).getBgpPeersForVpc(vpc);
+
+        virtualNetworkApplianceManagerImpl.finalizeNetworkRulesForNetwork(cmds, router, Network.Provider.VirtualRouter, guestNetworkId);
+
+        Mockito.verify(_commandSetupHelper).createBgpPeersCommands(bgpPeers, router, cmds, network);
+    }
 }

server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java

@@ -73,6 +73,7 @@ import org.apache.cloudstack.api.command.user.vpc.UpdateVPCCmd;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
 import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.network.Ipv4GuestSubnetNetworkMap;
 import org.apache.cloudstack.network.RoutedIpv4Manager;
 import org.junit.After;
 import org.junit.Assert;
@@ -87,6 +88,7 @@ import org.springframework.test.util.ReflectionTestUtils;
 
 import java.lang.reflect.Field;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -98,6 +100,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyMap;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -541,6 +544,30 @@ public class VpcManagerImplTest {
         verify(routedIpv4Manager).getOrCreateIpv4SubnetForVpc(any(), anyString());
     }
 
+    @Test
+    public void testCreateRoutedVpcWithDynamicRouting() {
+        mockVpcDnsResources(true, false);
+        VpcVO vpc = Mockito.mock(VpcVO.class);
+        Mockito.when(vpcDao.persist(any(), anyMap())).thenReturn(vpc);
+        Mockito.when(vpc.getUuid()).thenReturn("uuid");
+        doReturn(true).when(routedIpv4Manager).isRoutedVpc(any());
+        doReturn(true).when(routedIpv4Manager).isVpcVirtualRouterGateway(vpcOfferingVO);
+        doReturn(true).when(routedIpv4Manager).isDynamicRoutedVpc(vpcOfferingVO);
+        Ipv4GuestSubnetNetworkMap ipv4GuestSubnetNetworkMap = Mockito.mock(Ipv4GuestSubnetNetworkMap.class);
+        doReturn(ipv4GuestSubnetNetworkMap).when(routedIpv4Manager).getOrCreateIpv4SubnetForVpc(any(), anyInt());
+        List<Long> bgpPeerIds = Arrays.asList(11L, 12L);
+        try {
+            doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc);
+            manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, null, vpcDomain,
+                    ip4Dns[0], ip4Dns[1], null, null, true, 1500, 24, null, bgpPeerIds);
+        } catch (ResourceAllocationException e) {
+            Assert.fail(String.format("failure with exception: %s", e.getMessage()));
+        }
+
+        verify(routedIpv4Manager).getOrCreateIpv4SubnetForVpc(any(), anyInt());
+        verify(routedIpv4Manager).validateBgpPeers(any(), any(), any());
+    }
+
     @Test
     public void validateVpcPrivateGatewayAclIdTestNullAclVoThrowsInvalidParameterValueException() {
         Mockito.doReturn(null).when(networkACLDaoMock).findById(aclId);

server/src/test/java/com/cloud/storage/StorageManagerImplTest.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.storage;
 
+import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -42,6 +43,7 @@ import com.cloud.vm.dao.VMInstanceDao;
 
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.framework.config.ConfigDepot;
+import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao;
 import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand;
@@ -756,4 +758,81 @@ public class StorageManagerImplTest {
         String failureReason = storageManagerImpl.getStoragePoolMountFailureReason(error);
         Assert.assertEquals(failureReason, "An incorrect mount option was specified");
     }
+
+    private void overrideDefaultConfigValue(final ConfigKey configKey, final String name, final Object o) throws IllegalAccessException, NoSuchFieldException {
+        Field f = ConfigKey.class.getDeclaredField(name);
+        f.setAccessible(true);
+        f.set(configKey, o);
+    }
+
+    private Long testCheckPoolforSpaceForResizeSetup(StoragePoolVO pool, Long allocatedSizeWithTemplate) {
+        Long poolId = 10L;
+        Long zoneId = 2L;
+
+        Long capacityBytes = (long) (allocatedSizeWithTemplate / Double.valueOf(CapacityManager.StorageAllocatedCapacityDisableThreshold.defaultValue())
+                        / Double.valueOf(CapacityManager.StorageOverprovisioningFactor.defaultValue()));
+        Long maxAllocatedSizeForResize = (long) (capacityBytes * Double.valueOf(CapacityManager.StorageOverprovisioningFactor.defaultValue())
+                * Double.valueOf(CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.defaultValue()));
+
+        System.out.println("maxAllocatedSizeForResize = " + maxAllocatedSizeForResize);
+        System.out.println("allocatedSizeWithTemplate = " + allocatedSizeWithTemplate);
+
+        Mockito.when(pool.getId()).thenReturn(poolId);
+        Mockito.when(pool.getCapacityBytes()).thenReturn(capacityBytes);
+        Mockito.when(pool.getDataCenterId()).thenReturn(zoneId);
+        Mockito.when(storagePoolDao.findById(poolId)).thenReturn(pool);
+        Mockito.when(pool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem);
+
+        return maxAllocatedSizeForResize - allocatedSizeWithTemplate;
+    }
+
+    @Test
+    public void testCheckPoolforSpaceForResize1() {
+        StoragePoolVO pool = Mockito.mock(StoragePoolVO.class);
+        Long allocatedSizeWithTemplate = 100L * 1024 * 1024 * 1024;
+
+        Long maxAskingSize = testCheckPoolforSpaceForResizeSetup(pool, allocatedSizeWithTemplate);
+        Long totalAskingSize = maxAskingSize / 2;
+
+        boolean result = storageManagerImpl.checkPoolforSpace(pool, allocatedSizeWithTemplate, totalAskingSize, false);
+        Assert.assertFalse(result);
+    }
+
+    @Test
+    public void testCheckPoolforSpaceForResize2() {
+        StoragePoolVO pool = Mockito.mock(StoragePoolVO.class);
+        Long allocatedSizeWithTemplate = 100L * 1024 * 1024 * 1024;
+
+        Long maxAskingSize = testCheckPoolforSpaceForResizeSetup(pool, allocatedSizeWithTemplate);
+        Long totalAskingSize = maxAskingSize / 2;
+
+        boolean result = storageManagerImpl.checkPoolforSpace(pool, allocatedSizeWithTemplate, totalAskingSize, true);
+        Assert.assertFalse(result);
+    }
+
+    @Test
+    public void testCheckPoolforSpaceForResize3() throws NoSuchFieldException, IllegalAccessException {
+        StoragePoolVO pool = Mockito.mock(StoragePoolVO.class);
+        Long allocatedSizeWithTemplate = 100L * 1024 * 1024 * 1024;
+
+        Long maxAskingSize = testCheckPoolforSpaceForResizeSetup(pool, allocatedSizeWithTemplate);
+        Long totalAskingSize = maxAskingSize + 1;
+        overrideDefaultConfigValue(StorageManagerImpl.AllowVolumeReSizeBeyondAllocation, "_defaultValue", "true");
+
+        boolean result = storageManagerImpl.checkPoolforSpace(pool, allocatedSizeWithTemplate, totalAskingSize, true);
+        Assert.assertFalse(result);
+    }
+
+    @Test
+    public void testCheckPoolforSpaceForResize4() throws NoSuchFieldException, IllegalAccessException {
+        StoragePoolVO pool = Mockito.mock(StoragePoolVO.class);
+        Long allocatedSizeWithTemplate = 100L * 1024 * 1024 * 1024;
+
+        Long maxAskingSize = testCheckPoolforSpaceForResizeSetup(pool, allocatedSizeWithTemplate);
+        Long totalAskingSize = maxAskingSize / 2;
+        overrideDefaultConfigValue(StorageManagerImpl.AllowVolumeReSizeBeyondAllocation, "_defaultValue", "true");
+
+        boolean result = storageManagerImpl.checkPoolforSpace(pool, allocatedSizeWithTemplate, totalAskingSize, true);
+        Assert.assertTrue(result);
+    }
 }

server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java

@@ -19,10 +19,13 @@ package com.cloud.storage;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.nullable;
 import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
@@ -38,14 +41,17 @@ import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 
+import com.cloud.server.ManagementService;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
 import org.apache.cloudstack.api.command.user.volume.CheckAndRepairVolumeCmd;
 import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd;
 import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd;
 import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd;
+import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd;
 import org.apache.cloudstack.backup.dao.BackupDao;
 import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
@@ -85,6 +91,7 @@ import org.mockito.junit.MockitoJUnitRunner;
 import org.springframework.test.util.ReflectionTestUtils;
 
 import com.cloud.api.query.dao.ServiceOfferingJoinDao;
+import com.cloud.configuration.ConfigurationManager;
 import com.cloud.configuration.Resource;
 import com.cloud.configuration.Resource.ResourceType;
 import com.cloud.dc.DataCenterVO;
@@ -210,6 +217,8 @@ public class VolumeApiServiceImplTest {
     private StoragePool storagePoolMock;
     private long storagePoolMockId = 1;
     @Mock
+    private DiskOfferingVO diskOfferingMock;
+    @Mock
     private DiskOfferingVO newDiskOfferingMock;
 
     @Mock
@@ -238,10 +247,20 @@ public class VolumeApiServiceImplTest {
     @Mock
     private StorageManager storageMgr;
 
+    @Mock
+    private ConfigurationManager _configMgr;
+
+    @Mock
+    private VolumeOrchestrationService _volumeMgr;
+
+    @Mock
+    private ManagementService managementService;
+
     private long accountMockId = 456l;
     private long volumeMockId = 12313l;
     private long vmInstanceMockId = 1123l;
     private long volumeSizeMock = 456789921939l;
+    private long newVolumeSizeMock = 456789930000l;
     private static long imageStoreId = 10L;
 
     private String projectMockUuid = "projectUuid";
@@ -250,6 +269,7 @@ public class VolumeApiServiceImplTest {
     private long projectMockAccountId = 132329390L;
 
     private long diskOfferingMockId = 100203L;
+    private long newDiskOfferingMockId = 100204L;
 
     private long offeringMockId = 31902L;
 
@@ -1820,4 +1840,92 @@ public class VolumeApiServiceImplTest {
 
         volumeApiServiceImpl.validationsForCheckVolumeOperation(volume);
     }
+
+    private void testResizeVolumeSetup() throws ExecutionException, InterruptedException {
+        Long poolId = 11L;
+
+        when(volumeDaoMock.findById(volumeMockId)).thenReturn(volumeVoMock);
+        when(volumeVoMock.getId()).thenReturn(volumeMockId);
+        when(volumeDaoMock.getHypervisorType(volumeMockId)).thenReturn(HypervisorType.KVM);
+        when(volumeVoMock.getState()).thenReturn(Volume.State.Ready);
+        when(volumeVoMock.getDiskOfferingId()).thenReturn(diskOfferingMockId);
+        when(_diskOfferingDao.findById(diskOfferingMockId)).thenReturn(diskOfferingMock);
+        when(_diskOfferingDao.findById(newDiskOfferingMockId)).thenReturn(newDiskOfferingMock);
+        when(newDiskOfferingMock.getRemoved()).thenReturn(null);
+        when(diskOfferingMock.getDiskSizeStrictness()).thenReturn(false);
+        when(newDiskOfferingMock.getDiskSizeStrictness()).thenReturn(false);
+        when(volumeVoMock.getInstanceId()).thenReturn(null);
+        when(volumeVoMock.getVolumeType()).thenReturn(Type.DATADISK);
+        when(newDiskOfferingMock.getDiskSize()).thenReturn(newVolumeSizeMock);
+
+        VolumeInfo volInfo = Mockito.mock(VolumeInfo.class);
+        when(volumeDataFactoryMock.getVolume(volumeMockId)).thenReturn(volInfo);
+        DataStore dataStore = Mockito.mock(DataStore.class);
+        when((volInfo.getDataStore())).thenReturn(dataStore);
+
+        when(volumeVoMock.getPoolId()).thenReturn(poolId);
+        StoragePoolVO storagePool = Mockito.mock(StoragePoolVO.class);
+        when(primaryDataStoreDaoMock.findById(poolId)).thenReturn(storagePool);
+
+        Mockito.lenient().doReturn(asyncCallFutureVolumeapiResultMock).when(volumeServiceMock).resize(any(VolumeInfo.class));
+        Mockito.doReturn(Mockito.mock(VolumeApiResult.class)).when(asyncCallFutureVolumeapiResultMock).get();
+    }
+
+    @Test
+    public void testResizeVolumeWithEnoughCapacity() throws ResourceAllocationException, ExecutionException, InterruptedException {
+        ResizeVolumeCmd cmd = new ResizeVolumeCmd();
+        ReflectionTestUtils.setField(cmd, "id", volumeMockId);
+        ReflectionTestUtils.setField(cmd, "newDiskOfferingId", newDiskOfferingMockId);
+
+        testResizeVolumeSetup();
+
+        when(storageMgr.storagePoolHasEnoughSpaceForResize(any(), nullable(Long.class), nullable(Long.class))).thenReturn(true);
+
+        try (MockedStatic<UsageEventUtils> ignored = Mockito.mockStatic(UsageEventUtils.class)) {
+            volumeApiServiceImpl.resizeVolume(cmd);
+
+            verify(volumeServiceMock).resize(any(VolumeInfo.class));
+        }
+    }
+
+    @Test
+    public void testResizeVolumeWithoutEnoughCapacity() throws ResourceAllocationException, ExecutionException, InterruptedException {
+        ResizeVolumeCmd cmd = new ResizeVolumeCmd();
+        ReflectionTestUtils.setField(cmd, "id", volumeMockId);
+        ReflectionTestUtils.setField(cmd, "newDiskOfferingId", newDiskOfferingMockId);
+        ReflectionTestUtils.setField(cmd, "autoMigrate", true);
+
+        testResizeVolumeSetup();
+
+        when(storageMgr.storagePoolHasEnoughSpaceForResize(any(), nullable(Long.class), nullable(Long.class))).thenReturn(false).thenReturn(true);
+        StoragePoolVO suitableStoragePool = Mockito.mock(StoragePoolVO.class);
+        Pair<List<? extends StoragePool>, List<? extends StoragePool>> poolsPair = new Pair<>(Arrays.asList(suitableStoragePool), Arrays.asList(suitableStoragePool));
+        when(managementService.listStoragePoolsForSystemMigrationOfVolume(anyLong(), anyLong(), anyLong(), anyLong(), anyLong(), anyBoolean(), anyBoolean())).thenReturn(poolsPair);
+        doReturn(volumeInfoMock).when(volumeApiServiceImpl).migrateVolume(any());
+        when(volumeInfoMock.getId()).thenReturn(volumeMockId);
+
+        try (MockedStatic<UsageEventUtils> ignored = Mockito.mockStatic(UsageEventUtils.class)) {
+            volumeApiServiceImpl.resizeVolume(cmd);
+
+            verify(volumeApiServiceImpl).migrateVolume(any());
+            verify(volumeServiceMock).resize(any(VolumeInfo.class));
+        }
+    }
+
+    @Test
+    public void testResizeVolumeInAllocateState() throws ResourceAllocationException, ExecutionException, InterruptedException {
+        ResizeVolumeCmd cmd = new ResizeVolumeCmd();
+        ReflectionTestUtils.setField(cmd, "id", volumeMockId);
+        ReflectionTestUtils.setField(cmd, "newDiskOfferingId", newDiskOfferingMockId);
+
+        testResizeVolumeSetup();
+
+        when(volumeVoMock.getState()).thenReturn(Volume.State.Allocated);
+
+        try (MockedStatic<UsageEventUtils> ignored = Mockito.mockStatic(UsageEventUtils.class)) {
+            volumeApiServiceImpl.resizeVolume(cmd);
+
+            verify(volumeServiceMock, times(0)).resize(any(VolumeInfo.class));
+        }
+    }
 }

server/src/test/java/com/cloud/user/AccountManagerImplTest.java

@@ -26,7 +26,9 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.event.ActionEventUtils;
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
+import org.apache.cloudstack.api.command.admin.account.UpdateAccountCmd;
 import org.apache.cloudstack.api.command.admin.user.DeleteUserCmd;
 
 import org.apache.cloudstack.acl.ControlledEntity;
@@ -90,6 +92,9 @@ public class AccountManagerImplTest extends AccountManagetImplTestBase {
     @Mock
     private UpdateUserCmd UpdateUserCmdMock;
 
+    @Mock
+    private UpdateAccountCmd UpdateAccountCmdMock;
+
     private long userVoIdMock = 111l;
     @Mock
     private UserVO userVoMock;
@@ -507,6 +512,46 @@ public class AccountManagerImplTest extends AccountManagetImplTestBase {
         Mockito.verify(userVoMock).setSecretKey(secretKey);
     }
 
+    @Test
+    public void validateAndUpdatUserApiKeyAccess() {
+        Mockito.doReturn("Enabled").when(UpdateUserCmdMock).getApiKeyAccess();
+        try (MockedStatic<ActionEventUtils> eventUtils = Mockito.mockStatic(ActionEventUtils.class)) {
+            Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(),
+                    Mockito.anyLong(),
+                    Mockito.anyString(), Mockito.anyString(),
+                    Mockito.anyLong(), Mockito.anyString())).thenReturn(1L);
+            accountManagerImpl.validateAndUpdateUserApiKeyAccess(UpdateUserCmdMock, userVoMock);
+        }
+
+        Mockito.verify(userVoMock).setApiKeyAccess(true);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void validateAndUpdatUserApiKeyAccessInvalidParameter() {
+        Mockito.doReturn("False").when(UpdateUserCmdMock).getApiKeyAccess();
+        accountManagerImpl.validateAndUpdateUserApiKeyAccess(UpdateUserCmdMock, userVoMock);
+    }
+
+    @Test
+    public void validateAndUpdatAccountApiKeyAccess() {
+        Mockito.doReturn("Inherit").when(UpdateAccountCmdMock).getApiKeyAccess();
+        try (MockedStatic<ActionEventUtils> eventUtils = Mockito.mockStatic(ActionEventUtils.class)) {
+            Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(),
+                    Mockito.anyLong(),
+                    Mockito.anyString(), Mockito.anyString(),
+                    Mockito.anyLong(), Mockito.anyString())).thenReturn(1L);
+            accountManagerImpl.validateAndUpdateAccountApiKeyAccess(UpdateAccountCmdMock, accountVoMock);
+        }
+
+        Mockito.verify(accountVoMock).setApiKeyAccess(null);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void validateAndUpdatAccountApiKeyAccessInvalidParameter() {
+        Mockito.doReturn("False").when(UpdateAccountCmdMock).getApiKeyAccess();
+        accountManagerImpl.validateAndUpdateAccountApiKeyAccess(UpdateAccountCmdMock, accountVoMock);
+    }
+
     @Test(expected = CloudRuntimeException.class)
     public void retrieveAndValidateAccountTestAccountNotFound() {
         Mockito.doReturn(accountMockId).when(userVoMock).getAccountId();

server/src/test/java/com/cloud/user/MockAccountManagerImpl.java

@@ -450,12 +450,12 @@ public class MockAccountManagerImpl extends ManagerBase implements Manager, Acco
     }
 
     @Override
-    public Map<String, String> getKeys(GetUserKeysCmd cmd) {
+    public Pair<Boolean, Map<String, String>> getKeys(GetUserKeysCmd cmd) {
         return null;
     }
 
     @Override
-    public Map<String, String> getKeys(Long userId) {
+    public Pair<Boolean, Map<String, String>> getKeys(Long userId) {
         return null;
     }
 

server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java

@@ -635,7 +635,7 @@ public class UserVmManagerImplTest {
         int expectedExceptionCounter = hypervisorTypeArray.length - 5;
 
         for(int i = 0; i < hypervisorTypeArray.length; i++) {
-            if (UserVmManagerImpl.ROOT_DISK_SIZE_OVERRIDE_SUPPORTING_HYPERVISORS.contains(hypervisorTypeArray[i])) {
+            if (hypervisorTypeArray[i].isFunctionalitySupported(Hypervisor.HypervisorType.Functionality.RootDiskSizeOverride)) {
                 userVmManagerImpl.verifyIfHypervisorSupportsRootdiskSizeOverride(hypervisorTypeArray[i]);
             } else {
                 try {

server/src/test/java/com/cloud/vm/snapshot/VMSnapshotManagerTest.java

@@ -136,7 +136,6 @@ public class VMSnapshotManagerTest {
     VMSnapshotDetailsDao _vmSnapshotDetailsDao;
     @Mock
     UserVmManager _userVmManager;
-    int _vmSnapshotMax = 10;
 
     private static final long TEST_VM_ID = 3L;
     private static final long SERVICE_OFFERING_ID = 1L;
@@ -194,8 +193,6 @@ public class VMSnapshotManagerTest {
 
         doNothing().when(_accountMgr).checkAccess(any(Account.class), any(AccessType.class), any(Boolean.class), any(ControlledEntity.class));
 
-        _vmSnapshotMgr._vmSnapshotMax = _vmSnapshotMax;
-
         _vmSnapshotMgr._serviceOfferingDao = _serviceOfferingDao;
         _vmSnapshotMgr._userVmDetailsDao = _userVmDetailsDao;
         _vmSnapshotMgr._vmSnapshotDetailsDao = _vmSnapshotDetailsDao;

ui/public/locales/en.json

@@ -32,6 +32,7 @@
 "label.accesskey": "Access key",
 "label.access.key": "Access key",
 "label.secret.key": "Secret key",
+"label.apikeyaccess": "Api Key Access",
 "label.account": "Account",
 "label.account.and.security.group": "Account - security group",
 "label.account.id": "Account ID",
@@ -882,6 +883,7 @@
 "label.edge": "Edge",
 "label.edge.zone": "Edge Zone",
 "label.edit": "Edit",
+"label.edit.account": "Edit Account",
 "label.edit.acl.list": "Edit ACL list",
 "label.edit.acl.rule": "Edit ACL rule",
 "label.edit.autoscale.vmprofile": "Edit AutoScale Instance Profile",
@@ -1291,6 +1293,7 @@
 "label.l3gatewayserviceuuid": "L3 Gateway Service UUID",
 "label.label": "Label",
 "label.last.updated": "Last update",
+"label.lastupdated": "Last update",
 "label.lastannotated": "Last annotation date",
 "label.lastheartbeat": "Last heartbeat",
 "label.lastsuccessfuljob": "Last successful job",
@@ -1380,6 +1383,7 @@
 "label.management.ips": "Management IP addresses",
 "label.management.server": "Management server",
 "label.management.servers": "Management servers",
+"label.management.server.peers": "Peers",
 "label.managementservers": "Number of management servers",
 "label.matchall": "Match all",
 "label.max": "Max.",
@@ -1668,6 +1672,13 @@
 "label.payload": "Payload",
 "label.payloadurl": "Payload URL",
 "label.pcidevice": "GPU",
+"label.peername": "Management Server",
+"label.peermsid": "Management Server Node ID",
+"label.peerrunid": "Process Timestamp",
+"label.peerserviceip": "Management IP",
+"label.peerserviceport": "Service Port",
+"label.peerstate": "Peer State",
+"label.peerstate.lastupdated": "Peer State Updated Time",
 "label.pending.jobs": "Pending Jobs",
 "label.per.account": "Per Account",
 "label.per.zone": "Per zone",
@@ -2036,7 +2047,7 @@
 "label.service.connectivity.distributedroutercapabilitycheckbox": "Distributed router",
 "label.service.connectivity.regionlevelvpccapabilitycheckbox": "Region level VPC",
 "label.service.group": "Service group",
-"label.serviceip": "Service IP",
+"label.serviceip": "Management IP",
 "label.service.lb.elasticlbcheckbox": "Elastic LB",
 "label.service.lb.inlinemodedropdown": "Mode",
 "label.service.lb.lbisolationdropdown": "LB isolation",
@@ -2151,6 +2162,7 @@
 "label.startport": "Start port",
 "label.startquota": "Quota value",
 "label.state": "State",
+"label.state.reported": "Reported State",
 "label.staticnat": "Static NAT",
 "label.static": "Static",
 "label.static.routes": "Static routes",
@@ -3540,6 +3552,7 @@
 "message.success.scale.kubernetes": "Successfully scaled Kubernetes cluster",
 "message.success.unmanage.instance": "Successfully unmanaged Instance",
 "message.success.unmanage.volume": "Successfully unmanaged Volume",
+"message.success.update.account": "Successfully updated Account",
 "message.success.update.bgp.peer": "Successfully updated BGP peer",
 "message.success.update.bucket": "Successfully updated bucket",
 "message.success.update.condition": "Successfully updated condition",

ui/src/components/view/InfoCard.vue

@@ -733,8 +733,18 @@
         </div>
       </div>
 
-      <div class="account-center-tags" v-if="showKeys">
+      <div class="account-center-tags" v-if="showKeys || resource.apikeyaccess">
         <a-divider/>
+      </div>
+      <div class="account-center-tags" v-if="resource.apikeyaccess && resource.account">
+        <div class="resource-detail-item">
+          <div class="resource-detail-item__label">{{ $t('label.apikeyaccess') }}</div>
+          <div class="resource-detail-item__details">
+            <status class="status" :text="resource.apikeyaccess" displayText/>
+          </div>
+        </div>
+      </div>
+      <div class="account-center-tags" v-if="showKeys">
         <div class="user-keys">
           <key-outlined />
           <strong>
@@ -1083,6 +1093,9 @@ export default {
       api('getUserKeys', { id: this.resource.id }).then(json => {
         this.showKeys = true
         this.newResource.secretkey = json.getuserkeysresponse.userkeys.secretkey
+        if (!this.isAdmin()) {
+          this.newResource.apikeyaccess = json.getuserkeysresponse.userkeys.apikeyaccess ? 'Enabled' : 'Disabled'
+        }
         this.$emit('change-resource', this.newResource)
       })
     },
@@ -1113,6 +1126,9 @@ export default {
         (this.resource.domainid === this.$store.getters.userInfo.domainid && this.resource.account === this.$store.getters.userInfo.account) ||
         (this.resource.project && this.resource.projectid === this.$store.getters.project.id)
     },
+    isAdmin () {
+      return ['Admin'].includes(this.$store.getters.userInfo.roletype)
+    },
     showInput () {
       this.inputVisible = true
       this.$nextTick(function () {

ui/src/components/view/SearchView.vue

@@ -311,7 +311,7 @@ export default {
           type = 'list'
         } else if (item === 'tags') {
           type = 'tag'
-        } else if (item === 'resourcetype') {
+        } else if (['resourcetype', 'apikeyaccess'].includes(item)) {
           type = 'autocomplete'
         } else if (item === 'isencrypted') {
           type = 'boolean'
@@ -424,6 +424,17 @@ export default {
         ]
         this.fields[resourceTypeIndex].loading = false
       }
+
+      if (arrayField.includes('apikeyaccess')) {
+        const apiKeyAccessIndex = this.fields.findIndex(item => item.name === 'apikeyaccess')
+        this.fields[apiKeyAccessIndex].loading = true
+        this.fields[apiKeyAccessIndex].opts = [
+          { value: 'Disabled' },
+          { value: 'Enabled' },
+          { value: 'Inherit' }
+        ]
+        this.fields[apiKeyAccessIndex].loading = false
+      }
     },
     async fetchDynamicFieldData (arrayField, searchKeyword) {
       const promises = []

ui/src/config/section/account.js

@@ -24,9 +24,15 @@ export default {
   icon: 'team-outlined',
   docHelp: 'adminguide/accounts.html',
   permission: ['listAccounts'],
-  searchFilters: ['name', 'accounttype', 'domainid'],
+  searchFilters: () => {
+    var filters = ['name', 'accounttype', 'domainid']
+    if (store.getters.userInfo.roletype === 'Admin') {
+      filters.push('apikeyaccess')
+    }
+    return filters
+  },
   columns: ['name', 'state', 'rolename', 'roletype', 'domainpath'],
-  details: ['name', 'id', 'rolename', 'roletype', 'domainpath', 'networkdomain', 'iptotal', 'vmtotal', 'volumetotal', 'receivedbytes', 'sentbytes', 'created'],
+  details: ['name', 'id', 'rolename', 'roletype', 'domainpath', 'networkdomain', 'apikeyaccess', 'iptotal', 'vmtotal', 'volumetotal', 'receivedbytes', 'sentbytes', 'created'],
   related: [{
     name: 'accountuser',
     title: 'label.users',
@@ -116,15 +122,8 @@ export default {
       icon: 'edit-outlined',
       label: 'label.action.edit.account',
       dataView: true,
-      args: ['newname', 'account', 'domainid', 'networkdomain', 'roleid'],
-      mapping: {
-        account: {
-          value: (record) => { return record.name }
-        },
-        domainid: {
-          value: (record) => { return record.domainid }
-        }
-      }
+      popup: true,
+      component: shallowRef(defineAsyncComponent(() => import('@/views/iam/EditAccount.vue')))
     },
     {
       api: 'updateResourceCount',

ui/src/config/section/infra/managementServers.js

@@ -43,6 +43,10 @@ export default {
       name: 'pending.jobs',
       component: shallowRef(defineAsyncComponent(() => import('@/views/infra/AsyncJobsTab.vue')))
     },
+    {
+      name: 'management.server.peers',
+      component: shallowRef(defineAsyncComponent(() => import('@/views/infra/ManagementServerPeerTab.vue')))
+    },
     {
       name: 'comments',
       component: shallowRef(defineAsyncComponent(() => import('@/components/view/AnnotationsTab.vue')))