Merge remote-tracking branch 'apache/4.18' into main

api/src/main/java/com/cloud/network/vpc/NetworkACLService.java

@@ -96,4 +96,6 @@ public interface NetworkACLService {
      * Updates a network item ACL to a new position. This method allows users to inform between which ACLs the given ACL will be placed. Therefore, the 'number' field will be filled out by the system in the best way possible to place the ACL accordingly.
      */
     NetworkACLItem moveNetworkAclRuleToNewPosition(MoveNetworkAclItemCmd moveNetworkAclItemCmd);
+
+    NetworkACLItem moveRuleToTheTopInACLList(NetworkACLItem ruleBeingMoved);
 }

api/src/test/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmdTest.java

@@ -17,26 +17,27 @@
 
 package org.apache.cloudstack.api.command.user.network;
 
+import org.apache.cloudstack.api.ResponseGenerator;
+import org.apache.cloudstack.api.ResponseObject;
+import org.apache.cloudstack.api.response.NetworkResponse;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.springframework.test.util.ReflectionTestUtils;
+
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.network.Network;
 import com.cloud.network.NetworkService;
 import com.cloud.offering.NetworkOffering;
 import com.cloud.utils.db.EntityManager;
-import junit.framework.TestCase;
-import org.apache.cloudstack.api.ResponseGenerator;
-import org.apache.cloudstack.api.ResponseObject;
-import org.apache.cloudstack.api.response.NetworkResponse;
-import org.junit.Assert;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.powermock.modules.junit4.PowerMockRunner;
-import org.springframework.test.util.ReflectionTestUtils;
 
-@RunWith(PowerMockRunner.class)
+@RunWith(MockitoJUnitRunner.class)
-public class CreateNetworkCmdTest extends TestCase {
+public class CreateNetworkCmdTest {
 
     @Mock
     public EntityManager _entityMgr;
@@ -46,66 +47,78 @@ public class CreateNetworkCmdTest extends TestCase {
 
     @InjectMocks
     CreateNetworkCmd cmd = new CreateNetworkCmd();
+
+    @Test
     public void testGetNetworkOfferingId() {
         Long networkOfferingId = 1L;
         ReflectionTestUtils.setField(cmd, "networkOfferingId", networkOfferingId);
         Assert.assertEquals(cmd.getNetworkOfferingId(), networkOfferingId);
     }
 
+    @Test
     public void testGetGateway() {
         String gateway = "10.10.10.1";
         ReflectionTestUtils.setField(cmd, "gateway", gateway);
         Assert.assertEquals(cmd.getGateway(), gateway);
     }
 
+    @Test
     public void testGetIsolatedPvlan() {
         String isolatedPvlan = "1234";
         ReflectionTestUtils.setField(cmd, "isolatedPvlan", isolatedPvlan);
         Assert.assertEquals(cmd.getIsolatedPvlan(), isolatedPvlan);
     }
 
+    @Test
     public void testGetAccountName() {
         String accountName = "admin";
         ReflectionTestUtils.setField(cmd, "accountName", accountName);
         Assert.assertEquals(cmd.getAccountName(), accountName);
     }
 
+    @Test
     public void testGetDomainId() {
         Long domainId = 1L;
         ReflectionTestUtils.setField(cmd, "domainId", domainId);
         Assert.assertEquals(cmd.getDomainId(), domainId);
     }
 
+    @Test
     public void testGetNetmask() {
         String netmask = "255.255.255.0";
         ReflectionTestUtils.setField(cmd, "netmask", netmask);
         Assert.assertEquals(cmd.getNetmask(), netmask);
     }
 
+    @Test
     public void testGetStartIp() {
         String startIp = "10.10.10.2";
         ReflectionTestUtils.setField(cmd, "startIp", startIp);
         Assert.assertEquals(cmd.getStartIp(), startIp);
     }
 
+    @Test
     public void testGetEndIp() {
         String endIp = "10.10.10.10";
         ReflectionTestUtils.setField(cmd, "endIp", endIp);
         Assert.assertEquals(cmd.getEndIp(), endIp);
     }
 
+    @Test
     public void testGetNetworkName() {
         String netName = "net-isolated";
         ReflectionTestUtils.setField(cmd, "name", netName);
         Assert.assertEquals(cmd.getNetworkName(), netName);
     }
 
+    @Test
     public void testGetDisplayTextWhenNotEmpty() {
         String description = "Isolated Network";
         ReflectionTestUtils.setField(cmd, "displayText", description);
         Assert.assertEquals(cmd.getDisplayText(), description);
     }
 
+    @Test
     public void testGetDisplayTextWhenEmpty() {
         String description = null;
         String netName = "net-isolated";
@@ -113,65 +126,74 @@ public class CreateNetworkCmdTest extends TestCase {
         Assert.assertEquals(cmd.getDisplayText(), netName);
     }
 
+    @Test
     public void testGetNetworkDomain() {
         String netDomain = "cs1cloud.internal";
         ReflectionTestUtils.setField(cmd, "networkDomain", netDomain);
         Assert.assertEquals(cmd.getNetworkDomain(), netDomain);
     }
 
+    @Test
     public void testGetProjectId() {
         Long projectId = 1L;
         ReflectionTestUtils.setField(cmd, "projectId", projectId);
         Assert.assertEquals(cmd.getProjectId(), projectId);
     }
 
+    @Test
     public void testGetAclType() {
         String aclType = "account";
         ReflectionTestUtils.setField(cmd, "aclType", aclType);
         Assert.assertEquals(cmd.getAclType(), aclType);
     }
 
+    @Test
     public void testGetSubdomainAccess() {
         Boolean subDomAccess = false;
         ReflectionTestUtils.setField(cmd, "subdomainAccess", subDomAccess);
         Assert.assertEquals(cmd.getSubdomainAccess(), subDomAccess);
     }
 
+    @Test
     public void testGetVpcId() {
         Long vpcId = 1L;
         ReflectionTestUtils.setField(cmd, "vpcId", vpcId);
         Assert.assertEquals(cmd.getVpcId(), vpcId);
     }
 
+    @Test
     public void testGetDisplayNetwork() {
         Boolean displayNet = true;
         ReflectionTestUtils.setField(cmd, "displayNetwork", displayNet);
         Assert.assertEquals(cmd.getDisplayNetwork(), displayNet);
     }
 
+    @Test
     public void testGetExternalId() {
         String externalId = "1";
         ReflectionTestUtils.setField(cmd, "externalId", externalId);
         Assert.assertEquals(cmd.getExternalId(), externalId);
     }
 
+    @Test
     public void testGetAssociatedNetworkId() {
         Long associatedNetId = 1L;
         ReflectionTestUtils.setField(cmd, "associatedNetworkId", associatedNetId);
         Assert.assertEquals(cmd.getAssociatedNetworkId(), associatedNetId);
     }
 
+    @Test
     public void testIsDisplayNullDefaultsToTrue() {
         Boolean displayNetwork = null;
         ReflectionTestUtils.setField(cmd, "displayNetwork", displayNetwork);
         Assert.assertTrue(cmd.isDisplay());
     }
 
+    @Test
     public void testGetPhysicalNetworkIdForInvalidNetOfferingId() {
         Long physicalNetworkId = 1L;
 
         ReflectionTestUtils.setField(cmd, "physicalNetworkId", physicalNetworkId);
-        Mockito.when(_entityMgr.findById(NetworkOffering.class, 1L)).thenReturn(null);
         try {
             cmd.getPhysicalNetworkId();
         } catch (Exception e) {
@@ -179,6 +201,7 @@ public class CreateNetworkCmdTest extends TestCase {
         }
     }
 
+    @Test
     public void testGetPhysicalNetworkIdForInvalidAssociatedNetId() {
         Long physicalNetworkId = 1L;
         Long networkOfferingId = 1L;
@@ -196,6 +219,7 @@ public class CreateNetworkCmdTest extends TestCase {
         }
     }
 
+    @Test
     public void testGetPhysicalNetworkIdForAssociatedNetIdForNonSharedNet() {
         Long physicalNetworkId = 1L;
         Long networkOfferingId = 1L;
@@ -215,6 +239,7 @@ public class CreateNetworkCmdTest extends TestCase {
         }
     }
 
+    @Test
     public void testGetPhysicalNetworkIdForNonSharedNet() {
         Long physicalNetworkId = 1L;
         Long networkOfferingId = 1L;
@@ -230,6 +255,7 @@ public class CreateNetworkCmdTest extends TestCase {
         }
     }
 
+    @Test
     public void testGetPhysicalNetworkIdForSharedNet() {
         Long physicalNetworkId = 1L;
         Long networkOfferingId = 1L;
@@ -245,6 +271,7 @@ public class CreateNetworkCmdTest extends TestCase {
         }
     }
 
+    @Test
     public void testGetZoneId() {
         Long physicalNetworkId = 1L;
         Long networkOfferingId = 1L;
@@ -258,30 +285,35 @@ public class CreateNetworkCmdTest extends TestCase {
         Assert.assertEquals(cmd.getZoneId(), zoneId);
     }
 
+    @Test
     public void testGetPublicMtuWhenNotSet() {
         Integer publicMtu = null;
         ReflectionTestUtils.setField(cmd, "publicMtu", publicMtu);
         Assert.assertEquals(NetworkService.DEFAULT_MTU, cmd.getPublicMtu());
     }
 
+    @Test
     public void testGetPublicMtuWhenSet() {
         Integer publicMtu = 1450;
         ReflectionTestUtils.setField(cmd, "publicMtu", publicMtu);
         Assert.assertEquals(cmd.getPublicMtu(), publicMtu);
     }
 
+    @Test
     public void testGetPrivateMtuWhenNotSet() {
         Integer privateMtu = null;
         ReflectionTestUtils.setField(cmd, "privateMtu", privateMtu);
         Assert.assertEquals(NetworkService.DEFAULT_MTU, cmd.getPrivateMtu());
     }
 
+    @Test
     public void testGetPrivateMtuWhenSet() {
         Integer privateMtu = 1250;
         ReflectionTestUtils.setField(cmd, "privateMtu", privateMtu);
         Assert.assertEquals(cmd.getPrivateMtu(), privateMtu);
     }
 
+    @Test
     public void testExecute() throws InsufficientCapacityException, ResourceAllocationException {
         ReflectionTestUtils.setField(cmd, "displayText", "testNetwork");
         ReflectionTestUtils.setField(cmd, "name", "testNetwork");

api/src/test/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmdTest.java

@@ -17,25 +17,25 @@
 
 package org.apache.cloudstack.api.command.user.network;
 
-import com.cloud.exception.InsufficientCapacityException;
-import com.cloud.network.Network;
-import com.cloud.network.NetworkService;
-import com.cloud.offering.NetworkOffering;
-import com.cloud.utils.db.EntityManager;
-import junit.framework.TestCase;
 import org.apache.cloudstack.api.ResponseGenerator;
 import org.apache.cloudstack.api.ResponseObject;
 import org.apache.cloudstack.api.response.NetworkResponse;
 import org.junit.Assert;
+import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.Mockito;
-import org.powermock.modules.junit4.PowerMockRunner;
+import org.mockito.junit.MockitoJUnitRunner;
 import org.springframework.test.util.ReflectionTestUtils;
 
-@RunWith(PowerMockRunner.class)
+import com.cloud.exception.InsufficientCapacityException;
-public class UpdateNetworkCmdTest extends TestCase {
+import com.cloud.network.Network;
+import com.cloud.network.NetworkService;
+import com.cloud.utils.db.EntityManager;
+
+@RunWith(MockitoJUnitRunner.class)
+public class UpdateNetworkCmdTest {
 
     @Mock
     NetworkService networkService;
@@ -45,115 +45,131 @@ public class UpdateNetworkCmdTest extends TestCase {
     @InjectMocks
     UpdateNetworkCmd cmd = new UpdateNetworkCmd();
 
+    @Test
     public void testGetId() {
         Long id = 1L;
         ReflectionTestUtils.setField(cmd, "id", id);
         Assert.assertEquals(cmd.getId(), id);
     }
 
+    @Test
     public void testGetNetworkName() {
         String name = "testNetwork";
         ReflectionTestUtils.setField(cmd, "name", name);
         Assert.assertEquals(cmd.getNetworkName(), name);
     }
 
+    @Test
     public void testGetDisplayText() {
         String displayText = "test network";
         ReflectionTestUtils.setField(cmd, "displayText", displayText);
         Assert.assertEquals(cmd.getDisplayText(), displayText);
     }
 
+    @Test
     public void testGetNetworkDomain() {
         String netDomain = "cs1cloud.internal";
         ReflectionTestUtils.setField(cmd, "networkDomain", netDomain);
         Assert.assertEquals(cmd.getNetworkDomain(), netDomain);
     }
 
+    @Test
     public void testGetNetworkOfferingId() {
         Long networkOfferingId = 1L;
         ReflectionTestUtils.setField(cmd, "networkOfferingId", networkOfferingId);
         Assert.assertEquals(cmd.getNetworkOfferingId(), networkOfferingId);
     }
 
+    @Test
     public void testGetChangeCidr() {
         Boolean changeCidr = true;
         ReflectionTestUtils.setField(cmd, "changeCidr", changeCidr);
         Assert.assertTrue(cmd.getChangeCidr());
     }
 
+    @Test
     public void testGetGuestVmCidr() {
         String guestVmCidr = "10.10.0.0/24";
         ReflectionTestUtils.setField(cmd, "guestVmCidr", guestVmCidr);
         Assert.assertEquals(cmd.getGuestVmCidr(), guestVmCidr);
     }
 
+    @Test
     public void testGetDisplayNetwork() {
         Boolean displayNetwork = true;
         ReflectionTestUtils.setField(cmd, "displayNetwork", displayNetwork);
         Assert.assertTrue(cmd.getDisplayNetwork());
     }
 
+    @Test
     public void testGetUpdateInSequenceIfNull() {
         Boolean updateInSequence = null;
         ReflectionTestUtils.setField(cmd, "updateInSequence", updateInSequence);
         Assert.assertFalse(cmd.getUpdateInSequence());
     }
 
+    @Test
     public void testGetUpdateInSequenceIfValidValuePassed() {
         Boolean updateInSequence = true;
         ReflectionTestUtils.setField(cmd, "updateInSequence", updateInSequence);
         Assert.assertTrue(cmd.getUpdateInSequence());
     }
 
+    @Test
     public void testGetForcedIfNull() {
         Boolean forced = null;
         ReflectionTestUtils.setField(cmd, "forced", forced);
         Assert.assertFalse(cmd.getUpdateInSequence());
     }
 
+    @Test
     public void testGetForcedIfValidValuePassed() {
         Boolean forced = true;
         ReflectionTestUtils.setField(cmd, "forced", forced);
         Assert.assertTrue(cmd.getForced());
     }
 
+    @Test
     public void testGetPublicMtu() {
         Integer publicMtu = 1450;
         ReflectionTestUtils.setField(cmd, "publicMtu", publicMtu);
         Assert.assertEquals(cmd.getPublicMtu(), publicMtu);
     }
 
+    @Test
     public void testGetPublicMtuIfNull() {
         Integer publicMtu = null;
         ReflectionTestUtils.setField(cmd, "publicMtu", publicMtu);
         Assert.assertNull(cmd.getPublicMtu());
     }
 
+    @Test
     public void testGetPrivateMtu() {
         Integer privateMtu = 1450;
         ReflectionTestUtils.setField(cmd, "privateMtu", privateMtu);
         Assert.assertEquals(cmd.getPrivateMtu(), privateMtu);
     }
 
+    @Test
     public void testGetPrivateMtuIfNull() {
         Integer privateMtu = null;
         ReflectionTestUtils.setField(cmd, "privateMtu", privateMtu);
         Assert.assertNull(cmd.getPrivateMtu());
     }
 
+    @Test
     public void testEventDescription() {
         long networkOfferingId = 1L;
         Network network = Mockito.mock(Network.class);
-        NetworkOffering offering = Mockito.mock(NetworkOffering.class);
         ReflectionTestUtils.setField(cmd, "networkOfferingId", networkOfferingId);
         ReflectionTestUtils.setField(cmd, "id", 1L);
         Mockito.when(networkService.getNetwork(Mockito.any(Long.class))).thenReturn(network);
         Mockito.when(network.getNetworkOfferingId()).thenReturn(networkOfferingId);
-        Mockito.when(_entityMgr.findById(NetworkOffering.class, networkOfferingId)).thenReturn(offering);
         String msg = cmd.getEventDescription();
         Assert.assertTrue(msg.contains("Updating network"));
     }
 
+    @Test
     public void testExecute() throws InsufficientCapacityException {
         long networkId = 1L;
         Integer publicmtu = 1200;

debian/cloudstack-usage.postinst

@@ -21,7 +21,12 @@ set -e
 
 case "$1" in
     configure)
-     
+        if ! getent passwd cloud >/dev/null; then
+            adduser --quiet --system --group --no-create-home --home /var/lib/cloudstack/management cloud
+        else
+            usermod -m -d /var/lib/cloudstack/management cloud || true
+        fi
+
         # Linking usage server db.properties to management server db.properties
         if [ -f "/etc/cloudstack/management/db.properties" ]; then
             echo "Replacing usage server's db.properties with a link to the management server's db.properties"

engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql

@@ -21,3 +21,13 @@
 
 -- create_public_parameter_on_roles. #6960
 ALTER TABLE `cloud`.`roles` ADD COLUMN `public_role` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'Indicates whether the role will be visible to all users (public) or only to root admins (private). If this parameter is not specified during the creation of the role its value will be defaulted to true (public).';
+
+-- Add Windows Server 2022 guest OS and mappings
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'KVM', 'default', 'Windows Server 2022 (64-bit)');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'VMware', '7.0', 'windows2019srvNext_64Guest');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'VMware', '7.0.1.0', 'windows2019srvNext_64Guest');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'VMware', '7.0.2.0', 'windows2019srvNext_64Guest');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'VMware', '7.0.3.0', 'windows2019srvNext_64Guest');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'VMware', '8.0', 'windows2019srvNext_64Guest');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'VMware', '8.0.0.1', 'windows2019srvNext_64Guest');
+CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'Xenserver', '8.2.0', 'Windows Server 2022 (64-bit)');

plugins/affinity-group-processors/non-strict-host-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessorTest.java

@@ -19,13 +19,15 @@
 
 package org.apache.cloudstack.affinity;
 
-import com.cloud.deploy.DataCenterDeployment;
+import static org.mockito.ArgumentMatchers.eq;
-import com.cloud.deploy.DeploymentPlan;
+import static org.mockito.ArgumentMatchers.nullable;
-import com.cloud.deploy.DeploymentPlanner.ExcludeList;
+import static org.mockito.Mockito.when;
-import com.cloud.vm.VMInstanceVO;
+
-import com.cloud.vm.VirtualMachine;
+import java.util.ArrayList;
-import com.cloud.vm.VirtualMachineProfile;
+import java.util.Arrays;
-import com.cloud.vm.dao.VMInstanceDao;
+import java.util.Date;
+import java.util.List;
+
 import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
 import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
 import org.junit.Assert;
@@ -35,19 +37,18 @@ import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.Spy;
-import org.powermock.modules.junit4.PowerMockRunner;
+import org.mockito.junit.MockitoJUnitRunner;
 import org.springframework.test.util.ReflectionTestUtils;
 
-import java.util.ArrayList;
+import com.cloud.deploy.DataCenterDeployment;
-import java.util.Arrays;
+import com.cloud.deploy.DeploymentPlan;
-import java.util.Date;
+import com.cloud.deploy.DeploymentPlanner.ExcludeList;
-import java.util.List;
+import com.cloud.vm.VMInstanceVO;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VirtualMachineProfile;
+import com.cloud.vm.dao.VMInstanceDao;
 
-import static org.mockito.ArgumentMatchers.eq;
+@RunWith(MockitoJUnitRunner.class)
-import static org.mockito.ArgumentMatchers.nullable;
-import static org.mockito.Mockito.when;
-
-@RunWith(PowerMockRunner.class)
 public class NonStrictHostAffinityProcessorTest {
 
     @Spy

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java

@@ -124,6 +124,7 @@ import com.cloud.network.security.SecurityGroupManager;
 import com.cloud.network.security.SecurityGroupService;
 import com.cloud.network.security.SecurityGroupVO;
 import com.cloud.network.security.SecurityRule;
+import com.cloud.network.vpc.NetworkACL;
 import com.cloud.offering.NetworkOffering;
 import com.cloud.offering.ServiceOffering;
 import com.cloud.offerings.NetworkOfferingServiceMapVO;
@@ -352,48 +353,50 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
         return  template;
     }
 
-    private boolean validateIsolatedNetwork(Network network, int clusterTotalNodeCount) {
+    protected void validateIsolatedNetworkIpRules(long ipId, FirewallRule.Purpose purpose, Network network, int clusterTotalNodeCount) {
-        if (Network.GuestType.Isolated.equals(network.getGuestType())) {
+        List<FirewallRuleVO> rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose);
-            if (Network.State.Allocated.equals(network.getState())) { // Allocated networks won't have IP and rules
+        for (FirewallRuleVO rule : rules) {
-                return true;
+            Integer startPort = rule.getSourcePortStart();
+            Integer endPort = rule.getSourcePortEnd();
+            if (LOGGER.isDebugEnabled()) {
+                LOGGER.debug(String.format("Validating rule with purpose: %s for network: %s with ports: %d-%d", purpose.toString(), network.getUuid(), startPort, endPort));
             }
-            IpAddress sourceNatIp = getSourceNatIp(network);
+            if (startPort <= KubernetesClusterActionWorker.CLUSTER_API_PORT && KubernetesClusterActionWorker.CLUSTER_API_PORT <= endPort) {
-            if (sourceNatIp == null) {
+                throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting %s rules to provision Kubernetes cluster for API access", network.getUuid(), purpose.toString().toLowerCase()));
-                throw new InvalidParameterValueException(String.format("Network ID: %s does not have a source NAT IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid()));
             }
-            List<FirewallRuleVO> rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.Firewall);
+            int expectedSshStart = KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT;
-            for (FirewallRuleVO rule : rules) {
+            int expectedSshEnd = expectedSshStart + clusterTotalNodeCount - 1;
-                Integer startPort = rule.getSourcePortStart();
+            if (Math.max(expectedSshStart, startPort) <= Math.min(expectedSshEnd, endPort)) {
-                Integer endPort = rule.getSourcePortEnd();
+                throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting %s rules to provision Kubernetes cluster for node VM SSH access", network.getUuid(), purpose.toString().toLowerCase()));
-                if (LOGGER.isDebugEnabled()) {
-                    LOGGER.debug("Network rule : " + startPort + " " + endPort);
-                }
-                if (startPort <= KubernetesClusterActionWorker.CLUSTER_API_PORT && KubernetesClusterActionWorker.CLUSTER_API_PORT <= endPort) {
-                    throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for API access", network.getUuid()));
-                }
-                if (startPort <= KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT && KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterTotalNodeCount <= endPort) {
-                    throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for node VM SSH access", network.getUuid()));
-                }
-            }
-            rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.PortForwarding);
-            for (FirewallRuleVO rule : rules) {
-                Integer startPort = rule.getSourcePortStart();
-                Integer endPort = rule.getSourcePortEnd();
-                if (LOGGER.isDebugEnabled()) {
-                    LOGGER.debug("Network rule : " + startPort + " " + endPort);
-                }
-                if (startPort <= KubernetesClusterActionWorker.CLUSTER_API_PORT && KubernetesClusterActionWorker.CLUSTER_API_PORT <= endPort) {
-                    throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for API access", network.getUuid()));
-                }
-                if (startPort <= KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT && KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterTotalNodeCount <= endPort) {
-                    throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for node VM SSH access", network.getUuid()));
-                }
             }
         }
-        return true;
     }
 
-    private boolean validateNetwork(Network network, int clusterTotalNodeCount) {
+    private void validateIsolatedNetwork(Network network, int clusterTotalNodeCount) {
+        if (!Network.GuestType.Isolated.equals(network.getGuestType())) {
+            return;
+        }
+        if (Network.State.Allocated.equals(network.getState())) { // Allocated networks won't have IP and rules
+            return;
+        }
+        IpAddress sourceNatIp = getSourceNatIp(network);
+        if (sourceNatIp == null) {
+            throw new InvalidParameterValueException(String.format("Network ID: %s does not have a source NAT IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid()));
+        }
+        validateIsolatedNetworkIpRules(sourceNatIp.getId(), FirewallRule.Purpose.Firewall, network, clusterTotalNodeCount);
+        validateIsolatedNetworkIpRules(sourceNatIp.getId(), FirewallRule.Purpose.PortForwarding, network, clusterTotalNodeCount);
+    }
+
+    protected void validateVpcTier(Network network) {
+        if (Network.State.Allocated.equals(network.getState())) { // Allocated networks won't have IP and rules
+            return;
+        }
+        if (network.getNetworkACLId() == NetworkACL.DEFAULT_DENY) {
+            throw new InvalidParameterValueException(String.format("Network ID: %s can not be used for Kubernetes cluster as it uses default deny ACL", network.getUuid()));
+        }
+    }
+
+    private void validateNetwork(Network network, int clusterTotalNodeCount) {
         NetworkOffering networkOffering = networkOfferingDao.findById(network.getNetworkOfferingId());
         if (networkOffering.isSystemOnly()) {
             throw new InvalidParameterValueException(String.format("Network ID: %s is for system use only", network.getUuid()));
@@ -401,7 +404,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
         if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.UserData)) {
             throw new InvalidParameterValueException(String.format("Network ID: %s does not support userdata that is required for Kubernetes cluster", network.getUuid()));
         }
-        if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall)) {
+        Long vpcId = network.getVpcId();
+        if (vpcId == null && !networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall)) {
             throw new InvalidParameterValueException(String.format("Network ID: %s does not support firewall that is required for Kubernetes cluster", network.getUuid()));
         }
         if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.PortForwarding)) {
@@ -410,8 +414,11 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
         if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) {
             throw new InvalidParameterValueException(String.format("Network ID: %s does not support DHCP that is required for Kubernetes cluster", network.getUuid()));
         }
+        if (network.getVpcId() != null) {
+            validateVpcTier(network);
+            return;
+        }
         validateIsolatedNetwork(network, clusterTotalNodeCount);
-        return true;
     }
 
     private boolean validateServiceOffering(final ServiceOffering serviceOffering, final KubernetesSupportedVersion version) {
@@ -736,9 +743,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
             network = networkDao.findById(networkId);
             if (Network.GuestType.Isolated.equals(network.getGuestType())) {
                 if (kubernetesClusterDao.listByNetworkId(network.getId()).isEmpty()) {
-                    if (!validateNetwork(network, controlNodesCount + nodesCount)) {
+                    validateNetwork(network, controlNodesCount + nodesCount);
-                        throw new InvalidParameterValueException(String.format("Network ID: %s is not suitable for Kubernetes cluster", network.getUuid()));
-                    }
                     networkModel.checkNetworkPermissions(owner, network);
                 } else {
                     throw new InvalidParameterValueException(String.format("Network ID: %s is already under use by another Kubernetes cluster", network.getUuid()));
@@ -803,6 +808,34 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
         }
     }
 
+    protected void validateKubernetesClusterScaleSize(final KubernetesClusterVO kubernetesCluster, final Long clusterSize, final int maxClusterSize, final DataCenter zone) {
+        if (clusterSize == null) {
+            return;
+        }
+        if (clusterSize == kubernetesCluster.getNodeCount()) {
+            return;
+        }
+        if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { // Cannot scale stopped cluster currently for cluster size
+            throw new PermissionDeniedException(String.format("Kubernetes cluster : %s is in %s state", kubernetesCluster.getName(), kubernetesCluster.getState().toString()));
+        }
+        if (clusterSize < 1) {
+            throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled for size, %d", kubernetesCluster.getName(), clusterSize));
+        }
+        if (clusterSize + kubernetesCluster.getControlNodeCount() > maxClusterSize) {
+            throw new InvalidParameterValueException(
+                    String.format("Maximum cluster size can not exceed %d. Please contact your administrator", maxClusterSize));
+        }
+        if (clusterSize > kubernetesCluster.getNodeCount()) { // Upscale
+            VMTemplateVO template = templateDao.findById(kubernetesCluster.getTemplateId());
+            if (template == null) {
+                throw new InvalidParameterValueException(String.format("Invalid template associated with Kubernetes cluster : %s",  kubernetesCluster.getName()));
+            }
+            if (CollectionUtils.isEmpty(templateJoinDao.newTemplateView(template, zone.getId(), true))) {
+                throw new InvalidParameterValueException(String.format("Template : %s associated with Kubernetes cluster : %s is not in Ready state for datacenter : %s", template.getName(), kubernetesCluster.getName(), zone.getName()));
+            }
+        }
+    }
+
     private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd cmd) {
         final Long kubernetesClusterId = cmd.getId();
         final Long serviceOfferingId = cmd.getServiceOfferingId();
@@ -844,8 +877,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
 
         int maxClusterSize = KubernetesMaxClusterSize.valueIn(kubernetesCluster.getAccountId());
         if (isAutoscalingEnabled != null && isAutoscalingEnabled) {
-            if (clusterSize != null || serviceOfferingId != null || nodeIds != null) {
+            if (clusterSize != null || nodeIds != null) {
-                throw new InvalidParameterValueException("Autoscaling can not be passed along with nodeids or clustersize or service offering");
+                throw new InvalidParameterValueException("Autoscaling can not be passed along with nodeids or clustersize");
             }
 
             if (!KubernetesVersionManagerImpl.versionSupportsAutoscaling(clusterVersion)) {
@@ -914,34 +947,14 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
                 }
             }
             final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId());
-            if (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() ||
+            if (KubernetesCluster.State.Running.equals(kubernetesCluster.getState()) && (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() ||
-                    serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed()) {
+                    serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed())) {
                 logAndThrow(Level.WARN, String.format("Kubernetes cluster cannot be scaled down for service offering. Service offering : %s offers lesser resources as compared to service offering : %s of Kubernetes cluster : %s",
                         serviceOffering.getName(), existingServiceOffering.getName(), kubernetesCluster.getName()));
             }
         }
 
-        if (clusterSize != null) {
+        validateKubernetesClusterScaleSize(kubernetesCluster, clusterSize, maxClusterSize, zone);
-            if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { // Cannot scale stopped cluster currently for cluster size
-                throw new PermissionDeniedException(String.format("Kubernetes cluster : %s is in %s state", kubernetesCluster.getName(), kubernetesCluster.getState().toString()));
-            }
-            if (clusterSize < 1) {
-                throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled for size, %d", kubernetesCluster.getName(), clusterSize));
-            }
-            if (clusterSize + kubernetesCluster.getControlNodeCount() > maxClusterSize) {
-                throw new InvalidParameterValueException(
-                    String.format("Maximum cluster size can not exceed %d. Please contact your administrator", maxClusterSize));
-            }
-            if (clusterSize > kubernetesCluster.getNodeCount()) { // Upscale
-                VMTemplateVO template = templateDao.findById(kubernetesCluster.getTemplateId());
-                if (template == null) {
-                    throw new InvalidParameterValueException(String.format("Invalid template associated with Kubernetes cluster : %s",  kubernetesCluster.getName()));
-                }
-                if (CollectionUtils.isEmpty(templateJoinDao.newTemplateView(template, zone.getId(), true))) {
-                    throw new InvalidParameterValueException(String.format("Template : %s associated with Kubernetes cluster : %s is not in Ready state for datacenter : %s", template.getName(), kubernetesCluster.getName(), zone.getName()));
-                }
-            }
-        }
     }
 
     private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesClusterCmd cmd) {

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java

@@ -17,9 +17,37 @@
 
 package com.cloud.kubernetes.cluster.actionworkers;
 
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.inject.Inject;
+
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.ca.CAManager;
+import org.apache.cloudstack.config.ApiServiceConfiguration;
+import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
 import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.dc.dao.VlanDao;
+import com.cloud.exception.InsufficientAddressCapacityException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.kubernetes.cluster.KubernetesCluster;
 import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO;
@@ -36,7 +64,10 @@ import com.cloud.network.IpAddressManager;
 import com.cloud.network.Network;
 import com.cloud.network.Network.GuestType;
 import com.cloud.network.NetworkModel;
+import com.cloud.network.NetworkService;
+import com.cloud.network.dao.IPAddressDao;
 import com.cloud.network.dao.NetworkDao;
+import com.cloud.network.vpc.VpcService;
 import com.cloud.projects.ProjectService;
 import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.storage.Storage;
@@ -64,37 +95,15 @@ import com.cloud.vm.VirtualMachineManager;
 import com.cloud.vm.VmDetailConstants;
 import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.UserVmDetailsDao;
-import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.ca.CAManager;
-import org.apache.cloudstack.config.ApiServiceConfiguration;
-import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
-import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-import javax.inject.Inject;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 
 public class KubernetesClusterActionWorker {
 
     public static final String CLUSTER_NODE_VM_USER = "cloud";
     public static final int CLUSTER_API_PORT = 6443;
+    public static final int DEFAULT_SSH_PORT = 22;
     public static final int CLUSTER_NODES_DEFAULT_START_SSH_PORT = 2222;
-    public static final int CLUSTER_NODES_DEFAULT_SSH_PORT_SG = 22;
+    public static final int CLUSTER_NODES_DEFAULT_SSH_PORT_SG = DEFAULT_SSH_PORT;
 
     public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup";
 
@@ -113,12 +122,16 @@ public class KubernetesClusterActionWorker {
     @Inject
     protected IpAddressManager ipAddressManager;
     @Inject
+    protected IPAddressDao ipAddressDao;
+    @Inject
     protected NetworkOrchestrationService networkMgr;
     @Inject
     protected NetworkDao networkDao;
     @Inject
     protected NetworkModel networkModel;
     @Inject
+    protected NetworkService networkService;
+    @Inject
     protected ServiceOfferingDao serviceOfferingDao;
     @Inject
     protected SSHKeyPairDao sshKeyPairDao;
@@ -140,6 +153,8 @@ public class KubernetesClusterActionWorker {
     protected LaunchPermissionDao launchPermissionDao;
     @Inject
     public ProjectService projectService;
+    @Inject
+    public VpcService vpcService;
 
     protected KubernetesClusterDao kubernetesClusterDao;
     protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
@@ -329,7 +344,79 @@ public class KubernetesClusterActionWorker {
         return ip;
     }
 
-    protected Pair<String, Integer> getKubernetesClusterServerIpSshPort(UserVm controlVm) {
+    protected IpAddress getNetworkSourceNatIp(Network network) {
+        List<? extends IpAddress> addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true);
+        if (CollectionUtils.isNotEmpty(addresses)) {
+            return addresses.get(0);
+        }
+        LOGGER.warn(String.format("No public IP addresses found for network : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName()));
+        return null;
+    }
+
+    protected IpAddress getVpcTierKubernetesPublicIp(Network network) {
+        KubernetesClusterDetailsVO detailsVO = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), ApiConstants.PUBLIC_IP_ID);
+        if (detailsVO == null || StringUtils.isEmpty(detailsVO.getValue())) {
+            return null;
+        }
+        IpAddress address = ipAddressDao.findByUuid(detailsVO.getValue());
+        if (address == null || network.getVpcId() != address.getVpcId()) {
+            LOGGER.warn(String.format("Public IP with ID: %s linked to the Kubernetes cluster: %s is not usable", detailsVO.getValue(), kubernetesCluster.getName()));
+            return null;
+        }
+        return address;
+    }
+
+    protected IpAddress acquireVpcTierKubernetesPublicIp(Network network) throws
+            InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException {
+        IpAddress ip = networkService.allocateIP(owner, kubernetesCluster.getZoneId(), network.getId(), null, null);
+        if (ip == null) {
+            return null;
+        }
+        ip = vpcService.associateIPToVpc(ip.getId(), network.getVpcId());
+        ip = ipAddressManager.associateIPToGuestNetwork(ip.getId(), network.getId(), false);
+        kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ApiConstants.PUBLIC_IP_ID, ip.getUuid(), false);
+        return ip;
+    }
+
+    protected Pair<String, Integer> getKubernetesClusterServerIpSshPortForIsolatedNetwork(Network network) {
+        String ip = null;
+        IpAddress address = getNetworkSourceNatIp(network);
+        if (address != null) {
+            ip = address.getAddress().addr();
+        }
+        return new Pair<>(ip, CLUSTER_NODES_DEFAULT_START_SSH_PORT);
+    }
+
+    protected Pair<String, Integer> getKubernetesClusterServerIpSshPortForSharedNetwork(UserVm controlVm) {
+        int port = DEFAULT_SSH_PORT;
+        controlVm = fetchControlVmIfMissing(controlVm);
+        if (controlVm == null) {
+            LOGGER.warn(String.format("Unable to retrieve control VM for Kubernetes cluster : %s", kubernetesCluster.getName()));
+            return new Pair<>(null, port);
+        }
+        return new Pair<>(controlVm.getPrivateIpAddress(), port);
+    }
+
+    protected Pair<String, Integer> getKubernetesClusterServerIpSshPortForVpcTier(Network network,
+                                                                                  boolean acquireNewPublicIpForVpcTierIfNeeded) throws
+            InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException {
+        int port = CLUSTER_NODES_DEFAULT_START_SSH_PORT;
+        IpAddress address = getVpcTierKubernetesPublicIp(network);
+        if (address != null) {
+            return new Pair<>(address.getAddress().addr(), port);
+        }
+        if (acquireNewPublicIpForVpcTierIfNeeded) {
+            address = acquireVpcTierKubernetesPublicIp(network);
+            if (address != null) {
+                return new Pair<>(address.getAddress().addr(), port);
+            }
+        }
+        LOGGER.warn(String.format("No public IP found for the the VPC tier: %s, Kubernetes cluster : %s", network, kubernetesCluster.getName()));
+        return new Pair<>(null, port);
+    }
+
+    protected Pair<String, Integer> getKubernetesClusterServerIpSshPort(UserVm controlVm, boolean acquireNewPublicIpForVpcTierIfNeeded) throws
+            InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException {
         int port = CLUSTER_NODES_DEFAULT_START_SSH_PORT;
         KubernetesClusterDetailsVO detail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS);
         if (detail != null && StringUtils.isNotEmpty(detail.getValue())) {
@@ -340,32 +427,27 @@ public class KubernetesClusterActionWorker {
             LOGGER.warn(String.format("Network for Kubernetes cluster : %s cannot be found", kubernetesCluster.getName()));
             return new Pair<>(null, port);
         }
+        if (network.getVpcId() != null) {
+            return getKubernetesClusterServerIpSshPortForVpcTier(network, acquireNewPublicIpForVpcTierIfNeeded);
+        }
         if (Network.GuestType.Isolated.equals(network.getGuestType())) {
-            List<? extends IpAddress> addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true);
+            return getKubernetesClusterServerIpSshPortForIsolatedNetwork(network);
-            if (CollectionUtils.isEmpty(addresses)) {
-                LOGGER.warn(String.format("No public IP addresses found for network : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName()));
-                return new Pair<>(null, port);
-            }
-            for (IpAddress address : addresses) {
-                if (address.isSourceNat()) {
-                    return new Pair<>(address.getAddress().addr(), port);
-                }
-            }
-            LOGGER.warn(String.format("No source NAT IP addresses found for network : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName()));
-            return new Pair<>(null, port);
         } else if (Network.GuestType.Shared.equals(network.getGuestType())) {
-            port = 22;
+            return getKubernetesClusterServerIpSshPortForSharedNetwork(controlVm);
-            controlVm = fetchControlVmIfMissing(controlVm);
-            if (controlVm == null) {
-                LOGGER.warn(String.format("Unable to retrieve control VM for Kubernetes cluster : %s", kubernetesCluster.getName()));
-                return new Pair<>(null, port);
-            }
-            return new Pair<>(controlVm.getPrivateIpAddress(), port);
         }
         LOGGER.warn(String.format("Unable to retrieve server IP address for Kubernetes cluster : %s", kubernetesCluster.getName()));
         return  new Pair<>(null, port);
     }
 
+    protected Pair<String, Integer> getKubernetesClusterServerIpSshPort(UserVm controlVm) {
+        try {
+            return getKubernetesClusterServerIpSshPort(controlVm, false);
+        } catch (InsufficientAddressCapacityException | ResourceAllocationException | ResourceUnavailableException e) {
+            LOGGER.debug("This exception should not have occurred", e);
+        }
+        return new Pair<>(null, CLUSTER_NODES_DEFAULT_START_SSH_PORT);
+    }
+
     protected void attachIsoKubernetesVMs(List<UserVm> clusterVMs, final KubernetesSupportedVersion kubernetesSupportedVersion) throws CloudRuntimeException {
         KubernetesSupportedVersion version = kubernetesSupportedVersion;
         if (kubernetesSupportedVersion == null) {

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java

@@ -19,16 +19,19 @@ package com.cloud.kubernetes.cluster.actionworkers;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 
 import org.apache.cloudstack.annotation.AnnotationService;
 import org.apache.cloudstack.annotation.dao.AnnotationDao;
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.log4j.Level;
 
 import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientAddressCapacityException;
 import com.cloud.exception.ManagementServerException;
 import com.cloud.exception.PermissionDeniedException;
 import com.cloud.exception.ResourceUnavailableException;
@@ -134,25 +137,15 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod
         }
     }
 
-    private void deleteKubernetesClusterNetworkRules() throws ManagementServerException {
+    protected void deleteKubernetesClusterIsolatedNetworkRules(Network network, List<Long> removedVmIds) throws ManagementServerException {
-        NetworkVO network = networkDao.findById(kubernetesCluster.getNetworkId());
+        IpAddress publicIp = getNetworkSourceNatIp(network);
-        if (network == null) {
-            return;
-        }
-        List<Long> removedVmIds = new ArrayList<>();
-        if (!CollectionUtils.isEmpty(clusterVMs)) {
-            for (KubernetesClusterVmMapVO clusterVM : clusterVMs) {
-                removedVmIds.add(clusterVM.getVmId());
-            }
-        }
-        IpAddress publicIp = getSourceNatIp(network);
         if (publicIp == null) {
             throw new ManagementServerException(String.format("No source NAT IP addresses found for network : %s", network.getName()));
         }
         try {
-            removeLoadBalancingRule(publicIp, network, owner, CLUSTER_API_PORT);
+            removeLoadBalancingRule(publicIp, network, owner);
         } catch (ResourceUnavailableException e) {
-            throw new ManagementServerException(String.format("Failed to KubernetesCluster load balancing rule for network : %s", network.getName()));
+            throw new ManagementServerException(String.format("Failed to KubernetesCluster load balancing rule for network : %s", network.getName()), e);
         }
         FirewallRule firewallRule = removeApiFirewallRule(publicIp);
         if (firewallRule == null) {
@@ -162,6 +155,19 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod
         if (firewallRule == null) {
             logMessage(Level.WARN, "Firewall rule for SSH access can't be removed", null);
         }
+        try {
+            removePortForwardingRules(publicIp, network, owner, removedVmIds);
+        } catch (ResourceUnavailableException e) {
+            throw new ManagementServerException(String.format("Failed to KubernetesCluster port forwarding rules for network : %s", network.getName()), e);
+        }
+    }
+
+    protected void deleteKubernetesClusterVpcTierRules(Network network, List<Long> removedVmIds) throws ManagementServerException {
+        IpAddress publicIp = getVpcTierKubernetesPublicIp(network);
+        if (publicIp == null) {
+            return;
+        }
+        removeVpcTierAclRules(network);
         try {
             removePortForwardingRules(publicIp, network, owner, removedVmIds);
         } catch (ResourceUnavailableException e) {
@@ -169,6 +175,22 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod
         }
     }
 
+    private void deleteKubernetesClusterNetworkRules() throws ManagementServerException {
+        NetworkVO network = networkDao.findById(kubernetesCluster.getNetworkId());
+        if (network == null) {
+            return;
+        }
+        List<Long> removedVmIds = new ArrayList<>();
+        if (!CollectionUtils.isEmpty(clusterVMs)) {
+            removedVmIds = clusterVMs.stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList());
+        }
+        if (network.getVpcId() != null) {
+            deleteKubernetesClusterVpcTierRules(network, removedVmIds);
+            return;
+        }
+        deleteKubernetesClusterIsolatedNetworkRules(network, removedVmIds);
+    }
+
     private void validateClusterVMsDestroyed() {
         if(clusterVMs!=null  && !clusterVMs.isEmpty()) { // Wait for few seconds to get all VMs really expunged
             final int maxRetries = 3;
@@ -200,6 +222,19 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod
         }
     }
 
+    private void releaseVpcTierPublicIpIfNeeded() throws InsufficientAddressCapacityException {
+        NetworkVO networkVO = networkDao.findById(kubernetesCluster.getNetworkId());
+        if (networkVO == null || networkVO.getVpcId() == null) {
+            return;
+        }
+        IpAddress address = getVpcTierKubernetesPublicIp(networkVO);
+        if (address == null) {
+            return;
+        }
+        networkService.releaseIpAddress(address.getId());
+        kubernetesClusterDetailsDao.removeDetail(kubernetesCluster.getId(), ApiConstants.PUBLIC_IP_ID);
+    }
+
     public boolean destroy() throws CloudRuntimeException {
         init();
         validateClusterSate();
@@ -258,6 +293,14 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod
                     updateKubernetesClusterEntryForGC();
                     throw new CloudRuntimeException(msg, e);
                 }
+                try {
+                    releaseVpcTierPublicIpIfNeeded();
+                } catch (InsufficientAddressCapacityException e) {
+                    String msg = String.format("Failed to release public IP for VPC tier used by Kubernetes cluster : %s", kubernetesCluster.getName());
+                    LOGGER.warn(msg, e);
+                    updateKubernetesClusterEntryForGC();
+                    throw new CloudRuntimeException(msg, e);
+                }
             }
         } else {
             String msg = String.format("Failed to destroy one or more VMs as part of Kubernetes cluster : %s cleanup", kubernetesCluster.getName());

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java

@@ -17,6 +17,31 @@
 
 package com.cloud.kubernetes.cluster.actionworkers;
 
+import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import javax.inject.Inject;
+
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
+import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd;
+import org.apache.cloudstack.api.command.user.vm.StartVMCmd;
+import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Level;
+
 import com.cloud.capacity.CapacityManager;
 import com.cloud.dc.ClusterDetailsDao;
 import com.cloud.dc.ClusterDetailsVO;
@@ -24,11 +49,14 @@ import com.cloud.dc.ClusterVO;
 import com.cloud.dc.DataCenter;
 import com.cloud.dc.dao.ClusterDao;
 import com.cloud.deploy.DeployDestination;
+import com.cloud.exception.InsufficientAddressCapacityException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.InsufficientServerCapacityException;
+import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.ManagementServerException;
 import com.cloud.exception.NetworkRuleConflictException;
 import com.cloud.exception.OperationTimedoutException;
+import com.cloud.exception.PermissionDeniedException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.host.Host;
 import com.cloud.host.HostVO;
@@ -48,9 +76,15 @@ import com.cloud.network.firewall.FirewallService;
 import com.cloud.network.lb.LoadBalancingRulesService;
 import com.cloud.network.rules.FirewallRule;
 import com.cloud.network.rules.FirewallRuleVO;
+import com.cloud.network.rules.LoadBalancer;
 import com.cloud.network.rules.PortForwardingRuleVO;
 import com.cloud.network.rules.RulesService;
 import com.cloud.network.rules.dao.PortForwardingRulesDao;
+import com.cloud.network.vpc.NetworkACL;
+import com.cloud.network.vpc.NetworkACLItem;
+import com.cloud.network.vpc.NetworkACLItemDao;
+import com.cloud.network.vpc.NetworkACLItemVO;
+import com.cloud.network.vpc.NetworkACLService;
 import com.cloud.offering.ServiceOffering;
 import com.cloud.resource.ResourceManager;
 import com.cloud.storage.Volume;
@@ -77,28 +111,6 @@ import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VmDetailConstants;
 import com.cloud.vm.dao.VMInstanceDao;
 
-import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.api.BaseCmd;
-import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
-import org.apache.cloudstack.api.command.user.vm.StartVMCmd;
-import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.Level;
-
-import javax.inject.Inject;
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
-
 public class KubernetesClusterResourceModifierActionWorker extends KubernetesClusterActionWorker {
 
     @Inject
@@ -114,6 +126,10 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
     @Inject
     protected FirewallService firewallService;
     @Inject
+    protected NetworkACLService networkACLService;
+    @Inject
+    protected  NetworkACLItemDao networkACLItemDao;
+    @Inject
     protected LoadBalancingRulesService lbService;
     @Inject
     protected RulesService rulesService;
@@ -402,19 +418,6 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
         return nodeVm;
     }
 
-    protected IpAddress getSourceNatIp(Network network) {
-        List<? extends IpAddress> addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true);
-        if (CollectionUtils.isEmpty(addresses)) {
-            return null;
-        }
-        for (IpAddress address : addresses) {
-            if (address.isSourceNat()) {
-                return address;
-            }
-        }
-        return null;
-    }
-
     protected void provisionFirewallRules(final IpAddress publicIp, final Account account, int startPort, int endPort) throws NoSuchFieldException,
             IllegalAccessException, ResourceUnavailableException, NetworkRuleConflictException {
         List<String> sourceCidrList = new ArrayList<String>();
@@ -447,52 +450,51 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
         firewallService.applyIngressFwRules(publicIp.getId(), account);
     }
 
+    protected void provisionPublicIpPortForwardingRule(IpAddress publicIp, Network network, Account account,
+                                                       final long vmId, final int sourcePort, final int destPort) throws NetworkRuleConflictException, ResourceUnavailableException {
+        final long publicIpId = publicIp.getId();
+        final long networkId = network.getId();
+        final long accountId = account.getId();
+        final long domainId = account.getDomainId();
+        Nic vmNic = networkModel.getNicInNetwork(vmId, networkId);
+        final Ip vmIp = new Ip(vmNic.getIPv4Address());
+        PortForwardingRuleVO pfRule = Transaction.execute((TransactionCallbackWithException<PortForwardingRuleVO, NetworkRuleConflictException>) status -> {
+            PortForwardingRuleVO newRule =
+                    new PortForwardingRuleVO(null, publicIpId,
+                            sourcePort, sourcePort,
+                            vmIp,
+                            destPort, destPort,
+                            "tcp", networkId, accountId, domainId, vmId);
+            newRule.setDisplay(true);
+            newRule.setState(FirewallRule.State.Add);
+            newRule = portForwardingRulesDao.persist(newRule);
+            return newRule;
+        });
+        rulesService.applyPortForwardingRules(publicIp.getId(), account);
+        if (LOGGER.isInfoEnabled()) {
+            LOGGER.info(String.format("Provisioned SSH port forwarding rule: %s from port %d to %d on %s to the VM IP : %s in Kubernetes cluster : %s", pfRule.getUuid(), sourcePort, destPort, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getName()));
+        }
+    }
+
     /**
      * To provision SSH port forwarding rules for the given Kubernetes cluster
      * for its given virtual machines
+     *
      * @param publicIp
      * @param network
      * @param account
-     * @param List<Long> clusterVMIds (when empty then method must be called while
+     * @param clusterVMIds (when empty then method must be called while
-     *                  down-scaling of the KubernetesCluster therefore no new rules
+     *                     down-scaling of the KubernetesCluster therefore no new rules
-     *                  to be added)
+     *                     to be added)
-     * @param firewallRuleSourcePortStart
      * @throws ResourceUnavailableException
      * @throws NetworkRuleConflictException
      */
     protected void provisionSshPortForwardingRules(IpAddress publicIp, Network network, Account account,
-                                                   List<Long> clusterVMIds, int firewallRuleSourcePortStart) throws ResourceUnavailableException,
+                                                   List<Long> clusterVMIds) throws ResourceUnavailableException,
             NetworkRuleConflictException {
         if (!CollectionUtils.isEmpty(clusterVMIds)) {
-            final long publicIpId = publicIp.getId();
-            final long networkId = network.getId();
-            final long accountId = account.getId();
-            final long domainId = account.getDomainId();
             for (int i = 0; i < clusterVMIds.size(); ++i) {
-                long vmId = clusterVMIds.get(i);
+                provisionPublicIpPortForwardingRule(publicIp, network, account, clusterVMIds.get(i), CLUSTER_NODES_DEFAULT_START_SSH_PORT + i, DEFAULT_SSH_PORT);
-                Nic vmNic = networkModel.getNicInNetwork(vmId, networkId);
-                final Ip vmIp = new Ip(vmNic.getIPv4Address());
-                final long vmIdFinal = vmId;
-                final int srcPortFinal = firewallRuleSourcePortStart + i;
-                PortForwardingRuleVO pfRule = Transaction.execute(new TransactionCallbackWithException<PortForwardingRuleVO, NetworkRuleConflictException>() {
-                    @Override
-                    public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException {
-                        PortForwardingRuleVO newRule =
-                                new PortForwardingRuleVO(null, publicIpId,
-                                        srcPortFinal, srcPortFinal,
-                                        vmIp,
-                                        22, 22,
-                                        "tcp", networkId, accountId, domainId, vmIdFinal);
-                        newRule.setDisplay(true);
-                        newRule.setState(FirewallRule.State.Add);
-                        newRule = portForwardingRulesDao.persist(newRule);
-                        return newRule;
-                    }
-                });
-                rulesService.applyPortForwardingRules(publicIp.getId(), account);
-                if (LOGGER.isInfoEnabled()) {
-                    LOGGER.info(String.format("Provisioned SSH port forwarding rule from port %d to 22 on %s to the VM IP : %s in Kubernetes cluster : %s", srcPortFinal, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getName()));
-                }
             }
         }
     }
@@ -551,19 +553,205 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
     }
 
     protected void removeLoadBalancingRule(final IpAddress publicIp, final Network network,
-                                           final Account account, final int port) throws ResourceUnavailableException {
+                                           final Account account) throws ResourceUnavailableException {
         List<LoadBalancerVO> rules = loadBalancerDao.listByIpAddress(publicIp.getId());
         for (LoadBalancerVO rule : rules) {
             if (rule.getNetworkId() == network.getId() &&
                     rule.getAccountId() == account.getId() &&
-                    rule.getSourcePortStart() == port &&
+                    rule.getSourcePortStart() == CLUSTER_API_PORT &&
-                    rule.getSourcePortEnd() == port) {
+                    rule.getSourcePortEnd() == CLUSTER_API_PORT) {
                 lbService.deleteLoadBalancerRule(rule.getId(), true);
                 break;
             }
         }
     }
 
+    protected void provisionVpcTierAllowPortACLRule(final Network network, int startPort, int endPorts) throws NoSuchFieldException,
+            IllegalAccessException, ResourceUnavailableException {
+        List<NetworkACLItemVO> aclItems = networkACLItemDao.listByACL(network.getNetworkACLId());
+        aclItems = aclItems.stream().filter(x -> !NetworkACLItem.State.Revoke.equals(x.getState())).collect(Collectors.toList());
+        CreateNetworkACLCmd rule = new CreateNetworkACLCmd();
+        rule = ComponentContext.inject(rule);
+        Map<String, Object> fieldValues = Map.of(
+                "protocol", "TCP",
+                "publicStartPort", startPort,
+                "publicEndPort", endPorts,
+                "trafficType", NetworkACLItem.TrafficType.Ingress.toString(),
+                "networkId", network.getId(),
+                "aclId", network.getNetworkACLId(),
+                "action", NetworkACLItem.Action.Allow.toString()
+        );
+        for (Map.Entry<String, Object> entry : fieldValues.entrySet()) {
+            Field field = rule.getClass().getDeclaredField(entry.getKey());
+            field.setAccessible(true);
+            field.set(rule, entry.getValue());
+        }
+        NetworkACLItem aclRule = networkACLService.createNetworkACLItem(rule);
+        networkACLService.moveRuleToTheTopInACLList(aclRule);
+        networkACLService.applyNetworkACL(aclRule.getAclId());
+    }
+
+    protected void removeVpcTierAllowPortACLRule(final Network network, int startPort, int endPort) throws NoSuchFieldException,
+            IllegalAccessException, ResourceUnavailableException {
+        List<NetworkACLItemVO> aclItems = networkACLItemDao.listByACL(network.getNetworkACLId());
+        aclItems = aclItems.stream().filter(x -> (x.getProtocol() != null &&
+                        x.getProtocol().equals("TCP") &&
+                        x.getSourcePortStart() != null &&
+                        x.getSourcePortStart().equals(startPort) &&
+                        x.getSourcePortEnd() != null &&
+                        x.getSourcePortEnd().equals(endPort) &&
+                        x.getAction().equals(NetworkACLItem.Action.Allow)))
+                .collect(Collectors.toList());
+
+        for (NetworkACLItemVO aclItem : aclItems) {
+            networkACLService.revokeNetworkACLItem(aclItem.getId());
+        }
+    }
+
+    protected void provisionLoadBalancerRule(final IpAddress publicIp, final Network network,
+                                             final Account account, final List<Long> clusterVMIds, final int port) throws NetworkRuleConflictException,
+            InsufficientAddressCapacityException {
+        LoadBalancer lb = lbService.createPublicLoadBalancerRule(null, "api-lb", "LB rule for API access",
+                port, port, port, port,
+                publicIp.getId(), NetUtils.TCP_PROTO, "roundrobin", network.getId(),
+                account.getId(), false, NetUtils.TCP_PROTO, true);
+
+        Map<Long, List<String>> vmIdIpMap = new HashMap<>();
+        for (int i = 0; i < kubernetesCluster.getControlNodeCount(); ++i) {
+            List<String> ips = new ArrayList<>();
+            Nic controlVmNic = networkModel.getNicInNetwork(clusterVMIds.get(i), kubernetesCluster.getNetworkId());
+            ips.add(controlVmNic.getIPv4Address());
+            vmIdIpMap.put(clusterVMIds.get(i), ips);
+        }
+        lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap, false);
+    }
+
+    protected void createFirewallRules(IpAddress publicIp, List<Long> clusterVMIds, boolean apiRule) throws ManagementServerException {
+        // Firewall rule for SSH access on each node VM
+        try {
+            int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1;
+            provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort);
+            if (LOGGER.isInfoEnabled()) {
+                LOGGER.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster : %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getName()));
+            }
+        } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) {
+            throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+        if (!apiRule) {
+            return;
+        }
+        // Firewall rule for API access for control node VMs
+        try {
+            provisionFirewallRules(publicIp, owner, CLUSTER_API_PORT, CLUSTER_API_PORT);
+            if (LOGGER.isInfoEnabled()) {
+                LOGGER.info(String.format("Provisioned firewall rule to open up port %d on %s for Kubernetes cluster %s",
+                        CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster.getName()));
+            }
+        } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) {
+            throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+    }
+
+    /**
+     * Setup network rules for Kubernetes cluster
+     * Open up firewall port CLUSTER_API_PORT, secure port on which Kubernetes
+     * API server is running. Also create load balancing rule to forward public
+     * IP traffic to control VMs' private IP.
+     * Open up  firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n
+     * for SSH access. Also create port-forwarding rule to forward public IP traffic to all
+     * @param network
+     * @param clusterVMIds
+     * @throws ManagementServerException
+     */
+    protected void setupKubernetesClusterIsolatedNetworkRules(IpAddress publicIp, Network network, List<Long> clusterVMIds, boolean apiRule) throws ManagementServerException {
+        createFirewallRules(publicIp, clusterVMIds, apiRule);
+
+        // Port forwarding rule for SSH access on each node VM
+        try {
+            provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds);
+        } catch (ResourceUnavailableException | NetworkRuleConflictException e) {
+            throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+
+        if (!apiRule) {
+            return;
+        }
+        // Load balancer rule for API access for control node VMs
+        try {
+            provisionLoadBalancerRule(publicIp, network, owner, clusterVMIds, CLUSTER_API_PORT);
+        } catch (NetworkRuleConflictException | InsufficientAddressCapacityException e) {
+            throw new ManagementServerException(String.format("Failed to provision load balancer rule for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+    }
+
+    protected void createVpcTierAclRules(Network network) throws ManagementServerException {
+        if (network.getNetworkACLId() == NetworkACL.DEFAULT_ALLOW) {
+            return;
+        }
+        // ACL rule for API access for control node VMs
+        try {
+            provisionVpcTierAllowPortACLRule(network, CLUSTER_API_PORT, CLUSTER_API_PORT);
+            if (LOGGER.isInfoEnabled()) {
+                LOGGER.info(String.format("Provisioned ACL rule to open up port %d on %s for Kubernetes cluster %s",
+                        CLUSTER_API_PORT, publicIpAddress, kubernetesCluster.getName()));
+            }
+        } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | InvalidParameterValueException | PermissionDeniedException e) {
+            throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+        try {
+            provisionVpcTierAllowPortACLRule(network, DEFAULT_SSH_PORT, DEFAULT_SSH_PORT);
+            if (LOGGER.isInfoEnabled()) {
+                LOGGER.info(String.format("Provisioned ACL rule to open up port %d on %s for Kubernetes cluster %s",
+                        DEFAULT_SSH_PORT, publicIpAddress, kubernetesCluster.getName()));
+            }
+        } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | InvalidParameterValueException | PermissionDeniedException e) {
+            throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+    }
+
+    protected void removeVpcTierAclRules(Network network) throws ManagementServerException {
+        if (network.getNetworkACLId() == NetworkACL.DEFAULT_ALLOW) {
+            return;
+        }
+        // ACL rule for API access for control node VMs
+        try {
+            removeVpcTierAllowPortACLRule(network, CLUSTER_API_PORT, CLUSTER_API_PORT);
+            if (LOGGER.isInfoEnabled()) {
+                LOGGER.info(String.format("Removed network ACL rule to open up port %d on %s for Kubernetes cluster %s",
+                        CLUSTER_API_PORT, publicIpAddress, kubernetesCluster.getName()));
+            }
+        } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException e) {
+            throw new ManagementServerException(String.format("Failed to remove network ACL rule for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+        // ACL rule for SSH access for all node VMs
+        try {
+            removeVpcTierAllowPortACLRule(network, DEFAULT_SSH_PORT, DEFAULT_SSH_PORT);
+            if (LOGGER.isInfoEnabled()) {
+                LOGGER.info(String.format("Removed network ACL rule to open up port %d on %s for Kubernetes cluster %s",
+                        DEFAULT_SSH_PORT, publicIpAddress, kubernetesCluster.getName()));
+            }
+        } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException e) {
+            throw new ManagementServerException(String.format("Failed to remove network ACL rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+    }
+
+    protected void setupKubernetesClusterVpcTierRules(IpAddress publicIp, Network network, List<Long> clusterVMIds) throws ManagementServerException {
+        // Create ACL rules
+        createVpcTierAclRules(network);
+        // Add port forwarding for API access
+        try {
+            provisionPublicIpPortForwardingRule(publicIp, network, owner, clusterVMIds.get(0), CLUSTER_API_PORT, CLUSTER_API_PORT);
+        } catch (ResourceUnavailableException | NetworkRuleConflictException e) {
+            throw new ManagementServerException(String.format("Failed to activate API port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+        // Add port forwarding rule for SSH access on each node VM
+        try {
+            provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds);
+        } catch (ResourceUnavailableException | NetworkRuleConflictException e) {
+            throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+    }
+
     protected String getKubernetesClusterNodeNamePrefix() {
         String prefix = kubernetesCluster.getName();
         if (!NetUtils.verifyDomainNameLabel(prefix, true)) {
@@ -669,7 +857,6 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
         } finally {
             // Deploying the autoscaler might fail but it can be deployed manually too, so no need to go to an alert state
             updateLoginUserDetails(null);
-            stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded);
         }
     }
 }

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java

@@ -28,6 +28,7 @@ import javax.inject.Inject;
 
 import org.apache.cloudstack.api.InternalIdentity;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Level;
 
 import com.cloud.dc.DataCenter;
@@ -57,7 +58,6 @@ import com.cloud.vm.UserVmVO;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.dao.VMInstanceDao;
-import org.apache.commons.lang3.StringUtils;
 
 public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModifierActionWorker {
 
@@ -115,6 +115,44 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif
         logTransitStateToFailedIfNeededAndThrow(logLevel, message, null);
     }
 
+    private void scaleKubernetesClusterIsolatedNetworkRules(final List<Long> clusterVMIds) throws ManagementServerException {
+        IpAddress publicIp = getNetworkSourceNatIp(network);
+        if (publicIp == null) {
+            throw new ManagementServerException(String.format("No source NAT IP addresses found for network : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName()));
+        }
+
+        // Remove existing SSH firewall rules
+        FirewallRule firewallRule = removeSshFirewallRule(publicIp);
+        if (firewallRule == null) {
+            throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned");
+        }
+        int existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd();
+        try {
+            removePortForwardingRules(publicIp, network, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, existingFirewallRuleSourcePortEnd);
+        } catch (ResourceUnavailableException e) {
+            throw new ManagementServerException(String.format("Failed to remove SSH port forwarding rules for removed VMs for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+        setupKubernetesClusterIsolatedNetworkRules(publicIp, network, clusterVMIds, false);
+    }
+
+    private void scaleKubernetesClusterVpcTierRules(final List<Long> clusterVMIds) throws ManagementServerException {
+        IpAddress publicIp = getVpcTierKubernetesPublicIp(network);
+        if (publicIp == null) {
+            throw new ManagementServerException(String.format("No public IP addresses found for VPC tier : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName()));
+        }
+        try {
+            removePortForwardingRules(publicIp, network, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1);
+        } catch (ResourceUnavailableException e) {
+            throw new ManagementServerException(String.format("Failed to remove SSH port forwarding rules for removed VMs for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+        // Add port forwarding rule for SSH access on each node VM
+        try {
+            provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds);
+        } catch (ResourceUnavailableException | NetworkRuleConflictException e) {
+            throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
+        }
+    }
+
     /**
      * Scale network rules for an existing Kubernetes cluster while scaling it
      * Open up firewall for SSH access from port NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n.
@@ -130,40 +168,11 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif
             }
             return;
         }
-        IpAddress publicIp = getSourceNatIp(network);
+        if (network.getVpcId() != null) {
-        if (publicIp == null) {
+            scaleKubernetesClusterVpcTierRules(clusterVMIds);
-            throw new ManagementServerException(String.format("No source NAT IP addresses found for network : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName()));
+            return;
-        }
-
-        // Remove existing SSH firewall rules
-        FirewallRule firewallRule = removeSshFirewallRule(publicIp);
-        if (firewallRule == null) {
-            throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned");
-        }
-        int existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd();
-        int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1;
-        // Provision new SSH firewall rules
-        try {
-            provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort);
-            if (LOGGER.isDebugEnabled()) {
-                LOGGER.debug(String.format("Provisioned  firewall rule to open up port %d to %d on %s in Kubernetes cluster %s",
-                        CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getName()));
-            }
-        } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException e) {
-            throw new ManagementServerException(String.format("Failed to activate SSH firewall rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
-        }
-
-        try {
-            removePortForwardingRules(publicIp, network, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, existingFirewallRuleSourcePortEnd);
-        } catch (ResourceUnavailableException e) {
-            throw new ManagementServerException(String.format("Failed to remove SSH port forwarding rules for removed VMs for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
-        }
-
-        try {
-            provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, CLUSTER_NODES_DEFAULT_START_SSH_PORT);
-        } catch (ResourceUnavailableException | NetworkRuleConflictException e) {
-            throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
         }
+        scaleKubernetesClusterIsolatedNetworkRules(clusterVMIds);
     }
 
     private KubernetesClusterVO updateKubernetesClusterEntry(final Long newSize, final ServiceOffering newServiceOffering) throws CloudRuntimeException {
@@ -406,6 +415,19 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif
         kubernetesCluster = updateKubernetesClusterEntry(clusterSize, null);
     }
 
+    private boolean isAutoscalingChanged() {
+        if (this.isAutoscalingEnabled == null) {
+            return false;
+        }
+        if (this.isAutoscalingEnabled != kubernetesCluster.getAutoscalingEnabled()) {
+            return true;
+        }
+        if (minSize != null && (!minSize.equals(kubernetesCluster.getMinSize()))) {
+            return true;
+        }
+        return maxSize != null && (!maxSize.equals(kubernetesCluster.getMaxSize()));
+    }
+
     public boolean scaleCluster() throws CloudRuntimeException {
         init();
         if (LOGGER.isInfoEnabled()) {
@@ -417,11 +439,17 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif
         if (existingServiceOffering == null) {
             logAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, service offering for the Kubernetes cluster not found!", kubernetesCluster.getName()));
         }
-
+        final boolean autscalingChanged = isAutoscalingChanged();
-        if (this.isAutoscalingEnabled != null) {
-            return autoscaleCluster(this.isAutoscalingEnabled, minSize, maxSize);
-        }
         final boolean serviceOfferingScalingNeeded = serviceOffering != null && serviceOffering.getId() != existingServiceOffering.getId();
+
+        if (autscalingChanged) {
+            boolean autoScaled = autoscaleCluster(this.isAutoscalingEnabled, minSize, maxSize);
+            if (autoScaled && serviceOfferingScalingNeeded) {
+                scaleKubernetesClusterOffering();
+            }
+            stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded);
+            return autoScaled;
+        }
         final boolean clusterSizeScalingNeeded = clusterSize != null && clusterSize != originalClusterSize;
         final long newVMRequired = clusterSize == null ? 0 : clusterSize - originalClusterSize;
         if (serviceOfferingScalingNeeded && clusterSizeScalingNeeded) {

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java

@@ -17,6 +17,27 @@
 
 package com.cloud.kubernetes.cluster.actionworkers;
 
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.InternalIdentity;
+import org.apache.cloudstack.framework.ca.Certificate;
+import org.apache.cloudstack.utils.security.CertUtils;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Level;
+
 import com.cloud.dc.DataCenter;
 import com.cloud.dc.Vlan;
 import com.cloud.dc.VlanVO;
@@ -25,7 +46,7 @@ import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientAddressCapacityException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.ManagementServerException;
-import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.kubernetes.cluster.KubernetesCluster;
@@ -40,7 +61,6 @@ import com.cloud.kubernetes.version.KubernetesVersionManagerImpl;
 import com.cloud.network.IpAddress;
 import com.cloud.network.Network;
 import com.cloud.network.addr.PublicIp;
-import com.cloud.network.rules.LoadBalancer;
 import com.cloud.offering.ServiceOffering;
 import com.cloud.storage.LaunchPermissionVO;
 import com.cloud.user.Account;
@@ -49,33 +69,11 @@ import com.cloud.uservm.UserVm;
 import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.net.Ip;
-import com.cloud.utils.net.NetUtils;
-import com.cloud.vm.Nic;
 import com.cloud.vm.ReservationContext;
 import com.cloud.vm.ReservationContextImpl;
 import com.cloud.vm.UserVmManager;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VmDetailConstants;
-import org.apache.cloudstack.api.BaseCmd;
-import org.apache.cloudstack.api.InternalIdentity;
-import org.apache.cloudstack.framework.ca.Certificate;
-import org.apache.cloudstack.utils.security.CertUtils;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.Level;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
 
 public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker {
 
@@ -378,91 +376,28 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
         return network;
     }
 
-    private void provisionLoadBalancerRule(final IpAddress publicIp, final Network network,
+    protected void setupKubernetesClusterNetworkRules(Network network, List<UserVm> clusterVMs) throws ManagementServerException {
-                                           final Account account, final List<Long> clusterVMIds, final int port) throws NetworkRuleConflictException,
-            InsufficientAddressCapacityException {
-        LoadBalancer lb = lbService.createPublicLoadBalancerRule(null, "api-lb", "LB rule for API access",
-                port, port, port, port,
-                publicIp.getId(), NetUtils.TCP_PROTO, "roundrobin", network.getId(),
-                account.getId(), false, NetUtils.TCP_PROTO, true);
-
-        Map<Long, List<String>> vmIdIpMap = new HashMap<>();
-        for (int i = 0; i < kubernetesCluster.getControlNodeCount(); ++i) {
-            List<String> ips = new ArrayList<>();
-            Nic controlVmNic = networkModel.getNicInNetwork(clusterVMIds.get(i), kubernetesCluster.getNetworkId());
-            ips.add(controlVmNic.getIPv4Address());
-            vmIdIpMap.put(clusterVMIds.get(i), ips);
-        }
-        lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap, false);
-    }
-
-    /**
-     * Setup network rules for Kubernetes cluster
-     * Open up firewall port CLUSTER_API_PORT, secure port on which Kubernetes
-     * API server is running. Also create load balancing rule to forward public
-     * IP traffic to control VMs' private IP.
-     * Open up  firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n
-     * for SSH access. Also create port-forwarding rule to forward public IP traffic to all
-     * @param network
-     * @param clusterVMs
-     * @throws ManagementServerException
-     */
-    private void setupKubernetesClusterNetworkRules(Network network, List<UserVm> clusterVMs) throws ManagementServerException {
         if (!Network.GuestType.Isolated.equals(network.getGuestType())) {
             if (LOGGER.isDebugEnabled()) {
                 LOGGER.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName()));
             }
             return;
         }
-        List<Long> clusterVMIds = new ArrayList<>();
+        List<Long> clusterVMIds = clusterVMs.stream().map(UserVm::getId).collect(Collectors.toList());
-        for (UserVm vm : clusterVMs) {
+        if (network.getVpcId() != null) {
-            clusterVMIds.add(vm.getId());
+            IpAddress publicIp = getVpcTierKubernetesPublicIp(network);
+            if (publicIp == null) {
+                throw new ManagementServerException(String.format("No public IP addresses found for VPC tier : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName()));
+            }
+            setupKubernetesClusterVpcTierRules(publicIp, network, clusterVMIds);
+            return;
         }
-        IpAddress publicIp = getSourceNatIp(network);
+        IpAddress publicIp = getNetworkSourceNatIp(network);
         if (publicIp == null) {
             throw new ManagementServerException(String.format("No source NAT IP addresses found for network : %s, Kubernetes cluster : %s",
-                network.getName(), kubernetesCluster.getName()));
+                    network.getName(), kubernetesCluster.getName()));
-        }
-
-        createFirewallRules(publicIp, clusterVMIds);
-
-        // Load balancer rule fo API access for control node VMs
-        try {
-            provisionLoadBalancerRule(publicIp, network, owner, clusterVMIds, CLUSTER_API_PORT);
-        } catch (NetworkRuleConflictException | InsufficientAddressCapacityException e) {
-            throw new ManagementServerException(String.format("Failed to provision load balancer rule for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
-        }
-
-        // Port forwarding rule fo SSH access on each node VM
-        try {
-            provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, CLUSTER_NODES_DEFAULT_START_SSH_PORT);
-        } catch (ResourceUnavailableException | NetworkRuleConflictException e) {
-            throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
-        }
-    }
-
-    private void createFirewallRules(IpAddress publicIp, List<Long> clusterVMIds) throws ManagementServerException {
-        // Firewall rule fo API access for control node VMs
-        try {
-            provisionFirewallRules(publicIp, owner, CLUSTER_API_PORT, CLUSTER_API_PORT);
-            if (LOGGER.isInfoEnabled()) {
-                LOGGER.info(String.format("Provisioned firewall rule to open up port %d on %s for Kubernetes cluster %s",
-                        CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster.getName()));
-            }
-        } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) {
-            throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
-        }
-
-        // Firewall rule fo SSH access on each node VM
-        try {
-            int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1;
-            provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort);
-            if (LOGGER.isInfoEnabled()) {
-                LOGGER.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster : %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getName()));
-            }
-        } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) {
-            throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
         }
+        setupKubernetesClusterIsolatedNetworkRules(publicIp, network, clusterVMIds, true);
     }
 
     private void startKubernetesClusterVMs() {
@@ -546,7 +481,12 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
         } catch (ManagementServerException e) {
             logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s as its network cannot be started", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
         }
-        Pair<String, Integer> publicIpSshPort = getKubernetesClusterServerIpSshPort(null);
+        Pair<String, Integer> publicIpSshPort = new Pair<>(null, null);
+        try {
+            publicIpSshPort = getKubernetesClusterServerIpSshPort(null, true);
+        } catch (InsufficientAddressCapacityException | ResourceAllocationException | ResourceUnavailableException e) {
+            logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s as failed to acquire public IP" , kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed);
+        }
         publicIpAddress = publicIpSshPort.first();
         if (StringUtils.isEmpty(publicIpAddress) &&
                 (Network.GuestType.Isolated.equals(network.getGuestType()) || kubernetesCluster.getControlNodeCount() > 1)) { // Shared network, single-control node cluster won't have an IP yet

plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java

@@ -0,0 +1,213 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.kubernetes.cluster;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import com.cloud.api.query.dao.TemplateJoinDao;
+import com.cloud.api.query.vo.TemplateJoinVO;
+import com.cloud.dc.DataCenter;
+import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.exception.PermissionDeniedException;
+import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker;
+import com.cloud.network.Network;
+import com.cloud.network.dao.FirewallRulesDao;
+import com.cloud.network.rules.FirewallRule;
+import com.cloud.network.rules.FirewallRuleVO;
+import com.cloud.network.vpc.NetworkACL;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.dao.VMTemplateDao;
+
+@RunWith(MockitoJUnitRunner.class)
+public class KubernetesClusterManagerImplTest {
+
+    @Mock
+    FirewallRulesDao firewallRulesDao;
+
+    @Mock
+    VMTemplateDao templateDao;
+
+    @Mock
+    TemplateJoinDao templateJoinDao;
+
+    @Spy
+    @InjectMocks
+    KubernetesClusterManagerImpl kubernetesClusterManager;
+
+    @Test
+    public void testValidateVpcTierAllocated() {
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(network.getState()).thenReturn(Network.State.Allocated);
+        kubernetesClusterManager.validateVpcTier(network);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateVpcTierDefaultDenyRule() {
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(network.getState()).thenReturn(Network.State.Implemented);
+        Mockito.when(network.getNetworkACLId()).thenReturn(NetworkACL.DEFAULT_DENY);
+        kubernetesClusterManager.validateVpcTier(network);
+    }
+
+    @Test
+    public void testValidateVpcTierValid() {
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(network.getState()).thenReturn(Network.State.Implemented);
+        Mockito.when(network.getNetworkACLId()).thenReturn(NetworkACL.DEFAULT_ALLOW);
+        kubernetesClusterManager.validateVpcTier(network);
+    }
+
+    @Test
+    public void validateIsolatedNetworkIpRulesNoRules() {
+        long ipId = 1L;
+        FirewallRule.Purpose purpose = FirewallRule.Purpose.Firewall;
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose)).thenReturn(new ArrayList<>());
+        kubernetesClusterManager.validateIsolatedNetworkIpRules(ipId, FirewallRule.Purpose.Firewall, network, 3);
+    }
+
+    private FirewallRuleVO createRule(int startPort, int endPort) {
+        FirewallRuleVO rule = new FirewallRuleVO(null, null, startPort, endPort, "tcp", 1, 1, 1, FirewallRule.Purpose.Firewall, List.of("0.0.0.0/0"), null, null, null, FirewallRule.TrafficType.Ingress);
+        return rule;
+    }
+
+    @Test
+    public void validateIsolatedNetworkIpRulesNoConflictingRules() {
+        long ipId = 1L;
+        FirewallRule.Purpose purpose = FirewallRule.Purpose.Firewall;
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose)).thenReturn(List.of(createRule(80, 80), createRule(443, 443)));
+        kubernetesClusterManager.validateIsolatedNetworkIpRules(ipId, FirewallRule.Purpose.Firewall, network, 3);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void validateIsolatedNetworkIpRulesApiConflictingRules() {
+        long ipId = 1L;
+        FirewallRule.Purpose purpose = FirewallRule.Purpose.Firewall;
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose)).thenReturn(List.of(createRule(6440, 6445), createRule(443, 443)));
+        kubernetesClusterManager.validateIsolatedNetworkIpRules(ipId, FirewallRule.Purpose.Firewall, network, 3);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void validateIsolatedNetworkIpRulesSshConflictingRules() {
+        long ipId = 1L;
+        FirewallRule.Purpose purpose = FirewallRule.Purpose.Firewall;
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose)).thenReturn(List.of(createRule(2200, KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT), createRule(443, 443)));
+        kubernetesClusterManager.validateIsolatedNetworkIpRules(ipId, FirewallRule.Purpose.Firewall, network, 3);
+    }
+
+    @Test
+    public void validateIsolatedNetworkIpRulesNearConflictingRules() {
+        long ipId = 1L;
+        FirewallRule.Purpose purpose = FirewallRule.Purpose.Firewall;
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose)).thenReturn(List.of(createRule(2220, 2221), createRule(2225, 2227), createRule(6440, 6442), createRule(6444, 6446)));
+        kubernetesClusterManager.validateIsolatedNetworkIpRules(ipId, FirewallRule.Purpose.Firewall, network, 3);
+    }
+
+    @Test
+    public void testValidateKubernetesClusterScaleSizeNullNewSizeNoError() {
+        kubernetesClusterManager.validateKubernetesClusterScaleSize(Mockito.mock(KubernetesClusterVO.class), null, 100, Mockito.mock(DataCenter.class));
+    }
+
+    @Test
+    public void testValidateKubernetesClusterScaleSizeSameNewSizeNoError() {
+        Long size = 2L;
+        KubernetesClusterVO clusterVO = Mockito.mock(KubernetesClusterVO.class);
+        Mockito.when(clusterVO.getNodeCount()).thenReturn(size);
+        kubernetesClusterManager.validateKubernetesClusterScaleSize(clusterVO, size, 100, Mockito.mock(DataCenter.class));
+    }
+
+    @Test(expected = PermissionDeniedException.class)
+    public void testValidateKubernetesClusterScaleSizeStoppedCluster() {
+        Long size = 2L;
+        KubernetesClusterVO clusterVO = Mockito.mock(KubernetesClusterVO.class);
+        Mockito.when(clusterVO.getNodeCount()).thenReturn(size);
+        Mockito.when(clusterVO.getState()).thenReturn(KubernetesCluster.State.Stopped);
+        kubernetesClusterManager.validateKubernetesClusterScaleSize(clusterVO, 3L, 100, Mockito.mock(DataCenter.class));
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateKubernetesClusterScaleSizeZeroNewSize() {
+        Long size = 2L;
+        KubernetesClusterVO clusterVO = Mockito.mock(KubernetesClusterVO.class);
+        Mockito.when(clusterVO.getState()).thenReturn(KubernetesCluster.State.Running);
+        Mockito.when(clusterVO.getNodeCount()).thenReturn(size);
+        kubernetesClusterManager.validateKubernetesClusterScaleSize(clusterVO, 0L, 100, Mockito.mock(DataCenter.class));
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateKubernetesClusterScaleSizeOverMaxSize() {
+        KubernetesClusterVO clusterVO = Mockito.mock(KubernetesClusterVO.class);
+        Mockito.when(clusterVO.getState()).thenReturn(KubernetesCluster.State.Running);
+        Mockito.when(clusterVO.getControlNodeCount()).thenReturn(1L);
+        kubernetesClusterManager.validateKubernetesClusterScaleSize(clusterVO, 4L, 4, Mockito.mock(DataCenter.class));
+    }
+
+    @Test
+    public void testValidateKubernetesClusterScaleSizeDownsacaleNoError() {
+        KubernetesClusterVO clusterVO = Mockito.mock(KubernetesClusterVO.class);
+        Mockito.when(clusterVO.getState()).thenReturn(KubernetesCluster.State.Running);
+        Mockito.when(clusterVO.getControlNodeCount()).thenReturn(1L);
+        Mockito.when(clusterVO.getNodeCount()).thenReturn(4L);
+        kubernetesClusterManager.validateKubernetesClusterScaleSize(clusterVO, 2L, 10, Mockito.mock(DataCenter.class));
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateKubernetesClusterScaleSizeUpscaleDeletedTemplate() {
+        KubernetesClusterVO clusterVO = Mockito.mock(KubernetesClusterVO.class);
+        Mockito.when(clusterVO.getState()).thenReturn(KubernetesCluster.State.Running);
+        Mockito.when(clusterVO.getControlNodeCount()).thenReturn(1L);
+        Mockito.when(clusterVO.getNodeCount()).thenReturn(2L);
+        Mockito.when(templateDao.findById(Mockito.anyLong())).thenReturn(null);
+        kubernetesClusterManager.validateKubernetesClusterScaleSize(clusterVO, 4L, 10, Mockito.mock(DataCenter.class));
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateKubernetesClusterScaleSizeUpscaleNotInZoneTemplate() {
+        KubernetesClusterVO clusterVO = Mockito.mock(KubernetesClusterVO.class);
+        Mockito.when(clusterVO.getState()).thenReturn(KubernetesCluster.State.Running);
+        Mockito.when(clusterVO.getControlNodeCount()).thenReturn(1L);
+        Mockito.when(clusterVO.getNodeCount()).thenReturn(2L);
+        Mockito.when(templateDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(VMTemplateVO.class));
+        Mockito.when(templateJoinDao.newTemplateView(Mockito.any(VMTemplateVO.class), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(null);
+        kubernetesClusterManager.validateKubernetesClusterScaleSize(clusterVO, 4L, 10, Mockito.mock(DataCenter.class));
+    }
+
+    @Test
+    public void testValidateKubernetesClusterScaleSizeUpscaleNoError() {
+        KubernetesClusterVO clusterVO = Mockito.mock(KubernetesClusterVO.class);
+        Mockito.when(clusterVO.getState()).thenReturn(KubernetesCluster.State.Running);
+        Mockito.when(clusterVO.getControlNodeCount()).thenReturn(1L);
+        Mockito.when(clusterVO.getNodeCount()).thenReturn(2L);
+        Mockito.when(templateDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(VMTemplateVO.class));
+        Mockito.when(templateJoinDao.newTemplateView(Mockito.any(VMTemplateVO.class), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(List.of(Mockito.mock(TemplateJoinVO.class)));
+        kubernetesClusterManager.validateKubernetesClusterScaleSize(clusterVO, 4L, 10, Mockito.mock(DataCenter.class));
+    }
+}

plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java

@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.kubernetes.cluster.actionworkers;
+
+import java.util.UUID;
+
+import org.apache.cloudstack.api.ApiConstants;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import com.cloud.kubernetes.cluster.KubernetesCluster;
+import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO;
+import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl;
+import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
+import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao;
+import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
+import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao;
+import com.cloud.network.IpAddress;
+import com.cloud.network.Network;
+import com.cloud.network.dao.IPAddressDao;
+import com.cloud.network.dao.IPAddressVO;
+
+@RunWith(MockitoJUnitRunner.class)
+public class KubernetesClusterActionWorkerTest {
+
+    @Mock
+    KubernetesClusterDao kubernetesClusterDao;
+
+    @Mock
+    KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
+
+    @Mock
+    KubernetesClusterDetailsDao kubernetesClusterDetailsDao;
+
+    @Mock
+    KubernetesSupportedVersionDao kubernetesSupportedVersionDao;
+
+    @Mock
+    KubernetesClusterManagerImpl kubernetesClusterManager;
+
+    @Mock
+    IPAddressDao ipAddressDao;
+
+    KubernetesClusterActionWorker actionWorker = null;
+
+    final static Long DEFAULT_ID = 1L;
+
+    @Before
+    public void setUp() throws Exception {
+        kubernetesClusterManager.kubernetesClusterDao = kubernetesClusterDao;
+        kubernetesClusterManager.kubernetesSupportedVersionDao = kubernetesSupportedVersionDao;
+        kubernetesClusterManager.kubernetesClusterDetailsDao = kubernetesClusterDetailsDao;
+        kubernetesClusterManager.kubernetesClusterVmMapDao = kubernetesClusterVmMapDao;
+        KubernetesCluster kubernetesCluster = Mockito.mock(KubernetesCluster.class);
+        Mockito.when(kubernetesCluster.getId()).thenReturn(DEFAULT_ID);
+        actionWorker = new KubernetesClusterActionWorker(kubernetesCluster, kubernetesClusterManager);
+        actionWorker.ipAddressDao = ipAddressDao;
+    }
+
+    @Test
+    public void testGetVpcTierKubernetesPublicIpNullDetail() {
+        IpAddress result = actionWorker.getVpcTierKubernetesPublicIp(Mockito.mock(Network.class));
+        Assert.assertNull(result);
+    }
+
+    private String mockClusterPublicIpDetail(boolean isNull) {
+        String uuid = isNull ? null : UUID.randomUUID().toString();
+        KubernetesClusterDetailsVO detailsVO = new KubernetesClusterDetailsVO(DEFAULT_ID, ApiConstants.PUBLIC_IP_ID, uuid, false);
+        Mockito.when(kubernetesClusterDetailsDao.findDetail(DEFAULT_ID, ApiConstants.PUBLIC_IP_ID)).thenReturn(detailsVO);
+        return uuid;
+    }
+
+    @Test
+    public void testGetVpcTierKubernetesPublicIpNullDetailValue() {
+        mockClusterPublicIpDetail(true);
+        IpAddress result = actionWorker.getVpcTierKubernetesPublicIp(Mockito.mock(Network.class));
+        Assert.assertNull(result);
+    }
+
+    private Network mockNetworkForGetVpcTierKubernetesPublicIpTest() {
+        Network network = Mockito.mock(Network.class);
+        Mockito.when(network.getVpcId()).thenReturn(DEFAULT_ID);
+        return network;
+    }
+
+    @Test
+    public void testGetVpcTierKubernetesPublicIpNullVpc() {
+        String uuid = mockClusterPublicIpDetail(false);
+        IPAddressVO address = Mockito.mock(IPAddressVO.class);
+        Mockito.when(ipAddressDao.findByUuid(uuid)).thenReturn(address);
+        IpAddress result = actionWorker.getVpcTierKubernetesPublicIp(mockNetworkForGetVpcTierKubernetesPublicIpTest());
+        Assert.assertNull(result);
+    }
+
+    @Test
+    public void testGetVpcTierKubernetesPublicIpDifferentVpc() {
+        String uuid = mockClusterPublicIpDetail(false);
+        IPAddressVO address = Mockito.mock(IPAddressVO.class);
+        Mockito.when(address.getVpcId()).thenReturn(2L);
+        Mockito.when(ipAddressDao.findByUuid(uuid)).thenReturn(address);
+        IpAddress result = actionWorker.getVpcTierKubernetesPublicIp(mockNetworkForGetVpcTierKubernetesPublicIpTest());
+        Assert.assertNull(result);
+    }
+
+    @Test
+    public void testGetVpcTierKubernetesPublicIpValid() {
+        String uuid = mockClusterPublicIpDetail(false);
+        IPAddressVO address = Mockito.mock(IPAddressVO.class);
+        Mockito.when(address.getVpcId()).thenReturn(DEFAULT_ID);
+        Mockito.when(ipAddressDao.findByUuid(uuid)).thenReturn(address);
+        IpAddress result = actionWorker.getVpcTierKubernetesPublicIp(mockNetworkForGetVpcTierKubernetesPublicIpTest());
+        Assert.assertNotNull(result);
+    }
+}

server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java

@@ -977,6 +977,15 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
         }
     }
 
+    @Override
+    public NetworkACLItem moveRuleToTheTopInACLList(NetworkACLItem ruleBeingMoved) {
+        List<NetworkACLItemVO> allRules = getAllAclRulesSortedByNumber(ruleBeingMoved.getAclId());
+        if (allRules.size() == 1) {
+            return ruleBeingMoved;
+        }
+        return moveRuleToTheTop(ruleBeingMoved, allRules);
+    }
+
     /**
      * Validates the consistency of the ACL; the validation process is the following.
      * <ul>
@@ -1028,7 +1037,7 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
 
     /**
      * Moves an ACL to the space between to other rules. If there is already enough room to accommodate the ACL rule being moved, we simply get the 'number' field from the previous ACL rule and add one, and then define this new value as the 'number' value for the ACL rule being moved.
-     * Otherwise, we will need to make room. This process is executed via {@link #updateAclRuleToNewPositionAndExecuteShiftIfNecessary(NetworkACLItemVO, int, List, int)}, which will create the space between ACL rules if necessary. This involves shifting ACL rules to accommodate the rule being moved.
+     * Otherwise, we will need to make room. This process is executed via {@link #updateAclRuleToNewPositionAndExecuteShiftIfNecessary(NetworkACLItem, int, List, int)}, which will create the space between ACL rules if necessary. This involves shifting ACL rules to accommodate the rule being moved.
      */
     protected NetworkACLItem moveRuleBetweenAclRules(NetworkACLItemVO ruleBeingMoved, List<NetworkACLItemVO> allAclRules, NetworkACLItemVO previousRule, NetworkACLItemVO nextRule) {
         if (previousRule.getNumber() + 1 != nextRule.getNumber()) {
@@ -1070,7 +1079,7 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
      *  Move the rule to the top of the ACL rule list. This means that the ACL rule being moved will receive the position '1'.
      *  Also, if necessary other ACL rules will have their 'number' field updated to create room for the new top rule.
      */
-    protected NetworkACLItem moveRuleToTheTop(NetworkACLItemVO ruleBeingMoved, List<NetworkACLItemVO> allAclRules) {
+    protected NetworkACLItem moveRuleToTheTop(NetworkACLItem ruleBeingMoved, List<NetworkACLItemVO> allAclRules) {
         return updateAclRuleToNewPositionAndExecuteShiftIfNecessary(ruleBeingMoved, 1, allAclRules, 0);
     }
 
@@ -1092,9 +1101,8 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
      *  <li> ACL C - number 4
      * </ul>
      */
-    protected NetworkACLItem updateAclRuleToNewPositionAndExecuteShiftIfNecessary(NetworkACLItemVO ruleBeingMoved, int newNumberFieldValue, List<NetworkACLItemVO> allAclRules,
+    protected NetworkACLItem updateAclRuleToNewPositionAndExecuteShiftIfNecessary(NetworkACLItem ruleBeingMoved, int newNumberFieldValue, List<NetworkACLItemVO> allAclRules,
             int indexToStartProcessing) {
-        ruleBeingMoved.setNumber(newNumberFieldValue);
         for (int i = indexToStartProcessing; i < allAclRules.size(); i++) {
             NetworkACLItemVO networkACLItemVO = allAclRules.get(i);
             if (networkACLItemVO.getId() == ruleBeingMoved.getId()) {

server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java

@@ -26,7 +26,9 @@ import static org.mockito.Mockito.times;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
 import org.apache.cloudstack.api.ServerApiException;
@@ -1264,6 +1266,8 @@ public class NetworkACLServiceImplTest {
     @Test
     public void updateAclRuleToNewPositionAndExecuteShiftIfNecessaryTest() {
         Mockito.when(previousAclRuleMock.getNumber()).thenReturn(10);
+        Mockito.when(previousAclRuleMock.getId()).thenReturn(50l);
+
 
         Mockito.when(nextAclRuleMock.getNumber()).thenReturn(11);
         Mockito.when(nextAclRuleMock.getId()).thenReturn(50l);
@@ -1290,18 +1294,33 @@ public class NetworkACLServiceImplTest {
         allAclRules.add(aclRuleBeingMovedMock);
 
         Mockito.doNothing().when(networkAclItemDaoMock).updateNumberFieldNetworkItem(Mockito.anyLong(), Mockito.anyInt());
-        Mockito.doReturn(null).when(networkAclItemDaoMock).findById(Mockito.anyLong());
 
-        networkAclServiceImpl.updateAclRuleToNewPositionAndExecuteShiftIfNecessary(aclRuleBeingMovedMock, 11, allAclRules, 1);
+        Map<Long, NetworkACLItemVO> updatedItems = new HashMap<>();
+        Mockito.doAnswer((Answer<Void>) invocation -> {
+            Long id = (Long)invocation.getArguments()[0];
+            int position = (int)invocation.getArguments()[1];
+            NetworkACLItemVO item = new NetworkACLItemVO();
+            item.setNumber(position);
+            updatedItems.put(id, item);
+            return null;
+        }).when(networkAclItemDaoMock).updateNumberFieldNetworkItem(Mockito.anyLong(), Mockito.anyInt());
+        Mockito.doAnswer((Answer<NetworkACLItemVO>) invocation -> {
+            Long id = (Long)invocation.getArguments()[0];
+            return updatedItems.get(id);
+        }).when(networkAclItemDaoMock).findById(Mockito.anyLong());
 
-        Mockito.verify(aclRuleBeingMovedMock).setNumber(11);
+        NetworkACLItem result = networkAclServiceImpl.updateAclRuleToNewPositionAndExecuteShiftIfNecessary(aclRuleBeingMovedMock, 11, allAclRules, 1);
-        Mockito.verify(nextAclRuleMock).setNumber(12);
+
+        Assert.assertNotNull(result);
+        Assert.assertEquals(11, result.getNumber());
+        Assert.assertEquals(11, updatedItems.get(aclRuleBeingMovedMock.getId()).getNumber());
+        Assert.assertEquals(12, updatedItems.get(nextAclRuleMock.getId()).getNumber());
         Mockito.verify(networkAclItemDaoMock).updateNumberFieldNetworkItem(1l, 11);
         Mockito.verify(networkAclItemDaoMock).updateNumberFieldNetworkItem(50l, 12);
 
-        Assert.assertEquals(13, networkACLItemVO12.getNumber());
+        Assert.assertEquals(networkACLItemVO12.getNumber() + 1, updatedItems.get(networkACLItemVO12.getId()).getNumber());
-        Assert.assertEquals(14, networkACLItemVO13.getNumber());
+        Assert.assertEquals(networkACLItemVO13.getNumber() + 1, updatedItems.get(networkACLItemVO13.getId()).getNumber());
-        Assert.assertEquals(15, networkACLItemVO14.getNumber());
+        Assert.assertEquals(networkACLItemVO14.getNumber() + 1, updatedItems.get(networkACLItemVO14.getId()).getNumber());
     }
 
     @Test

services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java

@@ -1997,6 +1997,16 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         }
     }
 
+    protected String getSnapshotFilepathForDelete(String path, String snapshotName) {
+        if (!path.endsWith(snapshotName)) {
+            return path + "/*" + snapshotName + "*";
+        }
+        if (s_logger.isDebugEnabled()) {
+            s_logger.debug(String.format("Snapshot file %s is present in the same name directory %s. Deleting the directory", snapshotName, path));
+        }
+        return path;
+    }
+
     protected Answer deleteSnapshot(final DeleteCommand cmd) {
         DataTO obj = cmd.getData();
         DataStoreTO dstore = obj.getDataStore();
@@ -2033,7 +2043,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
                 return new Answer(cmd, true, details);
             }
             // delete snapshot in the directory if exists
-            String lPath = absoluteSnapshotPath + "/*" + snapshotName + "*";
+            String lPath = getSnapshotFilepathForDelete(absoluteSnapshotPath, snapshotName);
             String result = deleteLocalFile(lPath);
             if (result != null) {
                 details = "failed to delete snapshot " + lPath + " , err=" + result;

services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java

@@ -91,4 +91,21 @@ public class NfsSecondaryStorageResourceTest {
         testLogAppender.assertMessagesLogged();
 
     }
+
+    private void performGetSnapshotFilepathForDeleteTest(String expected, String path, String name) {
+        Assert.assertEquals("Incorrect resultant snapshot delete path", expected, resource.getSnapshotFilepathForDelete(path, name));
+    }
+
+    @Test
+    public void testGetSnapshotFilepathForDelete() {
+        performGetSnapshotFilepathForDeleteTest("/snapshots/2/10/somename",
+                "/snapshots/2/10/somename",
+                "somename");
+        performGetSnapshotFilepathForDeleteTest("/snapshots/2/10/diffName/*diffname*",
+                "/snapshots/2/10/diffName",
+                "diffname");
+        performGetSnapshotFilepathForDeleteTest("/snapshots/2/10/*somename*",
+                "/snapshots/2/10",
+                "somename");
+    }
 }

test/integration/smoke/test_kubernetes_clusters.py

@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-""" Tests for Kubernetes supported version """
+""" Tests for Kubernetes cluster """
 
 #Import Local Modules
 from marvin.cloudstackTestCase import cloudstackTestCase
@@ -37,11 +37,16 @@ from marvin.cloudstackAPI import (listInfrastructure,
 from marvin.cloudstackException import CloudstackAPIException
 from marvin.codes import PASS, FAILED
 from marvin.lib.base import (Template,
+                             NetworkOffering,
                              Network,
                              ServiceOffering,
                              Account,
                              StoragePool,
-                             Configurations)
+                             Configurations,
+                             VpcOffering,
+                             VPC,
+                             NetworkACLList,
+                             NetworkACL)
 from marvin.lib.utils import (cleanup_resources,
                               validateList,
                               random_gen)
@@ -57,6 +62,11 @@ import time, io, yaml
 _multiprocess_shared_ = True
 
 k8s_cluster = None
+VPC_DATA = {
+    "cidr": "10.1.0.0/22",
+    "tier1_gateway": "10.1.1.1",
+    "tier_netmask": "255.255.255.0"
+}
 
 class TestKubernetesCluster(cloudstackTestCase):
 
@@ -75,6 +85,7 @@ class TestKubernetesCluster(cloudstackTestCase):
         cls.setup_failed = False
         cls._cleanup = []
         cls.kubernetes_version_ids = []
+        cls.vpcAllowAllAclDetailsMap = {}
 
         if cls.hypervisorNotSupported == False:
             cls.endpoint_url = Configurations.list(cls.apiclient, name="endpoint.url")[0].value
@@ -342,11 +353,7 @@ class TestKubernetesCluster(cloudstackTestCase):
         return
 
     def tearDown(self):
-        try:
+        super(TestKubernetesCluster, self).tearDown()
-            cleanup_resources(self.apiclient, self.cleanup)
-        except Exception as e:
-            raise Exception("Warning: Exception during cleanup : %s" % e)
-        return
 
     @attr(tags=["advanced", "smoke"], required_hardware="true")
     @skipTestIf("hypervisorNotSupported")
@@ -449,7 +456,7 @@ class TestKubernetesCluster(cloudstackTestCase):
         self.debug("Autoscaling Kubernetes cluster with ID: %s" % k8s_cluster.id)
         try:
             k8s_cluster = self.autoscaleKubernetesCluster(k8s_cluster.id, 1, 2)
-            self.verifyKubernetesClusterAutocale(k8s_cluster, 1, 2)
+            self.verifyKubernetesClusterAutoscale(k8s_cluster, 1, 2)
 
             up = self.waitForAutoscalerPodInRunningState(k8s_cluster.id)
             self.assertTrue(up, "Autoscaler pod failed to run")
@@ -576,6 +583,33 @@ class TestKubernetesCluster(cloudstackTestCase):
         self.debug("Deleting Kubernetes cluster with ID: %s" % k8s_cluster.id)
         return
 
+    @attr(tags=["advanced", "smoke"], required_hardware="true")
+    @skipTestIf("hypervisorNotSupported")
+    def test_10_vpc_tier_kubernetes_cluster(self):
+        """Test to deploy a Kubernetes cluster on VPC
+
+        # Validate the following:
+        # 1. Deploy a Kubernetes cluster on a VPC tier
+        # 2. Destroy it
+        """
+        if self.setup_failed == True:
+            self.fail("Setup incomplete")
+        global k8s_cluster
+        if k8s_cluster != None and k8s_cluster.id != None:
+            self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True)
+        self.createVpcOffering()
+        self.createVpcTierOffering()
+        self.deployVpc()
+        self.deployNetworkTier()
+        self.default_network = self.vpc_tier
+        k8s_cluster = self.getValidKubernetesCluster(1, 1)
+
+        self.debug("Deleting Kubernetes cluster with ID: %s" % k8s_cluster.id)
+        self.deleteKubernetesClusterAndVerify(k8s_cluster.id)
+        self.debug("Kubernetes cluster with ID: %s successfully deleted" % k8s_cluster.id)
+        k8s_cluster = None
+        return
+
     def createKubernetesCluster(self, name, version_id, size=1, control_nodes=1):
         createKubernetesClusterCmd = createKubernetesCluster.createKubernetesClusterCmd()
         createKubernetesClusterCmd.name = name
@@ -783,7 +817,7 @@ class TestKubernetesCluster(cloudstackTestCase):
         self.verifyKubernetesClusterState(cluster_response, 'Running')
         self.verifyKubernetesClusterSize(cluster_response, size, control_nodes)
 
-    def verifyKubernetesClusterAutocale(self, cluster_response, minsize, maxsize):
+    def verifyKubernetesClusterAutoscale(self, cluster_response, minsize, maxsize):
         """Check if Kubernetes cluster state and node sizes are valid after upgrade"""
 
         self.verifyKubernetesClusterState(cluster_response, 'Running')
@@ -816,3 +850,91 @@ class TestKubernetesCluster(cloudstackTestCase):
             'Stopped',
             "KubernetesCluster not stopped in DB, {}".format(db_cluster_state)
         )
+
+    def createVpcOffering(self):
+        off_service = self.services["vpc_offering"]
+        self.vpc_offering = VpcOffering.create(
+            self.apiclient,
+            off_service
+        )
+        self.cleanup.append(self.vpc_offering)
+        self.vpc_offering.update(self.apiclient, state='Enabled')
+
+    def createVpcTierOffering(self):
+        off_service = self.services["nw_offering_isolated_vpc"]
+        self.vpc_tier_offering = NetworkOffering.create(
+            self.apiclient,
+            off_service,
+            conservemode=False
+        )
+        self.cleanup.append(self.vpc_tier_offering)
+        self.vpc_tier_offering.update(self.apiclient, state='Enabled')
+
+    def deployAllowEgressDenyIngressVpcInternal(self, cidr):
+        service = self.services["vpc"]
+        service["cidr"] = cidr
+        vpc = VPC.create(
+            self.apiclient,
+            service,
+            vpcofferingid=self.vpc_offering.id,
+            zoneid=self.zone.id,
+            account=self.account.name,
+            domainid=self.account.domainid
+        )
+        self.cleanup.append(vpc)
+        acl = NetworkACLList.create(
+            self.apiclient,
+            services={},
+            name="allowegressdenyingress",
+            description="allowegressdenyingress",
+            vpcid=vpc.id
+        )
+        rule ={
+            "protocol": "all",
+            "traffictype": "egress",
+        }
+        NetworkACL.create(self.apiclient,
+            services=rule,
+            aclid=acl.id
+        )
+        rule["traffictype"] = "ingress"
+        rule["action"] = "deny"
+        NetworkACL.create(self.apiclient,
+            services=rule,
+            aclid=acl.id
+        )
+        self.vpcAllowAllAclDetailsMap[vpc.id] = acl.id
+        return vpc
+
+    def deployVpc(self):
+        self.vpc = self.deployAllowEgressDenyIngressVpcInternal(VPC_DATA["cidr"])
+
+    def deployNetworkTierInternal(self, network_offering_id, vpc_id, tier_gateway, tier_netmask, acl_id=None, tier_name=None):
+        if not acl_id and vpc_id in self.vpcAllowAllAclDetailsMap:
+            acl_id = self.vpcAllowAllAclDetailsMap[vpc_id]
+        service = self.services["ntwk"]
+        if tier_name:
+            service["name"] = tier_name
+            service["displaytext"] = "vpc-%s" % tier_name
+        network = Network.create(
+            self.apiclient,
+            service,
+            self.account.name,
+            self.account.domainid,
+            networkofferingid=network_offering_id,
+            vpcid=vpc_id,
+            zoneid=self.zone.id,
+            gateway=tier_gateway,
+            netmask=tier_netmask,
+            aclid=acl_id
+        )
+        self.cleanup.append(network)
+        return network
+
+    def deployNetworkTier(self):
+        self.vpc_tier = self.deployNetworkTierInternal(
+            self.vpc_tier_offering.id,
+            self.vpc.id,
+            VPC_DATA["tier1_gateway"],
+            VPC_DATA["tier_netmask"]
+        )

ui/src/config/section/compute.js

@@ -504,7 +504,7 @@ export default {
           message: 'message.kubernetes.cluster.scale',
           docHelp: 'plugins/cloudstack-kubernetes-service.html#scaling-kubernetes-cluster',
           dataView: true,
-          show: (record) => { return ['Created', 'Running'].includes(record.state) },
+          show: (record) => { return ['Created', 'Running', 'Stopped'].includes(record.state) },
           popup: true,
           component: shallowRef(defineAsyncComponent(() => import('@/views/compute/ScaleKubernetesCluster.vue')))
         },

ui/src/views/compute/ScaleKubernetesCluster.vue

@@ -28,6 +28,25 @@
         :rules="rules"
         @finish="handleSubmit"
         layout="vertical">
+        <a-form-item name="serviceofferingid" ref="serviceofferingid">
+          <template #label>
+            <tooltip-label :title="$t('label.serviceofferingid')" :tooltip="apiParams.serviceofferingid.description"/>
+          </template>
+          <a-select
+            id="offering-selection"
+            v-model:value="form.serviceofferingid"
+            showSearch
+            optionFilterProp="label"
+            :filterOption="(input, option) => {
+              return option.label.children.toLowerCase().indexOf(input.toLowerCase()) >= 0
+            }"
+            :loading="serviceOfferingLoading"
+            :placeholder="apiParams.serviceofferingid.description">
+            <a-select-option v-for="(opt, optIndex) in serviceOfferings" :key="optIndex" :label="opt.name || opt.description">
+              {{ opt.name || opt.description }}
+            </a-select-option>
+          </a-select>
+        </a-form-item>
         <a-form-item name="autoscalingenabled" ref="autoscalingenabled" v-if="apiParams.autoscalingenabled">
           <template #label>
             <tooltip-label :title="$t('label.cks.cluster.autoscalingenabled')" :tooltip="apiParams.autoscalingenabled.description"/>
@@ -53,26 +72,7 @@
           </a-form-item>
         </span>
         <span v-else>
-          <a-form-item name="serviceofferingid" ref="serviceofferingid">
+          <a-form-item name="size" ref="size" v-if="['Created', 'Running'].includes(resource.state)">
-            <template #label>
-              <tooltip-label :title="$t('label.serviceofferingid')" :tooltip="apiParams.serviceofferingid.description"/>
-            </template>
-            <a-select
-              id="offering-selection"
-              v-model:value="form.serviceofferingid"
-              showSearch
-              optionFilterProp="label"
-              :filterOption="(input, option) => {
-                return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-              }"
-              :loading="serviceOfferingLoading"
-              :placeholder="apiParams.serviceofferingid.description">
-              <a-select-option v-for="(opt, optIndex) in serviceOfferings" :key="optIndex" :label="opt.name || opt.description">
-                {{ opt.name || opt.description }}
-              </a-select-option>
-            </a-select>
-          </a-form-item>
-          <a-form-item name="size" ref="size">
             <template #label>
               <tooltip-label :title="$t('label.cks.cluster.size')" :tooltip="apiParams.size.description"/>
             </template>
@@ -152,6 +152,10 @@ export default {
       })
     },
     fetchData () {
+      if (this.resource.state === 'Running') {
+        this.fetchKubernetesClusterServiceOfferingData()
+        return
+      }
       this.fetchKubernetesVersionData()
     },
     isValidValueForKey (obj, key) {
@@ -163,13 +167,28 @@ export default {
     isObjectEmpty (obj) {
       return !(obj !== null && obj !== undefined && Object.keys(obj).length > 0 && obj.constructor === Object)
     },
+    fetchKubernetesClusterServiceOfferingData () {
+      const params = {}
+      if (!this.isObjectEmpty(this.resource)) {
+        params.id = this.resource.serviceofferingid
+      }
+      api('listServiceOfferings', params).then(json => {
+        var items = json?.listserviceofferingsresponse?.serviceoffering || []
+        if (this.arrayHasItems(items) && !this.isObjectEmpty(items[0])) {
+          this.minCpu = items[0].cpunumber
+          this.minMemory = items[0].memory
+        }
+      }).finally(() => {
+        this.fetchServiceOfferingData()
+      })
+    },
     fetchKubernetesVersionData () {
       const params = {}
       if (!this.isObjectEmpty(this.resource)) {
         params.id = this.resource.kubernetesversionid
       }
       api('listKubernetesSupportedVersions', params).then(json => {
-        const versionObjs = json.listkubernetessupportedversionsresponse.kubernetessupportedversion
+        const versionObjs = json?.listkubernetessupportedversionsresponse?.kubernetessupportedversion || []
         if (this.arrayHasItems(versionObjs) && !this.isObjectEmpty(versionObjs[0])) {
           this.minCpu = versionObjs[0].mincpunumber
           this.minMemory = versionObjs[0].minmemory
@@ -180,14 +199,16 @@ export default {
     },
     fetchServiceOfferingData () {
       this.serviceOfferings = []
-      const params = {}
+      const params = {
+        cpunumber: this.minCpu,
+        memory: this.minMemory
+      }
       this.serviceOfferingLoading = true
       api('listServiceOfferings', params).then(json => {
-        var items = json.listserviceofferingsresponse.serviceoffering
+        var items = json?.listserviceofferingsresponse?.serviceoffering || []
-        if (items != null) {
+        if (this.arrayHasItems(items)) {
           for (var i = 0; i < items.length; i++) {
-            if (items[i].iscustomized === false &&
+            if (items[i].iscustomized === false) {
-                items[i].cpunumber >= this.minCpu && items[i].memory >= this.minMemory) {
               this.serviceOfferings.push(items[i])
             }
           }
@@ -220,7 +241,7 @@ export default {
         if (this.isValidValueForKey(values, 'size') && values.size > 0) {
           params.size = values.size
         }
-        if (this.isValidValueForKey(values, 'serviceofferingid') && this.arrayHasItems(this.serviceOfferings) && this.autoscalingenabled == null) {
+        if (this.isValidValueForKey(values, 'serviceofferingid') && this.arrayHasItems(this.serviceOfferings)) {
           params.serviceofferingid = this.serviceOfferings[values.serviceofferingid].id
         }
         if (this.isValidValueForKey(values, 'minsize')) {