Merge release branch 4.7 to master

* 4.7:
  Fix unable to setup more than one Site2Site VPN Connection
  FIX S2S VPN rVPC: Check only redundant routers in state MASTER
  PEP8 of integration/smoke/test_vpc_vpn
  Add S2S VPN test for Redundant VPC
  Make integration/smoke/test_vpc_vpn Hypervisor independant
  FIX VPN: non-working ipsec commands
  [UI] MADNESS
  [DB] Add force_encap field to s2s_customer_gateway table
  [ROUTER] Add forceencaps field to python router ipsec config method
  [TEST] unittest needs rework
  [MARVIN] Add forceencap field to VpnCustomerGateway class in marvin base
  [CORE] Add Force UDP Encapsulation option to Site2Site VPN
  CLOUDSTACK-9186: Root admin cannot see VPC created by Domain admin user
  CLOUDSTACK-9192: UpdateVpnCustomerGateway is failing
  CLOUDSTACK-6485 prevent ip asignment of private gw iface
  CLOUDSTACK-9204 Do not error when staticroute is already gone
  make both check lines consistent
  CLOUDSTACK-9181 Prevent syntax error in checkrouter.sh
  CLOUDSTACK-9202 Bump ssh timeout

api/src/com/cloud/network/Site2SiteCustomerGateway.java

@@ -39,6 +39,8 @@ public interface Site2SiteCustomerGateway extends ControlledEntity, Identity, In
 
     public Boolean getDpd();
 
+    public Boolean getEncap();
+
     public Date getRemoved();
 
     String getName();

api/src/org/apache/cloudstack/api/ApiConstants.java

@@ -492,6 +492,7 @@ public class ApiConstants {
     public static final String IKE_LIFETIME = "ikelifetime";
     public static final String ESP_LIFETIME = "esplifetime";
     public static final String DPD = "dpd";
+    public static final String FORCE_ENCAP = "forceencap";
     public static final String FOR_VPC = "forvpc";
     public static final String SHRINK_OK = "shrinkok";
     public static final String NICIRA_NVP_DEVICE_ID = "nvpdeviceid";
@@ -642,4 +643,4 @@ public class ApiConstants {
     public enum VMDetails {
         all, group, nics, stats, secgrp, tmpl, servoff, diskoff, iso, volume, min, affgrp;
     }
-}
+}

api/src/org/apache/cloudstack/api/command/user/vpn/CreateVpnCustomerGatewayCmd.java

@@ -75,6 +75,9 @@ public class CreateVpnCustomerGatewayCmd extends BaseAsyncCmd {
     @Parameter(name = ApiConstants.DPD, type = CommandType.BOOLEAN, required = false, description = "If DPD is enabled for VPN connection")
     private Boolean dpd;
 
+    @Parameter(name = ApiConstants.FORCE_ENCAP, type = CommandType.BOOLEAN, required = false, description = "Force Encapsulation for NAT traversal")
+    private Boolean encap;
+
     @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "the account associated with the gateway. Must be used with the domainId parameter.")
     private String accountName;
 
@@ -129,6 +132,8 @@ public class CreateVpnCustomerGatewayCmd extends BaseAsyncCmd {
         return dpd;
     }
 
+    public Boolean getEncap() { return encap; }
+
     public String getAccountName() {
         return accountName;
     }

api/src/org/apache/cloudstack/api/command/user/vpn/UpdateVpnCustomerGatewayCmd.java

@@ -81,6 +81,9 @@ public class UpdateVpnCustomerGatewayCmd extends BaseAsyncCmd {
     @Parameter(name = ApiConstants.DPD, type = CommandType.BOOLEAN, required = false, description = "If DPD is enabled for VPN connection")
     private Boolean dpd;
 
+    @Parameter(name = ApiConstants.FORCE_ENCAP, type = CommandType.BOOLEAN, required = false, description = "Force encapsulation for Nat Traversal")
+    private Boolean encap;
+
     @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "the account associated with the gateway. Must be used with the domainId parameter.")
     private String accountName;
 
@@ -135,6 +138,8 @@ public class UpdateVpnCustomerGatewayCmd extends BaseAsyncCmd {
         return dpd;
     }
 
+    public Boolean getEncap() { return encap; }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/org/apache/cloudstack/api/response/Site2SiteCustomerGatewayResponse.java

@@ -74,6 +74,10 @@ public class Site2SiteCustomerGatewayResponse extends BaseResponse implements Co
     @Param(description = "if DPD is enabled for customer gateway")
     private Boolean dpd;
 
+    @SerializedName(ApiConstants.FORCE_ENCAP)
+    @Param(description = "if Force NAT Encapsulation is enabled for customer gateway")
+    private Boolean encap;
+
     @SerializedName(ApiConstants.ACCOUNT)
     @Param(description = "the owner")
     private String accountName;
@@ -142,6 +146,8 @@ public class Site2SiteCustomerGatewayResponse extends BaseResponse implements Co
         this.dpd = dpd;
     }
 
+    public void setEncap(Boolean encap) { this.encap = encap; }
+
     public void setRemoved(Date removed) {
         this.removed = removed;
     }

api/src/org/apache/cloudstack/api/response/Site2SiteVpnConnectionResponse.java

@@ -87,6 +87,11 @@ public class Site2SiteVpnConnectionResponse extends BaseResponse implements Cont
     //from CustomerGateway
     private Boolean dpd;
 
+    @SerializedName(ApiConstants.FORCE_ENCAP)
+    @Param(description = "if Force NAT Encapsulation is enabled for customer gateway")
+    //from CustomerGateway
+    private Boolean encap;
+
     @SerializedName(ApiConstants.STATE)
     @Param(description = "State of vpn connection")
     private String state;
@@ -175,6 +180,10 @@ public class Site2SiteVpnConnectionResponse extends BaseResponse implements Cont
         this.dpd = dpd;
     }
 
+    public void setEncap(Boolean encap) {
+        this.encap = encap;
+    }
+
     public void setState(String state) {
         this.state = state;
     }

client/WEB-INF/classes/resources/messages.properties

@@ -2259,3 +2259,4 @@ message.please.select.ssh.key.pair.use.with.this.vm=Please select a ssh key pair
 message.configure.firewall.rules.allow.traffic=Configure the rules to allow Traffic
 message.configure.firewall.rules.block.traffic=Configure the rules to block Traffic
 message.ldap.group.import=All The users from the given group name will be imported
+label.vpn.force.encapsulation=Force UDP Encapsulation of ESP Packets

client/WEB-INF/classes/resources/messages_nl_NL.properties

@@ -1726,6 +1726,7 @@ label.vpc=VPC
 label.VPN.connection=VPN Connectie
 label.vpn.customer.gateway=VPN Customer Gateway
 label.VPN.customer.gateway=VPN Customer Gateway
+label.vpn.force.encapsulation=Forceer UDP Encapsulatie van ESP Packets
 label.VPN.gateway=VPN Gateway
 label.vpn=VPN
 label.vsmctrlvlanid=Controle VLAN ID

core/src/com/cloud/agent/api/routing/Site2SiteVpnCfgCommand.java

@@ -34,6 +34,7 @@ public class Site2SiteVpnCfgCommand extends NetworkElementCommand {
     private long espLifetime;
     private boolean dpd;
     private boolean passive;
+    private boolean encap;
 
     @Override
     public boolean executeInSequence() {
@@ -45,7 +46,7 @@ public class Site2SiteVpnCfgCommand extends NetworkElementCommand {
     }
 
     public Site2SiteVpnCfgCommand(boolean create, String localPublicIp, String localPublicGateway, String localGuestCidr, String peerGatewayIp, String peerGuestCidrList,
-            String ikePolicy, String espPolicy, String ipsecPsk, Long ikeLifetime, Long espLifetime, Boolean dpd, boolean passive) {
+            String ikePolicy, String espPolicy, String ipsecPsk, Long ikeLifetime, Long espLifetime, Boolean dpd, boolean passive, boolean encap) {
         this.create = create;
         this.setLocalPublicIp(localPublicIp);
         this.setLocalPublicGateway(localPublicGateway);
@@ -59,6 +60,7 @@ public class Site2SiteVpnCfgCommand extends NetworkElementCommand {
         this.espLifetime = espLifetime;
         this.dpd = dpd;
         this.passive = passive;
+        this.encap = encap;
     }
 
     public boolean isCreate() {
@@ -117,6 +119,14 @@ public class Site2SiteVpnCfgCommand extends NetworkElementCommand {
         this.dpd = dpd;
     }
 
+    public Boolean getEncap() {
+        return encap;
+    }
+
+    public void setEncap(Boolean encap) {
+        this.encap = encap;
+    }
+
     public String getLocalPublicIp() {
         return localPublicIp;
     }

core/src/com/cloud/agent/resource/virtualnetwork/facade/Site2SiteVpnConfigItem.java

@@ -36,7 +36,7 @@ public class Site2SiteVpnConfigItem extends AbstractConfigItemFacade {
 
         final Site2SiteVpn site2siteVpn = new Site2SiteVpn(command.getLocalPublicIp(), command.getLocalGuestCidr(), command.getLocalPublicGateway(), command.getPeerGatewayIp(),
                 command.getPeerGuestCidrList(), command.getEspPolicy(), command.getIkePolicy(), command.getIpsecPsk(), command.getIkeLifetime(), command.getEspLifetime(), command.isCreate(), command.getDpd(),
-                command.isPassive());
+                command.isPassive(), command.getEncap());
         return generateConfigItems(site2siteVpn);
     }
 

core/src/com/cloud/agent/resource/virtualnetwork/model/Site2SiteVpn.java

@@ -23,7 +23,7 @@ public class Site2SiteVpn extends ConfigBase {
 
     private String localPublicIp, localGuestCidr, localPublicGateway, peerGatewayIp, peerGuestCidrList, espPolicy, ikePolicy, ipsecPsk;
     private Long ikeLifetime, espLifetime;
-    private boolean create, dpd, passive;
+    private boolean create, dpd, passive, encap;
 
     public Site2SiteVpn() {
         super(ConfigBase.SITE2SITEVPN);
@@ -31,7 +31,7 @@ public class Site2SiteVpn extends ConfigBase {
 
     public Site2SiteVpn(String localPublicIp, String localGuestCidr, String localPublicGateway, String peerGatewayIp, String peerGuestCidrList, String espPolicy,
  String ikePolicy,
-            String ipsecPsk, Long ikeLifetime, Long espLifetime, boolean create, Boolean dpd, boolean passive) {
+            String ipsecPsk, Long ikeLifetime, Long espLifetime, boolean create, Boolean dpd, boolean passive, boolean encap) {
         super(ConfigBase.SITE2SITEVPN);
         this.localPublicIp = localPublicIp;
         this.localGuestCidr = localGuestCidr;
@@ -46,6 +46,7 @@ public class Site2SiteVpn extends ConfigBase {
         this.create = create;
         this.dpd = dpd;
         this.passive = passive;
+        this.encap = encap;
     }
 
     public String getLocalPublicIp() {
@@ -152,4 +153,12 @@ public class Site2SiteVpn extends ConfigBase {
         this.passive = passive;
     }
 
+    public boolean getEncap() {
+        return encap;
+    }
+
+    public void setEncap(boolean encap) {
+        this.encap = encap;
+    }
+
 }

core/test/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java

@@ -494,17 +494,17 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer {
     public void testSite2SiteVpnCfgCommand() {
         _count = 0;
 
-        Site2SiteVpnCfgCommand cmd = new Site2SiteVpnCfgCommand(true, "64.10.1.10", "64.10.1.1", "192.168.1.1/16", "124.10.1.10", "192.168.100.1/24", "3des-sha1,aes128-sha1;modp1536", "3des-sha1,aes128-md5", "psk", Long.valueOf(1800), Long.valueOf(1800), true, false);
+        Site2SiteVpnCfgCommand cmd = new Site2SiteVpnCfgCommand(true, "64.10.1.10", "64.10.1.1", "192.168.1.1/16", "124.10.1.10", "192.168.100.1/24", "3des-sha1,aes128-sha1;modp1536", "3des-sha1,aes128-md5", "psk", Long.valueOf(1800), Long.valueOf(1800), true, false, false);
         cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME);
         Answer answer = _resource.executeRequest(cmd);
         assertTrue(answer.getResult());
 
-        cmd = new Site2SiteVpnCfgCommand(true, "64.10.1.10", "64.10.1.1", "192.168.1.1/16", "124.10.1.10", "192.168.100.1/24", "3des-sha1,aes128-sha1;modp1536", "3des-sha1,aes128-md5", "psk", Long.valueOf(1800), Long.valueOf(1800), false, true);
+        cmd = new Site2SiteVpnCfgCommand(true, "64.10.1.10", "64.10.1.1", "192.168.1.1/16", "124.10.1.10", "192.168.100.1/24", "3des-sha1,aes128-sha1;modp1536", "3des-sha1,aes128-md5", "psk", Long.valueOf(1800), Long.valueOf(1800), false, true, false);
         cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME);
         answer = _resource.executeRequest(cmd);
         assertTrue(answer.getResult());
 
-        cmd = new Site2SiteVpnCfgCommand(false, "64.10.1.10", "64.10.1.1", "192.168.1.1/16", "124.10.1.10", "192.168.100.1/24", "3des-sha1,aes128-sha1;modp1536", "3des-sha1,aes128-md5", "psk", Long.valueOf(1800), Long.valueOf(1800), false, true);
+        cmd = new Site2SiteVpnCfgCommand(false, "64.10.1.10", "64.10.1.1", "192.168.1.1/16", "124.10.1.10", "192.168.100.1/24", "3des-sha1,aes128-sha1;modp1536", "3des-sha1,aes128-md5", "psk", Long.valueOf(1800), Long.valueOf(1800), false, true, false);
         cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME);
         answer = _resource.executeRequest(cmd);
         assertTrue(answer.getResult());

engine/schema/src/com/cloud/network/dao/Site2SiteCustomerGatewayVO.java

@@ -70,6 +70,9 @@ public class Site2SiteCustomerGatewayVO implements Site2SiteCustomerGateway {
     @Column(name = "dpd")
     private boolean dpd;
 
+    @Column(name = "force_encap")
+    private boolean encap;
+
     @Column(name = "domain_id")
     private Long domainId;
 
@@ -83,7 +86,7 @@ public class Site2SiteCustomerGatewayVO implements Site2SiteCustomerGateway {
     }
 
     public Site2SiteCustomerGatewayVO(String name, long accountId, long domainId, String gatewayIp, String guestCidrList, String ipsecPsk, String ikePolicy,
-            String espPolicy, long ikeLifetime, long espLifetime, boolean dpd) {
+            String espPolicy, long ikeLifetime, long espLifetime, boolean dpd, boolean encap) {
         this.name = name;
         this.gatewayIp = gatewayIp;
         this.guestCidrList = guestCidrList;
@@ -93,6 +96,7 @@ public class Site2SiteCustomerGatewayVO implements Site2SiteCustomerGateway {
         this.ikeLifetime = ikeLifetime;
         this.espLifetime = espLifetime;
         this.dpd = dpd;
+        this.encap = encap;
         uuid = UUID.randomUUID().toString();
         this.accountId = accountId;
         this.domainId = domainId;
@@ -193,6 +197,15 @@ public class Site2SiteCustomerGatewayVO implements Site2SiteCustomerGateway {
         this.dpd = dpd;
     }
 
+    @Override
+    public Boolean getEncap() {
+        return encap;
+    }
+
+    public void setEncap(boolean encap) {
+        this.encap = encap;
+    }
+
     @Override
     public String getUuid() {
         return uuid;

server/src/com/cloud/api/ApiResponseHelper.java

@@ -3001,7 +3001,7 @@ public class ApiResponseHelper implements ResponseGenerator {
         response.setIkeLifetime(result.getIkeLifetime());
         response.setEspLifetime(result.getEspLifetime());
         response.setDpd(result.getDpd());
-
+        response.setEncap(result.getEncap());
         response.setRemoved(result.getRemoved());
         response.setObjectName("vpncustomergateway");
 
@@ -3041,6 +3041,7 @@ public class ApiResponseHelper implements ResponseGenerator {
                 response.setIkeLifetime(customerGateway.getIkeLifetime());
                 response.setEspLifetime(customerGateway.getEspLifetime());
                 response.setDpd(customerGateway.getDpd());
+                response.setEncap(customerGateway.getEncap());
             }
         }
 

server/src/com/cloud/network/router/CommandSetupHelper.java

@@ -857,9 +857,10 @@ public class CommandSetupHelper {
         final Long ikeLifetime = gw.getIkeLifetime();
         final Long espLifetime = gw.getEspLifetime();
         final Boolean dpd = gw.getDpd();
+        final Boolean encap = gw.getEncap();
 
         final Site2SiteVpnCfgCommand cmd = new Site2SiteVpnCfgCommand(isCreate, localPublicIp, localPublicGateway, localGuestCidr, peerGatewayIp, peerGuestCidrList, ikePolicy,
-                espPolicy, ipsecPsk, ikeLifetime, espLifetime, dpd, conn.isPassive());
+                espPolicy, ipsecPsk, ikeLifetime, espLifetime, dpd, conn.isPassive(), encap);
         cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId()));
         cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId()));
         cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());

server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -907,6 +907,9 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
             if (conns == null || conns.isEmpty()) {
                 continue;
             }
+            if (router.getIsRedundantRouter() && router.getRedundantState() != RedundantState.MASTER){
+                continue;
+            }
             if (router.getState() != VirtualMachine.State.Running) {
                 for (final Site2SiteVpnConnectionVO conn : conns) {
                     if (conn.getState() != Site2SiteVpnConnection.State.Error) {

server/src/com/cloud/network/vpc/VpcManagerImpl.java

@@ -1668,7 +1668,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
                         // A more permanent solution would be to define a type of 'gatewaynetwork'
                         // so that handling code is not mixed between the two
                         final NetworkVO gatewaynet = _ntwkDao.findById(privateNtwk.getId());
-                        gatewaynet.setVpcId(vpcId);
+                        gatewaynet.setVpcId(null);
                         _ntwkDao.persist(gatewaynet);
                     }
 

server/src/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java

@@ -218,6 +218,11 @@ public class Site2SiteVpnManagerImpl extends ManagerBase implements Site2SiteVpn
             dpd = false;
         }
 
+        Boolean encap = cmd.getEncap();
+        if (encap == null) {
+            encap = false;
+        }
+
         long accountId = owner.getAccountId();
         if (_customerGatewayDao.findByGatewayIpAndAccountId(gatewayIp, accountId) != null) {
             throw new InvalidParameterValueException("The customer gateway with ip " + gatewayIp + " already existed in the system!");
@@ -229,7 +234,7 @@ public class Site2SiteVpnManagerImpl extends ManagerBase implements Site2SiteVpn
         checkCustomerGatewayCidrList(peerCidrList);
 
         Site2SiteCustomerGatewayVO gw =
-            new Site2SiteCustomerGatewayVO(name, accountId, owner.getDomainId(), gatewayIp, peerCidrList, ipsecPsk, ikePolicy, espPolicy, ikeLifetime, espLifetime, dpd);
+            new Site2SiteCustomerGatewayVO(name, accountId, owner.getDomainId(), gatewayIp, peerCidrList, ipsecPsk, ikePolicy, espPolicy, ikeLifetime, espLifetime, dpd, encap);
         _customerGatewayDao.persist(gw);
         return gw;
     }
@@ -467,6 +472,11 @@ public class Site2SiteVpnManagerImpl extends ManagerBase implements Site2SiteVpn
             dpd = false;
         }
 
+        Boolean encap = cmd.getEncap();
+        if (encap == null) {
+            encap = false;
+        }
+
         checkCustomerGatewayCidrList(guestCidrList);
 
         long accountId = gw.getAccountId();
@@ -488,6 +498,7 @@ public class Site2SiteVpnManagerImpl extends ManagerBase implements Site2SiteVpn
         gw.setIkeLifetime(ikeLifetime);
         gw.setEspLifetime(espLifetime);
         gw.setDpd(dpd);
+        gw.setEncap(encap);
         _customerGatewayDao.persist(gw);
         return gw;
     }

setup/db/db/schema-470to471.sql

@@ -18,3 +18,4 @@
 --;
 -- Schema upgrade from 4.7.0 to 4.7.1;
 --;
+ALTER TABLE cloud.s2s_customer_gateway ADD COLUMN force_encap INT(1) NOT NULL DEFAULT 0 AFTER dpd;

systemvm/patches/debian/config/opt/cloud/bin/checkrouter.sh

@@ -17,21 +17,20 @@
 # under the License.
 
 STATUS=UNKNOWN
-INTERFACE=eth1
 ROUTER_TYPE=$(cat /etc/cloudstack/cmdline.json | grep type | awk '{print $2;}' | sed -e 's/[,\"]//g')
-if [ $ROUTER_TYPE = "router" ]
+if [ "$ROUTER_TYPE" = "router" ]
 then
 	ROUTER_STATE=$(ip addr | grep eth0 | grep inet | wc -l | xargs bash -c  'if [ $0 == 2 ]; then echo "MASTER"; else echo "BACKUP"; fi')
 	STATUS=$ROUTER_STATE
 else
-	ROUTER_STATE=$(ip addr | grep $INTERFACE | grep state | awk '{print $9;}')
+	ROUTER_STATE=$(ip addr | grep eth1 | grep state | awk '{print $9;}')
-	if [ $ROUTER_STATE = "UP" ]
+	if [ "$ROUTER_STATE" = "UP" ]
 	then
 	    STATUS=MASTER
-	elif [ $ROUTER_STATE = "DOWN" ]
+	elif [ "$ROUTER_STATE" = "DOWN" ]
 	then
 	    STATUS=BACKUP
 	fi
 fi
 
-echo "Status: ${STATUS}"
+echo "Status: ${STATUS}"

systemvm/patches/debian/config/opt/cloud/bin/configure.py

@@ -462,16 +462,20 @@ class CsSite2SiteVpn(CsDataBag):
             if m:
                 self.confips.append(m.group(1))
 
-        for public_ip in self.dbag:
+        for vpn in self.dbag:
-            if public_ip == "id":
+            if vpn == "id":
                 continue
-            dev = CsHelper.get_device(public_ip)
+
+            local_ip = self.dbag[vpn]['local_public_ip']
+            dev = CsHelper.get_device(local_ip)
+
             if dev == "":
-                logging.error("Request for ipsec to %s not possible because ip is not configured", public_ip)
+                logging.error("Request for ipsec to %s not possible because ip is not configured", local_ip)
                 continue
+
             CsHelper.start_if_stopped("ipsec")
-            self.configure_iptables(dev, self.dbag[public_ip])
+            self.configure_iptables(dev, self.dbag[vpn])
-            self.configure_ipsec(self.dbag[public_ip])
+            self.configure_ipsec(self.dbag[vpn])
 
         # Delete vpns that are no longer in the configuration
         for ip in self.confips:
@@ -527,6 +531,7 @@ class CsSite2SiteVpn(CsDataBag):
         file.addeq(" pfs=%s" % CsHelper.bool_to_yn(obj['dpd']))
         file.addeq(" keyingtries=2")
         file.addeq(" auto=start")
+        file.addeq(" forceencaps=%s" % CsHelper.bool_to_yn(obj['encap']))
         if obj['dpd']:
             file.addeq("  dpddelay=30")
             file.addeq("  dpdtimeout=120")
@@ -538,9 +543,9 @@ class CsSite2SiteVpn(CsDataBag):
             file.commit()
             logging.info("Configured vpn %s %s", leftpeer, rightpeer)
             CsHelper.execute("ipsec auto --rereadall")
-            CsHelper.execute("ipsec --add vpn-%s" % rightpeer)
+            CsHelper.execute("ipsec auto --add vpn-%s" % rightpeer)
             if not obj['passive']:
-                CsHelper.execute("ipsec --up vpn-%s" % rightpeer)
+                CsHelper.execute("ipsec auto --up vpn-%s" % rightpeer)
         os.chmod(vpnsecretsfile, 0o400)
 
     def convert_sec_to_h(self, val):

systemvm/patches/debian/config/opt/cloud/bin/cs_site2sitevpn.py

@@ -19,7 +19,7 @@ from pprint import pprint
 
 
 def merge(dbag, vpn):
-    key = vpn['local_public_ip']
+    key = vpn['peer_gateway_ip']
     op = vpn['create']
     if key in dbag.keys() and not op:
         del(dbag[key])

systemvm/patches/debian/config/opt/cloud/bin/cs_staticroutes.py

@@ -23,7 +23,10 @@ def merge(dbag, staticroutes):
         key = route['ip_address']
         revoke = route['revoke']
         if revoke:
-            del dbag[key]
+            try:
+                del dbag[key]
+            except KeyError:
+                pass
         else:
             dbag[key] = route
 

test/integration/smoke/test_vpc_vpn.py

@@ -19,7 +19,8 @@
 # Import Local Modules
 from marvin.codes import PASS, FAILED
 from marvin.cloudstackTestCase import cloudstackTestCase
-from marvin.lib.utils import (cleanup_resources,
+from marvin.lib.utils import (validateList,
+                              cleanup_resources,
                               get_process_status)
 
 from marvin.lib.base import (Domain,
@@ -71,13 +72,10 @@ class Services:
                 "firstname": "Test",
                 "lastname": "User",
                 "username": "test",
-                # Random characters are appended for unique
-                # username
                 "password": "password",
             },
             "host1": None,
             "host2": None,
-            "default_hypervisor": "kvm",
             "compute_offering": {
                 "name": "Tiny Instance",
                 "displaytext": "Tiny Instance",
@@ -132,6 +130,11 @@ class Services:
                 "egress_policy": "true",
             },
             "vpc_offering": {
+                "name": 'VPC off',
+                "displaytext": 'VPC off',
+                "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat',
+            },
+            "redundant_vpc_offering": {
                 "name": 'Redundant VPC off',
                 "displaytext": 'Redundant VPC off',
                 "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat',
@@ -224,15 +227,35 @@ class Services:
                     "requireshvm": "True",
                 },
 
-                "xen": {
+                "xenserver": {
                     "name": "tiny-xen",
                     "displaytext": "macchinina xen",
                     "format": "vhd",
                     "hypervisor": "xen",
-                    "ostype": "Other (64-bit)",
+                    "ostype": "Other PV (64-bit)",
                     "url": "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-xen.vhd.bz2",
                     "requireshvm": "True",
                 },
+
+                "hyperv": {
+                    "name": "tiny-hyperv",
+                    "displaytext": "macchinina xen",
+                    "format": "vhd",
+                    "hypervisor": "hyperv",
+                    "ostype": "Other PV (64-bit)",
+                    "url": "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-hyperv.vhd.zip",
+                    "requireshvm": "True",
+                },
+
+                "vmware": {
+                    "name": "tiny-vmware",
+                    "displaytext": "macchinina vmware",
+                    "format": "ova",
+                    "hypervisor": "vmware",
+                    "ostype": "Other PV (64-bit)",
+                    "url": "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-vmware.vmdk.bz2",
+                    "requireshvm": "True",
+                }
             }
         }
 
@@ -253,6 +276,7 @@ class TestVpcRemoteAccessVpn(cloudstackTestCase):
 
         cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
         cls.domain = get_domain(cls.apiclient)
+
         cls.compute_offering = ServiceOffering.create(
             cls.apiclient,
             cls.services["compute_offering"]
@@ -260,25 +284,30 @@ class TestVpcRemoteAccessVpn(cloudstackTestCase):
         cls.account = Account.create(
             cls.apiclient, services=cls.services["account"])
 
-        cls.hypervisor = cls.services["default_hypervisor"]
+        cls.hypervisor = testClient.getHypervisorInfo()
+
         cls.logger.debug("Downloading Template: %s from: %s" % (cls.services["template"][
-                         cls.hypervisor]["name"], cls.services["template"][cls.hypervisor]["url"]))
+                         cls.hypervisor.lower()], cls.services["template"][cls.hypervisor.lower()]["url"]))
-        cls.template = Template.register(cls.apiclient, cls.services["template"][
+        cls.template = Template.register(cls.apiclient, cls.services["template"][cls.hypervisor.lower(
-                                         cls.hypervisor], cls.zone.id, hypervisor=cls.hypervisor, account=cls.account.name, domainid=cls.domain.id)
+        )], cls.zone.id, hypervisor=cls.hypervisor.lower(), account=cls.account.name, domainid=cls.domain.id)
         cls.template.download(cls.apiclient)
 
         if cls.template == FAILED:
-            assert False, "get_template() failed to return template with description %s" % cls.services[
+            assert False, "get_template() failed to return template"
-                "compute_offering"]
 
-        cls.services["virtual_machine"][
+        cls.logger.debug("Successfully created account: %s, id: \
-            "hypervisor"] = cls.services["default_hypervisor"]
+                   %s" % (cls.account.name,
-        cls.cleanup = [cls.account]
+                          cls.account.id))
+
+        cls.cleanup = [cls.template, cls.account, cls.compute_offering]
+        return
 
     @attr(tags=["advanced"], required_hardware="true")
-    def test_vpc_remote_access_vpn(self):
+    def test_01_vpc_remote_access_vpn(self):
         """Test Remote Access VPN in VPC"""
 
+        self.logger.debug("Starting test: test_01_vpc_remote_access_vpn")
+
         # 0) Get the default network offering for VPC
         self.logger.debug("Retrieving default VPC offering")
         networkOffering = NetworkOffering.list(
@@ -334,8 +363,7 @@ class TestVpcRemoteAccessVpn(cloudstackTestCase):
                                        domainid=self.domain.id,
                                        serviceofferingid=self.compute_offering.id,
                                        networkids=ntwk.id,
-                                       hypervisor=self.services[
+                                       hypervisor=self.hypervisor
-                                           "virtual_machine"]["hypervisor"]
                                        )
             self.assert_(vm is not None, "VM failed to deploy")
             self.assert_(vm.state == 'Running', "VM is not running")
@@ -426,29 +454,34 @@ class TestVpcSite2SiteVpn(cloudstackTestCase):
 
         cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
         cls.domain = get_domain(cls.apiclient)
-        cls.service_offering = ServiceOffering.create(
+
+        cls.compute_offering = ServiceOffering.create(
             cls.apiclient,
             cls.services["compute_offering"]
         )
 
         cls.account = Account.create(
             cls.apiclient, services=cls.services["account"])
-        cls.hypervisor = cls.services["default_hypervisor"]
+
+        cls.hypervisor = testClient.getHypervisorInfo()
+
         cls.logger.debug("Downloading Template: %s from: %s" % (cls.services["template"][
-                         cls.hypervisor]["name"], cls.services["template"][cls.hypervisor]["url"]))
+                         cls.hypervisor.lower()], cls.services["template"][cls.hypervisor.lower()]["url"]))
-        cls.template = Template.register(cls.apiclient, cls.services["template"][
+        cls.template = Template.register(cls.apiclient, cls.services["template"][cls.hypervisor.lower(
-                                         cls.hypervisor], cls.zone.id, hypervisor=cls.hypervisor, account=cls.account.name, domainid=cls.domain.id)
+        )], cls.zone.id, hypervisor=cls.hypervisor.lower(), account=cls.account.name, domainid=cls.domain.id)
         cls.template.download(cls.apiclient)
 
         if cls.template == FAILED:
-            assert False, "get_template() failed to return template with description %s" % cls.services[
+            assert False, "get_template() failed to return template"
-                "compute_offering"]
 
-        cls.services["virtual_machine"][
+        cls.logger.debug("Successfully created account: %s, id: \
-            "hypervisor"] = cls.services["default_hypervisor"]
+                   %s" % (cls.account.name,
-        cls.cleanup = [cls.account]
+                          cls.account.id))
 
-    def get_ssh_client(self, virtual_machine, services, retries):
+        cls.cleanup = [cls.template, cls.account, cls.compute_offering]
+        return
+
+    def _get_ssh_client(self, virtual_machine, services, retries):
         """ Setup ssh client connection and return connection
         vm requires attributes public_ip, public_port, username, password """
 
@@ -468,7 +501,408 @@ class TestVpcSite2SiteVpn(cloudstackTestCase):
 
         return ssh_client
 
-    def create_natrule(self, vpc, vm, public_port, private_port, public_ip, network, services=None):
+    def _create_natrule(self, vpc, vm, public_port, private_port, public_ip, network, services=None):
+        self.logger.debug("Creating NAT rule in network for vm with public IP")
+        if not services:
+            self.services["natrule"]["privateport"] = private_port
+            self.services["natrule"]["publicport"] = public_port
+            self.services["natrule"]["startport"] = public_port
+            self.services["natrule"]["endport"] = public_port
+            services = self.services["natrule"]
+
+        nat_rule = NATRule.create(
+            apiclient=self.apiclient,
+            services=services,
+            ipaddressid=public_ip.ipaddress.id,
+            virtual_machine=vm,
+            networkid=network.id
+        )
+        self.assertIsNotNone(
+            nat_rule, "Failed to create NAT Rule for %s" % public_ip.ipaddress.ipaddress)
+        self.logger.debug(
+            "Adding NetworkACL rules to make NAT rule accessible")
+
+        vm.ssh_ip = nat_rule.ipaddress
+        vm.public_ip = nat_rule.ipaddress
+        vm.public_port = int(public_port)
+        return nat_rule
+
+    def _validate_vpc_offering(self, vpc_offering):
+
+        self.logger.debug("Check if the VPC offering is created successfully?")
+        vpc_offs = VpcOffering.list(
+            self.apiclient,
+            id=vpc_offering.id
+        )
+        offering_list = validateList(vpc_offs)
+        self.assertEqual(offering_list[0],
+                         PASS,
+                         "List VPC offerings should return a valid list"
+                         )
+        self.assertEqual(
+            vpc_offering.name,
+            vpc_offs[0].name,
+            "Name of the VPC offering should match with listVPCOff data"
+        )
+        self.logger.debug(
+            "VPC offering is created successfully - %s" %
+            vpc_offering.name)
+        return
+
+    def _create_vpc_offering(self, offering_name):
+
+        vpc_off = None
+        if offering_name is not None:
+
+            self.logger.debug("Creating VPC offering: %s", offering_name)
+            vpc_off = VpcOffering.create(
+                self.apiclient,
+                self.services[offering_name]
+            )
+
+            self._validate_vpc_offering(vpc_off)
+            self.cleanup.append(vpc_off)
+
+        return vpc_off
+
+    @attr(tags=["advanced"], required_hardware="true")
+    def test_01_vpc_site2site_vpn(self):
+        """Test Site 2 Site VPN Across VPCs"""
+        self.logger.debug("Starting test: test_01_vpc_site2site_vpn")
+        # 0) Get the default network offering for VPC
+        networkOffering = NetworkOffering.list(
+            self.apiclient, name="DefaultIsolatedNetworkOfferingForVpcNetworks")
+        self.assert_(networkOffering is not None and len(
+            networkOffering) > 0, "No VPC based network offering")
+
+        # Create and Enable VPC offering
+        vpc_offering = self._create_vpc_offering('vpc_offering')
+        self.assert_(vpc_offering is not None, "Failed to create VPC Offering")
+        vpc_offering.update(self.apiclient, state='Enabled')
+
+        vpc1 = None
+        # Create VPC 1
+        try:
+            vpc1 = VPC.create(
+                apiclient=self.apiclient,
+                services=self.services["vpc"],
+                networkDomain="vpc1.vpn",
+                vpcofferingid=vpc_offering.id,
+                zoneid=self.zone.id,
+                account=self.account.name,
+                domainid=self.domain.id
+            )
+        except Exception as e:
+            self.fail(e)
+        finally:
+            self.assert_(vpc1 is not None, "VPC1 creation failed")
+
+        self.logger.debug("VPC1 %s created" % vpc1.id)
+
+        vpc2 = None
+        # Create VPC 2
+        try:
+            vpc2 = VPC.create(
+                apiclient=self.apiclient,
+                services=self.services["vpc2"],
+                networkDomain="vpc2.vpn",
+                vpcofferingid=vpc_offering.id,
+                zoneid=self.zone.id,
+                account=self.account.name,
+                domainid=self.account.domainid
+            )
+        except Exception as e:
+            self.fail(e)
+        finally:
+            self.assert_(vpc2 is not None, "VPC2 creation failed")
+
+        self.logger.debug("VPC2 %s created" % vpc2.id)
+
+        default_acl = NetworkACLList.list(
+            self.apiclient, name="default_allow")[0]
+
+        ntwk1 = None
+        # Create network in VPC 1
+        try:
+            ntwk1 = Network.create(
+                apiclient=self.apiclient,
+                services=self.services["network_1"],
+                accountid=self.account.name,
+                domainid=self.account.domainid,
+                networkofferingid=networkOffering[0].id,
+                zoneid=self.zone.id,
+                vpcid=vpc1.id,
+                aclid=default_acl.id
+            )
+        except Exception as e:
+            self.fail(e)
+        finally:
+            self.assertIsNotNone(ntwk1, "Network failed to create")
+
+        self.logger.debug("Network %s created in VPC %s" % (ntwk1.id, vpc1.id))
+
+        ntwk2 = None
+        # Create network in VPC 2
+        try:
+            ntwk2 = Network.create(
+                apiclient=self.apiclient,
+                services=self.services["network_2"],
+                accountid=self.account.name,
+                domainid=self.account.domainid,
+                networkofferingid=networkOffering[0].id,
+                zoneid=self.zone.id,
+                vpcid=vpc2.id,
+                aclid=default_acl.id
+            )
+        except Exception as e:
+            self.fail(e)
+        finally:
+            self.assertIsNotNone(ntwk2, "Network failed to create")
+
+        self.logger.debug("Network %s created in VPC %s" % (ntwk2.id, vpc2.id))
+
+        vm1 = None
+        # Deploy a vm in network 2
+        try:
+            vm1 = VirtualMachine.create(self.apiclient, services=self.services["virtual_machine"],
+                                        templateid=self.template.id,
+                                        zoneid=self.zone.id,
+                                        accountid=self.account.name,
+                                        domainid=self.account.domainid,
+                                        serviceofferingid=self.compute_offering.id,
+                                        networkids=ntwk1.id,
+                                        hypervisor=self.hypervisor
+                                        )
+        except Exception as e:
+            self.fail(e)
+        finally:
+            self.assert_(vm1 is not None, "VM failed to deploy")
+            self.assert_(vm1.state == 'Running', "VM is not running")
+
+        self.logger.debug("VM %s deployed in VPC %s" % (vm1.id, vpc1.id))
+
+        vm2 = None
+        # Deploy a vm in network 2
+        try:
+            vm2 = VirtualMachine.create(self.apiclient, services=self.services["virtual_machine"],
+                                        templateid=self.template.id,
+                                        zoneid=self.zone.id,
+                                        accountid=self.account.name,
+                                        domainid=self.account.domainid,
+                                        serviceofferingid=self.compute_offering.id,
+                                        networkids=ntwk2.id,
+                                        hypervisor=self.hypervisor
+                                        )
+        except Exception as e:
+            self.fail(e)
+        finally:
+            self.assert_(vm2 is not None, "VM failed to deploy")
+            self.assert_(vm2.state == 'Running', "VM is not running")
+
+        self.debug("VM %s deployed in VPC %s" % (vm2.id, vpc2.id))
+
+        # 4) Enable Site-to-Site VPN for VPC
+        vpn1_response = Vpn.createVpnGateway(self.apiclient, vpc1.id)
+        self.assert_(
+            vpn1_response is not None, "Failed to enable VPN Gateway 1")
+        self.logger.debug("VPN gateway for VPC %s enabled" % vpc1.id)
+
+        vpn2_response = Vpn.createVpnGateway(self.apiclient, vpc2.id)
+        self.assert_(
+            vpn2_response is not None, "Failed to enable VPN Gateway 2")
+        self.logger.debug("VPN gateway for VPC %s enabled" % vpc2.id)
+
+        # 5) Add VPN Customer gateway info
+        src_nat_list = PublicIPAddress.list(
+            self.apiclient,
+            account=self.account.name,
+            domainid=self.account.domainid,
+            listall=True,
+            issourcenat=True,
+            vpcid=vpc1.id
+        )
+        ip1 = src_nat_list[0]
+        src_nat_list = PublicIPAddress.list(
+            self.apiclient,
+            account=self.account.name,
+            domainid=self.account.domainid,
+            listall=True,
+            issourcenat=True,
+            vpcid=vpc2.id
+        )
+        ip2 = src_nat_list[0]
+
+        services = self.services["vpncustomergateway"]
+        customer1_response = VpnCustomerGateway.create(
+            self.apiclient, services, "Peer VPC1", ip1.ipaddress, vpc1.cidr, self.account.name, self.domain.id)
+        self.debug("VPN customer gateway added for VPC %s enabled" % vpc1.id)
+        self.logger.debug(vars(customer1_response))
+
+        customer2_response = VpnCustomerGateway.create(
+            self.apiclient, services, "Peer VPC2", ip2.ipaddress, vpc2.cidr, self.account.name, self.domain.id)
+        self.debug("VPN customer gateway added for VPC %s enabled" % vpc2.id)
+        self.logger.debug(vars(customer2_response))
+
+        # 6) Connect two VPCs
+        vpnconn1_response = Vpn.createVpnConnection(
+            self.apiclient, customer1_response.id, vpn2_response['id'], True)
+        self.debug("VPN passive connection created for VPC %s" % vpc2.id)
+
+        vpnconn2_response = Vpn.createVpnConnection(
+            self.apiclient, customer2_response.id, vpn1_response['id'])
+        self.debug("VPN connection created for VPC %s" % vpc1.id)
+
+        self.assertEqual(
+            vpnconn2_response['state'], "Connected", "Failed to connect between VPCs!")
+
+        # acquire an extra ip address to use to ssh into vm2
+        try:
+            vm2.public_ip = PublicIPAddress.create(
+                apiclient=self.apiclient,
+                accountid=self.account.name,
+                zoneid=self.zone.id,
+                domainid=self.account.domainid,
+                services=self.services,
+                networkid=ntwk2.id,
+                vpcid=vpc2.id)
+        except Exception as e:
+            self.fail(e)
+        finally:
+            self.assert_(
+                vm2.public_ip is not None, "Failed to aqcuire public ip for vm2")
+
+        natrule = None
+        # Create port forward to be able to ssh into vm2
+        try:
+            natrule = self._create_natrule(
+                vpc2, vm2, 22, 22, vm2.public_ip, ntwk2)
+        except Exception as e:
+            self.fail(e)
+        finally:
+            self.assert_(
+                natrule is not None, "Failed to create portforward for vm2")
+            time.sleep(20)
+
+        # setup ssh connection to vm2
+        ssh_client = self._get_ssh_client(vm2, self.services, 10)
+
+        if ssh_client:
+            # run ping test
+            packet_loss = ssh_client.execute(
+                "/bin/ping -c 3 -t 10 " + vm1.nic[0].ipaddress + " |grep packet|cut -d ' ' -f 7| cut -f1 -d'%'")[0]
+            self.assert_(int(packet_loss) == 0, "Ping did not succeed")
+        else:
+            self.fail("Failed to setup ssh connection to %s" % vm2.public_ip)
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            cleanup_resources(cls.apiclient, cls.cleanup)
+        except Exception, e:
+            raise Exception("Cleanup failed with %s" % e)
+
+
+class TestRVPCSite2SiteVpn(cloudstackTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.logger = logging.getLogger('TestRVPCSite2SiteVPN')
+        cls.stream_handler = logging.StreamHandler()
+        cls.logger.setLevel(logging.DEBUG)
+        cls.logger.addHandler(cls.stream_handler)
+
+        testClient = super(TestRVPCSite2SiteVpn, cls).getClsTestClient()
+        cls.apiclient = testClient.getApiClient()
+        cls.services = Services().services
+
+        cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
+        cls.domain = get_domain(cls.apiclient)
+
+        cls.compute_offering = ServiceOffering.create(
+            cls.apiclient,
+            cls.services["compute_offering"]
+        )
+
+        cls.account = Account.create(
+            cls.apiclient, services=cls.services["account"])
+
+        cls.hypervisor = testClient.getHypervisorInfo()
+
+        cls.logger.debug("Downloading Template: %s from: %s" % (cls.services["template"][
+                         cls.hypervisor.lower()], cls.services["template"][cls.hypervisor.lower()]["url"]))
+        cls.template = Template.register(cls.apiclient, cls.services["template"][cls.hypervisor.lower(
+        )], cls.zone.id, hypervisor=cls.hypervisor.lower(), account=cls.account.name, domainid=cls.domain.id)
+        cls.template.download(cls.apiclient)
+
+        if cls.template == FAILED:
+            assert False, "get_template() failed to return template"
+
+        cls.logger.debug("Successfully created account: %s, id: \
+                   %s" % (cls.account.name,
+                          cls.account.id))
+
+        cls.cleanup = [cls.template, cls.account, cls.compute_offering]
+        return
+
+    def _validate_vpc_offering(self, vpc_offering):
+
+        self.logger.debug("Check if the VPC offering is created successfully?")
+        vpc_offs = VpcOffering.list(
+            self.apiclient,
+            id=vpc_offering.id
+        )
+        offering_list = validateList(vpc_offs)
+        self.assertEqual(offering_list[0],
+                         PASS,
+                         "List VPC offerings should return a valid list"
+                         )
+        self.assertEqual(
+            vpc_offering.name,
+            vpc_offs[0].name,
+            "Name of the VPC offering should match with listVPCOff data"
+        )
+        self.logger.debug(
+            "VPC offering is created successfully - %s" %
+            vpc_offering.name)
+        return
+
+    def _create_vpc_offering(self, offering_name):
+
+        vpc_off = None
+        if offering_name is not None:
+
+            self.logger.debug("Creating VPC offering: %s", offering_name)
+            vpc_off = VpcOffering.create(
+                self.apiclient,
+                self.services[offering_name]
+            )
+
+            self._validate_vpc_offering(vpc_off)
+            self.cleanup.append(vpc_off)
+
+        return vpc_off
+
+    def _get_ssh_client(self, virtual_machine, services, retries):
+        """ Setup ssh client connection and return connection
+        vm requires attributes public_ip, public_port, username, password """
+
+        try:
+            ssh_client = SshClient(
+                virtual_machine.public_ip,
+                services["virtual_machine"]["ssh_port"],
+                services["virtual_machine"]["username"],
+                services["virtual_machine"]["password"],
+                retries)
+
+        except Exception as e:
+            self.fail("Unable to create ssh connection: " % e)
+
+        self.assertIsNotNone(
+            ssh_client, "Failed to setup ssh connection to vm=%s on public_ip=%s" % (virtual_machine.name, virtual_machine.public_ip))
+
+        return ssh_client
+
+    def _create_natrule(self, vpc, vm, public_port, private_port, public_ip, network, services=None):
         self.logger.debug("Creating NAT rule in network for vm with public IP")
         if not services:
             self.services["natrule"]["privateport"] = private_port
@@ -495,8 +929,9 @@ class TestVpcSite2SiteVpn(cloudstackTestCase):
         return nat_rule
 
     @attr(tags=["advanced"], required_hardware="true")
-    def test_vpc_site2site_vpn(self):
+    def test_01_redundant_vpc_site2site_vpn(self):
-        """Test VPN in VPC"""
+        """Test Site 2 Site VPN Across redundant VPCs"""
+        self.logger.debug("Starting test: test_02_redundant_vpc_site2site_vpn")
 
         # 0) Get the default network offering for VPC
         networkOffering = NetworkOffering.list(
@@ -504,18 +939,22 @@ class TestVpcSite2SiteVpn(cloudstackTestCase):
         self.assert_(networkOffering is not None and len(
             networkOffering) > 0, "No VPC based network offering")
 
-        # 1) Create VPC offering
+        # Create and enable redundant VPC offering
-        vpcOffering = VpcOffering.list(self.apiclient, isdefault=True)
+        redundant_vpc_offering = self._create_vpc_offering(
-        self.assert_(vpcOffering is not None and len(
+            'redundant_vpc_offering')
-            vpcOffering) > 0, "No VPC offerings found")
+        self.assert_(redundant_vpc_offering is not None,
+                     "Failed to create redundant VPC Offering")
+
+        redundant_vpc_offering.update(self.apiclient, state='Enabled')
 
         # Create VPC 1
+        vpc1 = None
         try:
             vpc1 = VPC.create(
                 apiclient=self.apiclient,
                 services=self.services["vpc"],
                 networkDomain="vpc1.vpn",
-                vpcofferingid=vpcOffering[0].id,
+                vpcofferingid=redundant_vpc_offering.id,
                 zoneid=self.zone.id,
                 account=self.account.name,
                 domainid=self.domain.id
@@ -525,15 +964,16 @@ class TestVpcSite2SiteVpn(cloudstackTestCase):
         finally:
             self.assert_(vpc1 is not None, "VPC1 creation failed")
 
-        self.logger.debug("VPC1 %s created" % (vpc1.id))
+        self.logger.debug("VPC1 %s created" % vpc1.id)
 
         # Create VPC 2
+        vpc2 = None
         try:
             vpc2 = VPC.create(
                 apiclient=self.apiclient,
                 services=self.services["vpc2"],
                 networkDomain="vpc2.vpn",
-                vpcofferingid=vpcOffering[0].id,
+                vpcofferingid=redundant_vpc_offering.id,
                 zoneid=self.zone.id,
                 account=self.account.name,
                 domainid=self.account.domainid
@@ -543,12 +983,13 @@ class TestVpcSite2SiteVpn(cloudstackTestCase):
         finally:
             self.assert_(vpc2 is not None, "VPC2 creation failed")
 
-        self.logger.debug("VPC2 %s created" % (vpc2.id))
+        self.logger.debug("VPC2 %s created" % vpc2.id)
 
         default_acl = NetworkACLList.list(
             self.apiclient, name="default_allow")[0]
 
         # Create network in VPC 1
+        ntwk1 = None
         try:
             ntwk1 = Network.create(
                 apiclient=self.apiclient,
@@ -568,6 +1009,7 @@ class TestVpcSite2SiteVpn(cloudstackTestCase):
         self.logger.debug("Network %s created in VPC %s" % (ntwk1.id, vpc1.id))
 
         # Create network in VPC 2
+        ntwk2 = None
         try:
             ntwk2 = Network.create(
                 apiclient=self.apiclient,
@@ -587,16 +1029,16 @@ class TestVpcSite2SiteVpn(cloudstackTestCase):
         self.logger.debug("Network %s created in VPC %s" % (ntwk2.id, vpc2.id))
 
         # Deploy a vm in network 2
+        vm1 = None
         try:
             vm1 = VirtualMachine.create(self.apiclient, services=self.services["virtual_machine"],
                                         templateid=self.template.id,
                                         zoneid=self.zone.id,
                                         accountid=self.account.name,
                                         domainid=self.account.domainid,
-                                        serviceofferingid=self.service_offering.id,
+                                        serviceofferingid=self.compute_offering.id,
                                         networkids=ntwk1.id,
-                                        hypervisor=self.services[
+                                        hypervisor=self.hypervisor
-                                            "virtual_machine"]["hypervisor"]
                                         )
         except Exception as e:
             self.fail(e)
@@ -607,16 +1049,16 @@ class TestVpcSite2SiteVpn(cloudstackTestCase):
         self.logger.debug("VM %s deployed in VPC %s" % (vm1.id, vpc1.id))
 
         # Deploy a vm in network 2
+        vm2 = None
         try:
             vm2 = VirtualMachine.create(self.apiclient, services=self.services["virtual_machine"],
                                         templateid=self.template.id,
                                         zoneid=self.zone.id,
                                         accountid=self.account.name,
                                         domainid=self.account.domainid,
-                                        serviceofferingid=self.service_offering.id,
+                                        serviceofferingid=self.compute_offering.id,
                                         networkids=ntwk2.id,
-                                        hypervisor=self.services[
+                                        hypervisor=self.hypervisor
-                                            "virtual_machine"]["hypervisor"]
                                         )
         except Exception as e:
             self.fail(e)
@@ -697,8 +1139,9 @@ class TestVpcSite2SiteVpn(cloudstackTestCase):
                 vm2.public_ip is not None, "Failed to aqcuire public ip for vm2")
 
         # Create port forward to be able to ssh into vm2
+        natrule = None
         try:
-            natrule = self.create_natrule(
+            natrule = self._create_natrule(
                 vpc2, vm2, 22, 22, vm2.public_ip, ntwk2)
         except Exception as e:
             self.fail(e)
@@ -708,7 +1151,7 @@ class TestVpcSite2SiteVpn(cloudstackTestCase):
             time.sleep(20)
 
         # setup ssh connection to vm2
-        ssh_client = self.get_ssh_client(vm2, self.services, 10)
+        ssh_client = self._get_ssh_client(vm2, self.services, 10)
 
         if ssh_client:
             # run ping test

tools/marvin/marvin/lib/base.py

@@ -3572,6 +3572,8 @@ class VpnCustomerGateway:
             cmd.esplifetime = services["esplifetime"]
         if "dpd" in services:
             cmd.dpd = services["dpd"]
+        if "forceencap" in services:
+            cmd.forceencap = services["forceencap"]
         if account:
             cmd.account = account
         if domainid:
@@ -3599,6 +3601,8 @@ class VpnCustomerGateway:
             cmd.esplifetime = services["esplifetime"]
         if "dpd" in services:
             cmd.dpd = services["dpd"]
+        if "forceencap" in services:
+            cmd.forceencap = services["forceencap"]
         return(apiclient.updateVpnCustomerGateway(cmd))
 
     def delete(self, apiclient):

ui/dictionary2.jsp

@@ -1124,6 +1124,7 @@ under the License.
 'message.desc.create.ssh.key.pair': '<fmt:message key="message.desc.create.ssh.key.pair" />',
 'message.removed.ssh.key.pair': '<fmt:message key="message.removed.ssh.key.pair" />',
 'message.please.select.ssh.key.pair.use.with.this.vm': '<fmt:message key="message.please.select.ssh.key.pair.use.with.this.vm" />',
-'message.ldap.group.import': '<fmt:message key="message.ldap.group.import" />'
+'message.ldap.group.import': '<fmt:message key="message.ldap.group.import" />',
+'label.vpn.force.encapsulation': '<fmt:message key="label.vpn.force.encapsulation" />'
 });
 </script>

ui/scripts/docs.js

@@ -1118,6 +1118,10 @@ cloudStack.docs = {
         desc: 'Check this to make the virtual router query its IKE peer at regular intervals to ensure continued availability. It is recommended to have the same DPD setting on both sides of the VPN connection.',
         externalLink: ''
     },
+    helpVPNGatewayForceEncapsulation: {
+        desc: 'Force UDP encapsulation for ESP packets even if no NAT situation is detected. This may help to surmount restrictive firewalls. In order to force the peer to encapsulate packets, NAT detection payloads are faked',
+        externalLink: ''
+    },
     // Copy template
     helpCopyTemplateDestination: {
         desc: 'The zone to which you want to copy the template',
@@ -1329,4 +1333,4 @@ cloudStack.docs = {
     helpLdapLinkDomainAdmin: {
         desc: 'domain admin of the linked domain. Specify a username in GROUP/OU of LDAP'
     }
-};
+};

ui/scripts/network.js

@@ -6133,6 +6133,14 @@
                                         docID: 'helpVPNGatewayDeadPeerDetection',
                                         isBoolean: true,
                                         isChecked: false
+                                    },
+
+                                    forceencap: {
+                                        label: 'label.vpn.force.encapsulation',
+                                        docID: 'helpVPNGatewayForceEncapsulation',
+                                        docID: 'helpVPNGatewayForceEncapsulation',
+                                        isBoolean: true,
+                                        isChecked: false
                                     }
                                 }
                             },
@@ -6144,7 +6152,8 @@
                                     ipsecpsk: args.data.ipsecpsk,
                                     ikelifetime: args.data.ikelifetime,
                                     esplifetime: args.data.esplifetime,
-                                    dpd: (args.data.dpd == "on")
+                                    dpd: (args.data.dpd == "on"),
+                                    forceencap: (args.data.forceencap == "on")
                                 };
 
                                 var ikepolicy = args.data.ikeEncryption + '-' + args.data.ikeHash;
@@ -6200,7 +6209,8 @@
                                         ipsecpsk: args.data.ipsecpsk,
                                         ikelifetime: args.data.ikelifetime,
                                         esplifetime: args.data.esplifetime,
-                                        dpd: (args.data.dpd == "on")
+                                        dpd: (args.data.dpd == "on"),
+                                        forceencap: (args.data.forceencap == "on")
                                     };
 
                                     var ikepolicy = args.data.ikeEncryption + '-' + args.data.ikeHash;
@@ -6223,7 +6233,7 @@
                                         url: createURL('updateVpnCustomerGateway'),
                                         data: data,
                                         success: function(json) {
-                                            var jobId = json.updatecustomergatewayresponse.jobid;
+                                            var jobId = json.updatevpncustomergatewayresponse.jobid;
                                             args.response.success({
                                                 _custom: {
                                                     jobId: jobId,
@@ -6235,6 +6245,9 @@
                                                     }
                                                 }
                                             });
+                                        },
+                                        error: function(json) {
+                                            args.response.error(parseXMLHttpResponse(json));
                                         }
                                     });
                                 },
@@ -6469,6 +6482,13 @@
                                         converter: cloudStack.converters.toBooleanText
                                     },
 
+                                    forceencap: {
+                                        label: 'label.vpn.force.encapsulation',
+                                        isBoolean: true,
+                                        isEditable: true,
+                                        converter: cloudStack.converters.toBooleanText
+                                    },
+
                                     id: {
                                         label: 'label.id'
                                     },

ui/scripts/vpc.js

@@ -748,7 +748,8 @@
                         $.ajax({
                             url: createURL('listLoadBalancers'),
                             data: {
-                                networkid: args.context.networks[0].id
+                                networkid: args.context.networks[0].id,
+                                listAll: true
                             },
                             success: function(json) {
                                 var items = json.listloadbalancersresponse.loadbalancer;
@@ -1132,7 +1133,8 @@
                             async: false,
                             data: {
                                 associatednetworkid: args.context.networks[0].id,
-                                forloadbalancing: true
+                                forloadbalancing: true,
+                                listall: true
                             },
                             success: function(json) {
                                 var items = json.listpublicipaddressesresponse.publicipaddress;
@@ -2904,6 +2906,12 @@
                                                 return str ? 'Yes' : 'No';
                                             }
                                         },
+                                        forceencap: {
+                                            label: 'label.vpn.force.encapsulation',
+                                            converter: function(str) {
+                                                return str ? 'Yes' : 'No';
+                                            }
+                                        },
                                         state: {
                                             label: 'label.state'
                                         },

utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java

@@ -29,6 +29,8 @@ import com.trilead.ssh2.Session;
 
 public class SSHCmdHelper {
     private static final Logger s_logger = Logger.getLogger(SSHCmdHelper.class);
+    private static final int DEFAULT_CONNECT_TIMEOUT = 180000;
+    private static final int DEFAULT_KEX_TIMEOUT = 60000;
 
     public static com.trilead.ssh2.Connection acquireAuthorizedConnection(String ip, String username, String password) {
         return acquireAuthorizedConnection(ip, 22, username, password);
@@ -37,7 +39,7 @@ public class SSHCmdHelper {
     public static com.trilead.ssh2.Connection acquireAuthorizedConnection(String ip, int port, String username, String password) {
         com.trilead.ssh2.Connection sshConnection = new com.trilead.ssh2.Connection(ip, port);
         try {
-            sshConnection.connect(null, 60000, 60000);
+            sshConnection.connect(null, DEFAULT_CONNECT_TIMEOUT, DEFAULT_KEX_TIMEOUT);
             if (!sshConnection.authenticateWithPassword(username, password)) {
                 String[] methods = sshConnection.getRemainingAuthMethods(username);
                 StringBuffer mStr = new StringBuffer();

utils/src/main/java/com/cloud/utils/ssh/SshHelper.java

@@ -29,14 +29,14 @@ import com.trilead.ssh2.ChannelCondition;
 import com.cloud.utils.Pair;
 
 public class SshHelper {
-    private static final int DEFAULT_CONNECT_TIMEOUT = 60000;
+    private static final int DEFAULT_CONNECT_TIMEOUT = 180000;
     private static final int DEFAULT_KEX_TIMEOUT = 60000;
 
     private static final Logger s_logger = Logger.getLogger(SshHelper.class);
 
     public static Pair<Boolean, String> sshExecute(String host, int port, String user, File pemKeyFile, String password, String command) throws Exception {
 
-        return sshExecute(host, port, user, pemKeyFile, password, command, 60000, 60000, 120000);
+        return sshExecute(host, port, user, pemKeyFile, password, command, DEFAULT_CONNECT_TIMEOUT, DEFAULT_KEX_TIMEOUT, 120000);
     }
 
     public static void scpTo(String host, int port, String user, File pemKeyFile, String password, String remoteTargetDirectory, String localFile, String fileMode)