S2S VPN: CS-15642: Re-initiate the VPN connections after router reboot

Conflicts:

	server/src/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java

patches/systemvm/debian/config/opt/cloud/bin/ipsectunnel.sh

@@ -142,7 +142,7 @@ ipsec_tunnel_add() {
     sudo echo "  esp=$esppolicy" >> $vpnconffile &&
     sudo echo "  salifetime=${esplifetime}s" >> $vpnconffile &&
     sudo echo "  pfs=$pfs" >> $vpnconffile &&
-    sudo echo "  keyingtries=3" >> $vpnconffile &&
+    sudo echo "  keyingtries=2" >> $vpnconffile &&
     sudo echo "  auto=add" >> $vpnconffile &&
     sudo echo "$leftpeer $rightpeer: PSK \"$secret\"" > $vpnsecretsfile &&
     sudo chmod 0400 $vpnsecretsfile
@@ -162,8 +162,8 @@ ipsec_tunnel_add() {
 
   logger -t cloud "$(basename $0): done ipsec tunnel entry for right peer=$rightpeer right networks=$rightnets"
 
-  #20 seconds for checking if it's ready
+  #5 seconds for checking if it's ready
-  for i in {1..4}
+  for i in {1..5}
   do
     logger -t cloud "$(basename $0): checking connection status..."
     /opt/cloud/bin/checks2svpn.sh $rightpeer
@@ -172,7 +172,7 @@ ipsec_tunnel_add() {
     then
         break
     fi
-    sleep 5
+    sleep 1
   done
   if [ $result -eq 0 ]
   then

server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -909,6 +909,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
         }
     }
 
+    @DB
     protected void updateSite2SiteVpnConnectionState(List<DomainRouterVO> routers) {
         for (DomainRouterVO router : routers) {
             List<Site2SiteVpnConnectionVO> conns = _s2sVpnMgr.getConnectionsForRouter(router);
@@ -958,6 +959,11 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
                     continue;
                 }
                 for (Site2SiteVpnConnectionVO conn : conns) {
+                    Site2SiteVpnConnectionVO lock = _s2sVpnConnectionDao.acquireInLockTable(conn.getId());
+                    if (lock == null) {
+                        throw new CloudRuntimeException("Unable to acquire lock on " + lock);
+                    }
+                    try {
                         if (conn.getState() != Site2SiteVpnConnection.State.Connected &&
                                 conn.getState() != Site2SiteVpnConnection.State.Disconnected) {
                             continue;
@@ -979,6 +985,9 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
                             _alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER,
                                     router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context);
                         }
+                    } finally {
+                        _s2sVpnConnectionDao.releaseFromLockTable(lock.getId());
+                    }
                 }
             }
         }
@@ -2320,6 +2329,11 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
     ConcurrentOperationException, ResourceUnavailableException {
         s_logger.debug("Starting router " + router);
         if (_itMgr.start(router, params, user, caller, planToDeploy) != null) {
+            // We don't want the failure of VPN Connection affect the status of router, so we try to make connection only after router start successfully
+            Long vpcId = router.getVpcId();
+            if (vpcId != null) {
+                _s2sVpnMgr.reconnectDisconnectedVpnByVpc(vpcId);
+            }
             return _routerDao.findById(router.getId());
         } else {
             return null;

server/src/com/cloud/network/vpn/Site2SiteVpnManager.java

@@ -11,4 +11,5 @@ public interface Site2SiteVpnManager extends Site2SiteVpnService {
     void markDisconnectVpnConnByVpc(long vpcId);
     List<Site2SiteVpnConnectionVO> getConnectionsForRouter(DomainRouterVO router);
     boolean deleteCustomerGatewayByAccount(long accountId);
+    void reconnectDisconnectedVpnByVpc(Long vpcId);
 }

server/src/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java

@@ -53,6 +53,7 @@ import com.cloud.user.dao.AccountDao;
 import com.cloud.utils.Ternary;
 import com.cloud.utils.component.Inject;
 import com.cloud.utils.component.Manager;
+import com.cloud.utils.db.DB;
 import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericDao;
 import com.cloud.utils.db.JoinBuilder;
@@ -240,14 +241,20 @@ public class Site2SiteVpnManagerImpl implements Site2SiteVpnManager, Manager {
     }
 
     @Override
+    @DB
     public Site2SiteVpnConnection startVpnConnection(long id) throws ResourceUnavailableException {
-        Site2SiteVpnConnectionVO conn = _vpnConnectionDao.findById(id);
+        Site2SiteVpnConnectionVO conn = _vpnConnectionDao.acquireInLockTable(id);
+        if (conn == null) {
+            throw new CloudRuntimeException("Unable to acquire lock on " + conn);
+        }
+        try {
             if (conn.getState() != State.Pending && conn.getState() != State.Disconnected) {
-            throw new InvalidParameterValueException("Site to site VPN connection " + id + " not in correct state(pending or disconnected) to process!");
+                throw new InvalidParameterValueException("Site to site VPN connection with specified connectionId not in correct state(pending or disconnected) to process!");
             }
 
             conn.setState(State.Pending);
             _vpnConnectionDao.persist(conn);
+
             List <? extends Site2SiteVpnServiceProvider> elements = _networkMgr.getSite2SiteVpnElements();
             boolean result = true;
             for (Site2SiteVpnServiceProvider element : elements) {
@@ -262,6 +269,9 @@ public class Site2SiteVpnManagerImpl implements Site2SiteVpnManager, Manager {
             conn.setState(State.Error);
             _vpnConnectionDao.persist(conn);
             throw new ResourceUnavailableException("Failed to apply site-to-site VPN", Site2SiteVpnConnection.class, id);
+        } finally {
+            _vpnConnectionDao.releaseFromLockTable(conn.getId());
+        }
     }
 
     @Override
@@ -419,16 +429,22 @@ public class Site2SiteVpnManagerImpl implements Site2SiteVpnManager, Manager {
         return true;
     }
 
+    @DB
     private void stopVpnConnection(Long id) throws ResourceUnavailableException {
-        Site2SiteVpnConnectionVO conn = _vpnConnectionDao.findById(id);
+        Site2SiteVpnConnectionVO conn = _vpnConnectionDao.acquireInLockTable(id);
-        if (conn.getState() != State.Connected && conn.getState() != State.Error) {
+        if (conn == null) {
-            throw new InvalidParameterValueException("Site to site VPN connection " + id + " not in correct state(connected) to process disconnect!");
+            throw new CloudRuntimeException("Unable to acquire lock on " + conn);
         }
+        try {
+            if (conn.getState() != State.Connected && conn.getState() != State.Error) {
+                throw new InvalidParameterValueException("Site to site VPN connection with specified id is not in correct state(connected) to process disconnect!");
+            }
+
+            conn.setState(State.Disconnected);
+            _vpnConnectionDao.persist(conn);
             
             List <? extends Site2SiteVpnServiceProvider> elements = _networkMgr.getSite2SiteVpnElements();
             boolean result = true;
-        conn.setState(State.Disconnected);
-        _vpnConnectionDao.persist(conn);
             for (Site2SiteVpnServiceProvider element : elements) {
                 result = result & element.stopSite2SiteVpn(conn);
             }
@@ -438,6 +454,9 @@ public class Site2SiteVpnManagerImpl implements Site2SiteVpnManager, Manager {
                 _vpnConnectionDao.persist(conn);
                 throw new ResourceUnavailableException("Failed to apply site-to-site VPN", Site2SiteVpnConnection.class, id);
             }
+        } finally {
+            _vpnConnectionDao.releaseFromLockTable(conn.getId());
+        }
     }
 
     @Override
@@ -617,16 +636,25 @@ public class Site2SiteVpnManagerImpl implements Site2SiteVpnManager, Manager {
     }
     
     @Override
+    @DB
     public void markDisconnectVpnConnByVpc(long vpcId) {
         List<Site2SiteVpnConnectionVO> conns = _vpnConnectionDao.listByVpcId(vpcId);
         for (Site2SiteVpnConnectionVO conn : conns) {
             if (conn == null) {
                 continue;
             }
+            Site2SiteVpnConnectionVO lock = _vpnConnectionDao.acquireInLockTable(conn.getId());
+            if (lock == null) {
+                throw new CloudRuntimeException("Unable to acquire lock on " + conn);
+            }
+            try {
                 if (conn.getState() == Site2SiteVpnConnection.State.Connected) {
                     conn.setState(Site2SiteVpnConnection.State.Disconnected);
                     _vpnConnectionDao.persist(conn);
                 }
+            } finally {
+                _vpnConnectionDao.releaseFromLockTable(lock.getId());
+            }
         }
     }
 
@@ -651,4 +679,22 @@ public class Site2SiteVpnManagerImpl implements Site2SiteVpnManager, Manager {
         }
         return result;
     }
+
+    @Override
+    public void reconnectDisconnectedVpnByVpc(Long vpcId) {
+        List<Site2SiteVpnConnectionVO> conns = _vpnConnectionDao.listByVpcId(vpcId);
+        for (Site2SiteVpnConnectionVO conn : conns) {
+            if (conn == null) {
+                continue;
+            }
+            if (conn.getState() == Site2SiteVpnConnection.State.Disconnected) {
+                try {
+                    startVpnConnection(conn.getId());
+                } catch (ResourceUnavailableException e) {
+                    Site2SiteCustomerGatewayVO gw = _customerGatewayDao.findById(conn.getCustomerGatewayId());
+                    s_logger.warn("Site2SiteVpnManager: Fail to re-initiate VPN connection " + conn.getId() + " which connect to " + gw.getName());
+                }
+            }
+        }
+    }
 }