apache/cloudstack
1
0
Fork 0
mirror of https://github.com/apache/cloudstack.git synced 2025-10-26 08:42:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-799: Redundant router: Speed up RvR status update

Browse Source 
The basic idea behind this is, deploy a fix sized threadpool for updating RvR
status, then using producer/consumer model. There is a global configuration
router.check.poolsize(10 by default) to control the pool size.

Using pool size 100 for 1000 RvR is tested with simulator and works well.

Also we can adjust the global configuration option router.check.interval to e.g.
60s from default 30s to mitigate the issue.
This commit is contained in:
Sheng Yang 2012-11-29 16:57:11 -08:00
parent f1c15f0c2e
commit aa9701a10c
6 changed files with 97 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
server/src/com/cloud/configuration/Config.java
View File

@ -167,6 +167,7 @@ public enum Config {
RouterStatsInterval("Advanced", NetworkManager.class, Integer.class, "router.stats.interval", "300", "Interval (in seconds) to report router statistics.", null),
ExternalNetworkStatsInterval("Advanced", NetworkManager.class, Integer.class, "external.network.stats.interval", "300", "Interval (in seconds) to report external network statistics.", null),
RouterCheckInterval("Advanced", NetworkManager.class, Integer.class, "router.check.interval", "30", "Interval (in seconds) to report redundant router status.", null),
RouterCheckPoolSize("Advanced", NetworkManager.class, Integer.class, "router.check.poolsize", "10", "Numbers of threads using to check redundant router status.", null),
RouterTemplateId("Advanced", NetworkManager.class, Long.class, "router.template.id", "1", "Default ID for template.", null),
RouterExtraPublicNics("Advanced", NetworkManager.class, Integer.class, "router.extra.public.nics", "2", "specify extra public nics used for virtual router(up to 5)", "0-5"),
StartRetry("Advanced", AgentManager.class, Integer.class, "start.retry", "10", "Number of times to retry create and start commands", null),

1
server/src/com/cloud/network/dao/NetworkDao.java
View File

@ -110,4 +110,5 @@ public interface NetworkDao extends GenericDao<NetworkVO, Long> {
List<NetworkVO> listNetworksByAccount(long accountId, long zoneId, Network.GuestType type, boolean isSystem);
List<NetworkVO> listRedundantNetworks();
}

9
server/src/com/cloud/network/dao/NetworkDaoImpl.java
View File

@ -103,6 +103,7 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long> implements N
AllFieldsSearch.and("vpcId", AllFieldsSearch.entity().getVpcId(), Op.EQ);
SearchBuilder<NetworkOfferingVO> join1 = _ntwkOffDao.createSearchBuilder();
join1.and("isSystem", join1.entity().isSystemOnly(), Op.EQ);
join1.and("isRedundant", join1.entity().getRedundantRouter(), Op.EQ);
AllFieldsSearch.join("offerings", join1, AllFieldsSearch.entity().getNetworkOfferingId(), join1.entity().getId(), JoinBuilder.JoinType.INNER);
AllFieldsSearch.done();
@ -574,4 +575,12 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long> implements N
List<NetworkVO> networks = search(sc, null);
return networks;
}
@Override
public List<NetworkVO> listRedundantNetworks() {
SearchCriteria<NetworkVO> sc = AllFieldsSearch.create();
sc.setJoinParameters("offerings", "isRedundant", true);
return listBy(sc, null);
}
}

94
server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
View File

@ -28,9 +28,14 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@ -339,6 +344,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
int _routerStatsInterval = 300;
int _routerCheckInterval = 30;
int _rvrStatusUpdatePoolSize = 10;
protected ServiceOfferingVO _offering;
private String _dnsBasicZoneUpdates = "all";
private Set<String> _guestOSNeedGatewayOnNonDefaultNetwork = new HashSet<String>();
@ -353,8 +359,11 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
ScheduledExecutorService _executor;
ScheduledExecutorService _checkExecutor;
ScheduledExecutorService _networkStatsUpdateExecutor;
ExecutorService _rvrStatusUpdateExecutor;
Account _systemAcct;
BlockingQueue<Long> _vrUpdateQueue = null;
@Override
public boolean sendSshKeysToHost(Long hostId, String pubKey, String prvKey) {
@ -582,7 +591,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
_executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterMonitor"));
_checkExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterStatusMonitor"));
_networkStatsUpdateExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NetworkStatsUpdater"));
final ComponentLocator locator = ComponentLocator.getCurrentLocator();
final Map<String, String> configs = _configDao.getConfiguration("AgentManager", params);
@ -609,7 +618,18 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
value = configs.get("router.check.interval");
_routerCheckInterval = NumbersUtil.parseInt(value, 30);
value = configs.get("router.check.poolsize");
_rvrStatusUpdatePoolSize = NumbersUtil.parseInt(value, 10);
/*
* We assume that one thread can handle 20 requests in 1 minute in normal situation, so here we give the queue size up to 50 minutes.
* It's mostly for buffer, since each time CheckRouterTask running, it would add all the redundant networks in the queue immediately
*/
_vrUpdateQueue = new LinkedBlockingQueue<Long>(_rvrStatusUpdatePoolSize * 1000);
_rvrStatusUpdateExecutor = Executors.newFixedThreadPool(_rvrStatusUpdatePoolSize, new NamedThreadFactory("RedundantRouterStatusMonitor"));
_instance = configs.get("instance.name");
if (_instance == null) {
_instance = "DEFAULT";
@ -707,6 +727,9 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
if (_routerCheckInterval > 0) {
_checkExecutor.scheduleAtFixedRate(new CheckRouterTask(), _routerCheckInterval, _routerCheckInterval, TimeUnit.SECONDS);
for (int i = 0; i < _rvrStatusUpdatePoolSize; i++) {
_rvrStatusUpdateExecutor.execute(new RvRStatusUpdateTask());
}
} else {
s_logger.debug("router.check.interval - " + _routerCheckInterval+ " so not scheduling the redundant router checking thread");
}
@ -1042,14 +1065,11 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
if (host == null || host.getStatus() != Status.Up) {
router.setRedundantState(RedundantState.UNKNOWN);
updated = true;
} else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) {
/* Only cover hosts managed by this management server */
continue;
} else if (privateIP != null) {
final CheckRouterCommand command = new CheckRouterCommand();
command.setAccessDetail(NetworkElementCommand.ROUTER_IP, getRouterControlIp(router.getId()));
command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
command.setWait(60);
command.setWait(30);
final Answer origAnswer = _agentMgr.easySend(router.getHostId(), command);
CheckRouterAnswer answer = null;
if (origAnswer instanceof CheckRouterAnswer) {
@ -1131,11 +1151,11 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
return priority;
}
protected class CheckRouterTask implements Runnable {
protected class RvRStatusUpdateTask implements Runnable {
public CheckRouterTask() {
public RvRStatusUpdateTask() {
}
/*
* In order to make fail-over works well at any time, we have to ensure:
* 1. Backup router's priority = Master's priority - DELTA + 1
@ -1198,9 +1218,9 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
String title = "More than one redundant virtual router is in MASTER state! Router " + router.getHostName() + " and router " + dupRouter.getHostName();
String context = "Virtual router (name: " + router.getHostName() + ", id: " + router.getId() + " and router (name: "
+ dupRouter.getHostName() + ", id: " + router.getId() + ") are both in MASTER state! If the problem persist, restart both of routers. ";
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context);
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER, dupRouter.getDataCenterIdToDeployIn(), dupRouter.getPodIdToDeployIn(), title, context);
s_logger.warn(context);
} else {
networkRouterMaps.put(routerGuestNtwkId, router);
}
@ -1209,19 +1229,61 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
}
}
@Override
public void run() {
while (true) {
try {
Long networkId = _vrUpdateQueue.take();
List <DomainRouterVO> routers = _routerDao.listByNetworkAndRole(networkId, Role.VIRTUAL_ROUTER);
if (routers.size() != 2) {
continue;
}
/*
* We update the router pair which the lower id router owned by this mgmt server, in order
* to prevent duplicate update of router status from cluster mgmt servers
*/
DomainRouterVO router = routers.get(0);
if (routers.get(1).getId() < router.getId()) {
router = routers.get(1);
}
HostVO host = _hostDao.findById(router.getHostId());
if (host == null || host.getManagementServerId() == null ||
host.getManagementServerId() != ManagementServerNode.getManagementServerId()) {
continue;
}
updateRoutersRedundantState(routers);
checkDuplicateMaster(routers);
checkSanity(routers);
} catch (Exception ex) {
s_logger.error("Fail to complete the RvRStatusUpdateTask! ", ex);
}
}
}
}
protected class CheckRouterTask implements Runnable {
public CheckRouterTask() {
}
@Override
public void run() {
try {
final List<DomainRouterVO> routers = _routerDao.listIsolatedByHostId(null);
s_logger.debug("Found " + routers.size() + " routers. ");
s_logger.debug("Found " + routers.size() + " routers to update status. ");
updateRoutersRedundantState(routers);
updateSite2SiteVpnConnectionState(routers);
/* FIXME assumed the a pair of redundant routers managed by same mgmt server,
* then the update above can get the latest status */
checkDuplicateMaster(routers);
checkSanity(routers);
final List<NetworkVO> networks = _networkDao.listRedundantNetworks();
s_logger.debug("Found " + networks.size() + " networks to update RvR status. ");
for (NetworkVO network : networks) {
if (!_vrUpdateQueue.offer(network.getId(), 500, TimeUnit.MILLISECONDS)) {
s_logger.warn("Cannot insert into virtual router update queue! Adjustment of router.check.interval and router.check.poolsize maybe needed.");
break;
}
}
} catch (Exception ex) {
s_logger.error("Fail to complete the CheckRouterTask! ", ex);
}

6
server/test/com/cloud/vpc/dao/MockNetworkDaoImpl.java
View File

@ -351,4 +351,10 @@ public class MockNetworkDaoImpl extends GenericDaoBase<NetworkVO, Long> implemen
return null;
}
@Override
public List<NetworkVO> listRedundantNetworks() {
// TODO Auto-generated method stub
return null;
}
}

2
setup/db/db/schema-40to410.sql
View File

@ -52,6 +52,8 @@ CREATE TABLE `cloud`.`template_s3_ref` (
INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.enable', 'false', 'enable s3');
INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'NetworkManager', 'router.check.poolsize' , '10', 'Numbers of threads using to check redundant router status.');
ALTER TABLE `cloud`.`snapshots` ADD COLUMN `s3_id` bigint unsigned COMMENT 'S3 to which this snapshot will be stored';
ALTER TABLE `cloud`.`snapshots` ADD CONSTRAINT `fk_snapshots__s3_id` FOREIGN KEY `fk_snapshots__s3_id` (`s3_id`) REFERENCES `s3` (`id`);