bug 11307: Mark router as to-be-stopped, rather than force stop it.

Force stop the router would release all the resources it used, but router may
still running. Add a column "stop_pending" in the database, and stop it when the
router come back.

Admin would able to choose to force destroy such router, then recover the
network using restartNetwork command with cleanup=false.
This commit is contained in:
Sheng Yang 2011-09-14 02:58:44 -07:00
parent ebd67feae7
commit e330e97f4b
6 changed files with 195 additions and 44 deletions

View File

@ -38,4 +38,6 @@ public interface VirtualRouter extends VirtualMachine {
}
RedundantState getRedundantState();
String getGuestIpAddress();
boolean isStopPending();
void setStopPending(boolean stopPending);
}

View File

@ -64,6 +64,9 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter {
@Enumerated(EnumType.STRING)
private RedundantState redundantState;
@Column(name="stop_pending")
boolean stopPending;
@Column(name="role")
@Enumerated(EnumType.STRING)
private Role role = Role.DHCP_FIREWALL_LB_PASSWD_USERDATA;
@ -80,13 +83,16 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter {
boolean isRedundantRouter,
int priority,
boolean isPriorityBumpUp,
RedundantState redundantState, boolean haEnabled) {
RedundantState redundantState,
boolean haEnabled,
boolean stopPending) {
super(id, serviceOfferingId, name, name, Type.DomainRouter, templateId, hypervisorType, guestOSId, domainId, accountId, haEnabled);
this.networkId = networkId;
this.isRedundantRouter = isRedundantRouter;
this.priority = priority;
this.redundantState = redundantState;
this.isPriorityBumpUp = isPriorityBumpUp;
this.stopPending = stopPending;
}
public DomainRouterVO(long id,
@ -102,13 +108,16 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter {
int priority,
boolean isPriorityBumpUp,
RedundantState redundantState,
boolean haEnabled, VirtualMachine.Type vmType) {
boolean haEnabled,
boolean stopPending,
VirtualMachine.Type vmType) {
super(id, serviceOfferingId, name, name, vmType, templateId, hypervisorType, guestOSId, domainId, accountId, haEnabled);
this.networkId = networkId;
this.isRedundantRouter = isRedundantRouter;
this.priority = priority;
this.redundantState = redundantState;
this.isPriorityBumpUp = isPriorityBumpUp;
this.stopPending = stopPending;
}
public void setPublicIpAddress(String publicIpAddress) {
@ -209,4 +218,13 @@ public class DomainRouterVO extends VMInstanceVO implements VirtualRouter {
this.isPriorityBumpUp = isPriorityBumpUp;
}
@Override
public boolean isStopPending() {
return this.stopPending;
}
@Override
public void setStopPending(boolean stopPending) {
this.stopPending = stopPending;
}
}

View File

@ -504,7 +504,7 @@ public class ElasticLoadBalancerManagerImpl implements
elbVm = new DomainRouterVO(id, _elasticLbVmOffering.getId(), VirtualMachineName.getSystemVmName(id, _instance, _elbVmNamePrefix), template.getId(), template.getHypervisorType(), template.getGuestOSId(),
owner.getDomainId(), owner.getId(), guestNetwork.getId(), false, 0, false, RedundantState.UNKNOWN, _elasticLbVmOffering.getOfferHA(), VirtualMachine.Type.ElasticLoadBalancerVm);
owner.getDomainId(), owner.getId(), guestNetwork.getId(), false, 0, false, RedundantState.UNKNOWN, _elasticLbVmOffering.getOfferHA(), false, VirtualMachine.Type.ElasticLoadBalancerVm);
elbVm.setRole(Role.LB);
elbVm = _itMgr.allocate(elbVm, template, _elasticLbVmOffering, networks, plan, null, owner);
//TODO: create usage stats

View File

@ -35,14 +35,20 @@ import javax.naming.ConfigurationException;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
import com.cloud.agent.Listener;
import com.cloud.agent.AgentManager.OnError;
import com.cloud.agent.api.AgentControlAnswer;
import com.cloud.agent.api.AgentControlCommand;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.BumpUpPriorityCommand;
import com.cloud.agent.api.CheckRouterAnswer;
import com.cloud.agent.api.CheckRouterCommand;
import com.cloud.agent.api.Command;
import com.cloud.agent.api.ModifySshKeysCommand;
import com.cloud.agent.api.NetworkUsageAnswer;
import com.cloud.agent.api.NetworkUsageCommand;
import com.cloud.agent.api.RebootAnswer;
import com.cloud.agent.api.StartupCommand;
import com.cloud.agent.api.StopAnswer;
import com.cloud.agent.api.check.CheckSshAnswer;
import com.cloud.agent.api.check.CheckSshCommand;
@ -93,6 +99,7 @@ import com.cloud.event.EventTypes;
import com.cloud.event.dao.EventDao;
import com.cloud.exception.AgentUnavailableException;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.ConnectionException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.InsufficientServerCapacityException;
import com.cloud.exception.InsufficientVirtualNetworkCapcityException;
@ -205,7 +212,7 @@ import com.cloud.vm.dao.VMInstanceDao;
* VirtualNetworkApplianceManagerImpl manages the different types of virtual network appliances available in the Cloud Stack.
*/
@Local(value = { VirtualNetworkApplianceManager.class, VirtualNetworkApplianceService.class })
public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplianceManager, VirtualNetworkApplianceService, VirtualMachineGuru<DomainRouterVO> {
public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplianceManager, VirtualNetworkApplianceService, VirtualMachineGuru<DomainRouterVO>, Listener {
private static final Logger s_logger = Logger.getLogger(VirtualNetworkApplianceManagerImpl.class);
String _name;
@ -627,6 +634,8 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
trafficSentinelHostname = configs.get("traffic.sentinel.hostname");
_agentMgr.registerForHostEvents(this, true, false, false);
s_logger.info("DomainRouterManager is configured.");
return true;
@ -786,12 +795,13 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
} else {
String privateIP = router.getPrivateIpAddress();
HostVO host = _hostDao.findById(router.getHostId());
/* Only cover hosts managed by this management server */
if (host == null || host.getStatus() != Status.Up ||
host.getManagementServerId() != ManagementServerNode.getManagementServerId()) {
if (host == null || host.getStatus() != Status.Up) {
router.setRedundantState(RedundantState.UNKNOWN);
updated = true;
} else if (host.getManagementServerId() != ManagementServerNode.getManagementServerId()) {
/* Only cover hosts managed by this management server */
continue;
}
if (privateIP != null) {
} else if (privateIP != null) {
final CheckRouterCommand command = new CheckRouterCommand();
command.setAccessDetail(NetworkElementCommand.ROUTER_IP, router.getPrivateIpAddress());
command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
@ -961,29 +971,27 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
public static boolean isAdmin(short accountType) {
return ((accountType == Account.ACCOUNT_TYPE_ADMIN) || (accountType == Account.ACCOUNT_TYPE_DOMAIN_ADMIN) || (accountType == Account.ACCOUNT_TYPE_READ_ONLY_ADMIN) || (accountType == Account.ACCOUNT_TYPE_RESOURCE_DOMAIN_ADMIN));
}
private int DEFAULT_FIRST_PRIORITY = 100;
private int DEFAULT_PRIORITY = 100;
private int DEFAULT_DELTA = 2;
protected int getPriority(Network guestNetwork, List<DomainRouterVO> routers) throws InsufficientVirtualNetworkCapcityException {
protected int getUpdatedPriority(Network guestNetwork, List<DomainRouterVO> routers, DomainRouterVO exclude) throws InsufficientVirtualNetworkCapcityException {
int priority;
if (routers.size() == 0) {
priority = DEFAULT_FIRST_PRIORITY;
priority = DEFAULT_PRIORITY;
} else {
int maxPriority = 0;
for (DomainRouterVO r : routers) {
int p = 0;
if (!r.getIsRedundantRouter()) {
throw new CloudRuntimeException("Redundant router is mixed with single router in one network!");
}
p = r.getPriority();
if (r.getIsPriorityBumpUp()) {
p += DEFAULT_DELTA;
}
//FIXME Assume the maxPriority one should be running or just created.
if (p > maxPriority) {
maxPriority = p;
if (r.getId() != exclude.getId() && getRealPriority(r) > maxPriority) {
maxPriority = getRealPriority(r);
}
}
if (maxPriority == 0) {
return DEFAULT_PRIORITY;
}
if (maxPriority < 20) {
s_logger.error("Current maximum priority is too low!");
throw new InsufficientVirtualNetworkCapcityException("Current maximum priority is too low as " + maxPriority + "!",
@ -1080,12 +1088,13 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
if (routers.size() >= 5) {
s_logger.error("Too much redundant routers!");
}
// Priority would be recalculated when start up the redundant router
int priority = 0;
if (isRedundant) {
priority = getPriority(guestNetwork, routers);
priority = DEFAULT_PRIORITY;
}
router = new DomainRouterVO(id, _offering.getId(), VirtualMachineName.getRouterName(id, _instance), template.getId(), template.getHypervisorType(), template.getGuestOSId(),
owner.getDomainId(), owner.getId(), guestNetwork.getId(), isRedundant, priority, false, RedundantState.UNKNOWN, _offering.getOfferHA());
owner.getDomainId(), owner.getId(), guestNetwork.getId(), isRedundant, priority, false, RedundantState.UNKNOWN, _offering.getOfferHA(), false);
router = _itMgr.allocate(router, template, _offering, networks, plan, null, owner);
// Creating stats entry for router
UserStatisticsVO stats = _userStatsDao.findBy(owner.getId(), dcId, router.getNetworkId(), null, router.getId(), router.getType().toString());
@ -1270,7 +1279,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
VMTemplateVO template = _templateDao.findRoutingTemplate(dest.getCluster().getHypervisorType());
router = new DomainRouterVO(id, _offering.getId(), VirtualMachineName.getRouterName(id, _instance), template.getId(), template.getHypervisorType(), template.getGuestOSId(),
owner.getDomainId(), owner.getId(), guestNetwork.getId(), false, 0, false, RedundantState.UNKNOWN, _offering.getOfferHA());
owner.getDomainId(), owner.getId(), guestNetwork.getId(), false, 0, false, RedundantState.UNKNOWN, _offering.getOfferHA(), false);
router.setRole(Role.DHCP_USERDATA);
router = _itMgr.allocate(router, template, _offering, networks, plan, null, owner);
routers.add(router);
@ -1374,7 +1383,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
buf.append(" redundant_router=1");
List<DomainRouterVO> routers = _routerDao.listByNetworkAndRole(network.getId(), Role.DHCP_FIREWALL_LB_PASSWD_USERDATA);
try {
int priority = getPriority(network, routers);
int priority = getUpdatedPriority(network, routers, router);
router.setPriority(priority);
} catch (InsufficientVirtualNetworkCapcityException e) {
s_logger.error("Failed to get update priority!", e);
@ -1808,6 +1817,15 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
continue;
}
if (router.isStopPending()) {
if (_hostDao.findById(router.getHostId()).getStatus() == Status.Up) {
throw new ResourceUnavailableException("Unable to process due to the stop pending router " + router.getInstanceName() + " haven't been stopped after it's host coming back!",
VirtualRouter.class, router.getId());
}
s_logger.warn("Unable to add virtual machine " + profile.getVirtualMachine() + " to the router " + router + " as the router is to be stopped");
continue;
}
//for basic zone:
//1) send vm data/password information only to the dhcp in the same pod
//2) send dhcp/dns information to all routers in the cloudstack only when _dnsBasicZoneUpdates is set to "all" value
@ -1922,7 +1940,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
String msg = "Unable to add new VM into network on disconnected router ";
if (!connectedRouters.isEmpty()) {
// These disconnected ones are out of sync now, stop them for synchronization
stopDisconnectedRouters(disconnectedRouters, true, msg);
handleSingleWorkingRedundantRouter(connectedRouters, disconnectedRouters, msg);
} else if (!disconnectedRouters.isEmpty()) {
for (VirtualRouter router : disconnectedRouters) {
if (s_logger.isDebugEnabled()) {
@ -2306,27 +2324,51 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
return true;
}
protected void stopDisconnectedRouters(List<? extends VirtualRouter> routers, boolean force, String reason)
protected void handleSingleWorkingRedundantRouter(List<? extends VirtualRouter> connectedRouters, List<? extends VirtualRouter> disconnectedRouters, String reason) throws ResourceUnavailableException
{
if (routers.isEmpty()) {
if (connectedRouters.isEmpty() || disconnectedRouters.isEmpty()) {
return;
}
for (VirtualRouter router : routers) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("About to stop the router " + router.getInstanceName() + " due to: " + reason);
}
String title = "Virtual router " + router.getInstanceName() + " would be stopped, due to " + reason;
String context = "Virtual router (name: " + router.getInstanceName() + ", id: " + router.getId() + ") would be stopped, due to: " + reason;
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER,
router.getDataCenterIdToDeployIn(), router.getPodIdToDeployIn(), title, context);
if (router.getIsRedundantRouter()) {
try {
stopRouter(router.getId(), force);
} catch (ConcurrentOperationException e) {
s_logger.warn("Fail to stop router " + router.getInstanceName(), e);
} catch (ResourceUnavailableException e) {
s_logger.warn("Fail to stop router " + router.getInstanceName(), e);
if (connectedRouters.size() != 1 || disconnectedRouters.size() != 1) {
s_logger.warn("How many redundant routers do we have?? ");
return;
}
if (!connectedRouters.get(0).getIsRedundantRouter()) {
throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", DataCenter.class, connectedRouters.get(0).getDataCenterIdToDeployIn());
}
if (!disconnectedRouters.get(0).getIsRedundantRouter()) {
throw new ResourceUnavailableException("Who is calling this with non-redundant router or non-domain router?", DataCenter.class, disconnectedRouters.get(0).getDataCenterIdToDeployIn());
}
DomainRouterVO connectedRouter = (DomainRouterVO)connectedRouters.get(0);
DomainRouterVO disconnectedRouter = (DomainRouterVO)disconnectedRouters.get(0);
if (s_logger.isDebugEnabled()) {
s_logger.debug("About to stop the router " + disconnectedRouter.getInstanceName() + " due to: " + reason);
}
String title = "Virtual router " + disconnectedRouter.getInstanceName() + " would be stopped after connecting back, due to " + reason;
String context = "Virtual router (name: " + disconnectedRouter.getInstanceName() + ", id: " + disconnectedRouter.getId() + ") would be stopped after connecting back, due to: " + reason;
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER,
disconnectedRouter.getDataCenterIdToDeployIn(), disconnectedRouter.getPodIdToDeployIn(), title, context);
disconnectedRouter.setStopPending(true);
disconnectedRouter = this.persist((DomainRouterVO)disconnectedRouter);
int connRouterPR = getRealPriority((DomainRouterVO)connectedRouter);
int disconnRouterPR = getRealPriority((DomainRouterVO)disconnectedRouter);
if (connRouterPR < disconnRouterPR) {
//connRouterPR < disconnRouterPR, they won't equal at anytime
if (!connectedRouter.getIsPriorityBumpUp()) {
final BumpUpPriorityCommand command = new BumpUpPriorityCommand();
command.setAccessDetail(NetworkElementCommand.ROUTER_IP, connectedRouter.getPrivateIpAddress());
command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, connectedRouter.getInstanceName());
final Answer answer = _agentMgr.easySend(connectedRouter.getHostId(), command);
if (!answer.getResult()) {
s_logger.error("Failed to bump up " + connectedRouter.getInstanceName() + "'s priority! " + answer.getDetails());
}
} else {
String t = "Can't bump up virtual router " + connectedRouter.getInstanceName() + "'s priority due to it's already bumped up!";
_alertMgr.sendAlert(AlertManager.ALERT_TYPE_DOMAIN_ROUTER,
connectedRouter.getDataCenterIdToDeployIn(), connectedRouter.getPodIdToDeployIn(), t, t);
}
}
}
@ -2344,6 +2386,15 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
String msg = "Unable to associate ip addresses on disconnected router ";
for (VirtualRouter router : routers) {
if (router.getState() == State.Running) {
if (router.isStopPending()) {
if (_hostDao.findById(router.getHostId()).getStatus() == Status.Up) {
throw new ResourceUnavailableException("Unable to process due to the stop pending router " + router.getInstanceName() + " haven't been stopped after it's host coming back!",
VirtualRouter.class, router.getId());
}
s_logger.debug("Router " + router.getInstanceName() + " is stop pending, so not sending apply firewall rules commands to the backend");
continue;
}
Commands cmds = new Commands(OnError.Continue);
// Have to resend all already associated ip addresses
createAssociateIPCommands(router, ipAddress, cmds, 0);
@ -2372,7 +2423,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
if (!connectedRouters.isEmpty()) {
// These disconnected ones are out of sync now, stop them for synchronization
stopDisconnectedRouters(disconnectedRouters, true, msg);
handleSingleWorkingRedundantRouter(connectedRouters, disconnectedRouters, msg);
} else if (!disconnectedRouters.isEmpty()) {
for (VirtualRouter router : disconnectedRouters) {
if (s_logger.isDebugEnabled()) {
@ -2397,6 +2448,15 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
boolean result = true;
for (VirtualRouter router : routers) {
if (router.getState() == State.Running) {
if (router.isStopPending()) {
if (_hostDao.findById(router.getHostId()).getStatus() == Status.Up) {
throw new ResourceUnavailableException("Unable to process due to the stop pending router " + router.getInstanceName() + " haven't been stopped after it's host coming back!",
VirtualRouter.class, router.getId());
}
s_logger.debug("Router " + router.getInstanceName() + " is stop pending, so not sending apply firewall rules commands to the backend");
continue;
}
if (rules != null && !rules.isEmpty()) {
try {
if (rules.get(0).getPurpose() == Purpose.LoadBalancing) {
@ -2441,7 +2501,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
if (!connectedRouters.isEmpty()) {
// These disconnected ones are out of sync now, stop them for synchronization
stopDisconnectedRouters(disconnectedRouters, true, msg);
handleSingleWorkingRedundantRouter(connectedRouters, disconnectedRouters, msg);
} else if (!disconnectedRouters.isEmpty()) {
for (VirtualRouter router : disconnectedRouters) {
if (s_logger.isDebugEnabled()) {
@ -2533,6 +2593,15 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
for (VirtualRouter router : routers) {
if (router.getState() == State.Running) {
s_logger.debug("Applying " + rules.size() + " static nat in network " + network);
if (router.isStopPending()) {
if (_hostDao.findById(router.getHostId()).getStatus() == Status.Up) {
throw new ResourceUnavailableException("Unable to process due to the stop pending router " + router.getInstanceName() + " haven't been stopped after it's host coming back!",
VirtualRouter.class, router.getId());
}
s_logger.debug("Router " + router.getInstanceName() + " is stop pending, so not sending apply firewall rules commands to the backend");
continue;
}
try {
result = applyStaticNat(router, rules);
connectedRouters.add(router);
@ -2556,7 +2625,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
if (!connectedRouters.isEmpty()) {
// These disconnected ones are out of sync now, stop them for synchronization
stopDisconnectedRouters(disconnectedRouters, true, msg);
handleSingleWorkingRedundantRouter(connectedRouters, disconnectedRouters, msg);
} else if (!disconnectedRouters.isEmpty()) {
for (VirtualRouter router : disconnectedRouters) {
if (s_logger.isDebugEnabled()) {
@ -2596,4 +2665,64 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString());
cmds.addCommand(cmd);
}
@Override
public int getTimeout() {
return -1;
}
@Override
public boolean isRecurring() {
return false;
}
@Override
public boolean processAnswers(long agentId, long seq, Answer[] answers) {
return false;
}
@Override
public boolean processCommands(long agentId, long seq, Command[] commands) {
return false;
}
@Override
public void processConnect(HostVO host, StartupCommand cmd, boolean forRebalance) throws ConnectionException {
UserContext context = UserContext.current();
context.setAccountId(1);
List<DomainRouterVO> routers = _routerDao.listVirtualByHostId(host.getId());
for (DomainRouterVO router : routers) {
if (router.isStopPending()) {
State state = router.getState();
if (state != State.Stopped && state != State.Destroyed) {
try {
stopRouter(router.getId(), false);
} catch (ResourceUnavailableException e) {
s_logger.warn("Fail to stop router " + router.getInstanceName(), e);
throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName());
} catch (ConcurrentOperationException e) {
s_logger.warn("Fail to stop router " + router.getInstanceName(), e);
throw new ConnectionException(false, "Fail to stop router " + router.getInstanceName());
}
}
router.setStopPending(false);
router = _routerDao.persist(router);
}
}
}
@Override
public AgentControlAnswer processControlCommand(long agentId, AgentControlCommand cmd) {
return null;
}
@Override
public boolean processDisconnect(long agentId, Status state) {
return false;
}
@Override
public boolean processTimeout(long agentId, long seq) {
return false;
}
}

View File

@ -942,6 +942,7 @@ CREATE TABLE `cloud`.`domain_router` (
`priority` int(4) unsigned COMMENT 'priority of router in the redundant router mode',
`is_priority_bumpup` int(1) unsigned NOT NULL COMMENT 'if the priority has been bumped up',
`redundant_state` varchar(64) NOT NULL COMMENT 'the state of redundant virtual router',
`stop_pending` int(1) unsigned NOT NULL COMMENT 'if this router would be stopped after we can connect to it',
`role` varchar(64) NOT NULL COMMENT 'type of role played by this router',
PRIMARY KEY (`id`),
CONSTRAINT `fk_domain_router__id` FOREIGN KEY `fk_domain_router__id` (`id`) REFERENCES `vm_instance`(`id`) ON DELETE CASCADE

View File

@ -14,6 +14,7 @@ ALTER TABLE `cloud`.`user_vm_details` ADD CONSTRAINT `fk_user_vm_details__vm_id`
ALTER TABLE `cloud`.`domain_router` ADD COLUMN `is_priority_bumpup` int(1) unsigned NOT NULL DEFAULT 0 COMMENT 'if the priority has been bumped up';
ALTER TABLE `cloud`.`domain_router` ADD COLUMN `stop_pending` int(1) unsigned NOT NULL DEFAULT 0 COMMENT 'if this router would be stopped after we can connect to it';
DELETE FROM `cloud`.`configuration` where name='vmware.guest.nic.device.type';