diff --git a/api/src/com/cloud/agent/api/routing/CreateLBApplianceCommand.java b/api/src/com/cloud/agent/api/routing/CreateLoadBalancerApplianceCommand.java similarity index 50% rename from api/src/com/cloud/agent/api/routing/CreateLBApplianceCommand.java rename to api/src/com/cloud/agent/api/routing/CreateLoadBalancerApplianceCommand.java index 008db121201..e55b2aa0e3e 100644 --- a/api/src/com/cloud/agent/api/routing/CreateLBApplianceCommand.java +++ b/api/src/com/cloud/agent/api/routing/CreateLoadBalancerApplianceCommand.java @@ -18,19 +18,41 @@ package com.cloud.agent.api.routing; -/** NetworkElementCommand to spin a VPX instance on the Netscaler SDX load balancer appliance */ +/** NetworkElementCommand to spin a load balancer appliance */ -//TODO: fill in Nitro API parameters +public class CreateLoadBalancerApplianceCommand extends NetworkElementCommand { -public class CreateLBApplianceCommand extends NetworkElementCommand { - - String lbApplianceIP = null; + String ip; + String netmask; + String gateway; + String username; + String password; - public CreateLBApplianceCommand(String lbIp) { - this.lbApplianceIP = lbIp; + public CreateLoadBalancerApplianceCommand(String ip, String netmask, String gateway, String username, String password) { + this.ip = ip; + this.netmask = netmask; + this.gateway = gateway; + this.username = username; + this.password = password; } - String getLoadBalancerIP() { - return lbApplianceIP; + public String getLoadBalancerIP() { + return ip; } -} + + public String getNetmask() { + return netmask; + } + + public String getGateway() { + return gateway; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } +} \ No newline at end of file diff --git a/api/src/com/cloud/agent/api/routing/DestroyLBApplianceCommand.java b/api/src/com/cloud/agent/api/routing/DestroyLoadBalancerApplianceCommand.java similarity index 79% rename from api/src/com/cloud/agent/api/routing/DestroyLBApplianceCommand.java rename to api/src/com/cloud/agent/api/routing/DestroyLoadBalancerApplianceCommand.java index 1acd9afb674..164d14264f9 100644 --- a/api/src/com/cloud/agent/api/routing/DestroyLBApplianceCommand.java +++ b/api/src/com/cloud/agent/api/routing/DestroyLoadBalancerApplianceCommand.java @@ -22,6 +22,15 @@ package com.cloud.agent.api.routing; -public class DestroyLBApplianceCommand extends NetworkElementCommand { +public class DestroyLoadBalancerApplianceCommand extends NetworkElementCommand { + String ip; + + public DestroyLoadBalancerApplianceCommand(String ip) { + this.ip = ip; + } + + public String getLoadBalancerIP() { + return ip; + } } diff --git a/core/src/com/cloud/network/resource/DestroyLBApplianceAnswer.java b/core/src/com/cloud/network/resource/CreateLoadBalancerApplianceAnswer.java similarity index 53% rename from core/src/com/cloud/network/resource/DestroyLBApplianceAnswer.java rename to core/src/com/cloud/network/resource/CreateLoadBalancerApplianceAnswer.java index d7f4f31af70..66117a3eeb2 100644 --- a/core/src/com/cloud/network/resource/DestroyLBApplianceAnswer.java +++ b/core/src/com/cloud/network/resource/CreateLoadBalancerApplianceAnswer.java @@ -20,9 +20,30 @@ package com.cloud.network.resource; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; +import com.cloud.resource.ServerResource; -public class DestroyLBApplianceAnswer extends Answer { - public DestroyLBApplianceAnswer(Command cmd, boolean success) { - +public class CreateLoadBalancerApplianceAnswer extends Answer { + String deviceName; + String providerName; + ServerResource serverResource; + + public CreateLoadBalancerApplianceAnswer(Command cmd, boolean success, String details, String deviceName, String providerName, ServerResource serverResource) { + this.deviceName = deviceName; + this.providerName = providerName; + this.serverResource = serverResource; + this.result = success; + this.details = details; + } + + public String getDeviceName() { + return deviceName; + } + + public String getProviderName() { + return providerName; + } + + public ServerResource getServerResource() { + return serverResource; } } diff --git a/core/src/com/cloud/network/resource/CreateLBApplianceAnswer.java b/core/src/com/cloud/network/resource/DestroyLoadBalancerApplianceAnswer.java similarity index 80% rename from core/src/com/cloud/network/resource/CreateLBApplianceAnswer.java rename to core/src/com/cloud/network/resource/DestroyLoadBalancerApplianceAnswer.java index b15aa74bdf2..bcd09cef35e 100644 --- a/core/src/com/cloud/network/resource/CreateLBApplianceAnswer.java +++ b/core/src/com/cloud/network/resource/DestroyLoadBalancerApplianceAnswer.java @@ -21,9 +21,9 @@ package com.cloud.network.resource; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; -public class CreateLBApplianceAnswer extends Answer { - public CreateLBApplianceAnswer(Command cmd, boolean success) { - +public class DestroyLoadBalancerApplianceAnswer extends Answer { + public DestroyLoadBalancerApplianceAnswer(Command cmd, boolean success, String details) { + this.result = success; + this.details = details; } - } diff --git a/core/src/com/cloud/network/resource/NetscalerResource.java b/core/src/com/cloud/network/resource/NetscalerResource.java index 1667b949910..496c380aa00 100644 --- a/core/src/com/cloud/network/resource/NetscalerResource.java +++ b/core/src/com/cloud/network/resource/NetscalerResource.java @@ -24,8 +24,8 @@ import javax.naming.ConfigurationException; import com.cloud.agent.IAgentControl; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; -import com.cloud.agent.api.routing.CreateLBApplianceCommand; -import com.cloud.agent.api.routing.DestroyLBApplianceCommand; +import com.cloud.agent.api.routing.CreateLoadBalancerApplianceCommand; +import com.cloud.agent.api.routing.DestroyLoadBalancerApplianceCommand; import com.cloud.agent.api.ExternalNetworkResourceUsageAnswer; import com.cloud.agent.api.ExternalNetworkResourceUsageCommand; import com.cloud.agent.api.MaintainAnswer; @@ -49,6 +49,7 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.exception.ExecutionException; import com.cloud.utils.net.NetUtils; import com.google.gson.Gson; +import com.vmware.vim25.VirtualMachinePowerState; import com.citrix.netscaler.nitro.service.nitro_service; import com.citrix.netscaler.nitro.resource.base.base_response; @@ -59,12 +60,15 @@ import com.citrix.netscaler.nitro.resource.config.network.*; import com.citrix.netscaler.nitro.resource.config.ns.*; import com.citrix.netscaler.nitro.resource.config.basic.server_service_binding; import com.citrix.netscaler.nitro.resource.stat.lb.lbvserver_stats; +import com.citrix.sdx.nitro.resource.config.ns; +import com.citrix.sdx.nitro.resource.config.mps; import org.apache.log4j.Logger; class NitroError { static final int NS_RESOURCE_EXISTS = 273; static final int NS_RESOURCE_NOT_EXISTS=258; static final int NS_NO_SERIVCE = 344; + static final int NS_OPERATION_NOT_PERMITTED = 257; } public class NetscalerResource implements ServerResource { @@ -88,7 +92,12 @@ public class NetscalerResource implements ServerResource { protected Gson _gson; private String _objectNamePathSep = "-"; - nitro_service _netscalerService ; + // interface to interact with VPX and MPX devices + com.citrix.netscaler.nitro.service.nitro_service _netscalerService ; + + // interface to interact with service VM of the SDX appliance + com.citrix.sdx.nitro.service.nitro_service _netscalerSdxService; + Long _timeout = new Long(100000); base_response apiCallResult; @@ -133,12 +142,12 @@ public class NetscalerResource implements ServerResource { if (_publicInterface == null) { throw new ConfigurationException("Unable to find public interface in the configuration parameters"); } - + _privateInterface = (String) params.get("privateinterface"); if (_privateInterface == null) { throw new ConfigurationException("Unable to find private interface in the configuration parameters"); } - + _numRetries = NumbersUtil.parseInt((String) params.get("numretries"), 2); _guid = (String)params.get("guid"); @@ -151,17 +160,17 @@ public class NetscalerResource implements ServerResource { throw new ConfigurationException("Unable to find the device name in the configuration parameters"); } - if (_deviceName.equalsIgnoreCase("NetscalerSDXLoadBalancer")) { - _isSdx = true; - } + _isSdx = _deviceName.equalsIgnoreCase("NetscalerSDXLoadBalancer"); _inline = Boolean.parseBoolean((String) params.get("inline")); - // validate device configration parameters + // validate device configuration parameters login(); - checkLoadBalancingFeatureEnabled(); - validateInterfaces(_publicInterface, _privateInterface); validateDeviceType(_deviceName); + validateInterfaces(_publicInterface, _privateInterface); + + //enable load balancing feature + enableLoadBalancingFeature(); return true; } catch (Exception e) { @@ -171,12 +180,21 @@ public class NetscalerResource implements ServerResource { private void login() throws ExecutionException { try { - _netscalerService = new nitro_service(_ip, "https"); - _netscalerService.set_credential(_username, _password); - _netscalerService.set_timeout(_timeout); - apiCallResult = _netscalerService.login(); - if (apiCallResult.errorcode != 0) { - throw new ExecutionException ("Failed to log in to Netscaler device at " + _ip + " due to error " + apiCallResult.errorcode + " and message " + apiCallResult.message); + if (!_isSdx) { + _netscalerService = new nitro_service(_ip, "https"); + _netscalerService.set_credential(_username, _password); + _netscalerService.set_timeout(_timeout); + apiCallResult = _netscalerService.login(); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException ("Failed to log in to Netscaler device at " + _ip + " due to error " + apiCallResult.errorcode + " and message " + apiCallResult.message); + } + } else { + _netscalerSdxService = new com.citrix.sdx.nitro.service.nitro_service(_ip, "https"); + _netscalerSdxService.set_credential(_username, _password); + com.citrix.sdx.nitro.resource.base.login login = _netscalerSdxService.login(); + if (login == null) { + throw new ExecutionException ("Failed to log in to Netscaler device at " + _ip + " due to error " + apiCallResult.errorcode + " and message " + apiCallResult.message); + } } } catch (nitro_exception e) { throw new ExecutionException("Failed to log in to Netscaler device at " + _ip + " due to " + e.getMessage()); @@ -185,7 +203,10 @@ public class NetscalerResource implements ServerResource { } } - private void checkLoadBalancingFeatureEnabled() throws ExecutionException { + private void enableLoadBalancingFeature() throws ExecutionException { + if (_isSdx) { + return; + } try { String[] features = _netscalerService.get_enabled_features(); if (features != null) { @@ -195,20 +216,73 @@ public class NetscalerResource implements ServerResource { } } } - throw new ExecutionException("Load balancing feature is not enabled on the device. Please enable the load balancing feature and add the device."); + + // enable load balancing on the device + String[] feature = new String[1]; + feature[0] = "LB"; + apiCallResult = _netscalerService.enable_features(feature); + if (apiCallResult.errorcode != 0) { + throw new ExecutionException("Enabling load balancing feature on the device failed."); + } } catch (nitro_exception e) { - throw new ExecutionException("Failed to verify load balancing is enalbed due to error " + apiCallResult.errorcode + " and message " + e.getMessage()); + throw new ExecutionException("Enabling load balancing feature on the device failed due to " + e.getMessage()); } catch (Exception e) { - throw new ExecutionException("Failed to verify load balancing is enalbed due to " + e.getMessage()); + throw new ExecutionException("Enabling load balancing feature on the device failed due to " + e.getMessage()); } } private void validateInterfaces(String publicInterface, String privateInterface) throws ExecutionException { - //FIXME verify the device type (VPX, MPX, SDX) specified indeed matches with actual device type + try { + if (_isSdx) { + return; + } else { + Interface publicIf = Interface.get(_netscalerService, publicInterface); + Interface privateIf = Interface.get(_netscalerService, publicInterface); + if (publicIf != null || privateIf != null) { + return; + } else { + throw new ExecutionException("Invalid interface name specified for public/private interfaces."); + } + } + } catch (nitro_exception e) { + if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { + throw new ExecutionException("Invalid interface name specified for public and private interfaces."); + } else { + throw new ExecutionException("Failed to verify public interface and private intefaces are valid due to " + e.getMessage()); + } + } catch (Exception e) { + throw new ExecutionException("Failed to verify public interface and private intefaces are valid due to " + e.getMessage()); + } } private void validateDeviceType(String deviceType) throws ExecutionException { - //FIXME validate public and private interface strings as well + try { + if (!_isSdx) { + nshardware nsHw = com.citrix.netscaler.nitro.resource.config.ns.nshardware.get(_netscalerService); + if (nsHw == null) { + throw new ExecutionException("Failed to get the hardware description of the Netscaler device at " + _ip); + } else { + if ((_deviceName.equalsIgnoreCase("NetscalerMPXLoadBalancer") && nsHw.get_hwdescription().contains("MPX")) + || (_deviceName.equalsIgnoreCase("NetscalerVPXLoadBalancer") && nsHw.get_hwdescription().contains("NetScaler Virtual Appliance"))) { + return; + } + throw new ExecutionException("Netscalar device type specified does not match with the actuall device type."); + } + } else { + mps serviceVM = mps.get(_netscalerSdxService); + if (serviceVM != null) { + if (serviceVM.get_platform().contains("SDX") || serviceVM.get_product().contains("SDX")) { + return; + } else { + throw new ExecutionException("Netscalar device type specified does not match with the actuall device type."); + } + } else { + throw new ExecutionException("Failed to get the hardware details of the Netscaler device at " + _ip); + } + } + } catch (Exception e) { + throw new ExecutionException("Failed to verify device type specified when matching with actuall device type due to " + e.getMessage()); + } } @Override @@ -228,7 +302,7 @@ public class NetscalerResource implements ServerResource { public Answer executeRequest(Command cmd) { return executeRequest(cmd, _numRetries); } - + private Answer executeRequest(Command cmd, int numRetries) { if (cmd instanceof ReadyCommand) { return execute((ReadyCommand) cmd); @@ -239,11 +313,11 @@ public class NetscalerResource implements ServerResource { } else if (cmd instanceof LoadBalancerConfigCommand) { return execute((LoadBalancerConfigCommand) cmd, numRetries); } else if (cmd instanceof ExternalNetworkResourceUsageCommand) { - return execute((ExternalNetworkResourceUsageCommand) cmd); - } else if (cmd instanceof CreateLBApplianceCommand) { - return execute((CreateLBApplianceCommand) cmd, numRetries); - } else if (cmd instanceof DestroyLBApplianceCommand) { - return execute((DestroyLBApplianceCommand) cmd, numRetries); + return execute((ExternalNetworkResourceUsageCommand) cmd, numRetries); + } else if (cmd instanceof CreateLoadBalancerApplianceCommand) { + return execute((CreateLoadBalancerApplianceCommand) cmd, numRetries); + } else if (cmd instanceof DestroyLoadBalancerApplianceCommand) { + return execute((DestroyLoadBalancerApplianceCommand) cmd, numRetries); } else { return Answer.createUnsupportedCommandAnswer(cmd); } @@ -270,31 +344,34 @@ public class NetscalerResource implements ServerResource { long guestVlanTag = Long.valueOf(ip.getVlanId()); String vlanSelfIp = ip.getVlanGateway(); String vlanNetmask = ip.getVlanNetmask(); - - // Check and delete any existing guest VLAN with this tag, self IP, and netmask - deleteGuestVlan(guestVlanTag, vlanSelfIp, vlanNetmask); - + if (ip.isAdd()) { // Add a new guest VLAN and its subnet and bind it to private interface addGuestVlanAndSubnet(guestVlanTag, vlanSelfIp, vlanNetmask); + } else { + // Check and delete guest VLAN with this tag, self IP, and netmask + deleteGuestVlan(guestVlanTag, vlanSelfIp, vlanNetmask); } - + saveConfiguration(); results[i++] = ip.getPublicIp() + " - success"; + String action = ip.isAdd() ? "associate" : "remove"; + if (s_logger.isDebugEnabled()) { + s_logger.debug("Netscaler load balancer " + _ip + " successfully executed IPAssocCommand to " + action + " IP " + ip); + } } } catch (ExecutionException e) { - s_logger.error("Failed to execute IPAssocCommand due to " + e); - + s_logger.error("Netscaler loadbalancer " + _ip+ " failed to execute IPAssocCommand due to " + e.getMessage()); if (shouldRetry(numRetries)) { return retry(cmd, numRetries); } else { results[i++] = IpAssocAnswer.errorResult; } - } - + } + return new IpAssocAnswer(cmd, results); } - + private synchronized Answer execute(LoadBalancerConfigCommand cmd, int numRetries) { try { if (_isSdx) { @@ -440,7 +517,7 @@ public class NetscalerResource implements ServerResource { } } } - removeLBVirtualServer(nsVirtualServerName); + removeLBVirtualServer(nsVirtualServerName); } } @@ -467,26 +544,122 @@ public class NetscalerResource implements ServerResource { } } - private synchronized Answer execute(CreateLBApplianceCommand cmd, int numRetries) { - assert(_isSdx) : "CreateLBApplianceCommand can only be sent to SDX device"; - // FIXME: use nitro API to spin a new VPX instance on SDX - return new CreateLBApplianceAnswer(cmd, true); - } + private synchronized Answer execute(CreateLoadBalancerApplianceCommand cmd, int numRetries) { - private synchronized Answer execute(DestroyLBApplianceCommand cmd, int numRetries) { - assert(_isSdx) : "DestroyLBApplianceCommand can only be sent to SDX device"; - // FIXME: use nitro API to destroy VPX instance on SDX - return new DestroyLBApplianceAnswer(cmd, true); - } + if (!_isSdx) { + return Answer.createUnsupportedCommandAnswer(cmd); + } - private synchronized ExternalNetworkResourceUsageAnswer execute(ExternalNetworkResourceUsageCommand cmd) { try { - return getPublicIpBytesSentAndReceived(cmd); - } catch (ExecutionException e) { - return new ExternalNetworkResourceUsageAnswer(cmd, e); + String vpxName = "Cloud-VPX-"+cmd.getLoadBalancerIP(); + String ip = cmd.getLoadBalancerIP(); + ns ns_obj = new ns(); + ns_obj.set_name(vpxName); + ns_obj.set_ip_address(cmd.getLoadBalancerIP()); + ns_obj.set_netmask(cmd.getNetmask()); + ns_obj.set_gateway(cmd.getGateway()); + ns_obj.set_username(cmd.getUsername()); + ns_obj.set_password(cmd.getPassword()); + + // configure VPX instances with defaults + ns_obj.set_feature_license("Standard"); + ns_obj.set_memory_total(new Double(2048)); + ns_obj.set_throughput(new Double(1000)); + ns_obj.set_pps(new Double(1000000)); + ns_obj.set_nsroot_profile("NS_nsroot_profile"); + ns_obj.set_number_of_ssl_cores(0); + ns_obj.set_image_name("NSVPX-XEN-9.3-52.4_nc.xva"); + ns_obj.set_if_10_1(new Boolean(true)); + + // create new VPX instance + ns newVpx = ns.add(_netscalerSdxService, ns_obj); + + if (newVpx == null) { + new Answer(cmd, new ExecutionException("Failed to create VPX instance on the netscaler SDX device " + _ip)); + } + + // wait for VPX instance to start-up + long startTick = System.currentTimeMillis(); + long startWaitMins = 200000; + while(!newVpx.get_ns_state().equalsIgnoreCase("up") && System.currentTimeMillis() - startTick < startWaitMins) { + try { + Thread.sleep(1000); + } catch(InterruptedException e) { + } + ns refreshNsObj = new ns(); + refreshNsObj.set_id(newVpx.get_id()); + newVpx = ns.get(_netscalerSdxService, refreshNsObj); + } + + // if vpx instance never came up then error out + if (!newVpx.get_ns_state().equalsIgnoreCase("up")) { + new Answer(cmd, new ExecutionException("Failed to start VPX instance " + vpxName + " created on the netscaler SDX device " + _ip)); + } + if (s_logger.isInfoEnabled()) { + s_logger.info("Successfully provisioned VPX instance " + vpxName + " on the Netscaler SDX device " + _ip); + } + return new CreateLoadBalancerApplianceAnswer(cmd, true, "provisioned VPX instance", "NetscalerVPXLoadBalancer", "Netscaler", new NetscalerResource()); + } catch (Exception e) { + + if (shouldRetry(numRetries)) { + return retry(cmd, numRetries); + } + + return new CreateLoadBalancerApplianceAnswer(cmd, false, "failed to provisioned VPX instance", null, null, null); } } - + + private synchronized Answer execute(DestroyLoadBalancerApplianceCommand cmd, int numRetries) { + String vpxName = "Cloud-VPX-"+cmd.getLoadBalancerIP(); + if (!_isSdx) { + return Answer.createUnsupportedCommandAnswer(cmd); + } + + try { + ns vpxToDelete =null; + ns[] vpxInstances = ns.get(_netscalerSdxService); + for (ns vpx : vpxInstances) { + if (vpx.get_name().equals(vpxName)) { + vpxToDelete = vpx; + break; + } + } + + if (vpxToDelete == null) { + String msg = "There is no VPX instance " + vpxName + " on the Netscaler SDX device " + _ip + " to delete"; + s_logger.warn(msg); + return new DestroyLoadBalancerApplianceAnswer(cmd, true, msg); + } + + // destroy the VPX instance + vpxToDelete = ns.delete(_netscalerSdxService, vpxToDelete); + String msg = "Deleted VPX instance " + vpxName + " on Netscaler SDX " + _ip + " successfully."; + s_logger.info(msg); + return new DestroyLoadBalancerApplianceAnswer(cmd, true,msg); + } catch (Exception e) { + if (shouldRetry(numRetries)) { + return retry(cmd, numRetries); + } + return new DestroyLoadBalancerApplianceAnswer(cmd, false, "Failed to delete VPX instance " + vpxName + " on Netscaler SDX " + _ip); + } + } + + private synchronized Answer execute(ExternalNetworkResourceUsageCommand cmd, int numRetries) { + try { + if (!_isSdx) { + return getPublicIpBytesSentAndReceived(cmd); + } else { + return Answer.createUnsupportedCommandAnswer(cmd); + } + } catch (ExecutionException e) { + if (shouldRetry(numRetries)) { + return retry(cmd, numRetries); + } else { + return new ExternalNetworkResourceUsageAnswer(cmd, e); + } + } + } + private void addGuestVlanAndSubnet(long vlanTag, String vlanSelfIp, String vlanNetmask) throws ExecutionException { try { if (!nsVlanExists(vlanTag)) { @@ -523,7 +696,16 @@ public class NetscalerResource implements ServerResource { vlanBinding.set_ifnum(_privateInterface); vlanBinding.set_tagged(true); vlanBinding.set_id(vlanTag); - apiCallResult = vlan_interface_binding.add(_netscalerService, vlanBinding); + try { + apiCallResult = vlan_interface_binding.add(_netscalerService, vlanBinding); + } catch (nitro_exception e) { + // FIXME: Vlan binding (subsequent unbind) to an interfaces will fail on the VPX created on Xen server and on + // NetScaler SDX appliance till the VPX fix to handle VLAN's is released. Relaxing this restriction NetScaler until then + if (!(_deviceName.equalsIgnoreCase("NetscalerVPXLoadBalancer") && e.getErrorCode() == NitroError.NS_OPERATION_NOT_PERMITTED)) { + throw new ExecutionException("Failed to bind vlan to the interface while implementing guest network on the Netscaler device due to " + e.getMessage()); + } + } + if (apiCallResult.errorcode != 0) { throw new ExecutionException("Failed to bind vlan with tag:" + vlanTag + " with the interface " + _privateInterface + " due to " + apiCallResult.message); } @@ -531,9 +713,9 @@ public class NetscalerResource implements ServerResource { throw new ExecutionException("Failed to configure Netscaler device for vlan with tag " + vlanTag + " as vlan already exisits"); } } catch (nitro_exception e) { - throw new ExecutionException("Failed to implement guest network on the Netscaler device"); + throw new ExecutionException("Failed to implement guest network on the Netscaler device due to " + e.getMessage()); } catch (Exception e) { - throw new ExecutionException("Failed to implement guest network on the Netscaler device"); + throw new ExecutionException("Failed to implement guest network on the Netscaler device " + e.getMessage()); } } @@ -549,7 +731,15 @@ public class NetscalerResource implements ServerResource { vlanIfBinding.set_id(vlanTag); vlanIfBinding.set_ifnum(_privateInterface); vlanIfBinding.set_tagged(true); - apiCallResult = vlan_interface_binding.delete(_netscalerService, vlanIfBinding); + try { + apiCallResult = vlan_interface_binding.delete(_netscalerService, vlanIfBinding); + } catch (nitro_exception e) { + // FIXME: Vlan binding (subsequent unbind) to an interfaces will fail on the VPX created on Xen server and on + // NetScaler SDX appliance till the VPX fix to handle VLAN's is released. Relaxing this restriction NetScaler until then + if (!(_deviceName.equalsIgnoreCase("NetscalerVPXLoadBalancer") && e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS)) { + throw new ExecutionException("Failed to unbind vlan from the interface while shutdown of guest network on the Netscaler device due to " + e.getMessage()); + } + } if (apiCallResult.errorcode != 0) { throw new ExecutionException("Failed to unbind vlan:" + vlanTag + " with the private interface due to " + apiCallResult.message); } @@ -638,13 +828,13 @@ public class NetscalerResource implements ServerResource { if (e.getErrorCode() == NitroError.NS_NO_SERIVCE) { return false; } else { - throw new ExecutionException(e.getMessage()); + throw new ExecutionException("Failed to verify service " + serviceName + " exists due to " + e.getMessage()); } } catch (Exception e) { - throw new ExecutionException(e.getMessage()); + throw new ExecutionException("Failed to verify service " + serviceName + " exists due to " + e.getMessage()); } } - + private boolean nsServiceBindingExists(String lbVirtualServer, String serviceName) throws ExecutionException { try { com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding[] serviceBindings = com.citrix.netscaler.nitro.resource.config.lb.lbvserver_service_binding.get(_netscalerService, lbVirtualServer); @@ -657,12 +847,12 @@ public class NetscalerResource implements ServerResource { } return false; } catch (nitro_exception e) { - throw new ExecutionException(e.getMessage()); + throw new ExecutionException("Failed to verify lb vserver " + lbVirtualServer + "and service " + serviceName + " binding exists due to " + e.getMessage()); } catch (Exception e) { - throw new ExecutionException(e.getMessage()); + throw new ExecutionException("Failed to verify lb vserver " + lbVirtualServer + "and service " + serviceName + " binding exists due to " + e.getMessage()); } } - + private void deleteServersInGuestVlan(long vlanTag, String vlanSelfIp, String vlanNetmask) throws ExecutionException { try { com.citrix.netscaler.nitro.resource.config.basic.server[] serverList = com.citrix.netscaler.nitro.resource.config.basic.server.get(_netscalerService); @@ -716,6 +906,8 @@ public class NetscalerResource implements ServerResource { lbMethod = "ROUNDROBIN"; } else if (lbMethod.equals("leastconn")) { lbMethod = "LEASTCONNECTION"; + } else if (lbMethod.equals("source")) { + lbMethod = "SOURCEIPHASH"; } else { throw new ExecutionException("Got invalid load balancing algorithm: " + lbMethod); } @@ -739,30 +931,28 @@ public class NetscalerResource implements ServerResource { apiCallResult = lbvserver.add(_netscalerService,vserver); } if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to create new virtual server:" + virtualServerName+ " due to " + apiCallResult.message); - } + throw new ExecutionException("Failed to create new load balancing virtual server:" + virtualServerName + " due to " + apiCallResult.message); + } if (s_logger.isDebugEnabled()) { s_logger.debug("Created load balancing virtual server " + virtualServerName + " on the Netscaler device"); } } catch (nitro_exception e) { - if (e.getErrorCode() != NitroError.NS_RESOURCE_EXISTS) { - throw new ExecutionException("Failed to create new virtual server:" + virtualServerName + " due to " + e.getMessage()); - } + throw new ExecutionException("Failed to create new virtual server:" + virtualServerName + " due to " + e.getMessage()); } catch (Exception e) { throw new ExecutionException("Failed to create new virtual server:" + virtualServerName + " due to " + e.getMessage()); } } - + private void removeLBVirtualServer (String virtualServerName) throws ExecutionException { try { lbvserver vserver = lbvserver.get(_netscalerService, virtualServerName); if (vserver == null) { - throw new ExecutionException("Failed to find virtual server with name:" + virtualServerName); + return; } apiCallResult = lbvserver.delete(_netscalerService, vserver); if (apiCallResult.errorcode != 0) { - throw new ExecutionException("Failed to remove virtual server:" + virtualServerName + " due to " + apiCallResult.message); + throw new ExecutionException("Failed to delete virtual server:" + virtualServerName + " due to " + apiCallResult.message); } } catch (nitro_exception e) { if (e.getErrorCode() == NitroError.NS_RESOURCE_NOT_EXISTS) { @@ -774,7 +964,7 @@ public class NetscalerResource implements ServerResource { throw new ExecutionException("Failed to remove virtual server:" + virtualServerName +" due to " + e.getMessage()); } } - + private void saveConfiguration() throws ExecutionException { try { apiCallResult = nsconfig.save(_netscalerService); @@ -790,11 +980,14 @@ public class NetscalerResource implements ServerResource { private ExternalNetworkResourceUsageAnswer getPublicIpBytesSentAndReceived(ExternalNetworkResourceUsageCommand cmd) throws ExecutionException { ExternalNetworkResourceUsageAnswer answer = new ExternalNetworkResourceUsageAnswer(cmd); - + try { - lbvserver_stats[] stats = lbvserver_stats.get(_netscalerService); + if (stats == null || stats.length == 0) { + return answer; + } + for (lbvserver_stats stat_entry : stats) { String lbvserverName = stat_entry.get_name(); lbvserver vserver = lbvserver.get(_netscalerService, lbvserverName); @@ -808,21 +1001,21 @@ public class NetscalerResource implements ServerResource { bytesSentAndReceived[1] += stat_entry.get_totalresponsebytes(); if (bytesSentAndReceived[0] >= 0 && bytesSentAndReceived[1] >= 0) { - answer.ipBytes.put(lbVirtualServerIp, bytesSentAndReceived); + answer.ipBytes.put(lbVirtualServerIp, bytesSentAndReceived); } } } catch (Exception e) { - s_logger.error(e); + s_logger.error("Failed to get bytes sent and recived statistics due to " + e); throw new ExecutionException(e.getMessage()); } - + return answer; } private Answer retry(Command cmd, int numRetries) { int numRetriesRemaining = numRetries - 1; - s_logger.error("Retrying " + cmd.getClass().getSimpleName() + ". Number of retries remaining: " + numRetriesRemaining); - return executeRequest(cmd, numRetriesRemaining); + s_logger.warn("Retrying " + cmd.getClass().getSimpleName() + ". Number of retries remaining: " + numRetriesRemaining); + return executeRequest(cmd, numRetriesRemaining); } private boolean shouldRetry(int numRetries) { @@ -838,17 +1031,17 @@ public class NetscalerResource implements ServerResource { } private String generateNSVirtualServerName(String srcIp, long srcPort, String protocol) { - return genObjectName("cloud-VirtualServer", protocol, srcIp, srcPort); + return genObjectName("Cloud-VirtualServer", protocol, srcIp, srcPort); } - + private String generateNSServerName(String serverIP) { - return genObjectName("cloud-server", serverIP); + return genObjectName("Cloud-Server-", serverIP); } private String generateNSServiceName(String ip, long port) { - return genObjectName("cloud-Service", ip, port); + return genObjectName("Cloud-Service", ip, port); } - + private String genObjectName(Object... args) { String objectName = ""; for (int i = 0; i < args.length; i++) { @@ -859,7 +1052,7 @@ public class NetscalerResource implements ServerResource { } return objectName; } - + @Override public IAgentControl getAgentControl() { return null; @@ -894,9 +1087,9 @@ public class NetscalerResource implements ServerResource { public boolean stop() { return true; } - + @Override public void disconnected() { return; } -} +} \ No newline at end of file diff --git a/deps/cloud-netscaler-sdx.jar b/deps/cloud-netscaler-sdx.jar new file mode 100755 index 00000000000..25a7496d2f4 Binary files /dev/null and b/deps/cloud-netscaler-sdx.jar differ diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index 91eb0dbae8f..2ef081c8d21 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -3138,7 +3138,7 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura tags = cleanupTags(tags); Map lbServiceCapabilityMap = serviceCapabilityMap.get(Service.Lb); - boolean dedicatedLb = true; + boolean dedicatedLb = false; if ((lbServiceCapabilityMap != null) && (!lbServiceCapabilityMap.isEmpty())) { String isolationCapability = lbServiceCapabilityMap.get(Capability.SupportedLBIsolation); dedicatedLb = isolationCapability.contains("dedicated"); diff --git a/server/src/com/cloud/network/ExternalLoadBalancerDeviceManager.java b/server/src/com/cloud/network/ExternalLoadBalancerDeviceManager.java index 9ce5d38aa56..02e9a5ea7c6 100644 --- a/server/src/com/cloud/network/ExternalLoadBalancerDeviceManager.java +++ b/server/src/com/cloud/network/ExternalLoadBalancerDeviceManager.java @@ -27,6 +27,7 @@ import com.cloud.host.Host; import com.cloud.network.rules.FirewallRule; import com.cloud.resource.ServerResource; import com.cloud.utils.component.Manager; +import com.cloud.utils.exception.ExecutionException; /* ExternalLoadBalancerDeviceManager provides a abstract implementation for managing a external load balancer in device agnostic manner. * Device specific managers for external load balancers (like F5 and Netscaler) should be implemented as pluggable service extending @@ -98,6 +99,7 @@ public interface ExternalLoadBalancerDeviceManager extends Manager{ * @throws ResourceUnavailableException * @throws InsufficientCapacityException */ - public boolean manageGuestNetworkWithExternalLoadBalancer(boolean add, Network guestConfig) throws ResourceUnavailableException, InsufficientCapacityException; + public boolean manageGuestNetworkWithExternalLoadBalancer(boolean add, Network guestConfig) throws ResourceUnavailableException, + InsufficientCapacityException; } diff --git a/server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java b/server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java index 52a1882312f..9c8cebc8b47 100644 --- a/server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java +++ b/server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java @@ -38,7 +38,8 @@ import com.cloud.agent.api.ExternalNetworkResourceUsageAnswer; import com.cloud.agent.api.ExternalNetworkResourceUsageCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupExternalLoadBalancerCommand; -import com.cloud.agent.api.routing.CreateLBApplianceCommand; +import com.cloud.agent.api.routing.CreateLoadBalancerApplianceCommand; +import com.cloud.agent.api.routing.DestroyLoadBalancerApplianceCommand; import com.cloud.agent.api.routing.IpAssocCommand; import com.cloud.agent.api.routing.LoadBalancerConfigCommand; import com.cloud.agent.api.routing.NetworkElementCommand; @@ -67,14 +68,15 @@ import com.cloud.host.dao.HostDetailsDao; import com.cloud.network.ExternalLoadBalancerDeviceVO.LBDeviceAllocationState; import com.cloud.network.ExternalLoadBalancerDeviceVO.LBDeviceState; import com.cloud.network.ExternalNetworkDeviceManager.NetworkDevice; -import com.cloud.network.Network.Capability; import com.cloud.network.Network.Service; import com.cloud.network.Networks.TrafficType; +import com.cloud.network.dao.ExternalFirewallDeviceDao; import com.cloud.network.dao.ExternalLoadBalancerDeviceDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.InlineLoadBalancerNicMapDao; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkExternalFirewallDao; import com.cloud.network.dao.NetworkExternalLoadBalancerDao; import com.cloud.network.dao.NetworkServiceMapDao; import com.cloud.network.dao.PhysicalNetworkDao; @@ -82,9 +84,11 @@ import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; import com.cloud.network.dao.PhysicalNetworkServiceProviderVO; import com.cloud.network.lb.LoadBalancingRule; import com.cloud.network.lb.LoadBalancingRule.LbDestination; +import com.cloud.network.resource.CreateLoadBalancerApplianceAnswer; +import com.cloud.network.resource.DestroyLoadBalancerApplianceAnswer; +import com.cloud.network.router.VirtualRouter.Role; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRuleVO; -import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.StaticNatRule; import com.cloud.network.rules.StaticNatRuleImpl; import com.cloud.network.rules.FirewallRule.Purpose; @@ -174,6 +178,10 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase NetworkExternalLoadBalancerDao _networkLBDao; @Inject NetworkServiceMapDao _ntwkSrvcProviderDao; + @Inject + NetworkExternalFirewallDao _networkExternalFirewallDao; + @Inject + ExternalFirewallDeviceDao _externalFirewallDeviceDao; ScheduledExecutorService _executor; private int _externalNetworkStatsInterval; @@ -337,6 +345,150 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase return physicalNetworkId + "-" + deviceName + "-" + ip; } + @Override + public ExternalLoadBalancerDeviceVO getExternalLoadBalancerForNetwork(Network network) { + NetworkExternalLoadBalancerVO lbDeviceForNetwork = _networkExternalLBDao.findByNetworkId(network.getId()); + if (lbDeviceForNetwork != null) { + long lbDeviceId = lbDeviceForNetwork.getExternalLBDeviceId(); + ExternalLoadBalancerDeviceVO lbDeviceVo = _externalLoadBalancerDeviceDao.findById(lbDeviceId); + assert(lbDeviceVo != null); + return lbDeviceVo; + } + return null; + } + + public void setExternalLoadBalancerForNetwork(Network network, long externalLBDeviceID) { + NetworkExternalLoadBalancerVO lbDeviceForNetwork = new NetworkExternalLoadBalancerVO(network.getId(), externalLBDeviceID); + _networkExternalLBDao.persist(lbDeviceForNetwork); + } + + @DB + protected ExternalLoadBalancerDeviceVO allocateLoadBalancerForNetwork(Network guestConfig) throws InsufficientCapacityException { + boolean retry = true; + boolean tryLbProvisioning = false; + ExternalLoadBalancerDeviceVO lbDevice = null; + long physicalNetworkId = guestConfig.getPhysicalNetworkId(); + NetworkOfferingVO offering = _networkOfferingDao.findById(guestConfig.getNetworkOfferingId()); + String provider = _ntwkSrvcProviderDao.getProviderForServiceInNetwork(guestConfig.getId(), Service.Lb); + + while (retry) { + GlobalLock deviceMapLock = GlobalLock.getInternLock("LoadBalancerAllocLock"); + Transaction txn = Transaction.currentTxn(); + try { + if (deviceMapLock.lock(120)) { + try { + boolean dedicatedLB = offering.getDedicatedLB(); // does network offering supports a dedicated load balancer? + long lbDeviceId; + + txn.start(); + try { + // FIXME: should the device allocation be done during network implement phase or do a + // lazy allocation when first rule for the network is configured?? + + // find a load balancer device for this network as per the network offering + lbDevice = findSuitableLoadBalancerForNetwork(guestConfig, dedicatedLB); + lbDeviceId = lbDevice.getId(); + + // persist the load balancer device id that will be used for this network. Once a network + // is implemented on a LB device then later on all rules will be programmed on to same device + NetworkExternalLoadBalancerVO networkLB = new NetworkExternalLoadBalancerVO(guestConfig.getId(), lbDeviceId); + _networkExternalLBDao.persist(networkLB); + + // mark device to be either dedicated or shared use + lbDevice.setAllocationState(dedicatedLB ? LBDeviceAllocationState.Dedicated : LBDeviceAllocationState.Shared); + _externalLoadBalancerDeviceDao.update(lbDeviceId, lbDevice); + + txn.commit(); + + // allocated load balancer for the network, so skip retry + tryLbProvisioning = false; + retry = false; + } catch (InsufficientCapacityException exception) { + // if already attempted to provision load balancer then throw out of capacity exception, + if (tryLbProvisioning) { + retry = false; + //TODO: throwing warning instead of error for now as its possible another provider can service this network + s_logger.warn("There are no load balancer device with the capacity for implementing this network"); + throw exception; + } else { + tryLbProvisioning = true; // if possible provision a LB appliance in to the physical network + } + } + } finally { + deviceMapLock.unlock(); + if (lbDevice == null) { + txn.rollback(); + } + } + } + } finally { + deviceMapLock.releaseRef(); + } + + // there are no LB devices or there is no free capacity on the devices in the physical network so provision a new LB appliance + if (tryLbProvisioning) { + // check if LB appliance can be dynamically provisioned + List providerLbDevices = _externalLoadBalancerDeviceDao.listByProviderAndDeviceAllocationState(physicalNetworkId, provider, LBDeviceAllocationState.Provider); + if ((providerLbDevices != null) && (!providerLbDevices.isEmpty())) { + for (ExternalLoadBalancerDeviceVO lbProviderDevice : providerLbDevices) { + if (lbProviderDevice.getState() == LBDeviceState.Enabled) { + + // FIXME: acquire a private IP from Pod network Guru needed for LB appliance to be provisioned + // and a public IP from the public network guru, for self-if on the public subnet + String lbIP = null; + String netmask = null; + String gateway = null; + String username = null; + String password = null; + String publicIf = null; + String privateIf = null; + + /* + String lbIP = "10.223.103.254"; + String netmask = "255.255.248.0"; + String gateway = "10.223.102.1"; + String username = "nsroot1"; + String password = "nsroot1"; + String publicIf = "0/1"; + String privateIf = "0/2"; + */ + + // send CreateLoadBalancerApplianceCommand to the host capable of provisioning + CreateLoadBalancerApplianceCommand lbProvisionCmd = new CreateLoadBalancerApplianceCommand(lbIP, netmask, gateway, username, password); + CreateLoadBalancerApplianceAnswer answer = null; + try { + answer = (CreateLoadBalancerApplianceAnswer) _agentMgr.easySend(lbProviderDevice.getHostId(), lbProvisionCmd); + if (answer == null || !answer.getResult()) { + s_logger.error("Could not provision load balancer instance on the load balancer device " + lbProviderDevice.getId()); + continue; + } + } catch (Exception agentException) { + s_logger.error("Could not provision load balancer instance on the load balancer device " + lbProviderDevice.getId() + " due to " + agentException.getMessage()); + continue; + } + + //we have provisioned load balancer so add the appliance as cloudstack provisioned external load balancer + String dedicatedLb = offering.getDedicatedLB()?"true":"false"; + String url = "https://" + lbIP + "?publicinterface="+publicIf+"&privateinterface="+privateIf+"&lbdevicededicated="+dedicatedLb;; + ExternalLoadBalancerDeviceVO lbAppliance = addExternalLoadBalancer(physicalNetworkId, url, "nsroot", "nsroot", answer.getDeviceName(), answer.getServerResource()); + + if (lbAppliance != null) { + // mark the load balancer as cloudstack managed + ExternalLoadBalancerDeviceVO managedLb = _externalLoadBalancerDeviceDao.findById(lbAppliance.getId()); + managedLb.setIsManagedDevice(true); + // set parent host id on which lb appliance is provisioned + managedLb.setParentHostId(lbProviderDevice.getHostId()); + _externalLoadBalancerDeviceDao.update(lbAppliance.getId(), managedLb); + } + } + } + } + } + } + + return lbDevice; + } + @Override public ExternalLoadBalancerDeviceVO findSuitableLoadBalancerForNetwork(Network network, boolean dedicatedLb) throws InsufficientCapacityException { long physicalNetworkId = network.getPhysicalNetworkId(); @@ -411,38 +563,91 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase } } - // there are no devices which capcity + // there are no devices which capacity throw new InsufficientNetworkCapacityException("Unable to find a load balancing provider with sufficient capcity " + " to implement the network", Network.class, network.getId()); } + @DB + protected boolean freeLoadBalancerForNetwork(Network guestConfig) { + Transaction txn = Transaction.currentTxn(); + GlobalLock deviceMapLock = GlobalLock.getInternLock("LoadBalancerAllocLock"); + + try { + if (deviceMapLock.lock(120)) { + txn.start(); + // since network is shutdown remove the network mapping to the load balancer device + NetworkExternalLoadBalancerVO networkLBDevice = _networkExternalLBDao.findByNetworkId(guestConfig.getId()); + long lbDeviceId = networkLBDevice.getExternalLBDeviceId(); + _networkExternalLBDao.remove(networkLBDevice.getId()); + + List ntwksMapped = _networkExternalLBDao.listByLoadBalancerDeviceId(networkLBDevice.getExternalLBDeviceId()); + ExternalLoadBalancerDeviceVO lbDevice = _externalLoadBalancerDeviceDao.findById(lbDeviceId); + boolean lbInUse = !(ntwksMapped == null || ntwksMapped.isEmpty()); + boolean lbCloudManaged = lbDevice.getIsManagedDevice(); + + if (!lbInUse && !lbCloudManaged) { + // this is the last network mapped to the load balancer device so set device allocation state to be free + lbDevice.setAllocationState(LBDeviceAllocationState.Free); + _externalLoadBalancerDeviceDao.update(lbDevice.getId(), lbDevice); + } + + //commit the changes before sending agent command to destroy cloudstack managed LB + txn.commit(); + + if (!lbInUse && lbCloudManaged) { + // send DestroyLoadBalancerApplianceCommand to the host where load balancer appliance + // is provisioned as this is the last network using it + Host lbHost = _hostDao.findById(lbDevice.getHostId()); + String lbIP = lbHost.getPrivateIpAddress(); + DestroyLoadBalancerApplianceCommand lbDeleteCmd = new DestroyLoadBalancerApplianceCommand(lbIP); + DestroyLoadBalancerApplianceAnswer answer = null; + try { + answer = (DestroyLoadBalancerApplianceAnswer) _agentMgr.easySend(lbDevice.getParentHostId(), lbDeleteCmd); + if (answer == null || !answer.getResult()) { + s_logger.warn("Failed to destoy load balancer appliance used by the network" + guestConfig.getId() + " due to " + answer.getDetails()); + } + } catch (Exception e) { + s_logger.warn("Failed to destroy load balancer appliance used by the network" + guestConfig.getId() + " due to " + e.getMessage()); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Successfully destroyed load balancer appliance used for the network" + guestConfig.getId()); + } + } + deviceMapLock.unlock(); + return true; + } else { + s_logger.error("Failed to release load balancer device for the network" + guestConfig.getId() + "as failed to acquire lock "); + return false; + } + } catch (Exception exception) { + txn.rollback(); + s_logger.error("Failed to release load balancer device for the network" + guestConfig.getId() + " due to " + exception.getMessage()); + }finally { + deviceMapLock.releaseRef(); + } + + return false; + } + HostVO getFirewallProviderForNetwork(Network network) { + HostVO fwHost = null; + + // get the firewall provider (could be either virtual router or external firewall device) for the network String fwProvider = _ntwkSrvcProviderDao.getProviderForServiceInNetwork(network.getId(), Service.Firewall); if (fwProvider.equalsIgnoreCase("VirtualRouter")) { - //FIXME: get the host Id of the host on which the virtual router is running + //FIXME: use network service provider container framework support to implement on virtual router } else { - //FIXME: external firewall host object + NetworkExternalFirewallVO fwDeviceForNetwork = _networkExternalFirewallDao.findByNetworkId(network.getId()); + assert (fwDeviceForNetwork != null) : "Why firewall provider is not ready for the network to apply static nat rules?"; + long fwDeviceId = fwDeviceForNetwork.getExternalFirewallDeviceId(); + ExternalFirewallDeviceVO fwDevice = _externalFirewallDeviceDao.findById(fwDeviceId); + fwHost = _hostDao.findById(fwDevice.getHostId()); } - return null; - } - - @Override - public ExternalLoadBalancerDeviceVO getExternalLoadBalancerForNetwork(Network network) { - NetworkExternalLoadBalancerVO lbDeviceForNetwork = _networkExternalLBDao.findByNetworkId(network.getId()); - if (lbDeviceForNetwork != null) { - long lbDeviceId = lbDeviceForNetwork.getExternalLBDeviceId(); - ExternalLoadBalancerDeviceVO lbDeviceVo = _externalLoadBalancerDeviceDao.findById(lbDeviceId); - assert(lbDeviceVo != null); - return lbDeviceVo; - } - return null; - } - - public void setExternalLoadBalancerForNetwork(Network network, long externalLBDeviceID) { - NetworkExternalLoadBalancerVO lbDeviceForNetwork = new NetworkExternalLoadBalancerVO(network.getId(), externalLBDeviceID); - _networkExternalLBDao.persist(lbDeviceForNetwork); + return fwHost; } private boolean externalLoadBalancerIsInline(HostVO externalLoadBalancer) { @@ -458,6 +663,18 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase return _nicDao.persist(nic); } + private NicVO getPlaceholderNic(Network network) { + List guestIps = _nicDao.listByNetworkId(network.getId()); + for (NicVO guestIp : guestIps) { + // only external firewall and external load balancer will create NicVO with PlaceHolder reservation strategy + if (guestIp.getReservationStrategy().equals(ReservationStrategy.PlaceHolder) && guestIp.getVmType() == null + && guestIp.getReserver() == null && !guestIp.getIp4Address().equals(network.getGateway())) { + return guestIp; + } + } + return null; + } + private void applyStaticNatRuleForInlineLBRule(DataCenterVO zone, Network network, HostVO firewallHost, boolean revoked, String publicIp, String privateIp) throws ResourceUnavailableException { List staticNatRules = new ArrayList(); IPAddressVO ipVO = _ipAddressDao.listByDcIdIpAddress(zone.getId(), publicIp).get(0); @@ -491,8 +708,24 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase long zoneId = network.getDataCenterId(); DataCenterVO zone = _dcDao.findById(zoneId); + List loadBalancingRules = new ArrayList(); + + for (FirewallRule rule : rules) { + if (rule.getPurpose().equals(Purpose.LoadBalancing)) { + loadBalancingRules.add((LoadBalancingRule) rule); + } + } + + if (loadBalancingRules == null || loadBalancingRules.isEmpty()) { + return true; + } + ExternalLoadBalancerDeviceVO lbDeviceVO = getExternalLoadBalancerForNetwork(network); - assert(lbDeviceVO != null) : "There is no device assigned to this network how apply rules ended up here??"; + if (lbDeviceVO == null) { + s_logger.warn("There is no external load balancer device assigned to this network either network is not implement are already shutdown so just returning"); + return true; + } + HostVO externalLoadBalancer = _hostDao.findById(lbDeviceVO.getHostId()); boolean externalLoadBalancerIsInline = externalLoadBalancerIsInline(externalLoadBalancer); @@ -502,14 +735,6 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase return true; } - List loadBalancingRules = new ArrayList(); - - for (FirewallRule rule : rules) { - if (rule.getPurpose().equals(Purpose.LoadBalancing)) { - loadBalancingRules.add((LoadBalancingRule) rule); - } - } - List loadBalancersToApply = new ArrayList(); for (int i = 0; i < loadBalancingRules.size(); i++) { LoadBalancingRule rule = loadBalancingRules.get(i); @@ -541,7 +766,7 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase s_logger.error(msg); throw new ResourceUnavailableException(msg, DataCenter.class, network.getDataCenterId()); } - + // If a NIC doesn't exist for the load balancing IP address, create one loadBalancingIpNic = _nicDao.findByIp4Address(loadBalancingIpAddress); if (loadBalancingIpNic == null) { @@ -551,7 +776,7 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase // Save a mapping between the source IP address and the load balancing IP address NIC mapping = new InlineLoadBalancerNicMapVO(rule.getId(), srcIp, loadBalancingIpNic.getId()); _inlineLoadBalancerNicMapDao.persist(mapping); - + // On the firewall provider for the network, create a static NAT rule between the source IP address and the load balancing IP address applyStaticNatRuleForInlineLBRule(zone, network, firewallProviderHost, revoked, srcIp, loadBalancingIpNic.getIp4Address()); } else { @@ -561,13 +786,13 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase if (mapping != null) { // Find the NIC that the mapping refers to loadBalancingIpNic = _nicDao.findById(mapping.getNicId()); - + // On the firewall provider for the network, delete the static NAT rule between the source IP address and the load balancing IP address applyStaticNatRuleForInlineLBRule(zone, network, firewallProviderHost, revoked, srcIp, loadBalancingIpNic.getIp4Address()); - + // Delete the mapping between the source IP address and the load balancing IP address _inlineLoadBalancerNicMapDao.expunge(mapping.getId()); - + // Delete the NIC _nicDao.expunge(loadBalancingIpNic.getId()); } else { @@ -575,11 +800,11 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase continue; } } - + // Change the source IP address for the load balancing rule to be the load balancing IP address srcIp = loadBalancingIpNic.getIp4Address(); } - + if (destinations != null && !destinations.isEmpty()) { LoadBalancerTO loadBalancer = new LoadBalancerTO(srcIp, srcPort, protocol, algorithm, revoked, false, destinations); loadBalancersToApply.add(loadBalancer); @@ -605,118 +830,27 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase } @Override - @DB public boolean manageGuestNetworkWithExternalLoadBalancer(boolean add, Network guestConfig) throws ResourceUnavailableException, InsufficientCapacityException { if (guestConfig.getTrafficType() != TrafficType.Guest) { - s_logger.trace("External load balancer can only be user for guest networks."); + s_logger.trace("External load balancer can only be used for guest networks."); return false; } long zoneId = guestConfig.getDataCenterId(); DataCenterVO zone = _dcDao.findById(zoneId); HostVO externalLoadBalancer = null; - String provider = _ntwkSrvcProviderDao.getProviderForServiceInNetwork(guestConfig.getId(), Service.Lb); - long physicalNetworkId = guestConfig.getPhysicalNetworkId(); if (add) { - boolean retry = true; - boolean provisionLB = false; - - while (retry) { - GlobalLock deviceMapLock = GlobalLock.getInternLock("LoadBalancerAllocLock"); - Transaction txn = Transaction.currentTxn(); - try { - if (deviceMapLock.lock(120)) { - try { - NetworkOfferingVO offering = _networkOfferingDao.findById(guestConfig.getNetworkOfferingId()); - long lbDeviceId; - txn.start(); - - // FIXME: should the device allocation be done during network implement phase or do a lazy allocation - // when first rule for the network is configured?? - - boolean dedicatedLB = offering.getDedicatedLB(); - try { - // find a load balancer device for this network as per the network offering - ExternalLoadBalancerDeviceVO lbDevice = findSuitableLoadBalancerForNetwork(guestConfig, dedicatedLB); - lbDeviceId = lbDevice.getId(); - - // persist the load balancer device id that will be used for this network. Once a network - // is implemented on a LB device then later on all rules will be programmed on to same device - NetworkExternalLoadBalancerVO networkLB = new NetworkExternalLoadBalancerVO(guestConfig.getId(), lbDeviceId); - _networkExternalLBDao.persist(networkLB); - - // mark device to be in use - lbDevice.setAllocationState(dedicatedLB ? LBDeviceAllocationState.Dedicated : LBDeviceAllocationState.Shared); - _externalLoadBalancerDeviceDao.update(lbDeviceId, lbDevice); - - // return the HostVO for the lb device - externalLoadBalancer = _hostDao.findById(lbDevice.getHostId()); - txn.commit(); - - // got the load balancer for the network, so skip retry - provisionLB = false; - retry = false; - } catch (InsufficientCapacityException exception) { - if (provisionLB) { - retry = false; - //TODO: throwing warning instead of error for now as its possible another provider can service this network - s_logger.warn("There are no load balancer device with the capacity for implementing this network"); - throw exception; // if already attempted once throw out of capacity exception, - } - provisionLB = true; // if possible provision a LB appliance in the physical network - } - } finally { - deviceMapLock.unlock(); - if (externalLoadBalancer == null) { - txn.rollback(); - } - } - } - } finally { - deviceMapLock.releaseRef(); - } - - // there are no LB devices or there is no free capacity on the devices in the physical network so provision a new LB appliance - if (provisionLB) { - // check if LB appliance can be dynamically provisioned - List providerLbDevices = _externalLoadBalancerDeviceDao.listByProviderAndDeviceAllocationState(physicalNetworkId, provider, LBDeviceAllocationState.Provider); - if ((providerLbDevices != null) && (!providerLbDevices.isEmpty())) { - for (ExternalLoadBalancerDeviceVO lbProviderDevice : providerLbDevices) { - if (lbProviderDevice.getState() == LBDeviceState.Enabled) { - // acquire a private IP needed for LB appliance to be provisioned - // TODO: get the ip from the pool of private IP's configured for physical network - String lbIP = null; - - //TODO: get the configuration details needed to provision LB instances - String username = null; - String password = null; - String publiInterface = null; - String privateInterface = null; - - // send CreateLBApplianceCommand to the host capable of provisioning - CreateLBApplianceCommand lbProvisionCmd = new CreateLBApplianceCommand(lbIP); - Answer answer = _agentMgr.easySend(lbProviderDevice.getHostId(), lbProvisionCmd); - - if (answer == null || !answer.getResult()) { - s_logger.trace("Could not provision load balancer instance on the load balancer device " + lbProviderDevice.getId()); - continue; - } - - //add the appliance as external load balancer - //addExternalLoadBalancer(); - //add the appliance to pool of the load balancers - } - } - } - } - } + ExternalLoadBalancerDeviceVO lbDeviceVO = allocateLoadBalancerForNetwork(guestConfig); + externalLoadBalancer = _hostDao.findById(lbDeviceVO.getHostId()); + s_logger.debug("Allocated external load balancer device:" + lbDeviceVO.getId() + " for the network: " + guestConfig.getId()); } else { // find the load balancer device allocated for the network - ExternalLoadBalancerDeviceVO lbDeviceVO = getExternalLoadBalancerForNetwork(guestConfig); + ExternalLoadBalancerDeviceVO lbDeviceVO = getExternalLoadBalancerForNetwork(guestConfig); if (lbDeviceVO == null) { - s_logger.warn("network shutdwon requested on external load balancer, which did no implement the network. Did the network implement failed half way throug?"); - return true; //bail out nothing to do here + s_logger.warn("network shutdwon requested on external load balancer, which did not implement the network." + + " Either network implement failed half way through or already network shutdown is completed. So just returning."); + return true; } externalLoadBalancer = _hostDao.findById(lbDeviceVO.getHostId()); @@ -725,10 +859,24 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase // Send a command to the external load balancer to implement or shutdown the guest network long guestVlanTag = Long.parseLong(guestConfig.getBroadcastUri().getHost()); - String selfIp = NetUtils.long2Ip(NetUtils.ip2Long(guestConfig.getGateway()) + 1); + String selfIp = null; String guestVlanNetmask = NetUtils.cidr2Netmask(guestConfig.getCidr()); Integer networkRate = _networkMgr.getNetworkRate(guestConfig.getId(), null); + if (add) { + // Acquire a self-ip address from the guest network IP address range + selfIp = _networkMgr.acquireGuestIpAddress(guestConfig, null); + if (selfIp == null) { + String msg = "failed to acquire guest IP address so not implementing the network on the external load balancer "; + s_logger.error(msg); + throw new InsufficientNetworkCapacityException(msg, Network.class, guestConfig.getId()); + } + } else { + // get the self-ip used by the load balancer + NicVO selfipNic = getPlaceholderNic(guestConfig); + selfIp = selfipNic.getIp4Address(); + } + IpAddressTO ip = new IpAddressTO(guestConfig.getAccountId(), null, add, false, true, String.valueOf(guestVlanTag), selfIp, guestVlanNetmask, null, null, networkRate, false); IpAddressTO[] ips = new IpAddressTO[1]; ips[0] = ip; @@ -740,38 +888,30 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase String answerDetails = (answer != null) ? answer.getDetails() : "answer was null"; String msg = "External load balancer was unable to " + action + " the guest network on the external load balancer in zone " + zone.getName() + " due to " + answerDetails; s_logger.error(msg); - throw new ResourceUnavailableException(msg, DataCenter.class, zoneId); + throw new ResourceUnavailableException(msg, Network.class, guestConfig.getId()); } - List reservedIpAddressesForGuestNetwork = _nicDao.listIpAddressInNetwork(guestConfig.getId()); - if (add && (!reservedIpAddressesForGuestNetwork.contains(selfIp))) { + if (add) { // Insert a new NIC for this guest network to reserve the self IP savePlaceholderNic(guestConfig, selfIp); - } + } else { + // release the self-ip obtained from guest network + NicVO selfipNic = getPlaceholderNic(guestConfig); + _nicDao.remove(selfipNic.getId()); - if (!add) { - Transaction txn = Transaction.currentTxn(); - txn.start(); - - // since network is shutdown remove the network mapping to the load balancer device - NetworkExternalLoadBalancerVO networkLBDevice = _networkExternalLBDao.findByNetworkId(guestConfig.getId()); - _networkExternalLBDao.remove(networkLBDevice.getId()); - - // if this is the last network mapped to the load balancer device then set device allocation state to be free - List ntwksMapped = _networkExternalLBDao.listByLoadBalancerDeviceId(networkLBDevice.getExternalLBDeviceId()); - if (ntwksMapped == null || ntwksMapped.isEmpty()) { - ExternalLoadBalancerDeviceVO lbDevice = _externalLoadBalancerDeviceDao.findById(networkLBDevice.getExternalLBDeviceId()); - lbDevice.setAllocationState(LBDeviceAllocationState.Free); - _externalLoadBalancerDeviceDao.update(lbDevice.getId(), lbDevice); - - //TODO: if device is cloud managed then take action + // release the load balancer allocated for the network + boolean releasedLB = freeLoadBalancerForNetwork(guestConfig); + if (!releasedLB) { + String msg = "Failed to release the external load balancer used for the network: " + guestConfig.getId(); + s_logger.error(msg); } - txn.commit(); } - Account account = _accountDao.findByIdIncludingRemoved(guestConfig.getAccountId()); - String action = add ? "implemented" : "shut down"; - s_logger.debug("External load balancer has " + action + " the guest network for account " + account.getAccountName() + "(id = " + account.getAccountId() + ") with VLAN tag " + guestVlanTag); + if (s_logger.isDebugEnabled()) { + Account account = _accountDao.findByIdIncludingRemoved(guestConfig.getAccountId()); + String action = add ? "implemented" : "shut down"; + s_logger.debug("External load balancer has " + action + " the guest network for account " + account.getAccountName() + "(id = " + account.getAccountId() + ") with VLAN tag " + guestVlanTag); + } return true; } @@ -820,7 +960,7 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase } } } catch (Exception e) { - s_logger.warn("Problems while getting external network usage", e); + s_logger.warn("Problems while getting external load balancer device usage", e); } finally { scanLock.releaseRef(); } @@ -831,38 +971,33 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase for (DataCenterVO zone : _dcDao.listAll()) { List domainRoutersInZone = _routerDao.listByDataCenter(zone.getId()); + if (domainRoutersInZone == null) { + continue; + } + for (DomainRouterVO domainRouter : domainRoutersInZone) { long accountId = domainRouter.getAccountId(); long zoneId = zone.getId(); + List networksForAccount = _networkDao.listBy(accountId, zoneId, Network.GuestType.Isolated); - + if (networksForAccount == null) { + continue; + } + for (NetworkVO network : networksForAccount) { if (!_networkMgr.networkIsConfiguredForExternalNetworking(zoneId, network.getId())) { s_logger.debug("Network " + network.getId() + " is not configured for external networking, so skipping usage check."); continue; } - HostVO externalFirewall = getFirewallProviderForNetwork(network); ExternalLoadBalancerDeviceVO lbDeviceVO = getExternalLoadBalancerForNetwork(network); - assert(lbDeviceVO != null); - HostVO externalLoadBalancer = _hostDao.findById(lbDeviceVO.getHostId()); - s_logger.debug("Collecting external network stats for network " + network.getId()); - - ExternalNetworkResourceUsageCommand cmd = new ExternalNetworkResourceUsageCommand(); - - // Get network stats from the external firewall - ExternalNetworkResourceUsageAnswer firewallAnswer = null; - if (externalFirewall != null) { - firewallAnswer = (ExternalNetworkResourceUsageAnswer) _agentMgr.easySend(externalFirewall.getId(), cmd); - if (firewallAnswer == null || !firewallAnswer.getResult()) { - String details = (firewallAnswer != null) ? firewallAnswer.getDetails() : "details unavailable"; - String msg = "Unable to get external firewall stats for " + zone.getName() + " due to: " + details + "."; - s_logger.error(msg); - continue; - } + if (lbDeviceVO == null) { + continue; } - // Get network stats from the external load balancer + // Get network stats from the external load balancer + HostVO externalLoadBalancer = _hostDao.findById(lbDeviceVO.getHostId()); + ExternalNetworkResourceUsageCommand cmd = new ExternalNetworkResourceUsageCommand(); ExternalNetworkResourceUsageAnswer lbAnswer = null; if (externalLoadBalancer != null) { lbAnswer = (ExternalNetworkResourceUsageAnswer) _agentMgr.easySend(externalLoadBalancer.getId(), cmd); @@ -878,12 +1013,12 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase s_logger.debug("Skipping stats update for account with ID " + accountId); continue; } - - if (!manageStatsEntries(true, accountId, zoneId, network, externalFirewall, firewallAnswer, externalLoadBalancer, lbAnswer)) { + + if (!manageStatsEntries(true, accountId, zoneId, network, externalLoadBalancer, lbAnswer)) { continue; } - - manageStatsEntries(false, accountId, zoneId, network, externalFirewall, firewallAnswer, externalLoadBalancer, lbAnswer); + + manageStatsEntries(false, accountId, zoneId, network, externalLoadBalancer, lbAnswer); } } } @@ -910,10 +1045,8 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase return _userStatsDao.update(userStats.getId(), userStats); } - - /* - * Creates a new stats entry for the specified parameters, if one doesn't already exist. - */ + + //Creates a new stats entry for the specified parameters, if one doesn't already exist. private boolean createStatsEntry(long accountId, long zoneId, long networkId, String publicIp, long hostId) { HostVO host = _hostDao.findById(hostId); UserStatisticsVO userStats = _userStatsDao.findBy(accountId, zoneId, networkId, publicIp, hostId, host.getType().toString()); @@ -923,10 +1056,8 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase return true; } } - - /* - * Updates an existing stats entry with new data from the specified usage answer. - */ + + // Updates an existing stats entry with new data from the specified usage answer. private boolean updateStatsEntry(long accountId, long zoneId, long networkId, String publicIp, long hostId, ExternalNetworkResourceUsageAnswer answer) { AccountVO account = _accountDao.findById(accountId); DataCenterVO zone = _dcDao.findById(zoneId); @@ -989,7 +1120,7 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase s_logger.warn("Unable to find user stats entry for " + statsEntryIdentifier); return false; } - + if (updateBytes(userStats, newCurrentBytesSent, newCurrentBytesReceived)) { s_logger.debug("Successfully updated stats for " + statsEntryIdentifier); return true; @@ -997,72 +1128,28 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase s_logger.debug("Failed to update stats for " + statsEntryIdentifier); return false; } - } - + } + private boolean createOrUpdateStatsEntry(boolean create, long accountId, long zoneId, long networkId, String publicIp, long hostId, ExternalNetworkResourceUsageAnswer answer) { if (create) { return createStatsEntry(accountId, zoneId, networkId, publicIp, hostId); } else { return updateStatsEntry(accountId, zoneId, networkId, publicIp, hostId, answer); - } + } } - + /* * Creates/updates all necessary stats entries for an account and zone. * Stats entries are created for source NAT IP addresses, static NAT rules, port forwarding rules, and load balancing rules */ private boolean manageStatsEntries(boolean create, long accountId, long zoneId, Network network, - HostVO externalFirewall, ExternalNetworkResourceUsageAnswer firewallAnswer, HostVO externalLoadBalancer, ExternalNetworkResourceUsageAnswer lbAnswer) { String accountErrorMsg = "Failed to update external network stats entry. Details: account ID = " + accountId; Transaction txn = Transaction.open(Transaction.CLOUD_DB); try { txn.start(); String networkErrorMsg = accountErrorMsg + ", network ID = " + network.getId(); - - boolean sharedSourceNat = false; - Map sourceNatCapabilities = _networkMgr.getNetworkServiceCapabilities(network.getId(), Service.SourceNat); - if (sourceNatCapabilities != null) { - String supportedSourceNatTypes = sourceNatCapabilities.get(Capability.SupportedSourceNatTypes).toLowerCase(); - if (supportedSourceNatTypes.contains("zone")) { - sharedSourceNat = true; - } - } - - if (!sharedSourceNat) { - // Manage the entry for this network's source NAT IP address - List sourceNatIps = _ipAddressDao.listByAssociatedNetwork(network.getId(), true); - if (sourceNatIps.size() == 1) { - String publicIp = sourceNatIps.get(0).getAddress().addr(); - if (!createOrUpdateStatsEntry(create, accountId, zoneId, network.getId(), publicIp, externalFirewall.getId(), firewallAnswer)) { - throw new ExecutionException(networkErrorMsg + ", source NAT IP = " + publicIp); - } - } - - // Manage one entry for each static NAT rule in this network - List staticNatIps = _ipAddressDao.listStaticNatPublicIps(network.getId()); - for (IPAddressVO staticNatIp : staticNatIps) { - String publicIp = staticNatIp.getAddress().addr(); - if (!createOrUpdateStatsEntry(create, accountId, zoneId, network.getId(), publicIp, externalFirewall.getId(), firewallAnswer)) { - throw new ExecutionException(networkErrorMsg + ", static NAT rule public IP = " + publicIp); - } - } - - // Manage one entry for each port forwarding rule in this network - List portForwardingRules = _portForwardingRulesDao.listByNetwork(network.getId()); - for (PortForwardingRuleVO portForwardingRule : portForwardingRules) { - String publicIp = _networkMgr.getIp(portForwardingRule.getSourceIpAddressId()).getAddress().addr(); - if (!createOrUpdateStatsEntry(create, accountId, zoneId, network.getId(), publicIp, externalFirewall.getId(), firewallAnswer)) { - throw new ExecutionException(networkErrorMsg + ", port forwarding rule public IP = " + publicIp); - } - } - } else { - // Manage the account-wide entry for the external firewall - if (!createOrUpdateStatsEntry(create, accountId, zoneId, network.getId(), null, externalFirewall.getId(), firewallAnswer)) { - throw new ExecutionException(networkErrorMsg); - } - } - + // If an external load balancer is added, manage one entry for each load balancing rule in this network if (externalLoadBalancer != null && lbAnswer != null) { List loadBalancers = _loadBalancerDao.listByNetworkId(network.getId()); @@ -1070,7 +1157,7 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase String publicIp = _networkMgr.getIp(loadBalancer.getSourceIpAddressId()).getAddress().addr(); if (!createOrUpdateStatsEntry(create, accountId, zoneId, network.getId(), publicIp, externalLoadBalancer.getId(), lbAnswer)) { throw new ExecutionException(networkErrorMsg + ", load balancing rule public IP = " + publicIp); - } + } } } return txn.commit();