From 989c6e69149a7534ccb883f8b594367b8ba2d575 Mon Sep 17 00:00:00 2001 From: Rajesh Battala Date: Fri, 11 Oct 2013 12:13:41 +0530 Subject: [PATCH] Modified cloud-early-config to configure, when the VR boots up in HyperV Environment Implemented commands that are required for VR to bootup and Vm deployment to work Modified hyperv agent code, to deploy VR with Boot Args, boot args passed to VR using KVP Exchange Component. Fix for VR to boot up and get configured with boot args, Fixed issue in VolumeOrchestrator Implemented SetFirewallRulesCommand in HyperV Resource Implemented VR network commands to provide the necessary services from VR Fixed hyperv localstorage path encode url issue. encode is converting space to '+' --- .../orchestration/VolumeOrchestrator.java | 5 +- .../ServerResource/HypervResource/WmiCalls.cs | 90 +- .../resource/HypervDirectConnectResource.java | 1323 ++++++++++++++++- ...oudStackPrimaryDataStoreLifeCycleImpl.java | 1 + .../config/etc/init.d/cloud-early-config | 17 +- 5 files changed, 1411 insertions(+), 25 deletions(-) diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 0821c81f71a..8d841d8fd1b 100644 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -1222,8 +1222,11 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati public void updateVolumeDiskChain(long volumeId, String path, String chainInfo) { VolumeVO vol = _volsDao.findById(volumeId); boolean needUpdate = false; + // Volume path is not getting updated in the DB, need to find reason and fix the issue. + if (vol.getPath() == null) + return; if(!vol.getPath().equalsIgnoreCase(path)) - needUpdate = true; + needUpdate = true; if(chainInfo != null && (vol.getChainInfo() == null || !chainInfo.equalsIgnoreCase(vol.getChainInfo()))) needUpdate = true; diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCalls.cs b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCalls.cs index 9b00c828104..1b9e073e2d0 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCalls.cs +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/WmiCalls.cs @@ -167,6 +167,7 @@ namespace HypervResource int memSize = vmInfo.maxRam / 1048576; string errMsg = vmName; var diskDrives = vmInfo.disks; + var bootArgs = vmInfo.bootArgs; // assert errMsg = vmName + ": missing disk information, array empty or missing, agent expects *at least* one disk for a VM"; @@ -283,34 +284,93 @@ namespace HypervResource AddDiskDriveToVm(newVm, vhdFile, ideCtrllr, driveResourceType); } - // add nics - foreach (var nic in nicInfo) + // Add the Nics to the VM in the deviceId order. + for (int i = 0; i <= 2; i++) { - string mac = nic.mac; - string vlan = null; - string isolationUri = nic.isolationUri; - if (isolationUri != null && isolationUri.StartsWith("vlan://") && !isolationUri.Equals("vlan://untagged")) + foreach (var nic in nicInfo) { - vlan = isolationUri.Substring("vlan://".Length); - int tmp; - if (!int.TryParse(vlan, out tmp)) + + int nicid = nic.deviceId; + string mac = nic.mac; + string vlan = null; + string isolationUri = nic.isolationUri; + if (isolationUri != null && isolationUri.StartsWith("vlan://") && !isolationUri.Equals("vlan://untagged")) { - // TODO: double check exception type - errMsg = string.Format("Invalid VLAN value {0} for on vm {1} for nic uuid {2}", isolationUri, vmName, nic.uuid); - var ex = new WmiException(errMsg); - logger.Error(errMsg, ex); - throw ex; + vlan = isolationUri.Substring("vlan://".Length); + int tmp; + if (!int.TryParse(vlan, out tmp)) + { + // TODO: double check exception type + errMsg = string.Format("Invalid VLAN value {0} for on vm {1} for nic uuid {2}", isolationUri, vmName, nic.uuid); + var ex = new WmiException(errMsg); + logger.Error(errMsg, ex); + throw ex; + } + } + + if (nicid == i) + { + CreateNICforVm(newVm, mac, vlan); + break; } } - CreateNICforVm(newVm, mac, vlan); + } + + // pass the boot args for the VM using KVP component. + // We need to pass the boot args to system vm's to get them configured with cloudstack configuration. + // Add new user data + var vm = WmiCallsV2.GetComputerSystem(vmName); + if (bootArgs != null) + { + + String bootargs = bootArgs; + WmiCallsV2.AddUserData(vm, bootargs); + + + // Get existing KVP + //var vmSettings = WmiCallsV2.GetVmSettings(vm); + //var kvpInfo = WmiCallsV2.GetKvpSettings(vmSettings); + //logger.DebugFormat("Boot Args presisted on the VM are ", kvpInfo); + //WmiCallsV2.AddUserData(vm, bootargs); + + // Verify key added to subsystem + //kvpInfo = WmiCallsV2.GetKvpSettings(vmSettings); + + // HostExchangesItems are embedded objects in the sense that the object value is stored and not a reference to the object. + //kvpProps = kvpInfo.HostExchangeItems; + + } + // call patch systemvm iso only for systemvms + if (vmName.StartsWith("r-")) + { + patchSystemVmIso(vmName); } logger.DebugFormat("Starting VM {0}", vmName); SetState(newVm, RequiredState.Enabled); + + // we need to reboot to get the hv kvp daemon get started vr gets configured. + if (vmName.StartsWith("r-")) + { + System.Threading.Thread.Sleep(90000); + SetState(newVm, RequiredState.Reboot); + // wait for the second boot and then return with suces + System.Threading.Thread.Sleep(50000); + } logger.InfoFormat("Started VM {0}", vmName); return newVm; } + /// this method is to add a dvd drive and attach the systemvm iso. + /// + + public static void patchSystemVmIso(String vmName) + { + ComputerSystem vmObject = WmiCalls.GetComputerSystem(vmName); + AddDiskDriveToVm(vmObject, "", "1", IDE_ISO_DRIVE); + WmiCalls.AttachIso(vmName, "c:\\systemvm.iso"); + } + /// /// Create a disk and attach it to the vm /// diff --git a/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java b/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java index e22f284b6e4..6d6dc1f413d 100644 --- a/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java +++ b/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java @@ -16,11 +16,18 @@ // under the License. package com.cloud.hypervisor.hyperv.resource; +import java.io.File; import java.io.IOException; +import java.net.ConnectException; +import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; +import java.nio.channels.SocketChannel; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.UUID; import javax.naming.ConfigurationException; @@ -35,20 +42,67 @@ import org.apache.http.util.EntityUtils; import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckRouterAnswer; +import com.cloud.agent.api.CheckRouterCommand; +import com.cloud.agent.api.CheckS2SVpnConnectionsAnswer; +import com.cloud.agent.api.CheckS2SVpnConnectionsCommand; import com.cloud.agent.api.Command; +import com.cloud.agent.api.GetDomRVersionAnswer; +import com.cloud.agent.api.GetDomRVersionCmd; +import com.cloud.agent.api.NetworkUsageAnswer; +import com.cloud.agent.api.NetworkUsageCommand; import com.cloud.agent.api.PingCommand; import com.cloud.agent.api.PingRoutingCommand; +import com.cloud.agent.api.PingTestCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; import com.cloud.agent.api.StartupStorageCommand; import com.cloud.agent.api.UnsupportedAnswer; import com.cloud.agent.api.StartupRoutingCommand.VmState; +import com.cloud.agent.api.check.CheckSshAnswer; +import com.cloud.agent.api.check.CheckSshCommand; +import com.cloud.agent.api.routing.CreateIpAliasCommand; +import com.cloud.agent.api.routing.DeleteIpAliasCommand; +import com.cloud.agent.api.routing.DhcpEntryCommand; +import com.cloud.agent.api.routing.DnsMasqConfigCommand; +import com.cloud.agent.api.routing.IpAliasTO; +import com.cloud.agent.api.routing.IpAssocAnswer; +import com.cloud.agent.api.routing.IpAssocCommand; +import com.cloud.agent.api.routing.LoadBalancerConfigCommand; +import com.cloud.agent.api.routing.NetworkElementCommand; +import com.cloud.agent.api.routing.SavePasswordCommand; +import com.cloud.agent.api.routing.SetFirewallRulesAnswer; +import com.cloud.agent.api.routing.SetFirewallRulesCommand; +import com.cloud.agent.api.routing.SetPortForwardingRulesAnswer; +import com.cloud.agent.api.routing.SetPortForwardingRulesCommand; +import com.cloud.agent.api.routing.SetSourceNatAnswer; +import com.cloud.agent.api.routing.SetSourceNatCommand; +import com.cloud.agent.api.routing.SetStaticNatRulesAnswer; +import com.cloud.agent.api.routing.SetStaticNatRulesCommand; +import com.cloud.agent.api.routing.SetStaticRouteAnswer; +import com.cloud.agent.api.routing.SetStaticRouteCommand; +import com.cloud.agent.api.routing.Site2SiteVpnCfgCommand; +import com.cloud.agent.api.routing.VmDataCommand; +import com.cloud.agent.api.to.DhcpTO; +import com.cloud.agent.api.to.FirewallRuleTO; +import com.cloud.agent.api.to.IpAddressTO; +import com.cloud.agent.api.to.PortForwardingRuleTO; +import com.cloud.agent.api.to.StaticNatRuleTO; +import com.cloud.dc.DataCenter.NetworkType; import com.cloud.host.Host.Type; import com.cloud.hypervisor.Hypervisor; +import com.cloud.network.HAProxyConfigurator; +import com.cloud.network.LoadBalancerConfigurator; import com.cloud.network.Networks.RouterPrivateIpStrategy; +import com.cloud.network.rules.FirewallRule; import com.cloud.resource.ServerResource; import com.cloud.resource.ServerResourceBase; import com.cloud.serializer.GsonHelper; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; +import com.cloud.utils.net.NetUtils; +import com.cloud.utils.ssh.SshHelper; +import com.cloud.vm.VirtualMachineName; import com.google.gson.Gson; /** @@ -68,6 +122,11 @@ public class HypervDirectConnectResource extends ServerResourceBase implements private String _guid; private String _agentIp; private int _port = DEFAULT_AGENT_PORT; + protected final long _ops_timeout = 900000; // 15 minutes time out to time + + protected final int _retry = 24; + protected final int _sleep = 10000; + protected final int DEFAULT_DOMR_SSHPORT = 3922; private String _clusterGuid; @@ -296,6 +355,8 @@ public class HypervDirectConnectResource extends ServerResourceBase implements // Using java.net.URI, see // http://docs.oracle.com/javase/1.5.0/docs/api/java/net/URI.html URI agentUri = null; + Class clazz = cmd.getClass(); + Answer answer = null; try { String cmdName = cmd.getClass().getName(); agentUri = @@ -307,10 +368,53 @@ public class HypervDirectConnectResource extends ServerResourceBase implements s_logger.error(errMsg, e); return null; } - String ansStr = postHttpRequest(s_gson.toJson(cmd), agentUri); + if (clazz == CheckSshCommand.class) { + answer = execute((CheckSshCommand) cmd); + } else if (clazz == GetDomRVersionCmd.class) { + answer = execute((GetDomRVersionCmd)cmd); + } else if (cmd instanceof NetworkUsageCommand) { + answer = execute((NetworkUsageCommand) cmd); + } else if (clazz == IpAssocCommand.class) { + answer = execute((IpAssocCommand) cmd); + } else if (clazz == DnsMasqConfigCommand.class) { + return execute((DnsMasqConfigCommand) cmd); + } else if (clazz == CreateIpAliasCommand.class) { + return execute((CreateIpAliasCommand) cmd); + } else if (clazz == DhcpEntryCommand.class) { + answer = execute((DhcpEntryCommand) cmd); + } else if (clazz == VmDataCommand.class) { + answer = execute((VmDataCommand) cmd); + } else if (clazz == SavePasswordCommand.class) { + answer = execute((SavePasswordCommand) cmd); + } else if (clazz == SetFirewallRulesCommand.class) { + answer = execute((SetFirewallRulesCommand)cmd); + } else if (clazz == LoadBalancerConfigCommand.class) { + answer = execute((LoadBalancerConfigCommand) cmd); + } else if (clazz == DeleteIpAliasCommand.class) { + return execute((DeleteIpAliasCommand) cmd); + } else if (clazz == PingTestCommand.class) { + answer = execute((PingTestCommand) cmd); + } else if (clazz == SetStaticNatRulesCommand.class) { + answer = execute((SetStaticNatRulesCommand) cmd); + } else if (clazz == CheckRouterCommand.class) { + answer = execute((CheckRouterCommand) cmd); + } else if (clazz == SetPortForwardingRulesCommand.class) { + answer = execute((SetPortForwardingRulesCommand) cmd); + } else if (clazz == SetSourceNatCommand.class) { + answer = execute((SetSourceNatCommand) cmd); + } else if (clazz == Site2SiteVpnCfgCommand.class) { + answer = execute((Site2SiteVpnCfgCommand) cmd); + } else if (clazz == CheckS2SVpnConnectionsCommand.class) { + answer = execute((CheckS2SVpnConnectionsCommand) cmd); + } else if (clazz == SetStaticRouteCommand.class) { + answer = execute((SetStaticRouteCommand) cmd); + } + else { + // Else send the cmd to hyperv agent. + String ansStr = postHttpRequest(s_gson.toJson(cmd), agentUri); if (ansStr == null) { - return null; + return Answer.createUnsupportedCommandAnswer(cmd); } // Only Answer instances are returned by remote agents. // E.g. see Response.getAnswers() @@ -320,9 +424,1176 @@ public class HypervDirectConnectResource extends ServerResourceBase implements if (result.length > 0) { return result[0]; } + } + return answer; + } + + private SetStaticRouteAnswer execute(SetStaticRouteCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource SetStaticRouteCommand: " + s_gson.toJson(cmd)); + } + + boolean endResult = true; + + String controlIp = getRouterSshControlIp(cmd); + String args = ""; + String[] results = new String[cmd.getStaticRoutes().length]; + int i = 0; + + // Extract and build the arguments for the command to be sent to the VR. + String[][] rules = cmd.generateSRouteRules(); + StringBuilder sb = new StringBuilder(); + String[] srRules = rules[0]; + for (int j = 0; j < srRules.length; j++) { + sb.append(srRules[j]).append(','); + } + args += " -a " + sb.toString(); + + // Send over the command for execution, via ssh, to the VR. + try { + Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, + "/opt/cloud/bin/vpc_staticroute.sh " + args); + + if (s_logger.isDebugEnabled()) + s_logger.debug("Executing script on domain router " + controlIp + ": /opt/cloud/bin/vpc_staticroute.sh " + args); + + if (!result.first()) { + s_logger.error("SetStaticRouteCommand failure on setting one rule. args: " + args); + results[i++] = "Failed"; + endResult = false; + } else { + results[i++] = null; + } + } catch (Throwable e) { + s_logger.error("SetStaticRouteCommand(args: " + args + ") failed on setting one rule due to " + e); + results[i++] = "Failed"; + endResult = false; + } + return new SetStaticRouteAnswer(cmd, endResult, results); + + } + + protected CheckS2SVpnConnectionsAnswer execute(CheckS2SVpnConnectionsCommand cmd) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Executing resource CheckS2SVpnConnectionsCommand: " + s_gson.toJson(cmd)); + s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /opt/cloud/bin/checkbatchs2svpn.sh "); + } + + Pair result; + try { + String controlIp = getRouterSshControlIp(cmd); + String cmdline = "/opt/cloud/bin/checkbatchs2svpn.sh "; + for (String ip : cmd.getVpnIps()) { + cmdline += " " + ip; + } + + result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, cmdline); + + if (!result.first()) { + s_logger.error("check site-to-site vpn connections command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " failed, message: " + + result.second()); + + return new CheckS2SVpnConnectionsAnswer(cmd, false, result.second()); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("check site-to-site vpn connections command on domain router " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " completed"); + } + } catch (Throwable e) { + String msg = "CheckS2SVpnConnectionsCommand failed due to " + e; + s_logger.error(msg, e); + return new CheckS2SVpnConnectionsAnswer(cmd, false, "CheckS2SVpnConneciontsCommand failed"); + } + return new CheckS2SVpnConnectionsAnswer(cmd, true, result.second()); + } + + protected Answer execute(Site2SiteVpnCfgCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource Site2SiteVpnCfgCommand " + s_gson.toJson(cmd)); + } + + String routerIp = getRouterSshControlIp(cmd); + + String args = ""; + if (cmd.isCreate()) { + args += " -A"; + args += " -l "; + args += cmd.getLocalPublicIp(); + args += " -n "; + args += cmd.getLocalGuestCidr(); + args += " -g "; + args += cmd.getLocalPublicGateway(); + args += " -r "; + args += cmd.getPeerGatewayIp(); + args += " -N "; + args += cmd.getPeerGuestCidrList(); + args += " -e "; + args += "\"" + cmd.getEspPolicy() + "\""; + args += " -i "; + args += "\"" + cmd.getIkePolicy() + "\""; + args += " -t "; + args += Long.toString(cmd.getIkeLifetime()); + args += " -T "; + args += Long.toString(cmd.getEspLifetime()); + args += " -s "; + args += "\"" + cmd.getIpsecPsk() + "\""; + args += " -d "; + if (cmd.getDpd()) { + args += "1"; + } else { + args += "0"; + } + } else { + args += " -D"; + args += " -r "; + args += cmd.getPeerGatewayIp(); + args += " -n "; + args += cmd.getLocalGuestCidr(); + args += " -N "; + args += cmd.getPeerGuestCidrList(); + } + + Pair result; + try { + result = SshHelper.sshExecute(routerIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, + "/opt/cloud/bin/ipsectunnel.sh " + args); + + if (!result.first()) { + s_logger.error("Setup site2site VPN " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " failed, message: " + result.second()); + + return new Answer(cmd, false, "Setup site2site VPN falied due to " + result.second()); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("setup site 2 site vpn on router " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " completed"); + } + } catch (Throwable e) { + String msg = "Setup site2site VPN falied due to " + e.getMessage(); + s_logger.error(msg, e); + return new Answer(cmd, false, "Setup site2site VPN failed due to " + e.getMessage()); + } + return new Answer(cmd, true, result.second()); + } + + protected SetSourceNatAnswer execute(SetSourceNatCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource SetSourceNatCommand " + s_gson.toJson(cmd)); + } + + String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME); + String routerIp = getRouterSshControlIp(cmd); + IpAddressTO pubIp = cmd.getIpAddress(); + try { + int ethDeviceNum = findRouterEthDeviceIndex(routerName, routerIp, pubIp.getVifMacAddress()); + String args = ""; + args += " -A "; + args += " -l "; + args += pubIp.getPublicIp(); + + args += " -c "; + args += "eth" + ethDeviceNum; + + Pair result = SshHelper.sshExecute(routerIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, + "/opt/cloud/bin/vpc_snat.sh " + args); + + if (!result.first()) { + String msg = "SetupGuestNetworkCommand on domain router " + routerIp + " failed. message: " + result.second(); + s_logger.error(msg); + + return new SetSourceNatAnswer(cmd, false, msg); + } + + return new SetSourceNatAnswer(cmd, true, "success"); + } catch (Exception e) { + String msg = "Ip SNAT failure due to " + e.toString(); + s_logger.error(msg, e); + return new SetSourceNatAnswer(cmd, false, msg); + } + } + + // + // find mac address of a specified ethx device + // ip address show ethx | grep link/ether | sed -e 's/^[ \t]*//' | cut -d' ' -f2 + // returns + // eth0:xx.xx.xx.xx + + // + // list IP with eth devices + // ifconfig ethx |grep -B1 "inet addr" | awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' + // | awk -F: '{ print $1 ": " $3 }' + // + // returns + // eth0:xx.xx.xx.xx + // + // + + private int findRouterEthDeviceIndex(String domrName, String routerIp, String mac) throws Exception { + + s_logger.info("findRouterEthDeviceIndex. mac: " + mac); + + // TODO : this is a temporary very inefficient solution, will refactor it later + Pair result = SshHelper.sshExecute(routerIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, + "ls /proc/sys/net/ipv4/conf"); + + // when we dynamically plug in a new NIC into virtual router, it may take time to show up in guest OS + // we use a waiting loop here as a workaround to synchronize activities in systems + long startTick = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTick < 15000) { + if (result.first()) { + String[] tokens = result.second().split("\\s+"); + for (String token : tokens) { + if (!("all".equalsIgnoreCase(token) || "default".equalsIgnoreCase(token) || "lo".equalsIgnoreCase(token))) { + String cmd = String.format("ip address show %s | grep link/ether | sed -e 's/^[ \t]*//' | cut -d' ' -f2", token); + + if (s_logger.isDebugEnabled()) + s_logger.debug("Run domr script " + cmd); + Pair result2 = SshHelper.sshExecute(routerIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, + // TODO need to find the dev index inside router based on IP address + cmd); + if (s_logger.isDebugEnabled()) + s_logger.debug("result: " + result2.first() + ", output: " + result2.second()); + + if (result2.first() && result2.second().trim().equalsIgnoreCase(mac.trim())) + return Integer.parseInt(token.substring(3)); + } + } + } + + s_logger.warn("can not find intereface associated with mac: " + mac + ", guest OS may still at loading state, retry..."); + + try { + Thread.currentThread().sleep(1000); + } catch (InterruptedException e) { + } + } + + return -1; + } + + protected Answer execute(SetPortForwardingRulesCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource SetPortForwardingRulesCommand: " + s_gson.toJson(cmd)); + } + + String controlIp = getRouterSshControlIp(cmd); + String args = ""; + String[] results = new String[cmd.getRules().length]; + int i = 0; + + boolean endResult = true; + for (PortForwardingRuleTO rule : cmd.getRules()) { + args += rule.revoked() ? " -D " : " -A "; + args += " -P " + rule.getProtocol().toLowerCase(); + args += " -l " + rule.getSrcIp(); + args += " -p " + rule.getStringSrcPortRange(); + args += " -r " + rule.getDstIp(); + args += " -d " + rule.getStringDstPortRange(); + + try { + Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "/root/firewall.sh " + args); + + if (s_logger.isDebugEnabled()) + s_logger.debug("Executing script on domain router " + controlIp + ": /root/firewall.sh " + args); + + if (!result.first()) { + s_logger.error("SetPortForwardingRulesCommand failure on setting one rule. args: " + args); + results[i++] = "Failed"; + endResult = false; + } else { + results[i++] = null; + } + } catch (Throwable e) { + s_logger.error("SetPortForwardingRulesCommand(args: " + args + ") failed on setting one rule due to " + e.getMessage()); + results[i++] = "Failed"; + endResult = false; + } + } + + return new SetPortForwardingRulesAnswer(cmd, results, endResult); + } + + protected Answer execute(CheckRouterCommand cmd) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Executing resource CheckRouterCommand: " + s_gson.toJson(cmd)); + s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /opt/cloud/bin/checkrouter.sh "); + } + + Pair result; + try { + + String controlIp = getRouterSshControlIp(cmd); + result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, + "/opt/cloud/bin/checkrouter.sh "); + + if (!result.first()) { + s_logger.error("check router command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " failed, message: " + result.second()); + + return new CheckRouterAnswer(cmd, "CheckRouter failed due to " + result.second()); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("check router command on domain router " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " completed"); + } + } catch (Throwable e) { + String msg = "CheckRouterCommand failed due to " + e.getMessage(); + s_logger.error(msg, e); + return new CheckRouterAnswer(cmd, msg); + } + return new CheckRouterAnswer(cmd, result.second(), true); + } + + protected Answer execute(SetStaticNatRulesCommand cmd) { + + if (cmd.getVpcId() != null) { + //return SetVPCStaticNatRules(cmd); + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource SetFirewallRuleCommand: " + s_gson.toJson(cmd)); + } + + String args = null; + String[] results = new String[cmd.getRules().length]; + int i = 0; + boolean endResult = true; + for (StaticNatRuleTO rule : cmd.getRules()) { + // 1:1 NAT needs instanceip;publicip;domrip;op + args = rule.revoked() ? " -D " : " -A "; + + args += " -l " + rule.getSrcIp(); + args += " -r " + rule.getDstIp(); + + if (rule.getProtocol() != null) { + args += " -P " + rule.getProtocol().toLowerCase(); + } + + args += " -d " + rule.getStringSrcPortRange(); + args += " -G "; + + try { + String controlIp = getRouterSshControlIp(cmd); + Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "/root/firewall.sh " + args); + + if (s_logger.isDebugEnabled()) + s_logger.debug("Executing script on domain router " + controlIp + ": /root/firewall.sh " + args); + + if (!result.first()) { + s_logger.error("SetStaticNatRulesCommand failure on setting one rule. args: " + args); + results[i++] = "Failed"; + endResult = false; + } else { + results[i++] = null; + } + } catch (Throwable e) { + s_logger.error("SetStaticNatRulesCommand (args: " + args + ") failed on setting one rule due to " + e.getMessage()); + results[i++] = "Failed"; + endResult = false; + } + } + return new SetStaticNatRulesAnswer(cmd, results, endResult); + } + + protected Answer execute(PingTestCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource PingTestCommand: " + s_gson.toJson(cmd)); + } + String controlIp = cmd.getRouterIp(); + String args = " -c 1 -n -q " + cmd.getPrivateIp(); + try { + Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "/bin/ping" + args); + if (result.first()) + return new Answer(cmd); + } catch (Exception e) { + s_logger.error("Unable to execute ping command on DomR (" + controlIp + "), domR may not be ready yet. failure due to " + + e.getMessage()); + } + return new Answer(cmd, false, "PingTestCommand failed"); + } + + protected Answer execute(final DeleteIpAliasCommand cmd) { + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + List revokedIpAliasTOs = cmd.getDeleteIpAliasTos(); + List activeIpAliasTOs = cmd.getCreateIpAliasTos(); + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing deleteIpAlias command: " + s_gson.toJson(cmd)); + } + String args = ""; + for (IpAliasTO ipAliasTO : revokedIpAliasTOs) { + args = args + ipAliasTO.getAlias_count() + ":" + ipAliasTO.getRouterip() + ":" + ipAliasTO.getNetmask() + "-"; + } + args = args + "- "; + for (IpAliasTO ipAliasTO : activeIpAliasTOs) { + args = args + ipAliasTO.getAlias_count() + ":" + ipAliasTO.getRouterip() + ":" + ipAliasTO.getNetmask() + "-"; + } + if (s_logger.isDebugEnabled()) { + s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/deleteIpAlias " + args); + } + + try { + String controlIp = getRouterSshControlIp(cmd); + Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, + "/root/deleteIpAlias.sh " + args); + + if (!result.first()) { + s_logger.error("deleteIpAlias command on domr " + controlIp + " failed, message: " + result.second()); + + return new Answer(cmd, false, "deleteIpAlias failed due to " + result.second()); + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("deleteIpAlias command on domain router " + controlIp + " completed"); + } + + } catch (Throwable e) { + String msg = "deleteIpAlias failed due to " + e.getMessage(); + s_logger.error(msg, e); + return new Answer(cmd, false, msg); + } + + return new Answer(cmd); + } + + protected Answer execute(final LoadBalancerConfigCommand cmd) { + + if (cmd.getVpcId() != null) { + //return VPCLoadBalancerConfig(cmd); + } + + File keyFile = getSystemVMKeyFile(); + + String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + String controlIp = getRouterSshControlIp(cmd); + + assert (controlIp != null); + + LoadBalancerConfigurator cfgtr = new HAProxyConfigurator(); + String[] config = cfgtr.generateConfiguration(cmd); + + String[][] rules = cfgtr.generateFwRules(cmd); + String tmpCfgFilePath = "/tmp/" + routerIp.replace('.', '_') + ".cfg"; + String tmpCfgFileContents = ""; + for (int i = 0; i < config.length; i++) { + tmpCfgFileContents += config[i]; + tmpCfgFileContents += "\n"; + } + + try { + SshHelper.scpTo(controlIp, DEFAULT_DOMR_SSHPORT, "root", keyFile, null, "/tmp/", tmpCfgFileContents.getBytes(), routerIp.replace('.', '_') + ".cfg", null); + + try { + String[] addRules = rules[LoadBalancerConfigurator.ADD]; + String[] removeRules = rules[LoadBalancerConfigurator.REMOVE]; + String[] statRules = rules[LoadBalancerConfigurator.STATS]; + + String args = ""; + args += "-i " + routerIp; + args += " -f " + tmpCfgFilePath; + + StringBuilder sb = new StringBuilder(); + if (addRules.length > 0) { + for (int i = 0; i < addRules.length; i++) { + sb.append(addRules[i]).append(','); + } + + args += " -a " + sb.toString(); + } + + sb = new StringBuilder(); + if (removeRules.length > 0) { + for (int i = 0; i < removeRules.length; i++) { + sb.append(removeRules[i]).append(','); + } + + args += " -d " + sb.toString(); + } + + sb = new StringBuilder(); + if (statRules.length > 0) { + for (int i = 0; i < statRules.length; i++) { + sb.append(statRules[i]).append(','); + } + + args += " -s " + sb.toString(); + } + + Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "scp " + tmpCfgFilePath + + " /etc/haproxy/haproxy.cfg.new"); + + if (!result.first()) { + s_logger.error("Unable to copy haproxy configuration file"); + return new Answer(cmd, false, "LoadBalancerConfigCommand failed due to uanble to copy haproxy configuration file"); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Run command on domain router " + routerIp + ", /root/loadbalancer.sh " + args); + } + + result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "/root/loadbalancer.sh " + args); + + if (!result.first()) { + String msg = "LoadBalancerConfigCommand on domain router " + routerIp + " failed. message: " + result.second(); + s_logger.error(msg); + + return new Answer(cmd, false, msg); + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("LoadBalancerConfigCommand on domain router " + routerIp + " completed"); + } + } finally { + SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "rm " + tmpCfgFilePath); + } + + return new Answer(cmd); + } catch (Throwable e) { + s_logger.error("Unexpected exception: " + e.toString(), e); + return new Answer(cmd, false, "LoadBalancerConfigCommand failed due to " + e.getMessage()); + } + } + + protected Answer execute(SavePasswordCommand cmd) { + if (s_logger.isInfoEnabled()) { + + s_logger.info("Executing resource SavePasswordCommand. vmName: " + cmd.getVmName() + ", vmIp: " + cmd.getVmIpAddress() + ", password: " + + StringUtils.getMaskedPasswordForDisplay(cmd.getPassword())); + } + + String controlIp = getRouterSshControlIp(cmd); + final String password = cmd.getPassword(); + final String vmIpAddress = cmd.getVmIpAddress(); + + // Run save_password_to_domr.sh + String args = " -v " + vmIpAddress; + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Run command on domain router " + controlIp + ", /root/savepassword.sh " + args + " -p " + StringUtils.getMaskedPasswordForDisplay(cmd.getPassword())); + } + + args += " -p " + password; + + try { + + Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "/root/savepassword.sh " + args); + + if (!result.first()) { + s_logger.error("savepassword command on domain router " + controlIp + " failed, message: " + result.second()); + + return new Answer(cmd, false, "SavePassword failed due to " + result.second()); + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("savepassword command on domain router " + controlIp + " completed"); + } + + } catch (Throwable e) { + String msg = "SavePasswordCommand failed due to " + e; + s_logger.error(msg, e); + return new Answer(cmd, false, msg); + } + return new Answer(cmd); + } + + protected SetFirewallRulesAnswer execute(SetFirewallRulesCommand cmd) { + String controlIp = getRouterSshControlIp(cmd); + String[] results = new String[cmd.getRules().length]; + FirewallRuleTO[] allrules = cmd.getRules(); + FirewallRule.TrafficType trafficType = allrules[0].getTrafficType(); + String egressDefault = cmd.getAccessDetail(NetworkElementCommand.FIREWALL_EGRESS_DEFAULT); + + String[][] rules = cmd.generateFwRules(); + String args = ""; + args += " -F "; + if (trafficType == FirewallRule.TrafficType.Egress){ + args+= " -E "; + if (egressDefault.equals("true")) { + args+= " -P 1 "; + } else if (egressDefault.equals("System")) { + args+= " -P 2 "; + } else { + args+= " -P 0 "; + } + } + + StringBuilder sb = new StringBuilder(); + String[] fwRules = rules[0]; + if (fwRules.length > 0) { + for (int i = 0; i < fwRules.length; i++) { + sb.append(fwRules[i]).append(','); + } + args += " -a " + sb.toString(); + } + + try { + Pair result = null; + + if (trafficType == FirewallRule.TrafficType.Egress){ + result = SshHelper.sshExecute(controlIp, + DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), + null, "/root/firewallRule_egress.sh " + args); + } else { + result = SshHelper.sshExecute(controlIp, + DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), + null, "/root/firewall_rule.sh " + args); + } + + if (s_logger.isDebugEnabled()) { + if (trafficType == FirewallRule.TrafficType.Egress){ + s_logger.debug("Executing script on domain router " + controlIp + + ": /root/firewallRule_egress.sh " + args); + } else { + s_logger.debug("Executing script on domain router " + controlIp + + ": /root/firewall_rule.sh " + args); + } + } + + + if (!result.first()) { + s_logger.error("SetFirewallRulesCommand failure on setting one rule. args: " + + args); + //FIXME - in the future we have to process each rule separately; now we temporarily set every rule to be false if single rule fails + for (int i=0; i < results.length; i++) { + results[i] = "Failed"; + } + + return new SetFirewallRulesAnswer(cmd, false, results); + } + } catch (Throwable e) { + s_logger.error("SetFirewallRulesCommand(args: " + args + + ") failed on setting one rule due to " + ,e); + //FIXME - in the future we have to process each rule separately; now we temporarily set every rule to be false if single rule fails + for (int i=0; i < results.length; i++) { + results[i] = "Failed"; + } + return new SetFirewallRulesAnswer(cmd, false, results); + } + + return new SetFirewallRulesAnswer(cmd, true, results); + } + + + protected Answer execute(VmDataCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource VmDataCommand: " + s_gson.toJson(cmd)); + } + + String routerPrivateIpAddress = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + String controlIp = getRouterSshControlIp(cmd); + + String vmIpAddress = cmd.getVmIpAddress(); + List vmData = cmd.getVmData(); + String[] vmDataArgs = new String[vmData.size() * 2 + 4]; + vmDataArgs[0] = "routerIP"; + vmDataArgs[1] = routerPrivateIpAddress; + vmDataArgs[2] = "vmIP"; + vmDataArgs[3] = vmIpAddress; + int i = 4; + for (String[] vmDataEntry : vmData) { + String folder = vmDataEntry[0]; + String file = vmDataEntry[1]; + String contents = (vmDataEntry[2] != null) ? vmDataEntry[2] : "none"; + + vmDataArgs[i] = folder + "," + file; + vmDataArgs[i + 1] = contents; + i += 2; + } + + String content = encodeDataArgs(vmDataArgs); + String tmpFileName = UUID.randomUUID().toString(); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Run vm_data command on domain router " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", data: " + content); + } + + try { + SshHelper.scpTo(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "/tmp", content.getBytes(), tmpFileName, null); + + try { + Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, + "/root/userdata.py " + tmpFileName); + + if (!result.first()) { + s_logger.error("vm_data command on domain router " + controlIp + " failed. messge: " + result.second()); + return new Answer(cmd, false, "VmDataCommand failed due to " + result.second()); + } + } finally { + + SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "rm /tmp/" + tmpFileName); + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("vm_data command on domain router " + controlIp + " completed"); + } + + } catch (Throwable e) { + String msg = "VmDataCommand failed due to " + e; + s_logger.error(msg, e); + return new Answer(cmd, false, msg); + } + return new Answer(cmd); + } + + private String encodeDataArgs(String[] dataArgs) { + StringBuilder sb = new StringBuilder(); + + for (String arg : dataArgs) { + sb.append(arg); + sb.append("\n"); + } + + return sb.toString(); + } + + + protected Answer execute(DhcpEntryCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource DhcpEntryCommand: " + s_gson.toJson(cmd)); + } + + // ssh -p 3922 -o StrictHostKeyChecking=no -i $cert root@$domr "/root/edithosts.sh $mac $ip $vm $dfltrt $ns $staticrt" >/dev/null + + String args = " -m " + cmd.getVmMac(); + if (cmd.getVmIpAddress() != null) { + args += " -4 " + cmd.getVmIpAddress(); + } + args += " -h " + cmd.getVmName(); + + if (cmd.getDefaultRouter() != null) { + args += " -d " + cmd.getDefaultRouter(); + } + + if (cmd.getDefaultDns() != null) { + args += " -n " + cmd.getDefaultDns(); + } + + if (cmd.getStaticRoutes() != null) { + args += " -s " + cmd.getStaticRoutes(); + } + + if (cmd.getVmIp6Address() != null) { + args += " -6 " + cmd.getVmIp6Address(); + args += " -u " + cmd.getDuid(); + } + + if (!cmd.isDefault()) { + args += " -N"; + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/edithosts.sh " + args); + } + + try { + String controlIp = getRouterSshControlIp(cmd); + Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, + "/root/edithosts.sh " + args); + + if (!result.first()) { + s_logger.error("dhcp_entry command on domR " + controlIp + " failed, message: " + result.second()); + + return new Answer(cmd, false, "DhcpEntry failed due to " + result.second()); + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("dhcp_entry command on domain router " + controlIp + " completed"); + } + + } catch (Throwable e) { + String msg = "DhcpEntryCommand failed due to " + e; + s_logger.error(msg, e); + return new Answer(cmd, false, msg); + } + + return new Answer(cmd); + } + + protected Answer execute(final CreateIpAliasCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing createIpAlias command: " + s_gson.toJson(cmd)); + } + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + List ipAliasTOs = cmd.getIpAliasList(); + String args=""; + for (IpAliasTO ipaliasto : ipAliasTOs) { + args = args + ipaliasto.getAlias_count()+":"+ipaliasto.getRouterip()+":"+ipaliasto.getNetmask()+"-"; + } + if (s_logger.isDebugEnabled()) { + s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/createIpAlias " + args); + } + + try { + String controlIp = getRouterSshControlIp(cmd); + Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, + "/root/createIpAlias.sh " + args); + + if (!result.first()) { + s_logger.error("CreateIpAlias command on domr " + controlIp + " failed, message: " + result.second()); + + return new Answer(cmd, false, "createipAlias failed due to " + result.second()); + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("createIpAlias command on domain router " + controlIp + " completed"); + } + + } catch (Throwable e) { + String msg = "createIpAlias failed due to " + e; + s_logger.error(msg, e); + return new Answer(cmd, false, msg); + } + + return new Answer(cmd); + } + + protected Answer execute(final DnsMasqConfigCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing dnsmasqConfig command: " + s_gson.toJson(cmd)); + } + String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + String controlIp = getRouterSshControlIp(cmd); + + assert(controlIp != null); + + List dhcpTos = cmd.getIps(); + String args =""; + for(DhcpTO dhcpTo : dhcpTos) { + args = args + dhcpTo.getRouterIp()+":"+dhcpTo.getGateway()+":"+dhcpTo.getNetmask()+":"+dhcpTo.getStartIpOfSubnet()+"-"; + } + //File keyFile = mgr.getSystemVMKeyFile(); + + try { + Pair result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "/root/dnsmasq.sh " + args); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Run command on domain router " + routerIp + ", /root/dnsmasq.sh"); + } + + if (!result.first()) { + s_logger.error("Unable update dnsmasq config file"); + return new Answer(cmd, false, "dnsmasq config update failed due to: " + result.second()); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("dnsmasq config command on domain router " + routerIp + " completed"); + } + }catch (Throwable e) { + String msg = "Dnsmasqconfig command failed due to " + e.getMessage(); + s_logger.error(msg, e); + return new Answer(cmd, false, msg); + } + + return new Answer(cmd); + } + + protected Answer execute(IpAssocCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource IPAssocCommand: " + s_gson.toJson(cmd)); + } + + int i = 0; + String[] results = new String[cmd.getIpAddresses().length]; + + try { + + IpAddressTO[] ips = cmd.getIpAddresses(); + String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME); + String controlIp = getRouterSshControlIp(cmd); + for (IpAddressTO ip : ips) { + assignPublicIpAddress(routerName, controlIp, ip.getPublicIp(), ip.isAdd(), ip.isFirstIP(), ip.isSourceNat(), ip.getBroadcastUri(), ip.getVlanGateway(), ip.getVlanNetmask(),ip.getVifMacAddress()); + results[i++] = ip.getPublicIp() + " - success"; + } + + for (; i < cmd.getIpAddresses().length; i++) { + results[i++] = IpAssocAnswer.errorResult; + } + } catch (Throwable e) { + s_logger.error("Unexpected exception: " + e.toString() + " will shortcut rest of IPAssoc commands", e); + + for (; i < cmd.getIpAddresses().length; i++) { + results[i++] = IpAssocAnswer.errorResult; + } + } + + return new IpAssocAnswer(cmd, results); + } + + protected void assignPublicIpAddress(final String vmName, final String privateIpAddress, final String publicIpAddress, final boolean add, final boolean firstIP, + final boolean sourceNat, final String vlanId, final String vlanGateway, final String vlanNetmask, final String vifMacAddress) throws Exception { + + //String publicNeworkName = HypervisorHostHelper.getPublicNetworkNamePrefix(vlanId); + //Pair publicNicInfo = vmMo.getNicDeviceIndex(publicNeworkName); + + if (s_logger.isDebugEnabled()) { + //s_logger.debug("Find public NIC index, public network name: " + publicNeworkName + ", index: " + publicNicInfo.first()); + } + + boolean addVif = false; + boolean removeVif = false; + if (add ) { // && publicNicInfo.first().intValue() == -1) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Plug new NIC to associate" + privateIpAddress + " to " + publicIpAddress); + } + + addVif = true; + } else if (!add && firstIP) { + removeVif = true; + + if (s_logger.isDebugEnabled()) { + //s_logger.debug("Unplug NIC " + publicNicInfo.first()); + } + } + +/* if (addVif) { + plugPublicNic(vmMo, vlanId, vifMacAddress); + publicNicInfo = vmMo.getNicDeviceIndex(publicNeworkName); + if (publicNicInfo.first().intValue() >= 0) { + networkUsage(privateIpAddress, "addVif", "eth" + publicNicInfo.first()); + } + } +*/ +/* if (publicNicInfo.first().intValue() < 0) { + String msg = "Failed to find DomR VIF to associate/disassociate IP with."; + s_logger.error(msg); + throw new InternalErrorException(msg); + } +*/ + String args = null; + + if (add) { + args = " -A "; + } else { + args = " -D "; + } + + if (sourceNat) { + args += " -s "; + } + if (firstIP) { + args += " -f "; + } + String cidrSize = Long.toString(NetUtils.getCidrSize(vlanNetmask)); + args += " -l "; + args += publicIpAddress + "/" + cidrSize; + + args += " -c "; + args += "eth" +"2"; // currently hardcoding to eth 2 (which is default public ipd)//publicNicInfo.first(); + + args += " -g "; + args += vlanGateway; + + if (addVif) { + args += " -n "; + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Run command on domain router " + privateIpAddress + ", /opt/cloud/bin/ipassoc.sh " + args); + } + + Pair result = SshHelper.sshExecute(privateIpAddress, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "/opt/cloud/bin/ipassoc.sh " + args); + + if (!result.first()) { + s_logger.error("ipassoc command on domain router " + privateIpAddress + " failed. message: " + result.second()); + throw new Exception("ipassoc failed due to " + result.second()); + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("ipassoc command on domain router " + privateIpAddress + " completed"); + } + } + + protected Answer execute(GetDomRVersionCmd cmd) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Executing resource GetDomRVersionCmd: " + s_gson.toJson(cmd)); + s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /opt/cloud/bin/get_template_version.sh "); + } + + Pair result; + try { + String controlIp = getRouterSshControlIp(cmd); + result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, + "/opt/cloud/bin/get_template_version.sh "); + + if (!result.first()) { + s_logger.error("GetDomRVersionCmd on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " failed, message: " + result.second()); + + return new GetDomRVersionAnswer(cmd, "GetDomRVersionCmd failed due to " + result.second()); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("GetDomRVersionCmd on domain router " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " completed"); + } + } catch (Throwable e) { + String msg = "GetDomRVersionCmd failed due to " + e; + s_logger.error(msg, e); + return new GetDomRVersionAnswer(cmd, msg); + } + String[] lines = result.second().split("&"); + if (lines.length != 2) { + return new GetDomRVersionAnswer(cmd, result.second()); + } + return new GetDomRVersionAnswer(cmd, result.second(), lines[0], lines[1]); + } + + + private static String getRouterSshControlIp(NetworkElementCommand cmd) { + String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP); + String routerGuestIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP); + String zoneNetworkType = cmd.getAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE); + + if(routerGuestIp != null && zoneNetworkType != null && NetworkType.valueOf(zoneNetworkType) == NetworkType.Basic) { + if(s_logger.isDebugEnabled()) + s_logger.debug("In Basic zone mode, use router's guest IP for SSH control. guest IP : " + routerGuestIp); + + return routerGuestIp; + } + + if(s_logger.isDebugEnabled()) + s_logger.debug("Use router's private IP for SSH control. IP : " + routerIp); + return routerIp; + } + + protected Answer execute(NetworkUsageCommand cmd) { + if ( cmd.isForVpc() ) { + //return VPCNetworkUsage(cmd); + } + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource NetworkUsageCommand "+ s_gson.toJson(cmd)); + } + if(cmd.getOption()!=null && cmd.getOption().equals("create") ){ + String result = networkUsage(cmd.getPrivateIP(), "create", null); + NetworkUsageAnswer answer = new NetworkUsageAnswer(cmd, "true", 0L, 0L); + return answer; + } + long[] stats = getNetworkStats(cmd.getPrivateIP()); + + NetworkUsageAnswer answer = new NetworkUsageAnswer(cmd, "", stats[0], stats[1]); + return answer; + } + private long[] getNetworkStats(String privateIP) { + String result = networkUsage(privateIP, "get", null); + long[] stats = new long[2]; + if (result != null) { + try { + String[] splitResult = result.split(":"); + int i = 0; + while (i < splitResult.length - 1) { + stats[0] += (new Long(splitResult[i++])).longValue(); + stats[1] += (new Long(splitResult[i++])).longValue(); + } + } catch (Throwable e) { + s_logger.warn("Unable to parse return from script return of network usage command: " + e.toString(), e); + } + } + return stats; + } + + + protected CheckSshAnswer execute(CheckSshCommand cmd) { + String vmName = cmd.getName(); + String privateIp = cmd.getIp(); + int cmdPort = cmd.getPort(); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Ping command port, " + privateIp + ":" + cmdPort); + } + + try { + String result = connect(cmd.getName(), privateIp, cmdPort); + if (result != null) { + s_logger.error("Can not ping System vm " + vmName + "due to:" + result); + return new CheckSshAnswer(cmd, "Can not ping System vm " + vmName + "due to:" + result); + } + } catch (Exception e) { + s_logger.error("Can not ping System vm " + vmName + "due to exception"); + return new CheckSshAnswer(cmd, e); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Ping command port succeeded for vm " + vmName); + } + + if (VirtualMachineName.isValidRouterName(vmName)) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Execute network usage setup command on " + vmName); + } + networkUsage(privateIp, "create", null); + } + + return new CheckSshAnswer(cmd); + } + + + protected String networkUsage(final String privateIpAddress, final String option, final String ethName) { + String args = null; + if (option.equals("get")) { + args = "-g"; + } else if (option.equals("create")) { + args = "-c"; + } else if (option.equals("reset")) { + args = "-r"; + } else if (option.equals("addVif")) { + args = "-a"; + args += ethName; + } else if (option.equals("deleteVif")) { + args = "-d"; + args += ethName; + } + + try { + if (s_logger.isTraceEnabled()) { + s_logger.trace("Executing /opt/cloud/bin/netusage.sh " + args + " on DomR " + privateIpAddress); + } + + Pair result = SshHelper.sshExecute(privateIpAddress, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "/opt/cloud/bin/netusage.sh " + args); + + if (!result.first()) { + return null; + } + + return result.second(); + } catch (Throwable e) { + s_logger.error("Unable to execute NetworkUsage command on DomR (" + privateIpAddress + "), domR may not be ready yet. failure due to " + + e); + } + return null; } + private File getSystemVMPatchIsoFile() { + // locate systemvm.iso + URL url = this.getClass().getClassLoader().getResource("vms/systemvm.iso"); + File isoFile = null; + if (url != null) { + isoFile = new File(url.getPath()); + } + + if(isoFile == null || !isoFile.exists()) { + isoFile = new File("/usr/share/cloudstack-common/vms/systemvm.iso"); + } + + assert(isoFile != null); + if(!isoFile.exists()) { + s_logger.error("Unable to locate systemvm.iso in your setup at " + isoFile.toString()); + } + return isoFile; + } + + public File getSystemVMKeyFile() { + URL url = this.getClass().getClassLoader().getResource("scripts/vm/systemvm/id_rsa.cloud"); + File keyFile = null; + if ( url != null ){ + keyFile = new File(url.getPath()); + } + if (keyFile == null || !keyFile.exists()) { + keyFile = new File("/usr/share/cloudstack-common/scripts/vm/systemvm/id_rsa.cloud"); + } + assert(keyFile != null); + if(!keyFile.exists()) { + s_logger.error("Unable to locate id_rsa.cloud in your setup at " + keyFile.toString()); + } + return keyFile; + } + + public static String postHttpRequest(final String jsonCmd, final URI agentUri) { // Using Apache's HttpClient for HTTP POST @@ -450,5 +1721,53 @@ public class HypervDirectConnectResource extends ServerResourceBase implements public void setRunLevel(final int level) { // TODO Auto-generated method stub } + protected String connect(final String vmName, final String ipAddress, final int port) { + long startTick = System.currentTimeMillis(); + + // wait until we have at least been waiting for _ops_timeout time or + // at least have tried _retry times, this is to coordinate with system + // VM patching/rebooting time that may need + int retry = _retry; + while (System.currentTimeMillis() - startTick <= _ops_timeout || --retry > 0) { + SocketChannel sch = null; + try { + s_logger.info("Trying to connect to " + ipAddress); + sch = SocketChannel.open(); + sch.configureBlocking(true); + sch.socket().setSoTimeout(5000); + + InetSocketAddress addr = new InetSocketAddress(ipAddress, port); + sch.connect(addr); + return null; + } catch (IOException e) { + s_logger.info("Could not connect to " + ipAddress + " due to " + e.toString()); + if (e instanceof ConnectException) { + // if connection is refused because of VM is being started, + // we give it more sleep time + // to avoid running out of retry quota too quickly + try { + Thread.sleep(5000); + } catch (InterruptedException ex) { + } + } + } finally { + if (sch != null) { + try { + sch.close(); + } catch (IOException e) { + } + } + } + + try { + Thread.sleep(1000); + } catch (InterruptedException ex) { + } + } + + s_logger.info("Unable to logon to " + ipAddress); + + return "Unable to connect"; + } } diff --git a/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index d916d454422..b304a008a09 100644 --- a/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -186,6 +186,7 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore Object localStorage = dsInfos.get("localStorage"); if (localStorage != null) { hostPath = hostPath.replaceFirst("/", ""); + hostPath = hostPath.replace("+"," "); } String userInfo = uri.getUserInfo(); int port = uri.getPort(); diff --git a/systemvm/patches/debian/config/etc/init.d/cloud-early-config b/systemvm/patches/debian/config/etc/init.d/cloud-early-config index d44f94d7d17..d4bf0eb3d92 100755 --- a/systemvm/patches/debian/config/etc/init.d/cloud-early-config +++ b/systemvm/patches/debian/config/etc/init.d/cloud-early-config @@ -130,12 +130,11 @@ get_boot_params() { vmware) vmtoolsd --cmd 'machine.id.get' > /var/cache/cloud/cmdline ;; - virtualpc) - # Hyper-V is recognized as virtualpc hypervisor type. Boot args are passed in the NTFS data-disk - mkdir -p $EXTRA_MOUNT - mount -t ntfs /dev/sdb1 $EXTRA_MOUNT - cp -f $EXTRA_MOUNT/cmdline /var/cache/cloud/cmdline - umount $EXTRA_MOUNT + virtualpc|hyperv) + # Hyper-V is recognized as virtualpc hypervisor type. Boot args are passed using KVP Daemon + #waiting for the hv_kvp_daemon to start up + #sleep 30 need to fix the race condition of hv_kvp_daemon and cloud-early-config + cp -f /var/opt/hyperv/.kvp_pool_0 /var/cache/cloud/cmdline ;; esac @@ -157,6 +156,10 @@ patch() { cdrom_dev=/dev/cdrom elif [ -e /dev/cdrom1 ]; then cdrom_dev=/dev/cdrom1 + elif [ -e /dev/cdrom2 ]; then + cdrom_dev=/dev/cdrom2 + elif [ -e /dev/cdrom3 ]; then + cdrom_dev=/dev/cdrom3 fi [ -f /var/cache/cloud/authorized_keys ] && privkey=/var/cache/cloud/authorized_keys @@ -718,7 +721,7 @@ setup_dnsmasq() { [ $ETH0_IP ] && echo "dhcp-option=6,$NS" >> /etc/dnsmasq.conf [ $ETH0_IP6 ] && echo "dhcp-option=option6:dns-server,$NS6" >> /etc/dnsmasq.conf #adding the name data-server to the /etc/hosts for allowing the access to user-data service and ssh-key reset in every subnet. - //removing the existing entires to avoid duplicates on restarts. +#removing the existing entires to avoid duplicates on restarts. sed -i '/data-server/d' /etc/hosts if [ -n "$ETH0_IP" ] then