Fixes to enable test to succeed. Small change to Marvin to be able to override retries

Signed-off-by: wilderrodrigues <wrodrigues@schubergphilis.com>
This commit is contained in:
Ian Southam 2015-07-02 08:19:15 -04:00 committed by wilderrodrigues
parent 1a93d700dd
commit b7b54f1680
4 changed files with 52 additions and 30 deletions

View File

@ -39,6 +39,7 @@ from cs.CsFile import CsFile
from cs.CsApp import CsApache, CsDnsmasq from cs.CsApp import CsApache, CsDnsmasq
from cs.CsMonitor import CsMonitor from cs.CsMonitor import CsMonitor
from cs.CsLoadBalancer import CsLoadBalancer from cs.CsLoadBalancer import CsLoadBalancer
from cs.CsConfig import CsConfig
class CsPassword(CsDataBag): class CsPassword(CsDataBag):

View File

@ -48,11 +48,14 @@ class CsAddress(CsDataBag):
def get_guest_if(self): def get_guest_if(self):
""" """
Return CsInterface object for the first guest interface Return CsInterface object for the lowest guest interface
""" """
ipr = []
for ip in self.get_ips(): for ip in self.get_ips():
if ip.is_guest(): if ip.is_guest():
return ip ipr.append(ip)
if len(ipr) > 0:
return sorted(ipr)[-1]
return None return None
def get_guest_ip(self): def get_guest_ip(self):
@ -407,10 +410,10 @@ class CsIP:
]) ])
if self.get_type() in ["public"]: if self.get_type() in ["public"]:
self.fw.append(["nat", "front", # self.fw.append(["nat", "front",
"-A POSTROUTING -o %s -j SNAT --to-source %s" % # "-A POSTROUTING -o %s -j SNAT --to-source %s" %
(self.dev, self.address['public_ip']) # (self.dev, self.address['public_ip'])
]) # ])
self.fw.append(["", "front", self.fw.append(["", "front",
"-A FORWARD -o %s -d %s -j ACL_INBOUND_%s" % (self.dev, self.address['network'], self.dev) "-A FORWARD -o %s -d %s -j ACL_INBOUND_%s" % (self.dev, self.address['network'], self.dev)
]) ])

View File

@ -30,16 +30,13 @@
# eth1 public ip # eth1 public ip
# eth2+ Guest networks # eth2+ Guest networks
# -------------------------------------------------------------------- # # -------------------------------------------------------------------- #
import sys
import os import os
from pprint import pprint
from CsDatabag import CsDataBag, CsCmdLine
import logging import logging
import CsHelper import CsHelper
from CsFile import CsFile from CsFile import CsFile
from CsConfig import CsConfig
from CsProcess import CsProcess from CsProcess import CsProcess
from CsApp import CsPasswdSvc from CsApp import CsPasswdSvc
from CsAddress import CsDevice
import socket import socket
from time import sleep from time import sleep
@ -64,6 +61,7 @@ class CsRedundant(object):
def __init__(self, config): def __init__(self, config):
self.cl = config.cmdline() self.cl = config.cmdline()
self.address = config.address() self.address = config.address()
self.config = config
def set(self): def set(self):
logging.debug("Router redundancy status is %s", self.cl.is_redundant()) logging.debug("Router redundancy status is %s", self.cl.is_redundant())
@ -83,13 +81,11 @@ class CsRedundant(object):
def _redundant_on(self): def _redundant_on(self):
guest = self.address.get_guest_if() guest = self.address.get_guest_if()
# No redundancy if there is no guest network # No redundancy if there is no guest network
if self.cl.is_master() or guest is None:
for obj in [o for o in self.address.get_ips() if o.is_public()]:
self.check_is_up(obj.get_device())
if guest is None: if guest is None:
self._redundant_off() self._redundant_off()
# Bring up the public Interface(s)
if self.cl.is_master():
for obj in [o for o in self.address.get_ips() if o.is_public()]:
print obj.get_device()
self.check_is_up(obj.get_device())
return return
CsHelper.mkdir(self.CS_RAMDISK_DIR, 0755, False) CsHelper.mkdir(self.CS_RAMDISK_DIR, 0755, False)
CsHelper.mount_tmpfs(self.CS_RAMDISK_DIR) CsHelper.mount_tmpfs(self.CS_RAMDISK_DIR)
@ -159,6 +155,12 @@ class CsRedundant(object):
if not proc.find(): if not proc.find():
CsHelper.service("keepalived", "restart") CsHelper.service("keepalived", "restart")
def release_lock(self):
try:
os.remove("/tmp/master_lock")
except OSError:
pass
def set_lock(self): def set_lock(self):
""" """
Make sure that master state changes happen sequentially Make sure that master state changes happen sequentially
@ -169,21 +171,21 @@ class CsRedundant(object):
for iter in range(0, iterations): for iter in range(0, iterations):
try: try:
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.bind('\0master_lock') s.bind('/tmp/master_lock')
return s return s
except socket.error, e: except socket.error, e:
error_code = e.args[0] error_code = e.args[0]
error_string = e.args[1] error_string = e.args[1]
print "Process already running (%d:%s). Exiting" % (error_code, error_string) print "Process already running (%d:%s). Exiting" % (error_code, error_string)
logging.info("Master is already running, waiting") logging.info("Master is already running, waiting")
sleep(1) sleep(time_between)
def set_fault(self): def set_fault(self):
""" Set fault mode on this router """ """ Set fault mode on this router """
if not self.cl.is_redundant(): if not self.cl.is_redundant():
logging.error("Set fault called on non-redundant router") logging.error("Set fault called on non-redundant router")
return return
s = self.set_lock() self.set_lock()
logging.info("Router switched to fault mode") logging.info("Router switched to fault mode")
ads = [o for o in self.address.get_ips() if o.is_public()] ads = [o for o in self.address.get_ips() if o.is_public()]
for o in ads: for o in ads:
@ -195,9 +197,10 @@ class CsRedundant(object):
CsHelper.service("dnsmasq", "stop") CsHelper.service("dnsmasq", "stop")
ads = [o for o in self.address.get_ips() if o.needs_vrrp()] ads = [o for o in self.address.get_ips() if o.needs_vrrp()]
for o in ads: for o in ads:
pwdsvc = CsPasswdSvc(o.get_gateway()).stop() CsPasswdSvc(o.get_gateway()).stop()
self.cl.set_fault_state() self.cl.set_fault_state()
self.cl.save() self.cl.save()
self.release_lock()
logging.info("Router switched to fault mode") logging.info("Router switched to fault mode")
def set_backup(self): def set_backup(self):
@ -210,22 +213,29 @@ class CsRedundant(object):
logging.error("Set backup called on node that is already backup") logging.error("Set backup called on node that is already backup")
return return
""" """
s = self.set_lock() self.set_lock()
logging.debug("Setting router to backup") logging.debug("Setting router to backup")
ads = [o for o in self.address.get_ips() if o.is_public()] ads = [o for o in self.address.get_ips() if o.is_public()]
dev = ''
for o in ads: for o in ads:
CsHelper.execute("ifconfig %s down" % o.get_device()) if dev == o.get_device():
continue
logging.info("Bringing public interface %s down" % o.get_device())
cmd2 = "ip link set %s up" % o.get_device()
CsHelper.execute(cmd2)
dev = o.get_device()
cmd = "%s -C %s" % (self.CONNTRACKD_BIN, self.CONNTRACKD_CONF) cmd = "%s -C %s" % (self.CONNTRACKD_BIN, self.CONNTRACKD_CONF)
CsHelper.execute("%s -d" % cmd) CsHelper.execute("%s -d" % cmd)
CsHelper.service("ipsec", "stop") CsHelper.service("ipsec", "stop")
CsHelper.service("xl2tpd", "stop") CsHelper.service("xl2tpd", "stop")
ads = [o for o in self.address.get_ips() if o.needs_vrrp()] ads = [o for o in self.address.get_ips() if o.needs_vrrp()]
for o in ads: for o in ads:
pwdsvc = CsPasswdSvc(o.get_gateway()).stop() CsPasswdSvc(o.get_gateway()).stop()
CsHelper.service("dnsmasq", "stop") CsHelper.service("dnsmasq", "stop")
# self._set_priority(self.CS_PRIO_DOWN) # self._set_priority(self.CS_PRIO_DOWN)
self.cl.set_master_state(False) self.cl.set_master_state(False)
self.cl.save() self.cl.save()
self.release_lock()
logging.info("Router switched to backup mode") logging.info("Router switched to backup mode")
def set_master(self): def set_master(self):
@ -238,15 +248,20 @@ class CsRedundant(object):
logging.error("Set master called on master node") logging.error("Set master called on master node")
return return
""" """
s = self.set_lock() self.set_lock()
logging.debug("Setting router to master") logging.debug("Setting router to master")
ads = [o for o in self.address.get_ips() if o.is_public()] ads = [o for o in self.address.get_ips() if o.is_public()]
dev = ''
for o in ads: for o in ads:
# cmd2 = "ip link set %s up" % self.getDevice() if dev == o.get_device():
CsHelper.execute("ifconfig %s down" % o.get_device()) continue
CsHelper.execute("ifconfig %s up" % o.get_device()) cmd2 = "ip link set %s up" % o.get_device()
CsHelper.execute("arping -I %s -A %s -c 1" % (o.get_device(), o.get_ip())) if CsDevice(o.get_device(), self.config).waitfordevice():
# FIXME Need to add in the default routes but I am unsure what the gateway is CsHelper.execute(cmd2)
dev = o.get_device()
logging.info("Bringing public interface %s up" % o.get_device())
else:
logging.error("Device %s was not ready could not bring it up" % o.get_device())
# ip route add default via $gw table Table_$dev proto static # ip route add default via $gw table Table_$dev proto static
cmd = "%s -C %s" % (self.CONNTRACKD_BIN, self.CONNTRACKD_CONF) cmd = "%s -C %s" % (self.CONNTRACKD_BIN, self.CONNTRACKD_CONF)
CsHelper.execute("%s -c" % cmd) CsHelper.execute("%s -c" % cmd)
@ -257,10 +272,11 @@ class CsRedundant(object):
CsHelper.service("xl2tpd", "restart") CsHelper.service("xl2tpd", "restart")
ads = [o for o in self.address.get_ips() if o.needs_vrrp()] ads = [o for o in self.address.get_ips() if o.needs_vrrp()]
for o in ads: for o in ads:
pwdsvc = CsPasswdSvc(o.get_gateway()).restart() CsPasswdSvc(o.get_gateway()).restart()
CsHelper.service("dnsmasq", "restart") CsHelper.service("dnsmasq", "restart")
self.cl.set_master_state(True) self.cl.set_master_state(True)
self.cl.save() self.cl.save()
self.release_lock()
logging.info("Router switched to master mode") logging.info("Router switched to master mode")
def _collect_ignore_ips(self): def _collect_ignore_ips(self):

View File

@ -551,7 +551,7 @@ class VirtualMachine:
def get_ssh_client( def get_ssh_client(
self, ipaddress=None, reconnect=False, port=None, self, ipaddress=None, reconnect=False, port=None,
keyPairFileLocation=None): keyPairFileLocation=None, retries=20):
"""Get SSH object of VM""" """Get SSH object of VM"""
# If NAT Rules are not created while VM deployment in Advanced mode # If NAT Rules are not created while VM deployment in Advanced mode
@ -570,6 +570,7 @@ class VirtualMachine:
self.ssh_port, self.ssh_port,
self.username, self.username,
self.password, self.password,
retries=retries,
keyPairFileLocation=keyPairFileLocation keyPairFileLocation=keyPairFileLocation
) )
self.ssh_client = self.ssh_client or is_server_ssh_ready( self.ssh_client = self.ssh_client or is_server_ssh_ready(
@ -577,6 +578,7 @@ class VirtualMachine:
self.ssh_port, self.ssh_port,
self.username, self.username,
self.password, self.password,
retries=retries,
keyPairFileLocation=keyPairFileLocation keyPairFileLocation=keyPairFileLocation
) )
return self.ssh_client return self.ssh_client