From ff1c6e78f457f346bf888491e34c5216d0f7d9d6 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 28 Jan 2020 06:05:59 +0100 Subject: [PATCH] router: Set up metadata/password/dhcp server on gateway IP instead of guest IP in RVR (#3477) When we create a vm in the network with redundant VRs, the lease file in the vm (for example /var/lib/dhcp/dhclient.eth0.leases) shows the dhcp-server-identifier is the guest ip (not vip/gateway) of master VR. That's the ip ipaddress where the vm fetch password and metadata from. if we stop the master VR (then backup will be master) or restart the network with cleanup (VRs will be created), the guest ip of master VR changes so vm are not able to get metadata/ssh-key using the ips in dhcp lease file. Setting up metadata/password/dhcp server on gateway instead of guest IP in redundant VRs will fix the issues. FIxes #3409 --- systemvm/debian/etc/apache2/vhost.template | 10 +++--- systemvm/debian/opt/cloud/bin/configure.py | 21 +++++++++--- systemvm/debian/opt/cloud/bin/cs/CsAddress.py | 4 +++ systemvm/debian/opt/cloud/bin/cs/CsApp.py | 15 ++++++--- systemvm/debian/opt/cloud/bin/cs/CsDhcp.py | 32 +++++++++++++------ systemvm/debian/opt/cloud/bin/cs/CsFile.py | 14 ++++++++ systemvm/debian/opt/cloud/bin/cs/CsHelper.py | 2 +- .../debian/opt/cloud/bin/cs/CsRedundant.py | 14 +++----- .../debian/opt/cloud/bin/passwd_server_ip.py | 11 +++++-- .../debian/opt/cloud/bin/setup/secstorage.sh | 8 ++--- 10 files changed, 91 insertions(+), 40 deletions(-) diff --git a/systemvm/debian/etc/apache2/vhost.template b/systemvm/debian/etc/apache2/vhost.template index 688239cd8c0..0226bb44eee 100644 --- a/systemvm/debian/etc/apache2/vhost.template +++ b/systemvm/debian/etc/apache2/vhost.template @@ -1,4 +1,4 @@ - + ServerAdmin webmaster@localhost DocumentRoot /var/www/html @@ -42,7 +42,7 @@ - + ServerAdmin webmaster@localhost DocumentRoot /var/www/html @@ -227,14 +227,14 @@ # Debian etch). See /usr/share/doc/apache2.2-common/NEWS.Debian.gz and # README.Debian.gz -Listen 10.1.1.1:80 +Listen 10.1.1.1:8180 # Server Name Indication for SSL named virtual hosts is currently not # supported by MSIE on Windows XP. - Listen 10.1.1.1:443 + Listen 10.1.1.1:8443 - Listen 10.1.1.1:443 + Listen 10.1.1.1:8443 diff --git a/systemvm/debian/opt/cloud/bin/configure.py b/systemvm/debian/opt/cloud/bin/configure.py index 1e0c057a871..cd0ef350ecb 100755 --- a/systemvm/debian/opt/cloud/bin/configure.py +++ b/systemvm/debian/opt/cloud/bin/configure.py @@ -58,11 +58,22 @@ class CsPassword(CsDataBag): except IOError: logging.debug("File %s does not exist" % self.TOKEN_FILE) - ips_cmd = "ip addr show | grep inet | awk '{print $2}'" - ips = CsHelper.execute(ips_cmd) - for ip in ips: - server_ip = ip.split('/')[0] - proc = CsProcess(['/opt/cloud/bin/passwd_server_ip.py', server_ip]) + server_ip = None + guest_ip = None + for interface in self.config.address().get_interfaces(): + if interface.ip_in_subnet(vm_ip): + if self.config.cl.is_redundant(): + server_ip = interface.get_gateway() + guest_ip = interface.get_ip() + else: + server_ip = interface.get_ip() + break + + if server_ip is not None: + if guest_ip is None: + proc = CsProcess(['/opt/cloud/bin/passwd_server_ip.py', server_ip]) + else: + proc = CsProcess(['/opt/cloud/bin/passwd_server_ip.py', server_ip + "," + guest_ip]) if proc.find(): url = "http://%s:8080/" % server_ip payload = {"ip": vm_ip, "password": password, "token": token} diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py index 8e678251fe3..ba9ee082740 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -661,6 +661,10 @@ class CsIP: if not found: self.delete(ip) + def get_gateway(self): + interface = CsInterface(self.address, self.config) + return interface.get_gateway() + def is_guest_gateway(self, bag, ip): """ Exclude the vrrp maintained addresses on a redundant router """ interface = CsInterface(bag, self.config) diff --git a/systemvm/debian/opt/cloud/bin/cs/CsApp.py b/systemvm/debian/opt/cloud/bin/cs/CsApp.py index 575ab2ac5b2..9f3375f7980 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsApp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsApp.py @@ -25,6 +25,7 @@ class CsApp: def __init__(self, ip): self.dev = ip.getDevice() self.ip = ip.get_ip_address() + self.gateway = ip.get_gateway() self.type = ip.get_type() self.fw = ip.fw self.config = ip.config @@ -44,10 +45,16 @@ class CsApache(CsApp): "/etc/apache2/sites-enabled/vhost-%s.conf" % self.ip) file = CsFile("/etc/apache2/sites-enabled/vhost-%s.conf" % (self.ip)) - file.search("", "\t" % (self.ip)) - file.search("", "\t" % (self.ip)) - file.search("Listen .*:80", "Listen %s:80" % (self.ip)) - file.search("Listen .*:443", "Listen %s:443" % (self.ip)) + if not self.config.cl.is_redundant(): + file.replaceIfFound("", "" % (self.ip)) + file.replaceIfFound("", "\t" % (self.ip)) + file.replaceIfFound("Listen .*:8180", "Listen %s:80" % (self.ip)) + file.replaceIfFound("Listen .*:8443", "Listen %s:443" % (self.ip)) + else: + file.replaceIfFound("", "" % (self.ip, self.gateway)) + file.replaceIfFound("", "\t" % (self.ip, self.gateway)) + file.replaceIfFound("Listen .*:8180", "Listen %s:80\nListen %s:80" % (self.ip, self.gateway)) + file.replaceIfFound("Listen .*:8443", "Listen %s:443\nListen %s:443" % (self.ip, self.gateway)) file.search("ServerName.*", "\tServerName %s.%s" % (self.config.cl.get_type(), self.config.get_domain())) if file.is_changed(): file.commit() diff --git a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py index 72c5daec0cc..d0b40e52de5 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py @@ -77,15 +77,25 @@ class CsDhcp(CsDataBag): def configure_server(self): # self.conf.addeq("dhcp-hostsfile=%s" % DHCP_HOSTS) idx = 0 + listen_address = ["127.0.0.1"] for i in self.devinfo: if not i['dnsmasq']: continue device = i['dev'] ip = i['ip'].split('/')[0] - sline = "dhcp-range=set:interface-%s-%s" % (device, idx) - line = "dhcp-range=set:interface-%s-%s,%s,static" % (device, idx, ip) - self.conf.search(sline, line) gn = CsGuestNetwork(device, self.config) + # Gateway + gateway = '' + if self.config.is_vpc(): + gateway = gn.get_gateway() + else: + gateway = i['gateway'] + sline = "dhcp-range=set:interface-%s-%s" % (device, idx) + if self.cl.is_redundant(): + line = "dhcp-range=set:interface-%s-%s,%s,static" % (device, idx, gateway) + else: + line = "dhcp-range=set:interface-%s-%s,%s,static" % (device, idx, ip) + self.conf.search(sline, line) sline = "dhcp-option=tag:interface-%s-%s,15" % (device, idx) line = "dhcp-option=tag:interface-%s-%s,15,%s" % (device, idx, gn.get_domain()) self.conf.search(sline, line) @@ -95,12 +105,6 @@ class CsDhcp(CsDataBag): dns_list = [x for x in gn.get_dns() if x] line = "dhcp-option=tag:interface-%s-%s,6,%s" % (device, idx, ','.join(dns_list)) self.conf.search(sline, line) - # Gateway - gateway = '' - if self.config.is_vpc(): - gateway = gn.get_gateway() - else: - gateway = i['gateway'] if gateway != '0.0.0.0': sline = "dhcp-option=tag:interface-%s-%s,3," % (device, idx) line = "dhcp-option=tag:interface-%s-%s,3,%s" % (device, idx, gateway) @@ -114,8 +118,18 @@ class CsDhcp(CsDataBag): sline = "dhcp-option=tag:interface-%s-%s,1," % (device, idx) line = "dhcp-option=tag:interface-%s-%s,1,%s" % (device, idx, netmask) self.conf.search(sline, line) + # Listen Address + if self.cl.is_redundant(): + listen_address.append(gateway) + else: + listen_address.append(ip) idx += 1 + # Listen Address + sline = "listen-address=" + line = "listen-address=%s" % (','.join(listen_address)) + self.conf.search(sline, line) + def delete_leases(self): macs_dhcphosts = [] try: diff --git a/systemvm/debian/opt/cloud/bin/cs/CsFile.py b/systemvm/debian/opt/cloud/bin/cs/CsFile.py index f3b2a272631..2ee631a89d6 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsFile.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsFile.py @@ -116,6 +116,20 @@ class CsFile: logging.debug("Searching for %s and replacing with %s" % (search, replace)) self.new_config = [w.replace(search, replace) for w in self.new_config] + def replaceIfFound(self, search, replace): + found = False + replace_filtered = replace + if re.search("PSK \"", replace): + replace_filtered = re.sub(r'".*"', '"****"', replace) + logging.debug("Searching for %s and replacing with %s if found" % (search, replace_filtered)) + for index, line in enumerate(self.new_config): + if line.lstrip().startswith("#"): + continue + if re.search(search, line): + if replace not in line: + self.new_config[index] = replace + "\n" + return False + def search(self, search, replace): found = False replace_filtered = replace diff --git a/systemvm/debian/opt/cloud/bin/cs/CsHelper.py b/systemvm/debian/opt/cloud/bin/cs/CsHelper.py index 278e1e6544f..e29891796b8 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsHelper.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsHelper.py @@ -113,7 +113,7 @@ def bool_to_yn(val): def get_device_info(): """ Returns all devices on system with their ipv4 ip netmask """ list = [] - for i in execute("ip addr show"): + for i in execute("ip addr show |grep -v secondary"): vals = i.strip().lstrip().rstrip().split() if vals[0] == "inet": to = {} diff --git a/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py b/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py index 25a4a1a9438..441723b29a6 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py @@ -245,8 +245,7 @@ class CsRedundant(object): interfaces = [interface for interface in self.address.get_interfaces() if interface.needs_vrrp()] for interface in interfaces: - CsPasswdSvc(interface.get_ip()).stop() - CsPasswdSvc(interface.get_gateway()).stop() + CsPasswdSvc(interface.get_gateway() + "," + interface.get_ip()).stop() self.cl.set_fault_state() self.cl.save() @@ -282,8 +281,7 @@ class CsRedundant(object): interfaces = [interface for interface in self.address.get_interfaces() if interface.needs_vrrp()] for interface in interfaces: - CsPasswdSvc(interface.get_ip()).stop() - CsPasswdSvc(interface.get_gateway()).stop() + CsPasswdSvc(interface.get_gateway() + "," + interface.get_ip()).stop() CsHelper.service("dnsmasq", "stop") @@ -341,8 +339,7 @@ class CsRedundant(object): interfaces = [interface for interface in self.address.get_interfaces() if interface.needs_vrrp()] for interface in interfaces: - CsPasswdSvc(interface.get_ip()).restart() - CsPasswdSvc(interface.get_gateway()).restart() + CsPasswdSvc(interface.get_gateway() + "," + interface.get_ip()).restart() CsHelper.service("dnsmasq", "restart") self.cl.set_master_state(True) @@ -408,9 +405,6 @@ class CsRedundant(object): cmdline = self.config.get_cmdline_instance() if not interface.is_added(): continue - if cmdline.get_type() == 'router': - str = " %s brd %s dev %s\n" % (cmdline.get_guest_gw(), interface.get_broadcast(), interface.get_device()) - else: - str = " %s brd %s dev %s\n" % (interface.get_gateway_cidr(), interface.get_broadcast(), interface.get_device()) + str = " %s brd %s dev %s\n" % (interface.get_gateway_cidr(), interface.get_broadcast(), interface.get_device()) lines.append(str) return lines diff --git a/systemvm/debian/opt/cloud/bin/passwd_server_ip.py b/systemvm/debian/opt/cloud/bin/passwd_server_ip.py index fc84910a117..85d76f65166 100755 --- a/systemvm/debian/opt/cloud/bin/passwd_server_ip.py +++ b/systemvm/debian/opt/cloud/bin/passwd_server_ip.py @@ -40,6 +40,7 @@ from SocketServer import ThreadingMixIn #, ForkingMixIn passMap = {} secureToken = None listeningAddress = '127.0.0.1' +allowAddresses = ['localhost', '127.0.0.1'] lock = threading.RLock() def getTokenFile(): @@ -139,7 +140,7 @@ class PasswordRequestHandler(BaseHTTPRequestHandler): self.send_response(200) self.end_headers() clientAddress = self.client_address[0] - if clientAddress not in ['localhost', '127.0.0.1', listeningAddress]: + if clientAddress not in allowAddresses: syslog.syslog('serve_password: non-localhost IP trying to save password: %s' % clientAddress) self.send_response(403) return @@ -170,8 +171,14 @@ def serve(HandlerClass = PasswordRequestHandler, ServerClass = ThreadedHTTPServer): global listeningAddress + global allowAddresses if len(sys.argv) > 1: - listeningAddress = sys.argv[1] + addresses = sys.argv[1].split(",") + if len(addresses) > 0: + listeningAddress = addresses[0] + allowAddresses.append(addresses[0]) + if len(addresses) > 1: + allowAddresses.append(addresses[1]) server_address = (listeningAddress, 8080) passwordServer = ServerClass(server_address, HandlerClass) diff --git a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh index d3a6d21cf65..13ed5c5d0ae 100755 --- a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh +++ b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh @@ -49,10 +49,10 @@ setup_secstorage() { setup_apache2 $ETH2_IP # Deprecated, should move to Cs Python all of it - sed -e "s///" \ - -e "s///" \ - -e "s/Listen .*:80/Listen $ETH2_IP:80/g" \ - -e "s/Listen .*:443/Listen $ETH2_IP:443/g" /etc/apache2/vhost.template > /etc/apache2/sites-enabled/vhost-${ETH2_IP}.conf + sed -e "s///" \ + -e "s///" \ + -e "s/Listen .*:8180/Listen $ETH2_IP:80/g" \ + -e "s/Listen .*:8443/Listen $ETH2_IP:443/g" /etc/apache2/vhost.template > /etc/apache2/sites-enabled/vhost-${ETH2_IP}.conf log_it "Setting up apache2 for post upload of volume/template" a2enmod proxy