From 0f75042566b7cc72666708485bdaa9337721e9fe Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Mon, 1 Feb 2016 15:57:26 +0100 Subject: [PATCH 01/11] Add iptables copnversion script. Source: https://raw.githubusercontent.com/sl0/conv/master/iptables_converter.py --- .../config/opt/cloud/bin/cs_iptables_save.py | 267 ++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py new file mode 100644 index 00000000000..4b64c2b1da9 --- /dev/null +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py @@ -0,0 +1,267 @@ +#!/usr/bin/python +# +# -*- coding: utf-8 -*- +# +""" +iptables_converter.py: + convert iptables commands within a script + into a correspondig iptables-save script + + default filename to read is rules, to read some other + file, append: -s filename + + output is written to stdout for maximum flexibilty + +Author: Johannes Hubertz +Date: 2015-03-17 +version: 0.9.8 +License: GNU General Public License version 3 or later + +Have Fun! +""" + +try: + from collections import UserDict +except ImportError: + from UserDict import UserDict +from optparse import OptionParser +import re +import sys + + +class ConverterError(): + """on accidential case of error show given reason""" + + def __init__(self, message): + """message to stdout to compatible testings 2.7 and 3.4""" + print (message) + sys.exit(1) + + +class Chains(UserDict): + """this is for one type of tables""" + + def __init__(self, name, tables): + """init Chains object""" + UserDict.__init__(self) + self.name = name + self.tables = tables + self.predef = tables + self.reset() # name, tables) + + def put_into_fgr(self, content): + """fill this line into this tabular""" + self.length += 1 + cha = "filter" + # act = "" + liste = content.split() + action = liste[0] + if "-t" in action: + liste.pop(0) # remove 1st: -t + fname = liste.pop(0) + legals = ["filter", "nat", "raw", "mangle"] + if fname not in legals: + msg = "Valid is one of %s, got: %s" % (legals, fname) + raise ValueError(msg) + action = liste[0] + content = "" # rebuild content from here + for elem in liste: + content = content + elem + " " + if len(liste) > 1: + chain_name = liste[1] + if "-F" in action: + self.reset() + return + if "-P" in action: + liste.pop(0) + cha = liste.pop(0) + new = liste.pop(0) + if new not in ["ACCEPT", "DROP", "REJECT"]: + msg = "Illegal policy: % s" % (new) + raise ValueError(msg) + self.poli[cha] = new + return + if "-X" in action: + predef = ['INPUT', 'FORWARD', 'OUTPUT', + 'PREROUTING', 'POSTROUTING'] + rem_chain_name = liste.pop(1) + if rem_chain_name in predef: + msg = "Cannot remove predefined chain" + raise ValueError(msg) + if rem_chain_name in self.data: + self.data[rem_chain_name] = [] # empty list + self.poli[rem_chain_name] = "-" # empty policy, no need + self.data.pop(rem_chain_name) + return + if "-N" in action: + new_chain_name = liste.pop(1) + existing = self.data.keys() + if new_chain_name in existing: + msg = "Chain %s already exists" % (new_chain_name) + raise ValueError(msg) + self.data[new_chain_name] = [] # empty list + self.poli[new_chain_name] = "-" # empty policy, no need + return + if "-I" in action: # or "-A" in action: + chain_name = liste[1] + existing = self.data.keys() + if chain_name not in existing: + msg = "invalid chain name: %s" % (chain_name) + raise ValueError(msg) + kette = self.data[chain_name] + if len(kette) > 0: + kette.insert(0, content) + else: + msg = "Empty chain %s allows append only!" % (chain_name) + raise ValueError(msg) + self.data[chain_name] = kette + return + if "-A" in action: # or "-I" in action: + chain_name = liste[1] + existing = self.data.keys() + if chain_name not in existing: + msg = "invalid chain name: %s" % (chain_name) + raise ValueError(msg) + kette = self.data[chain_name] + kette.append(content) + self.data[chain_name] = kette + return + msg = "Unknown filter command in input:", content + raise ValueError(msg) + + def reset(self): # name, tables): + """ + name is one of filter, nat, raw, mangle, + tables is a list of tables in that table-class + """ + self.poli = {} # empty dict + self.length = 0 + self.policy = "-" + for tabular in self.tables: + self.data[tabular] = [] + self.poli[tabular] = "ACCEPT" + + +class Tables(UserDict): + """ + some chaingroups in tables are predef: filter, nat, mangle, raw + """ + + def __init__(self, fname="reference-one"): + """init Tables Object is easy going""" + UserDict.__init__(self) + self.reset(fname) + + def reset(self, fname): + """all predefined Chains aka lists are setup as new here""" + filter = Chains("filter", ["INPUT", "FORWARD", "OUTPUT"]) + + mang = ["PREROUTING", "INPUT", "FORWARD", "OUTPUT", "POSTROUTING", ] + mangle = Chains("mangle", mang) + + # kernel 2.6.32 has no INPUT in NAT! + nat = Chains("nat", ["PREROUTING", "OUTPUT", "POSTROUTING"]) + + raw = Chains("raw", ["PREROUTING", "OUTPUT", ]) + + self.data["filter"] = filter + self.data["mangle"] = mangle + self.data["nat"] = nat + self.data["raw"] = raw + if len(fname) > 0: + self.linecounter = self.read_file(fname) + + def table_printout(self): + """printout nonempty tabulars in fixed sequence""" + for key in ["raw", "nat", "mangle", "filter"]: + len = self.data[key].length + if len > -1: + print("*%s" % (self.data[key].name)) + for chain in self.data[key].keys(): + poli = self.data[key].poli[chain] + print(":%s %s [0:0]" % (chain, poli)) + for chain in self.data[key].values(): + for elem in chain: + print(elem) + print("COMMIT") + + def put_into_tables(self, line): + """put line into matching Chains-object""" + liste = line.split() + liste.pop(0) # we always know, it's iptables + rest = "" + for elem in liste: # remove redirects and the like + if ">" not in elem: + rest = rest + elem + " " # string again with single blanks + action = liste.pop(0) # action is one of {N,F,A,I, etc.} + fam = "filter" + if "-t nat" in line: # nat filter group + fam = "nat" + elif "-t mangle" in line: # mangle filter group + fam = "mangle" + elif "-t raw" in line: # raw filter group + fam = "raw" + fam_dict = self.data[fam] # select the group dictionary + fam_dict.put_into_fgr(rest) # do action thers + + def read_file(self, fname): + """read file into Tables-object""" + self.linecounter = 0 + self.tblctr = 0 + try: + fil0 = open(fname, 'r') + for zeile in fil0: + line = str(zeile.strip()) + self.linecounter += 1 + if line.startswith('#'): + continue + for element in ['\$', '\(', '\)', ]: + if re.search(element, line): + m1 = "Line %d:\n%s\nplain files only, " % \ + (self.linecounter, line) + if element in ['\(', '\)', ]: + m2 = "unable to convert shell functions, abort" + else: + m2 = "unable to resolve shell variables, abort" + msg = m1 + m2 + raise ConverterError(msg) + for muster in ["^/sbin/iptables ", "^iptables "]: + if re.search(muster, line): + self.tblctr += 1 + self.put_into_tables(line) + fil0.close() + except ValueError as err: + print (fname + ": "), err + sys.exit(1) + except IOError as err: + print(fname + ": "), err.strerror + sys.exit(1) + if not fname == "reference-one": + print("# generated from: %s" % (fname)) + + +def main(): + """ + main parses options, filnames and the like + one option (-s) may be given: input-filename + if none given, it defaults to: rules + """ + usage = "usage: %prog --help | -h \n\n\t%prog: version 0.9.8" + usage = usage + "\tHave Fun!" + parser = OptionParser(usage) + parser.disable_interspersed_args() + parser.add_option("-s", "", dest="sourcefile", + help="file with iptables commands, default: rules\n") + (options, args) = parser.parse_args() + hlp = "\n\tplease use \"--help\" as argument, abort!\n" + if options.sourcefile is None: + options.sourcefile = "rules" + sourcefile = options.sourcefile + + chains = Tables(sourcefile) + chains.table_printout() + + +if __name__ == "__main__": + main() + sys.exit(0) \ No newline at end of file From b857f79fcb14b9594840e483cfcf0e4c0fac2f6b Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Mon, 1 Feb 2016 15:58:27 +0100 Subject: [PATCH 02/11] Restore iptables at once using iptables-restore instead of calling iptables numerous times --- .../debian/config/opt/cloud/bin/configure.py | 19 +-- .../config/opt/cloud/bin/cs/CsAddress.py | 31 ++-- .../config/opt/cloud/bin/cs/CsLoadBalancer.py | 14 +- .../config/opt/cloud/bin/cs/CsNetfilter.py | 69 +++++---- .../config/opt/cloud/bin/cs_iptables_save.py | 132 ++++++------------ 5 files changed, 112 insertions(+), 153 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index ab134fcfca7..0e57eb6579c 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -17,27 +17,16 @@ # specific language governing permissions and limitations # under the License. import sys -import os import base64 -from merge import DataBag -from pprint import pprint -import subprocess -import logging import re -import time -import shutil -import os.path -import os from fcntl import flock, LOCK_EX, LOCK_UN -from cs.CsDatabag import CsDataBag, CsCmdLine -import cs.CsHelper +from cs.CsDatabag import CsDataBag from cs.CsNetfilter import CsNetfilters from cs.CsDhcp import CsDhcp from cs.CsRedundant import * from cs.CsFile import CsFile -from cs.CsApp import CsApache, CsDnsmasq from cs.CsMonitor import CsMonitor from cs.CsLoadBalancer import CsLoadBalancer from cs.CsConfig import CsConfig @@ -281,7 +270,7 @@ class CsAcl(CsDataBag): rstr = "%s -m icmp --icmp-type %s" % (rstr, self.icmp_type) rstr = "%s %s -j %s" % (rstr, self.dport, self.action) rstr = rstr.replace(" ", " ").lstrip() - self.fw.append([self.table, self.count, rstr]) + self.fw.append([self.table, "", rstr]) def process(self): for item in self.dbag: @@ -495,7 +484,7 @@ class CsSite2SiteVpn(CsDataBag): self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 500 -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])]) self.fw.append(["", "front", "-A INPUT -i %s -p udp -m udp --dport 4500 -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])]) self.fw.append(["", "front", "-A INPUT -i %s -p esp -s %s -d %s -j ACCEPT" % (dev, obj['peer_gateway_ip'], obj['local_public_ip'])]) - self.fw.append(["nat", "front", "-A POSTROUTING -t nat -o %s -m mark --mark 0x525 -j ACCEPT" % dev]) + self.fw.append(["nat", "front", "-A POSTROUTING -o %s -m mark --mark 0x525 -j ACCEPT" % dev]) for net in obj['peer_guest_cidr_list'].lstrip().rstrip().split(','): self.fw.append(["mangle", "front", "-A FORWARD -s %s -d %s -j MARK --set-xmark 0x525/0xffffffff" % (obj['local_guest_cidr'], net)]) @@ -804,7 +793,7 @@ class CsForwardingRules(CsDataBag): rule['internal_ip'], self.portsToString(rule['internal_ports'], '-') ) - fw4 = "-j SNAT --to-source %s -A POSTROUTING -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \ + fw4 = "-A POSTROUTING -j SNAT --to-source %s -s %s -d %s/32 -o %s -p %s -m %s --dport %s" % \ ( self.getGuestIp(), self.getNetworkByIp(rule['internal_ip']), diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py index 1b39b385d4d..5f63c06d713 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsAddress.py @@ -15,9 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from CsDatabag import CsDataBag, CsCmdLine +from CsDatabag import CsDataBag from CsApp import CsApache, CsDnsmasq, CsPasswdSvc -import CsHelper import logging from netaddr import IPAddress, IPNetwork import CsHelper @@ -198,7 +197,7 @@ class CsInterface: return self.get_attr("add") def to_str(self): - pprint(self.address) + print(self.address) class CsDevice: @@ -383,8 +382,6 @@ class CsIP: self.fw.append(["mangle", "front", "-A FIREWALL_%s " % self.address['public_ip'] + "-m state --state RELATED,ESTABLISHED -j ACCEPT"]) - self.fw.append(["mangle", "", - "-A FIREWALL_%s DROP" % self.address['public_ip']]) self.fw.append(["mangle", "", "-A VPN_%s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.address['public_ip']]) self.fw.append(["mangle", "", @@ -402,8 +399,7 @@ class CsIP: self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"]) - self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % - self.dev]) + self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.dev]) self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"]) @@ -446,6 +442,13 @@ class CsIP: self.fw.append(["mangle", "front", "-A PREROUTING " + "-m state --state RELATED,ESTABLISHED " + "-j CONNMARK --restore-mark --nfmask 0xffffffff --ctmask 0xffffffff"]) + + self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"]) + self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"]) + self.fw.append(["", "front", "-A OUTPUT -j NETWORK_STATS"]) + + self.fw.append(["filter", "", "-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT"]) + if self.get_type() in ["guest"]: self.fw.append(["filter", "", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" % (self.address['network'], self.dev, self.dev)]) @@ -484,10 +487,6 @@ class CsIP: ]) if self.get_type() in ["public"]: - self.fw.append(["", "front", - "-A FORWARD -o %s -d %s -j ACL_INBOUND_%s" % ( - self.dev, self.address['network'], self.dev) - ]) self.fw.append( ["mangle", "", "-A FORWARD -j VPN_STATS_%s" % self.dev]) self.fw.append( @@ -495,11 +494,7 @@ class CsIP: self.fw.append( ["mangle", "", "-A VPN_STATS_%s -i %s -m mark --mark 0x524/0xffffffff" % (self.dev, self.dev)]) self.fw.append( - ["", "front", "-A FORWARD -j NETWORK_STATS_%s" % self.dev]) - - self.fw.append(["", "front", "-A FORWARD -j NETWORK_STATS"]) - self.fw.append(["", "front", "-A INPUT -j NETWORK_STATS"]) - self.fw.append(["", "front", "-A OUTPUT -j NETWORK_STATS"]) + ["", "front", "-A FORWARD -j NETWORK_STATS_eth1"]) self.fw.append(["", "", "-A NETWORK_STATS -i eth0 -o eth2 -p tcp"]) self.fw.append(["", "", "-A NETWORK_STATS -i eth2 -o eth0 -p tcp"]) @@ -508,9 +503,11 @@ class CsIP: self.fw.append(["filter", "", "-A INPUT -d 224.0.0.18/32 -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -d 225.0.0.50/32 -j ACCEPT"]) - + self.fw.append(["filter", "", "-A INPUT -i %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % self.dev]) + self.fw.append(["filter", "", "-A INPUT -i lo -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -p icmp -j ACCEPT"]) self.fw.append(["filter", "", "-A INPUT -i eth0 -p tcp -m tcp --dport 3922 -m state --state NEW,ESTABLISHED -j ACCEPT"]) + self.fw.append(["filter", "", "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT"]) self.fw.append(["filter", "", "-P INPUT DROP"]) self.fw.append(["filter", "", "-P FORWARD DROP"]) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py index d8f39dcd24a..0360ddc5aeb 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsLoadBalancer.py @@ -71,14 +71,16 @@ class CsLoadBalancer(CsDataBag): port = path[1] firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) - for rules in remove_rules: - path = rules.split(':') - ip = path[0] - port = path[1] - firewall.append(["filter", "", "-D INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) - for rules in stat_rules: path = rules.split(':') ip = path[0] port = path[1] firewall.append(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) + + for rules in remove_rules: + path = rules.split(':') + ip = path[0] + port = path[1] + if ["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)] in firewall: + firewall.remove(["filter", "", "-A INPUT -p tcp -m tcp -d %s --dport %s -m state --state NEW -j ACCEPT" % (ip, port)]) + diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py index 4b5b49231f2..47eb50bd6ce 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py @@ -15,10 +15,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import print_function + import CsHelper -from pprint import pprint -from CsDatabag import CsDataBag, CsCmdLine +from CsDatabag import CsCmdLine import logging +from cs_iptables_save import Tables class CsChain(object): @@ -81,6 +83,7 @@ class CsNetfilters(object): def __init__(self, load=True): self.rules = [] + self.iptablerules = [] self.table = CsTable() self.chain = CsChain() if load: @@ -91,7 +94,10 @@ class CsNetfilters(object): if i.startswith('*'): # Table self.table.add(i[1:]) if i.startswith(':'): # Chain - self.chain.add(self.table.last(), i[1:].split(' ')[0]) + string = i[1:].split(' ')[0] + cmd = "iptables -t %s -N %s" % (self.table.last(), string) + self.iptablerules.append(cmd) + self.chain.add(self.table.last(), string) if i.startswith('-A'): # Rule self.chain.add_rule(i.split()[1]) rule = CsNetfilter() @@ -125,10 +131,7 @@ class CsNetfilters(object): def get_unseen(self): del_list = [x for x in self.rules if x.unseen()] for r in del_list: - cmd = "iptables -t %s %s" % (r.get_table(), r.to_str(True)) - logging.debug("unseen cmd: %s ", cmd) - CsHelper.execute(cmd) - # print "Delete rule %s from table %s" % (r.to_str(True), r.get_table()) + self.delete(r) logging.info("Delete rule %s from table %s", r.to_str(True), r.get_table()) def compare(self, list): @@ -137,12 +140,16 @@ class CsNetfilters(object): # Ensure all inbound/outbound chains have a default drop rule if c.startswith("ACL_INBOUND") or c.startswith("ACL_OUTBOUND"): list.append(["filter", "", "-A %s -j DROP" % c]) - # PASS 1: Ensure all chains are present + # PASS 1: Ensure all chains are present and cleanup unused rules. for fw in list: new_rule = CsNetfilter() new_rule.parse(fw[2]) new_rule.set_table(fw[0]) - self.add_chain(new_rule) + self.has_rule(new_rule) + + self.del_standard() + self.get_unseen() + # PASS 2: Create rules for fw in list: new_rule = CsNetfilter() @@ -151,28 +158,32 @@ class CsNetfilters(object): if isinstance(fw[1], int): new_rule.set_count(fw[1]) - logging.debug("Checking if the rule already exists: rule=%s table=%s chain=%s", new_rule.get_rule(), new_rule.get_table(), new_rule.get_chain()) - if self.has_rule(new_rule): - logging.debug("Exists: rule=%s table=%s", fw[2], new_rule.get_table()) - else: - # print "Add rule %s in table %s" % ( fw[2], new_rule.get_table()) - logging.info("Add: rule=%s table=%s", fw[2], new_rule.get_table()) - # front means insert instead of append - cpy = fw[2] - if fw[1] == "front": - cpy = cpy.replace('-A', '-I') - if isinstance(fw[1], int): - cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1])) + logging.info("Add: rule=%s table=%s", fw[2], new_rule.get_table()) + # front means insert instead of append + cpy = fw[2] + if fw[1] == "front": + cpy = cpy.replace('-A', '-I') + if isinstance(fw[1], int): + cpy = cpy.replace("-A %s" % new_rule.get_chain(), '-I %s %s' % (new_rule.get_chain(), fw[1])) - CsHelper.execute("iptables -t %s %s" % (new_rule.get_table(), cpy)) - self.del_standard() - self.get_unseen() + self.iptablerules.append("iptables -t %s %s" % (new_rule.get_table(), cpy)) + self.apply_rules() - def add_chain(self, rule): - """ Add the given chain if it is not already present """ - if not self.has_chain(rule.get_table(), rule.get_chain()): - CsHelper.execute("iptables -t %s -N %s" % (rule.get_table(), rule.get_chain())) - self.chain.add(rule.get_table(), rule.get_chain()) + def apply_rules(self): + s = [] + for r in self.iptablerules: + if r not in s: + s.append(r) + + chains = Tables(s) + chains.table_printout() + + # COMMIT all rules. + result = CsHelper.execute("iptables-restore < /tmp/rules.save") + if result: + logging.info("iptables-restore result: %s", result) + else: + logging.info("iptables-restore result: success!") def del_standard(self): """ Del rules that are there but should not be deleted diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py index 4b64c2b1da9..f5e61ac05d8 100644 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_iptables_save.py @@ -19,14 +19,15 @@ License: GNU General Public License version 3 or later Have Fun! """ +from __future__ import print_function try: from collections import UserDict except ImportError: from UserDict import UserDict -from optparse import OptionParser import re import sys +import logging class ConverterError(): @@ -97,8 +98,8 @@ class Chains(UserDict): new_chain_name = liste.pop(1) existing = self.data.keys() if new_chain_name in existing: - msg = "Chain %s already exists" % (new_chain_name) - raise ValueError(msg) + logging.debug("Chain %s already exists" % new_chain_name) + return self.data[new_chain_name] = [] # empty list self.poli[new_chain_name] = "-" # empty policy, no need return @@ -106,22 +107,18 @@ class Chains(UserDict): chain_name = liste[1] existing = self.data.keys() if chain_name not in existing: - msg = "invalid chain name: %s" % (chain_name) - raise ValueError(msg) + self.data[chain_name] = [] + self.poli[chain_name] = "-" kette = self.data[chain_name] - if len(kette) > 0: - kette.insert(0, content) - else: - msg = "Empty chain %s allows append only!" % (chain_name) - raise ValueError(msg) + kette.insert(0, content.replace("-I", "-A")) self.data[chain_name] = kette return if "-A" in action: # or "-I" in action: chain_name = liste[1] existing = self.data.keys() if chain_name not in existing: - msg = "invalid chain name: %s" % (chain_name) - raise ValueError(msg) + self.data[chain_name] = [] + self.poli[chain_name] = "-" kette = self.data[chain_name] kette.append(content) self.data[chain_name] = kette @@ -147,12 +144,12 @@ class Tables(UserDict): some chaingroups in tables are predef: filter, nat, mangle, raw """ - def __init__(self, fname="reference-one"): + def __init__(self, rules): """init Tables Object is easy going""" UserDict.__init__(self) - self.reset(fname) + self.reset(rules) - def reset(self, fname): + def reset(self, rules): """all predefined Chains aka lists are setup as new here""" filter = Chains("filter", ["INPUT", "FORWARD", "OUTPUT"]) @@ -168,22 +165,23 @@ class Tables(UserDict): self.data["mangle"] = mangle self.data["nat"] = nat self.data["raw"] = raw - if len(fname) > 0: - self.linecounter = self.read_file(fname) + if rules is not None: + self.read_file(rules) def table_printout(self): """printout nonempty tabulars in fixed sequence""" - for key in ["raw", "nat", "mangle", "filter"]: - len = self.data[key].length - if len > -1: - print("*%s" % (self.data[key].name)) - for chain in self.data[key].keys(): - poli = self.data[key].poli[chain] - print(":%s %s [0:0]" % (chain, poli)) - for chain in self.data[key].values(): - for elem in chain: - print(elem) - print("COMMIT") + with open("/tmp/rules.save", 'w') as f: + for key in ["raw", "nat", "mangle", "filter"]: + len = self.data[key].length + if len > -1: + print("*%s" % (self.data[key].name), file=f) + for chain in self.data[key].keys(): + poli = self.data[key].poli[chain] + print(":%s %s [0:0]" % (chain, poli), file=f) + for chain in self.data[key].values(): + for elem in chain: + print(elem, file=f) + print("COMMIT", file=f) def put_into_tables(self, line): """put line into matching Chains-object""" @@ -204,64 +202,26 @@ class Tables(UserDict): fam_dict = self.data[fam] # select the group dictionary fam_dict.put_into_fgr(rest) # do action thers - def read_file(self, fname): + def read_file(self, rules): """read file into Tables-object""" self.linecounter = 0 self.tblctr = 0 - try: - fil0 = open(fname, 'r') - for zeile in fil0: - line = str(zeile.strip()) - self.linecounter += 1 - if line.startswith('#'): - continue - for element in ['\$', '\(', '\)', ]: - if re.search(element, line): - m1 = "Line %d:\n%s\nplain files only, " % \ - (self.linecounter, line) - if element in ['\(', '\)', ]: - m2 = "unable to convert shell functions, abort" - else: - m2 = "unable to resolve shell variables, abort" - msg = m1 + m2 - raise ConverterError(msg) - for muster in ["^/sbin/iptables ", "^iptables "]: - if re.search(muster, line): - self.tblctr += 1 - self.put_into_tables(line) - fil0.close() - except ValueError as err: - print (fname + ": "), err - sys.exit(1) - except IOError as err: - print(fname + ": "), err.strerror - sys.exit(1) - if not fname == "reference-one": - print("# generated from: %s" % (fname)) - - -def main(): - """ - main parses options, filnames and the like - one option (-s) may be given: input-filename - if none given, it defaults to: rules - """ - usage = "usage: %prog --help | -h \n\n\t%prog: version 0.9.8" - usage = usage + "\tHave Fun!" - parser = OptionParser(usage) - parser.disable_interspersed_args() - parser.add_option("-s", "", dest="sourcefile", - help="file with iptables commands, default: rules\n") - (options, args) = parser.parse_args() - hlp = "\n\tplease use \"--help\" as argument, abort!\n" - if options.sourcefile is None: - options.sourcefile = "rules" - sourcefile = options.sourcefile - - chains = Tables(sourcefile) - chains.table_printout() - - -if __name__ == "__main__": - main() - sys.exit(0) \ No newline at end of file + for zeile in rules: + line = str(zeile.strip()) + self.linecounter += 1 + if line.startswith('#'): + continue + for element in ['\$', '\(', '\)', ]: + if re.search(element, line): + m1 = "Line %d:\n%s\nplain files only, " % \ + (self.linecounter, line) + if element in ['\(', '\)', ]: + m2 = "unable to convert shell functions, abort" + else: + m2 = "unable to resolve shell variables, abort" + msg = m1 + m2 + raise ConverterError(msg) + for muster in ["^/sbin/iptables ", "^iptables "]: + if re.search(muster, line): + self.tblctr += 1 + self.put_into_tables(line) From 69e9348900e6337a16788a54283d5c9ee3b7a221 Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Wed, 3 Feb 2016 15:30:19 +0100 Subject: [PATCH 03/11] Remove duplicate spaces, and thus duplicate rules. --- systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py index 47eb50bd6ce..71127bdbfcc 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsNetfilter.py @@ -172,6 +172,7 @@ class CsNetfilters(object): def apply_rules(self): s = [] for r in self.iptablerules: + r.replace(' ', ' ') # Remove duplicate spaces if r not in s: s.append(r) From eb9706b6558d5113000a6c10d861308fbc8cd692 Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Wed, 3 Feb 2016 16:15:28 +0100 Subject: [PATCH 04/11] Wait for dnsmasq to finish restart --- systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py index 023b180cf27..48533ec82a3 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py @@ -54,7 +54,7 @@ class CsDhcp(CsDataBag): self.cloud.commit() # We restart DNSMASQ every time the configure.py is called in order to avoid lease problems. - CsHelper.service("dnsmasq", "restart") + CsHelper.execute2("service dnsmasq restart") def configure_server(self): # self.conf.addeq("dhcp-hostsfile=%s" % DHCP_HOSTS) From 18d5cd285536954403cf5eec64024e5909e55d41 Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Fri, 5 Feb 2016 21:04:06 +0100 Subject: [PATCH 05/11] Do not load previous firewall rules as we replace everyhing anyway --- systemvm/patches/debian/config/opt/cloud/bin/configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index 0e57eb6579c..840c7cb6017 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -988,7 +988,7 @@ def main(argv): lb.process() logging.debug("Configuring iptables rules") - nf = CsNetfilters() + nf = CsNetfilters(False) nf.compare(config.get_fw()) logging.debug("Configuring iptables rules done ...saving rules") From 709be45cf0f32bacc54ec2384be9e54bd62a5acf Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Fri, 5 Feb 2016 21:06:53 +0100 Subject: [PATCH 06/11] Check the existence of 'forceencap' parameter before use --- systemvm/patches/debian/config/opt/cloud/bin/configure.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index 840c7cb6017..f317b87b6ba 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -520,6 +520,8 @@ class CsSite2SiteVpn(CsDataBag): file.addeq(" pfs=%s" % CsHelper.bool_to_yn(obj['dpd'])) file.addeq(" keyingtries=2") file.addeq(" auto=start") + if not obj.has_key('encap'): + obj['encap']="false" file.addeq(" forceencaps=%s" % CsHelper.bool_to_yn(obj['encap'])) if obj['dpd']: file.addeq(" dpddelay=30") From 30741a3309da346f324a8f365cb6ef8e1aab39d8 Mon Sep 17 00:00:00 2001 From: Boris Schrijver Date: Fri, 5 Feb 2016 21:11:59 +0100 Subject: [PATCH 07/11] Split the cidr lists so we won't hit the iptables-resture limits --- .../debian/config/opt/cloud/bin/configure.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index f317b87b6ba..27e26a7587e 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -217,7 +217,23 @@ class CsAcl(CsDataBag): def process(self, direction, rule_list, base): count = base - for i in rule_list: + rule_list_splitted = [] + for rule in rule_list: + if ',' in rule['cidr']: + cidrs = rule['cidr'].split(',') + for cidr in cidrs: + new_rule = { + 'cidr': cidr, + 'last_port': rule['last_port'], + 'type': rule['type'], + 'first_port': rule['first_port'], + 'allowed': rule['allowed'] + } + rule_list_splitted.append(new_rule) + else: + rule_list_splitted.append(rule) + + for i in rule_list_splitted: r = self.AclRule(direction, self, i, self.config, count) r.create() count += 1 From ebb7cb690dd04263568cae0a93e4ff28e5ba157a Mon Sep 17 00:00:00 2001 From: Remi Bergsma Date: Sat, 30 Apr 2016 21:09:33 +0200 Subject: [PATCH 08/11] Resolve conflict as forceencap is already in master --- systemvm/patches/debian/config/opt/cloud/bin/configure.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/configure.py b/systemvm/patches/debian/config/opt/cloud/bin/configure.py index 27e26a7587e..1e4469c3ea5 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/configure.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/configure.py @@ -536,8 +536,6 @@ class CsSite2SiteVpn(CsDataBag): file.addeq(" pfs=%s" % CsHelper.bool_to_yn(obj['dpd'])) file.addeq(" keyingtries=2") file.addeq(" auto=start") - if not obj.has_key('encap'): - obj['encap']="false" file.addeq(" forceencaps=%s" % CsHelper.bool_to_yn(obj['encap'])) if obj['dpd']: file.addeq(" dpddelay=30") From 2bfb12cba994ecf37478a283c873043f2475818e Mon Sep 17 00:00:00 2001 From: Remi Bergsma Date: Wed, 23 Mar 2016 16:33:20 +0100 Subject: [PATCH 09/11] Have rVPCs use the router.redundant.vrrp.interval setting It defaults to 1, which is hardcoded in the template: ./cosmic/cosmic-core/systemvm/patches/debian/config/opt/cloud/templates/keepalived.conf.templ As non-VPC redundant routers use this setting, I think it makes sense to use it for rVPCs as well. We also need a change to pickup the cmd_line parameter and use it in the Python code that configures the router. --- .../network/router/VirtualNetworkApplianceManagerImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index ac8b86816a2..f7947d5c54b 100644 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -1598,6 +1598,9 @@ Configurable, StateListener Date: Mon, 14 Dec 2015 17:37:28 -0500 Subject: [PATCH 11/11] Fix Sync of template.properties in Swift --- .../ObjectInDataStoreManagerImpl.java | 1 + server/src/com/cloud/test/TestAppender.java | 178 +++++++++ services/secondary-storage/server/pom.xml | 3 - .../resource/NfsSecondaryStorageResource.java | 371 +++++++++++------- .../LocalNfsSecondaryStorageResourceTest.java | 44 +-- .../NfsSecondaryStorageResourceTest.java | 112 +++--- 6 files changed, 477 insertions(+), 232 deletions(-) create mode 100644 server/src/com/cloud/test/TestAppender.java diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index b1d6127e243..5117b7cb84f 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -156,6 +156,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { // template.properties // there } + ts.setInstallPath(installPath); ts.setState(ObjectInDataStoreStateMachine.State.Allocated); ts = templateDataStoreDao.persist(ts); diff --git a/server/src/com/cloud/test/TestAppender.java b/server/src/com/cloud/test/TestAppender.java new file mode 100644 index 00000000000..539cac1ee7b --- /dev/null +++ b/server/src/com/cloud/test/TestAppender.java @@ -0,0 +1,178 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ +package com.cloud.test; + +import com.google.common.base.Joiner; +import com.google.common.base.Objects; +import com.google.common.collect.ImmutableMap; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; +import org.springframework.util.Assert; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.lang.String.format; +import static org.apache.log4j.Level.ALL; +import static org.apache.log4j.Level.DEBUG; +import static org.apache.log4j.Level.ERROR; +import static org.apache.log4j.Level.FATAL; +import static org.apache.log4j.Level.INFO; +import static org.apache.log4j.Level.OFF; + +/** +* +* Tracks one or more patterns to determine whether or not they have been +* logged. It uses a streaming approach to determine whether or not a message +* has a occurred to prevent unnecessary memory consumption. Instances of this +* of this class are created using the {@link TestAppenderBuilder}. +* +* To use this class, register a one or more expected patterns by level as part +* of the test setup and retain an reference to the appender instance. After the +* expected logging events have occurred in the test case, call +* {@link TestAppender#assertMessagesLogged()} which will fail the test if any of the +* expected patterns were not logged. +* +*/ +public final class TestAppender extends AppenderSkeleton { + private final static String APPENDER_NAME = "test_appender"; + private final ImmutableMap> expectedPatternResults; + private TestAppender(final Map> expectedPatterns) { + super(); + expectedPatternResults = ImmutableMap.copyOf(expectedPatterns); + } + protected void append(LoggingEvent loggingEvent) { + checkArgument(loggingEvent != null, "append requires a non-null loggingEvent"); + final Level level = loggingEvent.getLevel(); + checkState(expectedPatternResults.containsKey(level), "level " + level + " not supported by append"); + for (final PatternResult patternResult : expectedPatternResults.get(level)) { + if (patternResult.getPattern().matcher(loggingEvent.getRenderedMessage()).matches()) { + patternResult.markFound(); + } + } + } + + public void close() { +// Do nothing ... + } + public boolean requiresLayout() { + return false; + } + public void assertMessagesLogged() { + final List unloggedPatterns = new ArrayList<>(); + for (final Map.Entry> expectedPatternResult : expectedPatternResults.entrySet()) { + for (final PatternResult patternResults : expectedPatternResult.getValue()) { + if (!patternResults.isFound()) { + unloggedPatterns.add(format("%1$s was not logged for level %2$s", + patternResults.getPattern().toString(), expectedPatternResult.getKey())); + } + } + } + if (!unloggedPatterns.isEmpty()) { + //Raise an assert + Assert.isTrue(false, Joiner.on(",").join(unloggedPatterns)); + } + } + + private static final class PatternResult { + private final Pattern pattern; + private boolean foundFlag = false; + private PatternResult(Pattern pattern) { + super(); + this.pattern = pattern; + } + public Pattern getPattern() { + return pattern; + } + public void markFound() { + // This operation is thread-safe because the value will only ever be switched from false to true. Therefore, + // multiple threads mutating the value for a pattern will not corrupt the value ... + foundFlag = true; + } + public boolean isFound() { + return foundFlag; + } + @Override + public boolean equals(Object thatObject) { + if (this == thatObject) { + return true; + } + if (thatObject == null || getClass() != thatObject.getClass()) { + return false; + } + PatternResult thatPatternResult = (PatternResult) thatObject; + return foundFlag == thatPatternResult.foundFlag && + Objects.equal(pattern, thatPatternResult.pattern); + } + @Override + public int hashCode() { + return Objects.hashCode(pattern, foundFlag); + } + @Override + public String toString() { + return format("Pattern Result [ pattern: %1$s, markFound: %2$s ]", pattern.toString(), foundFlag); + } + } + + public static final class TestAppenderBuilder { + private final Map> expectedPatterns; + public TestAppenderBuilder() { + super(); + expectedPatterns = new HashMap<>(); + expectedPatterns.put(ALL, new HashSet()); + expectedPatterns.put(DEBUG, new HashSet()); + expectedPatterns.put(ERROR, new HashSet()); + expectedPatterns.put(FATAL, new HashSet()); + expectedPatterns.put(INFO, new HashSet()); + expectedPatterns.put(OFF, new HashSet()); + } + public TestAppenderBuilder addExpectedPattern(final Level level, final String pattern) { + checkArgument(level != null, "addExpectedPattern requires a non-null level"); + checkArgument(!isNullOrEmpty(pattern), "addExpectedPattern requires a non-blank pattern"); + checkState(expectedPatterns.containsKey(level), "level " + level + " is not supported by " + getClass().getName()); + expectedPatterns.get(level).add(new PatternResult(Pattern.compile(pattern))); + return this; + } + public TestAppender build() { + return new TestAppender(expectedPatterns); + } + } + /** + * + * Attaches a {@link TestAppender} to a {@link Logger} and ensures that it is the only + * test appender attached to the logger. + * + * @param logger The logger which will be monitored by the test + * @param testAppender The test appender to attach to {@code logger} + */ + public static void safeAddAppender(Logger logger, TestAppender testAppender) { + logger.removeAppender(APPENDER_NAME); + logger.addAppender(testAppender); + } +} \ No newline at end of file diff --git a/services/secondary-storage/server/pom.xml b/services/secondary-storage/server/pom.xml index 417e0a60a7e..63aef271c5b 100644 --- a/services/secondary-storage/server/pom.xml +++ b/services/secondary-storage/server/pom.xml @@ -26,9 +26,6 @@ 4.7.2-SNAPSHOT ../pom.xml - - true - log4j diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 768a177cdc5..f512da040c8 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -16,91 +16,7 @@ // under the License. package org.apache.cloudstack.storage.resource; -import static com.cloud.utils.storage.S3.S3Utils.putFile; -import static com.cloud.utils.StringUtils.join; -import static java.lang.String.format; -import static java.util.Arrays.asList; -import static org.apache.commons.lang.StringUtils.substringAfterLast; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.math.BigInteger; -import java.net.InetAddress; -import java.net.URI; -import java.net.UnknownHostException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import javax.naming.ConfigurationException; - -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.storage.Storage; -import com.cloud.storage.template.TemplateConstants; -import com.cloud.utils.EncryptionUtil; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.codec.http.HttpContentCompressor; -import io.netty.handler.codec.http.HttpRequestDecoder; -import io.netty.handler.codec.http.HttpResponseEncoder; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.logging.LoggingHandler; -import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.cloudstack.storage.template.UploadEntity; -import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.NameValuePair; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URLEncodedUtils; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.log4j.Logger; - import com.amazonaws.services.s3.model.S3ObjectSummary; - -import org.apache.cloudstack.framework.security.keystore.KeystoreManager; -import org.apache.cloudstack.storage.command.CopyCmdAnswer; -import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.DeleteCommand; -import org.apache.cloudstack.storage.command.DownloadCommand; -import org.apache.cloudstack.storage.command.DownloadProgressCommand; -import org.apache.cloudstack.storage.command.UploadStatusAnswer; -import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; -import org.apache.cloudstack.storage.command.UploadStatusCommand; -import org.apache.cloudstack.storage.template.DownloadManager; -import org.apache.cloudstack.storage.template.DownloadManagerImpl; -import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser; -import org.apache.cloudstack.storage.template.UploadManager; -import org.apache.cloudstack.storage.template.UploadManagerImpl; -import org.apache.cloudstack.storage.to.SnapshotObjectTO; -import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; - import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckHealthAnswer; import com.cloud.agent.api.CheckHealthCommand; @@ -135,11 +51,13 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.exception.InternalErrorException; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.resource.ServerResourceBase; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; import com.cloud.storage.VMTemplateStorageResourceAssoc; @@ -149,24 +67,102 @@ import com.cloud.storage.template.Processor.FormatInfo; import com.cloud.storage.template.QCOW2Processor; import com.cloud.storage.template.RawImageProcessor; import com.cloud.storage.template.TARProcessor; +import com.cloud.storage.template.TemplateConstants; import com.cloud.storage.template.TemplateLocation; import com.cloud.storage.template.TemplateProp; import com.cloud.storage.template.VhdProcessor; import com.cloud.storage.template.VmdkProcessor; +import com.cloud.utils.EncryptionUtil; import com.cloud.utils.NumbersUtil; -import com.cloud.utils.storage.S3.S3Utils; import com.cloud.utils.SwiftUtil; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; +import com.cloud.utils.storage.S3.S3Utils; import com.cloud.vm.SecondaryStorageVm; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.http.HttpContentCompressor; +import io.netty.handler.codec.http.HttpRequestDecoder; +import io.netty.handler.codec.http.HttpResponseEncoder; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import org.apache.cloudstack.framework.security.keystore.KeystoreManager; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.command.DownloadProgressCommand; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; +import org.apache.cloudstack.storage.command.UploadStatusAnswer; +import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; +import org.apache.cloudstack.storage.command.UploadStatusCommand; +import org.apache.cloudstack.storage.template.DownloadManager; +import org.apache.cloudstack.storage.template.DownloadManagerImpl; +import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser; +import org.apache.cloudstack.storage.template.UploadEntity; +import org.apache.cloudstack.storage.template.UploadManager; +import org.apache.cloudstack.storage.template.UploadManagerImpl; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.log4j.Logger; import org.joda.time.DateTime; import org.joda.time.format.ISODateTimeFormat; +import javax.naming.ConfigurationException; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.net.InetAddress; +import java.net.URI; +import java.net.UnknownHostException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static com.cloud.utils.StringUtils.join; +import static com.cloud.utils.storage.S3.S3Utils.putFile; +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static org.apache.commons.lang.StringUtils.substringAfterLast; + public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource { - private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class); + public static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class); private static final String TEMPLATE_ROOT_DIR = "template/tmpl"; private static final String VOLUME_ROOT_DIR = "volumes"; @@ -499,10 +495,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String destFileFullPath = destFile.getAbsolutePath() + File.separator + fileName; s_logger.debug("copy snapshot " + srcFile.getAbsolutePath() + " to template " + destFileFullPath); Script.runSimpleBashScript("cp " + srcFile.getAbsolutePath() + " " + destFileFullPath); - String metaFileName = destFile.getAbsolutePath() + File.separator + "template.properties"; + String metaFileName = destFile.getAbsolutePath() + File.separator + _tmpltpp; File metaFile = new File(metaFileName); try { - _storage.create(destFile.getAbsolutePath(), "template.properties"); + _storage.create(destFile.getAbsolutePath(), _tmpltpp); try ( // generate template.properties file FileWriter writer = new FileWriter(metaFile); BufferedWriter bufferWriter = new BufferedWriter(writer); @@ -597,32 +593,14 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return answer; } s_logger.debug("starting copy template to swift"); - DataTO newTemplate = answer.getNewData(); - File templateFile = getFile(newTemplate.getPath(), ((NfsTO)srcDataStore).getUrl()); - SwiftTO swift = (SwiftTO)destDataStore; - String containterName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId()); - String swiftPath = SwiftUtil.putObject(swift, templateFile, containterName, templateFile.getName()); - //upload template.properties - File properties = new File(templateFile.getParent() + File.separator + _tmpltpp); - if (properties.exists()) { - SwiftUtil.putObject(swift, properties, containterName, _tmpltpp); - } + TemplateObjectTO newTemplate = (TemplateObjectTO)answer.getNewData(); + newTemplate.setDataStore(srcDataStore); + CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence()); + Answer result = copyFromNfsToSwift(newCpyCmd); - //clean up template data on staging area - try { - DeleteCommand deleteCommand = new DeleteCommand(newTemplate); - execute(deleteCommand); - } catch (Exception e) { - s_logger.debug("Failed to clean up staging area:", e); - } + cleanupStagingNfs(newTemplate); + return result; - TemplateObjectTO template = new TemplateObjectTO(); - template.setPath(swiftPath); - template.setSize(templateFile.length()); - template.setPhysicalSize(template.getSize()); - SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData; - template.setFormat(snapshot.getVolume().getFormat()); - return new CopyCmdAnswer(template); } else if (destDataStore instanceof S3TO) { //create template on the same data store CopyCmdAnswer answer = @@ -635,18 +613,27 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S newTemplate.setDataStore(srcDataStore); CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence()); Answer result = copyFromNfsToS3(newCpyCmd); - //clean up template data on staging area - try { - DeleteCommand deleteCommand = new DeleteCommand(newTemplate); - execute(deleteCommand); - } catch (Exception e) { - s_logger.debug("Failed to clean up staging area:", e); - } + + cleanupStagingNfs(newTemplate); + return result; } } - s_logger.debug("Failed to create templat from snapshot"); - return new CopyCmdAnswer("Unsupported prototcol"); + s_logger.debug("Failed to create template from snapshot"); + return new CopyCmdAnswer("Unsupported protocol"); + } + + /** + * clean up template data on staging area + * @param newTemplate: The template on the secondary storage that needs to be cleaned up + */ + protected void cleanupStagingNfs(TemplateObjectTO newTemplate) { + try { + DeleteCommand deleteCommand = new DeleteCommand(newTemplate); + execute(deleteCommand); + } catch (Exception e) { + s_logger.debug("Failed to clean up staging area:", e); + } } protected Answer copyFromNfsToImage(CopyCommand cmd) { @@ -759,22 +746,18 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String container = "T-" + cmd.getId(); String swiftPath = SwiftUtil.putObject(swiftTO, file, container, null); + long virtualSize = getVirtualSize(file, getTemplateFormat(file.getName())); + long size = file.length(); + String uniqueName = cmd.getName(); + //put metda file File uniqDir = _storage.createUniqDir(); - String metaFileName = uniqDir.getAbsolutePath() + File.separator + "template.properties"; - _storage.create(uniqDir.getAbsolutePath(), "template.properties"); - File metaFile = new File(metaFileName); - FileWriter writer = new FileWriter(metaFile); - BufferedWriter bufferWriter = new BufferedWriter(writer); - bufferWriter.write("uniquename=" + cmd.getName()); - bufferWriter.write("\n"); - bufferWriter.write("filename=" + fileName); - bufferWriter.write("\n"); - bufferWriter.write("size=" + file.length()); - bufferWriter.close(); - writer.close(); + String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp; + _storage.create(uniqDir.getAbsolutePath(), _tmpltpp); - SwiftUtil.putObject(swiftTO, metaFile, container, "template.properties"); + File metaFile = swiftWriteMetadataFile(metaFileName, uniqueName, fileName, size, virtualSize); + + SwiftUtil.putObject(swiftTO, metaFile, container, _tmpltpp); metaFile.delete(); uniqDir.delete(); String md5sum = null; @@ -785,7 +768,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } DownloadAnswer answer = - new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, file.length(), file.length(), md5sum); + new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, virtualSize, file.length(), md5sum); return answer; } catch (IOException e) { s_logger.debug("Failed to register template into swift", e); @@ -942,6 +925,118 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + /*** + *This method will create a file using the filenName and metaFileName. + *That file will contain the given attributes (unique name, file name, size, and virtualSize). + * + * @param metaFileName : The path of the metadata file + * @param filename :attribute: Filename of the template + * @param uniqueName :attribute: Unique name of the template + * @param size :attribute: physical size of the template + * @param virtualSize :attribute: virtual size of the template + * @return File representing the metadata file + * @throws IOException + */ + + protected File swiftWriteMetadataFile(String metaFileName, String uniqueName, String filename, long size, long virtualSize) throws IOException { + File metaFile = new File(metaFileName); + FileWriter writer = new FileWriter(metaFile); + BufferedWriter bufferWriter = new BufferedWriter(writer); + bufferWriter.write("uniquename=" + uniqueName); + bufferWriter.write("\n"); + bufferWriter.write("filename=" + filename); + bufferWriter.write("\n"); + bufferWriter.write("size=" + size); + bufferWriter.write("\n"); + bufferWriter.write("virtualsize=" + virtualSize); + bufferWriter.close(); + writer.close(); + return metaFile; + } + + /** + * Creates a template.properties for Swift with its correct unique name + * + * @param swift The swift object + * @param srcFile Source file on the staging NFS + * @param containerName Destination container + * @return true on successful write + */ + protected boolean swiftUploadMetadataFile(SwiftTO swift, File srcFile, String containerName) throws IOException { + + String uniqueName = FilenameUtils.getBaseName(srcFile.getName()); + + File uniqDir = _storage.createUniqDir(); + String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp; + _storage.create(uniqDir.getAbsolutePath(), _tmpltpp); + + long virtualSize = getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())); + + File metaFile = swiftWriteMetadataFile(metaFileName, + uniqueName, + srcFile.getName(), + srcFile.length(), + virtualSize); + + SwiftUtil.putObject(swift, metaFile, containerName, _tmpltpp); + metaFile.delete(); + uniqDir.delete(); + + return true; + } + + /** + * Copies data from NFS and uploads it into a Swift container + * + * @param cmd CopyComand + * @return CopyCmdAnswer + */ + protected Answer copyFromNfsToSwift(CopyCommand cmd) { + + final DataTO srcData = cmd.getSrcTO(); + final DataTO destData = cmd.getDestTO(); + + DataStoreTO srcDataStore = srcData.getDataStore(); + NfsTO srcStore = (NfsTO)srcDataStore; + DataStoreTO destDataStore = destData.getDataStore(); + File srcFile = getFile(srcData.getPath(), srcStore.getUrl()); + + SwiftTO swift = (SwiftTO)destDataStore; + + try { + + String containerName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId()); + String swiftPath = SwiftUtil.putObject(swift, srcFile, containerName, srcFile.getName()); + + + DataTO retObj = null; + if (destData.getObjectType() == DataObjectType.TEMPLATE) { + swiftUploadMetadataFile(swift, srcFile, containerName); + TemplateObjectTO newTemplate = new TemplateObjectTO(); + newTemplate.setPath(swiftPath); + newTemplate.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()))); + newTemplate.setPhysicalSize(srcFile.length()); + newTemplate.setFormat(getTemplateFormat(srcFile.getName())); + retObj = newTemplate; + } else if (destData.getObjectType() == DataObjectType.VOLUME) { + VolumeObjectTO newVol = new VolumeObjectTO(); + newVol.setPath(containerName); + newVol.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()))); + retObj = newVol; + } else if (destData.getObjectType() == DataObjectType.SNAPSHOT) { + SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); + newSnapshot.setPath(containerName); + retObj = newSnapshot; + } + + return new CopyCmdAnswer(retObj); + + } catch (Exception e) { + s_logger.error("failed to upload " + srcData.getPath(), e); + return new CopyCmdAnswer("failed to upload " + srcData.getPath() + e.toString()); + } + } + String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) { Script command = new Script("/bin/bash", s_logger); command.add("-c"); @@ -1458,13 +1553,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S Map tmpltInfos = new HashMap(); for (String container : containers) { if (container.startsWith("T-")) { - String[] files = SwiftUtil.list(swift, container, "template.properties"); + String[] files = SwiftUtil.list(swift, container, _tmpltpp); if (files.length != 1) { continue; } try { File tempFile = File.createTempFile("template", ".tmp"); - File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + "template.properties"); + File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + _tmpltpp); if (tmpFile == null) { continue; } @@ -1779,7 +1874,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } else { boolean found = false; for (File f : tmpltFiles) { - if (!found && f.getName().equals("template.properties")) { + if (!found && f.getName().equals(_tmpltpp)) { found = true; } diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java index 52bde6a11da..b33ce3b7474 100644 --- a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java @@ -18,29 +18,6 @@ */ package org.apache.cloudstack.storage.resource; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.Map; -import java.util.Properties; -import java.util.UUID; - -import javax.naming.ConfigurationException; - -import junit.framework.Assert; -import junit.framework.TestCase; - -import org.apache.log4j.Logger; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -import org.apache.cloudstack.storage.command.CopyCmdAnswer; -import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.DownloadCommand; -import org.apache.cloudstack.storage.to.TemplateObjectTO; - import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.ListTemplateAnswer; import com.cloud.agent.api.storage.ListTemplateCommand; @@ -51,7 +28,28 @@ import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.exception.CloudRuntimeException; +import junit.framework.Assert; +import junit.framework.TestCase; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.log4j.Logger; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.mockito.Mockito; +import javax.naming.ConfigurationException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; + +@Ignore public class LocalNfsSecondaryStorageResourceTest extends TestCase { private static Map testParams; diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java index 13ddb3531a8..e437a0f3b64 100644 --- a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java @@ -18,91 +18,67 @@ */ package org.apache.cloudstack.storage.resource; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URI; -import java.util.Map; -import java.util.Properties; - -import javax.naming.ConfigurationException; - -import junit.framework.Assert; -import junit.framework.TestCase; - +import com.cloud.test.TestAppender; +import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.log4j.Level; -import org.apache.log4j.Logger; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; -import com.cloud.utils.PropertiesUtil; -import com.cloud.utils.exception.CloudRuntimeException; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.StringWriter; -public class NfsSecondaryStorageResourceTest extends TestCase { - private static Map testParams; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; - private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResourceTest.class.getName()); +@RunWith(PowerMockRunner.class) +public class NfsSecondaryStorageResourceTest { - NfsSecondaryStorageResource resource; + private NfsSecondaryStorageResource resource; @Before - @Override - public void setUp() throws ConfigurationException { - s_logger.setLevel(Level.ALL); + public void setUp() { resource = new NfsSecondaryStorageResource(); - resource.setInSystemVM(true); - testParams = PropertiesUtil.toMap(loadProperties()); - resource.configureStorageLayerClass(testParams); - Object testLocalRoot = testParams.get("testLocalRoot"); - if (testLocalRoot != null) { - resource.setParentPath((String)testLocalRoot); - } } @Test - public void testMount() throws Exception { - String sampleUriStr = "cifs://192.168.1.128/CSHV3?user=administrator&password=1pass%40word1&foo=bar"; - URI sampleUri = new URI(sampleUriStr); + @PrepareForTest(NfsSecondaryStorageResource.class) + public void testSwiftWriteMetadataFile() throws Exception { + String expected = "uniquename=test\nfilename=testfile\nsize=100\nvirtualsize=1000"; - s_logger.info("Check HostIp parsing"); - String hostIpStr = resource.getUriHostIp(sampleUri); - Assert.assertEquals("Expected host IP " + sampleUri.getHost() + " and actual host IP " + hostIpStr + " differ.", sampleUri.getHost(), hostIpStr); + StringWriter stringWriter = new StringWriter(); + BufferedWriter bufferWriter = new BufferedWriter(stringWriter); + PowerMockito.whenNew(BufferedWriter.class).withArguments(any(FileWriter.class)).thenReturn(bufferWriter); - s_logger.info("Check option parsing"); - String expected = "user=administrator,password=1pass@word1,foo=bar,"; - String actualOpts = resource.parseCifsMountOptions(sampleUri); - Assert.assertEquals("Options should be " + expected + " and not " + actualOpts, expected, actualOpts); + resource.swiftWriteMetadataFile("testfile", "test", "testfile", 100, 1000); - // attempt a configured mount - final Map params = PropertiesUtil.toMap(loadProperties()); - String sampleMount = (String)params.get("testCifsMount"); - if (!sampleMount.isEmpty()) { - s_logger.info("functional test, mount " + sampleMount); - URI realMntUri = new URI(sampleMount); - String mntSubDir = resource.mountUri(realMntUri); - s_logger.info("functional test, umount " + mntSubDir); - resource.umount(resource.getMountingRoot() + mntSubDir, realMntUri); - } else { - s_logger.info("no entry for testCifsMount in " + "./conf/agent.properties - skip functional test"); - } + Assert.assertEquals(expected, stringWriter.toString()); } - public static Properties loadProperties() throws ConfigurationException { - Properties properties = new Properties(); - final File file = PropertiesUtil.findConfigFile("agent.properties"); - if (file == null) { - throw new ConfigurationException("Unable to find agent.properties."); - } - s_logger.info("agent.properties found at " + file.getAbsolutePath()); - try(FileInputStream fs = new FileInputStream(file);) { - properties.load(fs); - } catch (final FileNotFoundException ex) { - throw new CloudRuntimeException("Cannot find the file: " + file.getAbsolutePath(), ex); - } catch (final IOException ex) { - throw new CloudRuntimeException("IOException in reading " + file.getAbsolutePath(), ex); - } - return properties; - } + @Test + public void testCleanupStagingNfs() throws Exception{ + NfsSecondaryStorageResource spyResource = spy(resource); + RuntimeException exception = new RuntimeException(); + doThrow(exception).when(spyResource).execute(any(DeleteCommand.class)); + TemplateObjectTO mockTemplate = Mockito.mock(TemplateObjectTO.class); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.DEBUG, "Failed to clean up staging area:"); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(NfsSecondaryStorageResource.s_logger, testLogAppender); + + spyResource.cleanupStagingNfs(mockTemplate); + + testLogAppender.assertMessagesLogged(); + + } }