From c10c3245d1e8a43afe9eaf45ba12e86ceef1d32e Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Mon, 8 May 2017 13:13:51 +0200 Subject: [PATCH 1/4] vRouter: prevent fh leakage and use buffered writes in DataBags --- .../config/opt/cloud/bin/cs/CsConfig.py | 2 +- .../debian/config/opt/cloud/bin/merge.py | 27 ++++++++++--------- .../config/opt/cloud/bin/update_config.py | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py index e3b900912fd..e242a8fc32e 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py @@ -26,7 +26,7 @@ class CsConfig(object): A class to cache all the stuff that the other classes need """ __LOG_FILE = "/var/log/cloud.log" - __LOG_LEVEL = "DEBUG" + __LOG_LEVEL = "INFO" __LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s" cl = None diff --git a/systemvm/patches/debian/config/opt/cloud/bin/merge.py b/systemvm/patches/debian/config/opt/cloud/bin/merge.py index e6ca8586eea..0c85461f94e 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/merge.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/merge.py @@ -47,28 +47,29 @@ class DataBag: data = self.bdata if not os.path.exists(self.DPATH): os.makedirs(self.DPATH) - self.fpath = self.DPATH + '/' + self.key + '.json' + self.fpath = os.path.join(self.DPATH, self.key + '.json') + try: - handle = open(self.fpath) + with open(self.fpath, 'r') as _fh: + logging.debug("Loading data bag type %s", self.key) + data = json.load(_fh) except IOError: logging.debug("Creating data bag type %s", self.key) data.update({"id": self.key}) - else: - logging.debug("Loading data bag type %s", self.key) - data = json.load(handle) - handle.close() - self.dbag = data + finally: + self.dbag = data def save(self, dbag): try: - handle = open(self.fpath, 'w') + with open(self.fpath, 'w') as _fh: + logging.debug("Writing data bag type %s", self.key) + json.dump( + dbag, _fh, + sort_keys=True, + indent=2 + ) except IOError: logging.error("Could not write data bag %s", self.key) - else: - logging.debug("Writing data bag type %s", self.key) - logging.debug(dbag) - jsono = json.dumps(dbag, indent=4, sort_keys=True) - handle.write(jsono) def getDataBag(self): return self.dbag diff --git a/systemvm/patches/debian/config/opt/cloud/bin/update_config.py b/systemvm/patches/debian/config/opt/cloud/bin/update_config.py index 9ec3f872785..620d042ef1a 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/update_config.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/update_config.py @@ -27,7 +27,7 @@ import configure import json from cs.CsVmPassword import * -logging.basicConfig(filename='/var/log/cloud.log', level=logging.DEBUG, format='%(asctime)s %(filename)s %(funcName)s:%(lineno)d %(message)s') +logging.basicConfig(filename='/var/log/cloud.log', level=logging.INFO, format='%(asctime)s %(filename)s %(funcName)s:%(lineno)d %(message)s') # first commandline argument should be the file to process if (len(sys.argv) != 2): From 8f5275399e0319f1320aa2245acdd7aba3b5aa0f Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Mon, 8 May 2017 13:30:56 +0200 Subject: [PATCH 2/4] vRouter: vr_cfg: Keep the original cfg --- systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh b/systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh index d0eb1fccf79..e004127c2a0 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh +++ b/systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh @@ -90,8 +90,8 @@ do fi done < $cfg -#remove the configuration file, log file should have all the records as well -rm -f $cfg +# archive the configuration file +mv $cfg /var/cache/cloud/processed/ # Flush kernel conntrack table log_it "VR config: Flushing conntrack table" From a2eb0f23a9dd10dbbba9120d6efbf6d39e6003b2 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Mon, 8 May 2017 14:08:29 +0200 Subject: [PATCH 3/4] vRouter: vr_cfg: style consistency fixes --- .../debian/config/opt/cloud/bin/vr_cfg.sh | 45 +++++++------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh b/systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh index e004127c2a0..ab16b40d6d3 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh +++ b/systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh @@ -27,39 +27,28 @@ log_it() { echo "$(date) : $*" >> $log } -while getopts 'c:' OPTION -do - case $OPTION in - c) cfg="$OPTARG" - ;; - esac -done +while getopts 'c:' OPTION; do + case $OPTION in + c) cfg="$OPTARG" ;; +esac; done -while read line -do +while read line; do #comment - if [[ $line == \#* ]] - then - continue - fi + if [[ $line == \#* ]]; then + continue - if [ "$line" == "" ] - then + elif [ "$line" == "" ]; then read line version=$line log_it "VR config: configuation format version $version" #skip read line - continue - fi - if [ "$line" == " read line log_it "VR config: execution success " - continue - fi - if [ "$line" == "" ] - then + elif [ "$line" == "" ]; then read line file=$line log_it "VR config: creating file: $file" rm -f $file - while read -r line - do - if [ "$line" == "" ] - then + while read -r line; do + if [ "$line" == "" ]; then break fi echo $line >> $file done log_it "VR config: create file success" - continue + fi + done < $cfg # archive the configuration file From 330a0c7f2b33dbae608680d8a1198cdd76901e3f Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Wed, 7 Jun 2017 18:17:31 +0200 Subject: [PATCH 4/4] vRouter defer configure: Resolve merge conflicts --- .../patches/debian/config/opt/cloud/bin/update_config.py | 7 ++++--- systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/update_config.py b/systemvm/patches/debian/config/opt/cloud/bin/update_config.py index 620d042ef1a..172402744e9 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/update_config.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/update_config.py @@ -52,15 +52,16 @@ def process(do_merge=True): qf.setFile(sys.argv[1]) qf.do_merge = do_merge qf.load(None) - return qf def process_file(): print "[INFO] process_file" qf = process() - # Converge - finish_config() + # These can be safely deferred, dramatically speeding up loading times + if not (os.environ.get('DEFER_CONFIG', False) and sys.argv[1] in ('vm_dhcp_entry.json', 'vm_metadata.json')): + # Converge + finish_config() def process_vmpasswd(): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh b/systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh index ab16b40d6d3..a0be91cd341 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh +++ b/systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh @@ -32,6 +32,7 @@ while getopts 'c:' OPTION; do c) cfg="$OPTARG" ;; esac; done +export DEFER_CONFIG=true while read line; do #comment if [[ $line == \#* ]]; then @@ -78,6 +79,10 @@ done < $cfg # archive the configuration file mv $cfg /var/cache/cloud/processed/ +unset DEFER_CONFIG +# trigger finish_config() +/opt/cloud/bin/configure.py + # Flush kernel conntrack table log_it "VR config: Flushing conntrack table" conntrackd -d 2> /dev/null