Merge release branch 4.9 to master

* 4.9:
  vRouter defer configure: Resolve merge conflicts
  vRouter: vr_cfg: style consistency fixes
  vRouter: vr_cfg: Keep the original cfg
  vRouter: prevent fh leakage and use buffered writes in DataBags

systemvm/patches/debian/config/opt/cloud/bin/cs/CsConfig.py

@@ -26,7 +26,7 @@ class CsConfig(object):
     A class to cache all the stuff that the other classes need
     """
     __LOG_FILE = "/var/log/cloud.log"
-    __LOG_LEVEL = "DEBUG"
+    __LOG_LEVEL = "INFO"
     __LOG_FORMAT = "%(asctime)s %(levelname)-8s %(message)s"
     cl = None
 

systemvm/patches/debian/config/opt/cloud/bin/merge.py

@@ -47,28 +47,29 @@ class DataBag:
         data = self.bdata
         if not os.path.exists(self.DPATH):
             os.makedirs(self.DPATH)
-        self.fpath = self.DPATH + '/' + self.key + '.json'
+        self.fpath = os.path.join(self.DPATH, self.key + '.json')
+
         try:
-            handle = open(self.fpath)
+            with open(self.fpath, 'r') as _fh:
+                logging.debug("Loading data bag type %s", self.key)
+                data = json.load(_fh)
         except IOError:
             logging.debug("Creating data bag type %s", self.key)
             data.update({"id": self.key})
-        else:
-            logging.debug("Loading data bag type %s",  self.key)
-            data = json.load(handle)
-            handle.close()
-        self.dbag = data
+        finally:
+            self.dbag = data
 
     def save(self, dbag):
         try:
-            handle = open(self.fpath, 'w')
+            with open(self.fpath, 'w') as _fh:
+                logging.debug("Writing data bag type %s", self.key)
+                json.dump(
+                    dbag, _fh,
+                    sort_keys=True,
+                    indent=2
+                )
         except IOError:
             logging.error("Could not write data bag %s", self.key)
-        else:
-            logging.debug("Writing data bag type %s", self.key)
-            logging.debug(dbag)
-        jsono = json.dumps(dbag, indent=4, sort_keys=True)
-        handle.write(jsono)
 
     def getDataBag(self):
         return self.dbag

systemvm/patches/debian/config/opt/cloud/bin/update_config.py

@@ -27,7 +27,7 @@ import configure
 import json
 from cs.CsVmPassword import *
 
-logging.basicConfig(filename='/var/log/cloud.log', level=logging.DEBUG, format='%(asctime)s  %(filename)s %(funcName)s:%(lineno)d %(message)s')
+logging.basicConfig(filename='/var/log/cloud.log', level=logging.INFO, format='%(asctime)s  %(filename)s %(funcName)s:%(lineno)d %(message)s')
 
 # first commandline argument should be the file to process
 if (len(sys.argv) != 2):
@@ -52,15 +52,16 @@ def process(do_merge=True):
     qf.setFile(sys.argv[1])
     qf.do_merge = do_merge
     qf.load(None)
-
     return qf
 
 
 def process_file():
     print "[INFO] process_file"
     qf = process()
-    # Converge
-    finish_config()
+    # These can be safely deferred, dramatically speeding up loading times
+    if not (os.environ.get('DEFER_CONFIG', False) and sys.argv[1] in ('vm_dhcp_entry.json', 'vm_metadata.json')):
+        # Converge
+        finish_config()
 
 
 def process_vmpasswd():

systemvm/patches/debian/config/opt/cloud/bin/vr_cfg.sh

@@ -27,39 +27,29 @@ log_it() {
     echo "$(date) : $*" >> $log
 }
 
-while getopts 'c:' OPTION
-do
-  case $OPTION in
-      c) cfg="$OPTARG"
-          ;;
-  esac
-done
+while getopts 'c:' OPTION; do
+    case $OPTION in
+        c) cfg="$OPTARG" ;;
+esac; done
 
-while read line
-do
+export DEFER_CONFIG=true
+while read line; do
     #comment
-    if [[ $line == \#* ]]
-    then
-	    continue
-    fi
+    if [[ $line == \#* ]]; then
+        continue
 
-    if [ "$line" == "<version>" ]
-    then
+    elif [ "$line" == "<version>" ]; then
         read line
         version=$line
         log_it "VR config: configuation format version $version"
         #skip </version>
         read line
-        continue
-    fi
 
-    if [ "$line" == "<script>" ]
-    then
+    elif [ "$line" == "<script>" ]; then
         read line
         log_it "VR config: executing: $line"
         eval $line >> $log 2>&1
-        if [ $? -ne 0 ]
-        then
+        if [ $? -ne 0 ]; then
             log_it "VR config: executing failed: $line"
             # expose error info to mgmt server
             echo "VR config: execution failed: \"$line\", check $log in VR for details " 1>&2
@@ -68,30 +58,30 @@ do
         #skip </script>
         read line
         log_it "VR config: execution success "
-        continue
-    fi
 
-    if [ "$line" == "<file>" ]
-    then
+    elif [ "$line" == "<file>" ]; then
         read line
         file=$line
         log_it "VR config: creating file: $file"
         rm -f $file
-        while read -r line
-        do
-            if [ "$line" == "</file>" ]
-            then
+        while read -r line; do
+            if [ "$line" == "</file>" ]; then
                 break
             fi
             echo $line >> $file
         done
         log_it "VR config: create file success"
-        continue
+
     fi
+
 done < $cfg
 
-#remove the configuration file, log file should have all the records as well
-rm -f $cfg
+# archive the configuration file
+mv $cfg /var/cache/cloud/processed/
+
+unset DEFER_CONFIG
+# trigger finish_config()
+/opt/cloud/bin/configure.py
 
 # Flush kernel conntrack table
 log_it "VR config: Flushing conntrack table"