1. Use a common config merging class now in helpers

that will be the central point of config fetching
   for both the 'init' stage and the 'module' stages
   a. This helps those who want to understand exactly
      what configs are fetched and how the merging occurs.

cloudinit/helpers.py

@@ -28,7 +28,8 @@ import os
 
 from ConfigParser import (NoSectionError, NoOptionError, RawConfigParser)
 
-from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE)
+from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE,
+                                CFG_ENV_NAME)
 
 from cloudinit import log as logging
 from cloudinit import util
@@ -176,6 +177,85 @@ class Runners(object):
                 return (True, results)
 
 
+class ConfigMerger(object):
+    def __init__(self, paths=None, datasource=None,
+                 additional_fns=None, base_cfg=None):
+        self._paths = paths
+        self._ds = datasource
+        self._fns = additional_fns
+        self._base_cfg = base_cfg
+        # Created on first use
+        self._cfg = None
+
+    def _get_datasource_configs(self):
+        d_cfgs = []
+        if self._ds:
+            try:
+                ds_cfg = self._ds.get_config_obj()
+                if ds_cfg and isinstance(ds_cfg, (dict)):
+                    d_cfgs.append(ds_cfg)
+            except:
+                util.logexc(LOG, ("Failed loading of datasource"
+                                  " config object from %s"), self._ds)
+        return d_cfgs
+
+    def _get_env_configs(self):
+        e_cfgs = []
+        if CFG_ENV_NAME in os.environ:
+            e_fn = os.environ[CFG_ENV_NAME]
+            try:
+                e_cfgs.append(util.read_conf(e_fn))
+            except:
+                util.logexc(LOG, ('Failed loading of env. config'
+                                  ' from %s'), e_fn)
+        return e_cfgs
+
+    def _get_instance_configs(self):
+        i_cfgs = []
+        # If cloud-config was written, pick it up as
+        # a configuration file to use when running...
+        if not self._paths:
+            return i_cfgs
+        cc_fn = self._paths.get_ipath_cur('cloud_config')
+        if cc_fn and os.path.isfile(cc_fn):
+            try:
+                i_cfgs.append(util.read_conf(cc_fn))
+            except:
+                util.logexc(LOG, ('Failed loading of cloud-config'
+                                      ' from %s'), cc_fn)
+        return i_cfgs
+
+    def _read_cfg(self):
+        # Input config files override
+        # env config files which
+        # override instance configs
+        # which override datasource
+        # configs which override
+        # base configuration
+        cfgs = []
+        if self._fns:
+            for c_fn in self._fns:
+                try:
+                    cfgs.append(util.read_conf(c_fn))
+                except:
+                    util.logexc(LOG, ("Failed loading of configuration"
+                                       " from %s"), c_fn)
+
+        cfgs.extend(self._get_env_configs())
+        cfgs.extend(self._get_instance_configs())
+        cfgs.extend(self._get_datasource_configs())
+        if self._base_cfg:
+            cfgs.append(self._base_cfg)
+        return util.mergemanydict(cfgs)
+
+    @property
+    def cfg(self):
+        # None check to avoid empty case causing re-reading
+        if self._cfg is None:
+            self._cfg = self._read_cfg()
+        return self._cfg
+
+
 class ContentHandlers(object):
 
     def __init__(self):

cloudinit/stages.py

@@ -147,23 +147,16 @@ class Init(object):
             LOG.debug("Loaded 'init' config %s", self._cfg)
 
     def _read_cfg(self, extra_fns):
-        # Read extra files provided (if any)
-        i_cfgs = []
-        if extra_fns:
-            for fn in extra_fns:
-                try:
-                    fn_cfg = util.read_conf(fn)
-                    i_cfgs.append(fn_cfg)
-                except:
-                    util.logexc(LOG, ("Failed loading of additional"
-                                      " configuration from %s"), fn)
-        # Now read in the built-in + base
         try:
-            conf = util.get_base_cfg(builtin=util.get_builtin_cfg())
+            base_conf = util.get_base_cfg(builtin=util.get_builtin_cfg())
         except Exception:
-            conf = util.get_builtin_cfg()
-        i_cfgs.append(conf)
-        return util.mergemanydict(i_cfgs)
+            base_conf = util.get_builtin_cfg()
+        no_cfg_pths = helpers.Paths({}, self.datasource)
+        merger = helpers.ConfigMerger(paths=no_cfg_pths,
+                                      datasource=self.datasource,
+                                      additional_fns=extra_fns,
+                                      base_cfg=base_conf)
+        return merger.cfg
 
     def _restore_from_cache(self):
         pickled_fn = self.paths.get_ipath_cur('obj_pkl')
@@ -392,46 +385,24 @@ class Init(object):
 
 class Modules(object):
     def __init__(self, init, cfg_files=None):
-        self.datasource = init.datasource
-        self.cfg_files = cfg_files
-        self.base_cfg = init.cfg
         self.init = init
+        self.cfg_files = cfg_files
         # Created on first use
         self._cached_cfg = None
 
     @property
     def cfg(self):
-        # None check to avoid empty case
+        # None check to avoid empty case causing re-reading
         if self._cached_cfg is None:
-            self._cached_cfg = self._get_config()
+            merger = helpers.ConfigMerger(paths=self.init.paths,
+                                          datasource=self.init.datasource,
+                                          fns=self.cfg_files,
+                                          base_cfg=self.init.cfg)
+            self._cached_cfg = merger.cfg
             LOG.debug("Loading 'module' config %s", self._cached_cfg)
         # Only give out a copy so that others can't modify this...
         return copy.deepcopy(self._cached_cfg)
 
-    def _get_config(self):
-        t_cfgs = []
-        if self.cfg_files:
-            for fn in self.cfg_files:
-                try:
-                    t_cfgs.append(util.read_conf(fn))
-                except:
-                    util.logexc(LOG, ("Failed loading of configuration"
-                                       " from %s"), fn)
-
-        if self.datasource:
-            try:
-                d_cfg = self.datasource.get_config_obj()
-                if d_cfg:
-                    t_cfgs.append(d_cfg)
-            except:
-                util.logexc(LOG, ("Failed loading of datasource"
-                                  " config object from %s"), self.datasource)
-
-        if self.base_cfg:
-            t_cfgs.append(self.base_cfg)
-
-        return util.mergemanydict(t_cfgs)
-
     def _read_modules(self, name):
         module_list = []
         if name not in self.cfg: