merge in work form ds-rework

rework of DataSource loading.

The DataSources that are loaded are now controlled entirely via
configuration file of 'datasource_list', like:
   datasource_list: [ "NoCloud", "OVF", "Ec2" ]

Each item in that list is a "DataSourceCollection".  for each item
in the list, cloudinit will attempt to load:
  cloudinit.DataSource<item>
and, failing that,
  DataSource<item>

The module is required to have a method named 'get_datasource_list'
in it that takes a single list of "dependencies" and returns
a list of python classes inside the collection that can run needing
only those dependencies.

The dependencies are defines in DataSource.py. Currently:
  DEP_FILESYSTEM = "FILESYSTEM"
  DEP_NETWORK = "NETWORK"

When 'get_datasource_list' is called for the DataSourceOVF module with
[DEP_FILESYSTEM], then DataSourceOVF returns a single item list with a
reference to the 'DataSourceOVF' class.

When 'get_datasource_list' is called for the DataSourceOVF module with
[DEP_FILESYSTEM, DEP_NETWORK], it will return a single item list
with a reference to 'DataSourceOVFNet'.

cloudinit will then instanciate the class and call its 'get_data' method.
if the get_data method returns 'True', then it selects this class as the
selected Datasource.

ChangeLog

@@ -31,3 +31,7 @@
    via the config file, or user data config file
  - add support for posting data about the instance to a url (phone_home)
  - add minimal OVF transport (iso) support
+ - make DataSources that are attempted dynamic and configurable from
+   system config. changen "cloud_type: auto" as configuration for this
+   to 'datasource_list: [ "Ec2" ]'.  Each of the items in that list
+   must be modules that can be loaded by "DataSource<item>"

cloud-init.py

@@ -23,6 +23,7 @@ import sys
 import cloudinit
 import cloudinit.util as util
 import cloudinit.CloudConfig as CC
+import cloudinit.DataSource as ds
 import time
 import logging
 import errno
@@ -32,10 +33,19 @@ def warn(wstr):
 
 def main():
     cmds = ( "start", "start-local" )
+    deps = { "start" : ( ds.DEP_FILESYSTEM, ds.DEP_NETWORK ),
+             "start-local" : ( ds.DEP_FILESYSTEM, ) }
+
     cmd = ""
     if len(sys.argv) > 1:
         cmd = sys.argv[1]
 
+    cfg_path = None
+    if len(sys.argv) > 2:
+        # this is really for debugging only
+        # but you can invoke on development system with ./config/cloud.cfg
+        cfg_path = sys.argv[2]
+
     if not cmd in cmds:
         sys.stderr.write("bad command %s. use one of %s\n" % (cmd, cmds))
         sys.exit(1)
@@ -49,12 +59,17 @@ def main():
        warn("unable to open /proc/uptime\n")
        uptime = "na"
 
-    source_type = "all"
+    try:
-    if cmd == "start-local":
+        cfg = cloudinit.get_base_cfg(cfg_path)
-        source_type = "local"
+    except Exception as e:
+        warn("Failed to get base config. falling back to builtin: %s\n" % e)
+        try:
+            cfg = cloudinit.get_builtin_cfg()
+        except Exception as e:
+            warn("Unable to load builtin config\n")
+            raise
 
     try:
-        cfg = cloudinit.get_base_cfg()
         (outfmt, errfmt) = CC.get_output_cfg(cfg,"init")
         CC.redirect_output(outfmt, errfmt)
     except Exception as e:
@@ -80,7 +95,7 @@ def main():
     if cmd == "start-local":
         cloudinit.purge_cache()
 
-    cloud = cloudinit.CloudInit(source_type=source_type)
+    cloud = cloudinit.CloudInit(ds_deps=deps[cmd])
 
     try:
         cloud.get_data_source()

cloudinit/DataSource.py

@@ -16,16 +16,23 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import cloudinit
+
+DEP_FILESYSTEM = "FILESYSTEM"
+DEP_NETWORK = "NETWORK"
+
 import UserDataHandler as ud
 
 class DataSource:
     userdata = None
     metadata = None
     userdata_raw = None
+    log = None
 
-    def __init__(self):
+    def __init__(self, log=None):
-       pass
+        if not log:
+            import logging
+            log = logging.log
+        self.log = log
 
     def get_userdata(self):
         if self.userdata == None:
@@ -91,3 +98,46 @@ class DataSource:
                     return("ip-%s" % '-'.join(r))
             except: pass
         return toks[0]
+
+# return a list of classes that have the same depends as 'depends'
+# iterate through cfg_list, loading "DataSourceCollections" modules
+# and calling their "get_datasource_list".
+# return an ordered list of classes that match
+#
+# - modules must be named "DataSource<item>", where 'item' is an entry
+#   in cfg_list
+# - if pkglist is given, it will iterate try loading from that package
+#   ie, pkglist=[ "foo", "" ]
+#     will first try to load foo.DataSource<item>
+#     then DataSource<item>
+def list_sources(cfg_list, depends, pkglist=[]):
+    retlist = []
+    for ds_coll in cfg_list:
+        for pkg in pkglist:
+            if pkg: pkg="%s." % pkg
+            try:
+                mod = __import__("%sDataSource%s" % (pkg, ds_coll))
+                if pkg:
+                    mod = getattr(mod, "DataSource%s" % ds_coll)
+                lister = getattr(mod, "get_datasource_list")
+                retlist.extend(lister(depends))
+                break
+            except:
+                raise
+    return(retlist)
+
+# depends is a list of dependencies (DEP_FILESYSTEM)
+# dslist is a list of 2 item lists
+# dslist = [ 
+#   ( class, ( depends-that-this-class-needs ) )
+# }
+# it returns a list of 'class' that matched these deps exactly
+# it is a helper function for DataSourceCollections
+def list_from_depends(depends, dslist):
+    retlist = [ ]
+    depset = set(depends)
+    for elem in dslist:
+        (cls, deps) = elem
+        if depset == set(deps):
+            retlist.append(cls)
+    return(retlist)

cloudinit/DataSourceEc2.py

@@ -18,7 +18,7 @@
 
 import DataSource
 
-import cloudinit
+from cloudinit import seeddir
 import cloudinit.util as util
 import socket
 import urllib2
@@ -30,10 +30,7 @@ import errno
 
 class DataSourceEc2(DataSource.DataSource):
     api_ver  = '2009-04-04'
-    seeddir = cloudinit.seeddir + '/ec2'
+    seeddir = seeddir + '/ec2'
-
-    def __init__(self):
-        pass
 
     def __str__(self):
         return("DataSourceEc2")
@@ -43,7 +40,7 @@ class DataSourceEc2(DataSource.DataSource):
         if util.read_optional_seed(seedret,base=self.seeddir+ "/"):
             self.userdata_raw = seedret['user-data']
             self.metadata = seedret['meta-data']
-            cloudinit.log.debug("using seeded ec2 data in %s" % self.seeddir)
+            self.log.debug("using seeded ec2 data in %s" % self.seeddir)
             return True
         
         try:
@@ -105,13 +102,13 @@ class DataSourceEc2(DataSource.DataSource):
                 reason = "url error [%s]" % e.reason
     
             if x == 0:
-                cloudinit.log.warning("waiting for metadata service at %s\n" % url)
+                self.log.warning("waiting for metadata service at %s\n" % url)
 
-            cloudinit.log.warning("  %s [%02s/%s]: %s\n" %
+            self.log.warning("  %s [%02s/%s]: %s\n" %
                 (time.strftime("%H:%M:%S",time.gmtime()), x+1, sleeps, reason))
             time.sleep(sleeptime)
 
-        cloudinit.log.critical("giving up on md after %i seconds\n" %
+        self.log.critical("giving up on md after %i seconds\n" %
                   int(time.time()-starttime))
         return False
 
@@ -131,7 +128,7 @@ class DataSourceEc2(DataSource.DataSource):
             if entname == "ephemeral" and name == "ephemeral0":
                 found = device
         if found == None:
-            cloudinit.log.warn("unable to convert %s to a device" % name)
+            self.log.warn("unable to convert %s to a device" % name)
             return None
 
         # LP: #611137
@@ -154,7 +151,7 @@ class DataSourceEc2(DataSource.DataSource):
             for nto in tlist:
                 cand = "/dev/%s%s" % (nto, short[len(nfrom):])
                 if os.path.exists(cand):
-                    cloudinit.log.debug("remapped device name %s => %s" % (found,cand))
+                    self.log.debug("remapped device name %s => %s" % (found,cand))
                     return(cand)
         return ofound
 
@@ -165,3 +162,11 @@ class DataSourceEc2(DataSource.DataSource):
             (p4 not in self.metadata or self.metadata[p4] == "")):
             return True
         return False
+
+datasources = [ 
+  ( DataSourceEc2, ( DataSource.DEP_FILESYSTEM , DataSource.DEP_NETWORK ) ),
+]
+
+# return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return(DataSource.list_from_depends(depends, datasources))

cloudinit/DataSourceNoCloud.py

@@ -18,7 +18,7 @@
 
 import DataSource
 
-import cloudinit
+from cloudinit import seeddir
 import cloudinit.util as util
 import sys
 import os.path
@@ -32,10 +32,7 @@ class DataSourceNoCloud(DataSource.DataSource):
     supported_seed_starts = ( "/" , "file://" )
     seed = None
     cmdline_id = "ds=nocloud"
-    seeddir = cloudinit.seeddir + '/nocloud'
+    seeddir = seeddir + '/nocloud'
-
-    def __init__(self):
-        pass
 
     def __str__(self):
         mstr="DataSourceNoCloud"
@@ -57,7 +54,7 @@ class DataSourceNoCloud(DataSource.DataSource):
             if parse_cmdline_data(self.cmdline_id, md):
                 found.append("cmdline")
         except:
-            util.logexc(cloudinit.log,util.WARN)
+            util.logexc(self.log,util.WARN)
             return False
 
         # check to see if the seeddir has data.
@@ -66,7 +63,7 @@ class DataSourceNoCloud(DataSource.DataSource):
             md = util.mergedict(md,seedret['meta-data'])
             ud = seedret['user-data']
             found.append(self.seeddir)
-            cloudinit.log.debug("using seeded cache data in %s" % self.seeddir)
+            self.log.debug("using seeded cache data in %s" % self.seeddir)
 
         # there was no indication on kernel cmdline or data
         # in the seeddir suggesting this handler should be used.
@@ -83,14 +80,14 @@ class DataSourceNoCloud(DataSource.DataSource):
                     seedfound=proto
                     break
             if not seedfound:
-                cloudinit.log.debug("seed from %s not supported by %s" %
+                self.log.debug("seed from %s not supported by %s" %
                     (seedfrom, self.__class__))
                 return False
 
             # this could throw errors, but the user told us to do it
             # so if errors are raised, let them raise
             (md_seed,ud) = util.read_seeded(seedfrom)
-            cloudinit.log.debug("using seeded cache data from %s" % seedfrom)
+            self.log.debug("using seeded cache data from %s" % seedfrom)
 
             # values in the command line override those from the seed
             md = util.mergedict(md,md_seed)
@@ -143,4 +140,14 @@ def parse_cmdline_data(ds_id,fill,cmdline=None):
 class DataSourceNoCloudNet(DataSourceNoCloud):
     cmdline_id = "ds=nocloud-net"
     supported_seed_starts = ( "http://", "https://", "ftp://" )
-    seeddir = cloudinit.seeddir + '/nocloud-net'
+    seeddir = seeddir + '/nocloud-net'
+
+datasources = (
+  ( DataSourceNoCloud, ( DataSource.DEP_FILESYSTEM, ) ),
+  ( DataSourceNoCloudNet, 
+    ( DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK ) ),
+)
+
+# return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return(DataSource.list_from_depends(depends, datasources))

cloudinit/DataSourceOVF.py

@@ -18,7 +18,7 @@
 
 import DataSource
 
-import cloudinit
+from cloudinit import seeddir
 import cloudinit.util as util
 import sys
 import os.path
@@ -27,19 +27,19 @@ import errno
 from xml.dom import minidom
 from xml.dom import Node
 import base64
+import re
+import tempfile
+import subprocess
 
 class DataSourceOVF(DataSource.DataSource):
     seed = None
-    seeddir = cloudinit.seeddir + '/ovf'
+    seeddir = seeddir + '/ovf'
     environment = None
     cfg = { }
     userdata_raw = None
     metadata = None
     supported_seed_starts = ( "/" , "file://" )
 
-    def __init__(self):
-        pass
-
     def __str__(self):
         mstr="DataSourceOVF"
         mstr = mstr + " [seed=%s]" % self.seed
@@ -87,12 +87,12 @@ class DataSourceOVF(DataSource.DataSource):
                     seedfound = proto
                     break
             if not seedfound:
-                cloudinit.log.debug("seed from %s not supported by %s" %
+                self.log.debug("seed from %s not supported by %s" %
                     (seedfrom, self.__class__))
                 return False
 
             (md_seed,ud) = util.read_seeded(seedfrom)
-            cloudinit.log.debug("using seeded cache data from %s" % seedfrom)
+            self.log.debug("using seeded cache data from %s" % seedfrom)
 
             md = util.mergedict(md,md_seed)
             found.append(seedfrom)
@@ -119,7 +119,7 @@ class DataSourceOVF(DataSource.DataSource):
         return(self.cfg)
 
 class DataSourceOVFNet(DataSourceOVF):
-    seeddir = cloudinit.seeddir + '/ovf-net'
+    seeddir = seeddir + '/ovf-net'
     supported_seed_starts = ( "http://", "https://", "ftp://" )
 
 # this will return a dict with some content
@@ -280,6 +280,16 @@ def getProperties(environString):
 
     return(props)
 
+datasources = (
+  ( DataSourceOVF, ( DataSource.DEP_FILESYSTEM, ) ),
+  ( DataSourceOVFNet, 
+    ( DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK ) ),
+)
+
+# return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return(DataSource.list_from_depends(depends, datasources))
+
 if __name__ == "__main__":
     import sys
     envStr = open(sys.argv[1]).read()

cloudinit/__init__.py

@@ -27,7 +27,7 @@ cfg_env_name = "CLOUD_CFG"
 
 cfg_builtin = """
 log_cfgs: [ ]
-cloud_type: auto
+datasource_list: [ "NoCloud", "OVF", "Ec2" ]
 def_log_file: /var/log/cloud-init.log
 syslog_fix_perms: syslog:adm
 """
@@ -101,29 +101,17 @@ def logging_set_from_cfg(cfg):
     raise Exception("no valid logging found\n")
 
 
-import DataSourceEc2
+import DataSource
-import DataSourceNoCloud
-import DataSourceOVF
 import UserDataHandler
 
 class CloudInit:
-    datasource_map = {
-        "ec2" : DataSourceEc2.DataSourceEc2,
-        "nocloud" : DataSourceNoCloud.DataSourceNoCloud,
-        "nocloud-net" : DataSourceNoCloud.DataSourceNoCloudNet,
-        "ovf" : DataSourceOVF.DataSourceOVF,
-    }
-    datasource = None
-    auto_orders = {
-        "all": ( "nocloud-net", "ec2" ),
-        "local" : ( "nocloud", "ovf" ),
-    }
     cfg = None
     part_handlers = { }
     old_conffile = '/etc/ec2-init/ec2-config.cfg'
-    source_type = "all"
+    ds_deps = [ DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK ]
+    datasource = None
 
-    def __init__(self, source_type = "all", sysconfig=system_config):
+    def __init__(self, ds_deps = None, sysconfig=system_config):
         self.part_handlers = {
             'text/x-shellscript' : self.handle_user_script,
             'text/cloud-config' : self.handle_cloud_config,
@@ -131,15 +119,19 @@ class CloudInit:
             'text/part-handler' : self.handle_handler,
             'text/cloud-boothook' : self.handle_cloud_boothook
         }
+        if ds_deps != None:
+            self.ds_deps = ds_deps
         self.sysconfig=sysconfig
         self.cfg=self.read_cfg()
-        self.source_type = source_type
 
     def read_cfg(self):
         if self.cfg:
             return(self.cfg)
 
-        conf = util.get_base_cfg(self.sysconfig,cfg_builtin, parsed_cfgs)
+        try:
+            conf = util.get_base_cfg(self.sysconfig,cfg_builtin, parsed_cfgs)
+        except Exception as e:
+            conf = get_builtin_cfg()
 
         # support reading the old ConfigObj format file and merging
         # it into the yaml dictionary
@@ -182,9 +174,6 @@ class CloudInit:
         except:
             return False
         
-    def get_cloud_type(self):
-        pass
-
     def get_data_source(self):
         if self.datasource is not None: return True
 
@@ -192,21 +181,14 @@ class CloudInit:
             log.debug("restored from cache type %s" % self.datasource)
             return True
 
-        dslist=[ ]
+        cfglist=self.cfg['datasource_list']
-        cfglist=self.cfg['cloud_type']
+        dslist = list_sources(cfglist, self.ds_deps)
-        if cfglist == "auto":
+        dsnames = map(lambda f: f.__name__, dslist)
-            dslist = self.auto_orders[self.source_type]
+        log.debug("searching for data source in %s" % dsnames)
-        elif cfglist:
+        for cls in dslist:
-            for ds in cfglist.split(','):
+            ds = cls.__name__
-                dslist.append(strip(ds).tolower())
-            
-        log.debug("searching for data source in [%s]" % str(dslist))
-        for ds in dslist:
-            if ds not in self.datasource_map:
-                log.warn("data source %s not found in map" % ds)
-                continue
             try:
-                s = self.datasource_map[ds]()
+                s = cls(log)
                 if s.get_data():
                     self.datasource = s
                     self.datasource_name = ds
@@ -216,8 +198,9 @@ class CloudInit:
                 log.warn("get_data of %s raised %s" % (ds,e))
                 util.logexc(log)
                 pass
-        log.debug("did not find data source from %s" % dslist)
+        msg = "Did not find data source. searched classes: %s" % dsnames
-        raise DataSourceNotFoundException("Could not find data source")
+        log.debug(msg)
+        raise DataSourceNotFoundException(msg)
 
     def set_cur_instance(self):
         try:
@@ -532,8 +515,15 @@ def get_ipath_cur(name=None):
 def get_cpath(name=None):
     return("%s%s" % (varlibdir, pathmap[name]))
 
-def get_base_cfg():
+def get_base_cfg(cfg_path=None):
-    return(util.get_base_cfg(system_config,cfg_builtin,parsed_cfgs))
+    if cfg_path is None: cfg_path = system_config
+    return(util.get_base_cfg(cfg_path,cfg_builtin,parsed_cfgs))
+
+def get_builtin_cfg():
+    return(yaml.load(cfg_builtin))
 
 class DataSourceNotFoundException(Exception):
     pass
+
+def list_sources(cfg_list, depends):
+    return(DataSource.list_sources(cfg_list,depends, ["cloudinit", "" ]))

config/cloud.cfg

@@ -1,7 +1,7 @@
-cloud: auto
 user: ubuntu
 disable_root: 1
 preserve_hostname: False
+# datasource_list: [ "NoCloud", "OVF", "Ec2" ]
 
 cloud_init_modules:
  - resizefs