x/cloud-init
Code Issues Proposed changes

rework of DataSource loading.

Browse Source 
The DataSources that are loaded are now controlled entirely via
configuration file of 'datasource_list', like:
   datasource_list: [ "NoCloud", "OVF", "Ec2" ]

Each item in that list is a "DataSourceCollection".  for each item
in the list, cloudinit will attempt to load:
  cloudinit.DataSource<item>
and, failing that,
  DataSource<item>

The module is required to have a method named 'get_datasource_list'
in it that takes a single list of "dependencies" and returns
a list of python classes inside the collection that can run needing
only those dependencies.

The dependencies are defines in DataSource.py. Currently:
  DEP_FILESYSTEM = "FILESYSTEM"
  DEP_NETWORK = "NETWORK"

When 'get_datasource_list' is called for the DataSourceOVF module with
[DEP_FILESYSTEM], then DataSourceOVF returns a single item list with a
reference to the 'DataSourceOVF' class.

When 'get_datasource_list' is called for the DataSourceOVF module with
[DEP_FILESYSTEM, DEP_NETWORK], it will return a single item list
with a reference to 'DataSourceOVFNet'.

cloudinit will then instanciate the class and call its 'get_data' method.
if the get_data method returns 'True', then it selects this class as the
selected Datasource.
This commit is contained in:
Scott Moser 2011-01-26 18:43:50 +00:00
parent 0e4f748c36
commit cdfbdc65b6
8 changed files with 156 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ChangeLog
View File

@ -31,3 +31,7 @@
via the config file, or user data config file
- add support for posting data about the instance to a url (phone_home)
- add minimal OVF transport (iso) support
- make DataSources that are attempted dynamic and configurable from
system config. changen "cloud_type: auto" as configuration for this
to 'datasource_list: [ "Ec2" ]'. Each of the items in that list
must be modules that can be loaded by "DataSource<item>"

25
cloud-init.py
View File

@ -23,6 +23,7 @@ import sys
import cloudinit
import cloudinit.util as util
import cloudinit.CloudConfig as CC
import cloudinit.DataSource as ds
import time
import logging
import errno
@ -32,10 +33,19 @@ def warn(wstr):
def main():
cmds = ( "start", "start-local" )
deps = { "start" : ( ds.DEP_FILESYSTEM, ds.DEP_NETWORK ),
"start-local" : ( ds.DEP_FILESYSTEM, ) }
cmd = ""
if len(sys.argv) > 1:
cmd = sys.argv[1]
cfg_path = None
if len(sys.argv) > 2:
# this is really for debugging only
# but you can invoke on development system with ./config/cloud.cfg
cfg_path = sys.argv[2]
if not cmd in cmds:
sys.stderr.write("bad command %s. use one of %s\n" % (cmd, cmds))
sys.exit(1)
@ -49,12 +59,17 @@ def main():
warn("unable to open /proc/uptime\n")
uptime = "na"
source_type = "all"
if cmd == "start-local":
source_type = "local"
try:
cfg = cloudinit.get_base_cfg(cfg_path)
except Exception as e:
warn("Failed to get base config. falling back to builtin: %s\n" % e)
try:
cfg = cloudinit.get_builtin_cfg()
except Exception as e:
warn("Unable to load builtin config\n")
raise
try:
cfg = cloudinit.get_base_cfg()
(outfmt, errfmt) = CC.get_output_cfg(cfg,"init")
CC.redirect_output(outfmt, errfmt)
except Exception as e:
@ -80,7 +95,7 @@ def main():
if cmd == "start-local":
cloudinit.purge_cache()
cloud = cloudinit.CloudInit(source_type=source_type)
cloud = cloudinit.CloudInit(ds_deps=deps[cmd])
try:
cloud.get_data_source()

56
cloudinit/DataSource.py
View File

@ -16,16 +16,23 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import cloudinit
DEP_FILESYSTEM = "FILESYSTEM"
DEP_NETWORK = "NETWORK"
import UserDataHandler as ud
class DataSource:
userdata = None
metadata = None
userdata_raw = None
log = None
def __init__(self):
pass
def __init__(self, log=None):
if not log:
import logging
log = logging.log
self.log = log
def get_userdata(self):
if self.userdata == None:
@ -91,3 +98,46 @@ class DataSource:
return("ip-%s" % '-'.join(r))
except: pass
return toks[0]
# return a list of classes that have the same depends as 'depends'
# iterate through cfg_list, loading "DataSourceCollections" modules
# and calling their "get_datasource_list".
# return an ordered list of classes that match
#
# - modules must be named "DataSource<item>", where 'item' is an entry
# in cfg_list
# - if pkglist is given, it will iterate try loading from that package
# ie, pkglist=[ "foo", "" ]
# will first try to load foo.DataSource<item>
# then DataSource<item>
def list_sources(cfg_list, depends, pkglist=[]):
retlist = []
for ds_coll in cfg_list:
for pkg in pkglist:
if pkg: pkg="%s." % pkg
try:
mod = __import__("%sDataSource%s" % (pkg, ds_coll))
if pkg:
mod = getattr(mod, "DataSource%s" % ds_coll)
lister = getattr(mod, "get_datasource_list")
retlist.extend(lister(depends))
break
except:
raise
return(retlist)
# depends is a list of dependencies (DEP_FILESYSTEM)
# dslist is a list of 2 item lists
# dslist = [
# ( class, ( depends-that-this-class-needs ) )
# }
# it returns a list of 'class' that matched these deps exactly
# it is a helper function for DataSourceCollections
def list_from_depends(depends, dslist):
retlist = [ ]
depset = set(depends)
for elem in dslist:
(cls, deps) = elem
if depset == set(deps):
retlist.append(cls)
return(retlist)

27
cloudinit/DataSourceEc2.py
View File

@ -18,7 +18,7 @@
import DataSource
import cloudinit
from cloudinit import seeddir
import cloudinit.util as util
import socket
import urllib2
@ -30,10 +30,7 @@ import errno
class DataSourceEc2(DataSource.DataSource):
api_ver = '2009-04-04'
seeddir = cloudinit.seeddir + '/ec2'
def __init__(self):
pass
seeddir = seeddir + '/ec2'
def __str__(self):
return("DataSourceEc2")
@ -43,7 +40,7 @@ class DataSourceEc2(DataSource.DataSource):
if util.read_optional_seed(seedret,base=self.seeddir+ "/"):
self.userdata_raw = seedret['user-data']
self.metadata = seedret['meta-data']
cloudinit.log.debug("using seeded ec2 data in %s" % self.seeddir)
self.log.debug("using seeded ec2 data in %s" % self.seeddir)
return True
try:
@ -105,13 +102,13 @@ class DataSourceEc2(DataSource.DataSource):
reason = "url error [%s]" % e.reason
if x == 0:
cloudinit.log.warning("waiting for metadata service at %s\n" % url)
self.log.warning("waiting for metadata service at %s\n" % url)
cloudinit.log.warning(" %s [%02s/%s]: %s\n" %
self.log.warning(" %s [%02s/%s]: %s\n" %
(time.strftime("%H:%M:%S",time.gmtime()), x+1, sleeps, reason))
time.sleep(sleeptime)
cloudinit.log.critical("giving up on md after %i seconds\n" %
self.log.critical("giving up on md after %i seconds\n" %
int(time.time()-starttime))
return False
@ -131,7 +128,7 @@ class DataSourceEc2(DataSource.DataSource):
if entname == "ephemeral" and name == "ephemeral0":
found = device
if found == None:
cloudinit.log.warn("unable to convert %s to a device" % name)
self.log.warn("unable to convert %s to a device" % name)
return None
# LP: #611137
@ -154,7 +151,7 @@ class DataSourceEc2(DataSource.DataSource):
for nto in tlist:
cand = "/dev/%s%s" % (nto, short[len(nfrom):])
if os.path.exists(cand):
cloudinit.log.debug("remapped device name %s => %s" % (found,cand))
self.log.debug("remapped device name %s => %s" % (found,cand))
return(cand)
return ofound
@ -165,3 +162,11 @@ class DataSourceEc2(DataSource.DataSource):
(p4 not in self.metadata or self.metadata[p4] == "")):
return True
return False
datasources = [
( DataSourceEc2, ( DataSource.DEP_FILESYSTEM , DataSource.DEP_NETWORK ) ),
]
# return a list of data sources that match this set of dependencies
def get_datasource_list(depends):
return(DataSource.list_from_depends(depends, datasources))

27
cloudinit/DataSourceNoCloud.py
View File

@ -18,7 +18,7 @@
import DataSource
import cloudinit
from cloudinit import seeddir
import cloudinit.util as util
import sys
import os.path
@ -32,10 +32,7 @@ class DataSourceNoCloud(DataSource.DataSource):
supported_seed_starts = ( "/" , "file://" )
seed = None
cmdline_id = "ds=nocloud"
seeddir = cloudinit.seeddir + '/nocloud'
def __init__(self):
pass
seeddir = seeddir + '/nocloud'
def __str__(self):
mstr="DataSourceNoCloud"
@ -57,7 +54,7 @@ class DataSourceNoCloud(DataSource.DataSource):
if parse_cmdline_data(self.cmdline_id, md):
found.append("cmdline")
except:
util.logexc(cloudinit.log,util.WARN)
util.logexc(self.log,util.WARN)
return False
# check to see if the seeddir has data.
@ -66,7 +63,7 @@ class DataSourceNoCloud(DataSource.DataSource):
md = util.mergedict(md,seedret['meta-data'])
ud = seedret['user-data']
found.append(self.seeddir)
cloudinit.log.debug("using seeded cache data in %s" % self.seeddir)
self.log.debug("using seeded cache data in %s" % self.seeddir)
# there was no indication on kernel cmdline or data
# in the seeddir suggesting this handler should be used.
@ -83,14 +80,14 @@ class DataSourceNoCloud(DataSource.DataSource):
seedfound=proto
break
if not seedfound:
cloudinit.log.debug("seed from %s not supported by %s" %
self.log.debug("seed from %s not supported by %s" %
(seedfrom, self.__class__))
return False
# this could throw errors, but the user told us to do it
# so if errors are raised, let them raise
(md_seed,ud) = util.read_seeded(seedfrom)
cloudinit.log.debug("using seeded cache data from %s" % seedfrom)
self.log.debug("using seeded cache data from %s" % seedfrom)
# values in the command line override those from the seed
md = util.mergedict(md,md_seed)
@ -143,4 +140,14 @@ def parse_cmdline_data(ds_id,fill,cmdline=None):
class DataSourceNoCloudNet(DataSourceNoCloud):
cmdline_id = "ds=nocloud-net"
supported_seed_starts = ( "http://", "https://", "ftp://" )
seeddir = cloudinit.seeddir + '/nocloud-net'
seeddir = seeddir + '/nocloud-net'
datasources = (
( DataSourceNoCloud, ( DataSource.DEP_FILESYSTEM, ) ),
( DataSourceNoCloudNet,
( DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK ) ),
)
# return a list of data sources that match this set of dependencies
def get_datasource_list(depends):
return(DataSource.list_from_depends(depends, datasources))

23
cloudinit/DataSourceOVF.py
View File

@ -18,7 +18,7 @@
import DataSource
import cloudinit
from cloudinit import seeddir
import cloudinit.util as util
import sys
import os.path
@ -33,16 +33,13 @@ import subprocess
class DataSourceOVF(DataSource.DataSource):
seed = None
seeddir = cloudinit.seeddir + '/ovf'
seeddir = seeddir + '/ovf'
environment = None
cfg = { }
userdata_raw = None
metadata = None
supported_seed_starts = ( "/" , "file://" )
def __init__(self):
pass
def __str__(self):
mstr="DataSourceOVF"
mstr = mstr + " [seed=%s]" % self.seed
@ -90,12 +87,12 @@ class DataSourceOVF(DataSource.DataSource):
seedfound = proto
break
if not seedfound:
cloudinit.log.debug("seed from %s not supported by %s" %
self.log.debug("seed from %s not supported by %s" %
(seedfrom, self.__class__))
return False
(md_seed,ud) = util.read_seeded(seedfrom)
cloudinit.log.debug("using seeded cache data from %s" % seedfrom)
self.log.debug("using seeded cache data from %s" % seedfrom)
md = util.mergedict(md,md_seed)
found.append(seedfrom)
@ -122,7 +119,7 @@ class DataSourceOVF(DataSource.DataSource):
return(self.cfg)
class DataSourceOVFNet(DataSourceOVF):
seeddir = cloudinit.seeddir + '/ovf-net'
seeddir = seeddir + '/ovf-net'
supported_seed_starts = ( "http://", "https://", "ftp://" )
# this will return a dict with some content
@ -283,6 +280,16 @@ def getProperties(environString):
return(props)
datasources = (
( DataSourceOVF, ( DataSource.DEP_FILESYSTEM, ) ),
( DataSourceOVFNet,
( DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK ) ),
)
# return a list of data sources that match this set of dependencies
def get_datasource_list(depends):
return(DataSource.list_from_depends(depends, datasources))
if __name__ == "__main__":
import sys
envStr = open(sys.argv[1]).read()

70
cloudinit/__init__.py
View File

@ -27,7 +27,7 @@ cfg_env_name = "CLOUD_CFG"
cfg_builtin = """
log_cfgs: [ ]
cloud_type: auto
datasource_list: [ "NoCloud", "OVF", "Ec2" ]
def_log_file: /var/log/cloud-init.log
syslog_fix_perms: syslog:adm
"""
@ -101,29 +101,17 @@ def logging_set_from_cfg(cfg):
raise Exception("no valid logging found\n")
import DataSourceEc2
import DataSourceNoCloud
import DataSourceOVF
import DataSource
import UserDataHandler
class CloudInit:
datasource_map = {
"ec2" : DataSourceEc2.DataSourceEc2,
"nocloud" : DataSourceNoCloud.DataSourceNoCloud,
"nocloud-net" : DataSourceNoCloud.DataSourceNoCloudNet,
"ovf" : DataSourceOVF.DataSourceOVF,
}
datasource = None
auto_orders = {
"all": ( "nocloud-net", "ec2" ),
"local" : ( "nocloud", "ovf" ),
}
cfg = None
part_handlers = { }
old_conffile = '/etc/ec2-init/ec2-config.cfg'
source_type = "all"
ds_deps = [ DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK ]
datasource = None
def __init__(self, source_type = "all", sysconfig=system_config):
def __init__(self, ds_deps = None, sysconfig=system_config):
self.part_handlers = {
'text/x-shellscript' : self.handle_user_script,
'text/cloud-config' : self.handle_cloud_config,
@ -131,15 +119,19 @@ class CloudInit:
'text/part-handler' : self.handle_handler,
'text/cloud-boothook' : self.handle_cloud_boothook
}
if ds_deps != None:
self.ds_deps = ds_deps
self.sysconfig=sysconfig
self.cfg=self.read_cfg()
self.source_type = source_type
def read_cfg(self):
if self.cfg:
return(self.cfg)
conf = util.get_base_cfg(self.sysconfig,cfg_builtin, parsed_cfgs)
try:
conf = util.get_base_cfg(self.sysconfig,cfg_builtin, parsed_cfgs)
except Exception as e:
conf = get_builtin_cfg()
# support reading the old ConfigObj format file and merging
# it into the yaml dictionary
@ -182,9 +174,6 @@ class CloudInit:
except:
return False
def get_cloud_type(self):
pass
def get_data_source(self):
if self.datasource is not None: return True
@ -192,21 +181,14 @@ class CloudInit:
log.debug("restored from cache type %s" % self.datasource)
return True
dslist=[ ]
cfglist=self.cfg['cloud_type']
if cfglist == "auto":
dslist = self.auto_orders[self.source_type]
elif cfglist:
for ds in cfglist.split(','):
dslist.append(strip(ds).tolower())
log.debug("searching for data source in [%s]" % str(dslist))
for ds in dslist:
if ds not in self.datasource_map:
log.warn("data source %s not found in map" % ds)
continue
cfglist=self.cfg['datasource_list']
dslist = list_sources(cfglist, self.ds_deps)
dsnames = map(lambda f: f.__name__, dslist)
log.debug("searching for data source in %s" % dsnames)
for cls in dslist:
ds = cls.__name__
try:
s = self.datasource_map[ds]()
s = cls(log)
if s.get_data():
self.datasource = s
self.datasource_name = ds
@ -216,8 +198,9 @@ class CloudInit:
log.warn("get_data of %s raised %s" % (ds,e))
util.logexc(log)
pass
log.debug("did not find data source from %s" % dslist)
raise DataSourceNotFoundException("Could not find data source")
msg = "Did not find data source. searched classes: %s" % dsnames
log.debug(msg)
raise DataSourceNotFoundException(msg)
def set_cur_instance(self):
try:
@ -532,8 +515,15 @@ def get_ipath_cur(name=None):
def get_cpath(name=None):
return("%s%s" % (varlibdir, pathmap[name]))
def get_base_cfg():
return(util.get_base_cfg(system_config,cfg_builtin,parsed_cfgs))
def get_base_cfg(cfg_path=None):
if cfg_path is None: cfg_path = system_config
return(util.get_base_cfg(cfg_path,cfg_builtin,parsed_cfgs))
def get_builtin_cfg():
return(yaml.load(cfg_builtin))
class DataSourceNotFoundException(Exception):
pass
def list_sources(cfg_list, depends):
return(DataSource.list_sources(cfg_list,depends, ["cloudinit", "" ]))

2
config/cloud.cfg
View File

@ -1,7 +1,7 @@
cloud: auto
user: ubuntu
disable_root: 1
preserve_hostname: False
# datasource_list: [ "NoCloud", "OVF", "Ec2" ]
cloud_init_modules:
- resizefs