4d1d7a9bb5
1. Remove the usage of the path.join function now that all code should be going through the util file methods (and they can be mocked out as needed). 2. Adjust all occurences of the above join function to either not use it or replace it with the standard os.path.join (which can also be mocked out as needed) 3. Fix pylint from complaining about the tests folder 'helpers.py' not being found 4. Add a pylintrc file that is used instead of the options hidden in the 'run_pylint' tool.
429 lines
14 KiB
Python
429 lines
14 KiB
Python
# vi: ts=4 expandtab
|
|
#
|
|
# Copyright (C) 2012 Canonical Ltd.
|
|
# Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
|
|
# Copyright (C) 2012 Yahoo! Inc.
|
|
#
|
|
# Author: Scott Moser <scott.moser@canonical.com>
|
|
# Author: Juerg Haefliger <juerg.haefliger@hp.com>
|
|
# Author: Joshua Harlow <harlowja@yahoo-inc.com>
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 3, as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
from time import time
|
|
|
|
import contextlib
|
|
import io
|
|
import os
|
|
|
|
from ConfigParser import (NoSectionError, NoOptionError, RawConfigParser)
|
|
|
|
from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE,
|
|
CFG_ENV_NAME)
|
|
|
|
from cloudinit import log as logging
|
|
from cloudinit import util
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class LockFailure(Exception):
|
|
pass
|
|
|
|
|
|
class DummyLock(object):
|
|
pass
|
|
|
|
|
|
class DummySemaphores(object):
|
|
def __init__(self):
|
|
pass
|
|
|
|
@contextlib.contextmanager
|
|
def lock(self, _name, _freq, _clear_on_fail=False):
|
|
yield DummyLock()
|
|
|
|
def has_run(self, _name, _freq):
|
|
return False
|
|
|
|
def clear(self, _name, _freq):
|
|
return True
|
|
|
|
def clear_all(self):
|
|
pass
|
|
|
|
|
|
class FileLock(object):
|
|
def __init__(self, fn):
|
|
self.fn = fn
|
|
|
|
def __str__(self):
|
|
return "<%s using file %r>" % (util.obj_name(self), self.fn)
|
|
|
|
|
|
class FileSemaphores(object):
|
|
def __init__(self, sem_path):
|
|
self.sem_path = sem_path
|
|
|
|
@contextlib.contextmanager
|
|
def lock(self, name, freq, clear_on_fail=False):
|
|
try:
|
|
yield self._acquire(name, freq)
|
|
except:
|
|
if clear_on_fail:
|
|
self.clear(name, freq)
|
|
raise
|
|
|
|
def clear(self, name, freq):
|
|
sem_file = self._get_path(name, freq)
|
|
try:
|
|
util.del_file(sem_file)
|
|
except (IOError, OSError):
|
|
util.logexc(LOG, "Failed deleting semaphore %s", sem_file)
|
|
return False
|
|
return True
|
|
|
|
def clear_all(self):
|
|
try:
|
|
util.del_dir(self.sem_path)
|
|
except (IOError, OSError):
|
|
util.logexc(LOG, "Failed deleting semaphore directory %s",
|
|
self.sem_path)
|
|
|
|
def _acquire(self, name, freq):
|
|
# Check again if its been already gotten
|
|
if self.has_run(name, freq):
|
|
return None
|
|
# This is a race condition since nothing atomic is happening
|
|
# here, but this should be ok due to the nature of when
|
|
# and where cloud-init runs... (file writing is not a lock...)
|
|
sem_file = self._get_path(name, freq)
|
|
contents = "%s: %s\n" % (os.getpid(), time())
|
|
try:
|
|
util.write_file(sem_file, contents)
|
|
except (IOError, OSError):
|
|
util.logexc(LOG, "Failed writing semaphore file %s", sem_file)
|
|
return None
|
|
return FileLock(sem_file)
|
|
|
|
def has_run(self, name, freq):
|
|
if not freq or freq == PER_ALWAYS:
|
|
return False
|
|
sem_file = self._get_path(name, freq)
|
|
# This isn't really a good atomic check
|
|
# but it suffices for where and when cloudinit runs
|
|
if os.path.exists(sem_file):
|
|
return True
|
|
return False
|
|
|
|
def _get_path(self, name, freq):
|
|
sem_path = self.sem_path
|
|
if not freq or freq == PER_INSTANCE:
|
|
return os.path.join(sem_path, name)
|
|
else:
|
|
return os.path.join(sem_path, "%s.%s" % (name, freq))
|
|
|
|
|
|
class Runners(object):
|
|
def __init__(self, paths):
|
|
self.paths = paths
|
|
self.sems = {}
|
|
|
|
def _get_sem(self, freq):
|
|
if freq == PER_ALWAYS or not freq:
|
|
return None
|
|
sem_path = None
|
|
if freq == PER_INSTANCE:
|
|
# This may not exist,
|
|
# so thats why we still check for none
|
|
# below if say the paths object
|
|
# doesn't have a datasource that can
|
|
# provide this instance path...
|
|
sem_path = self.paths.get_ipath("sem")
|
|
elif freq == PER_ONCE:
|
|
sem_path = self.paths.get_cpath("sem")
|
|
if not sem_path:
|
|
return None
|
|
if sem_path not in self.sems:
|
|
self.sems[sem_path] = FileSemaphores(sem_path)
|
|
return self.sems[sem_path]
|
|
|
|
def run(self, name, functor, args, freq=None, clear_on_fail=False):
|
|
sem = self._get_sem(freq)
|
|
if not sem:
|
|
sem = DummySemaphores()
|
|
if not args:
|
|
args = []
|
|
if sem.has_run(name, freq):
|
|
LOG.debug("%s already ran (freq=%s)", name, freq)
|
|
return (False, None)
|
|
with sem.lock(name, freq, clear_on_fail) as lk:
|
|
if not lk:
|
|
raise LockFailure("Failed to acquire lock for %s" % name)
|
|
else:
|
|
LOG.debug("Running %s using lock (%s)", name, lk)
|
|
if isinstance(args, (dict)):
|
|
results = functor(**args)
|
|
else:
|
|
results = functor(*args)
|
|
return (True, results)
|
|
|
|
|
|
class ConfigMerger(object):
|
|
def __init__(self, paths=None, datasource=None,
|
|
additional_fns=None, base_cfg=None):
|
|
self._paths = paths
|
|
self._ds = datasource
|
|
self._fns = additional_fns
|
|
self._base_cfg = base_cfg
|
|
# Created on first use
|
|
self._cfg = None
|
|
|
|
def _get_datasource_configs(self):
|
|
d_cfgs = []
|
|
if self._ds:
|
|
try:
|
|
ds_cfg = self._ds.get_config_obj()
|
|
if ds_cfg and isinstance(ds_cfg, (dict)):
|
|
d_cfgs.append(ds_cfg)
|
|
except:
|
|
util.logexc(LOG, ("Failed loading of datasource"
|
|
" config object from %s"), self._ds)
|
|
return d_cfgs
|
|
|
|
def _get_env_configs(self):
|
|
e_cfgs = []
|
|
if CFG_ENV_NAME in os.environ:
|
|
e_fn = os.environ[CFG_ENV_NAME]
|
|
try:
|
|
e_cfgs.append(util.read_conf(e_fn))
|
|
except:
|
|
util.logexc(LOG, ('Failed loading of env. config'
|
|
' from %s'), e_fn)
|
|
return e_cfgs
|
|
|
|
def _get_instance_configs(self):
|
|
i_cfgs = []
|
|
# If cloud-config was written, pick it up as
|
|
# a configuration file to use when running...
|
|
if not self._paths:
|
|
return i_cfgs
|
|
cc_fn = self._paths.get_ipath_cur('cloud_config')
|
|
if cc_fn and os.path.isfile(cc_fn):
|
|
try:
|
|
i_cfgs.append(util.read_conf(cc_fn))
|
|
except:
|
|
util.logexc(LOG, ('Failed loading of cloud-config'
|
|
' from %s'), cc_fn)
|
|
return i_cfgs
|
|
|
|
def _read_cfg(self):
|
|
# Input config files override
|
|
# env config files which
|
|
# override instance configs
|
|
# which override datasource
|
|
# configs which override
|
|
# base configuration
|
|
cfgs = []
|
|
if self._fns:
|
|
for c_fn in self._fns:
|
|
try:
|
|
cfgs.append(util.read_conf(c_fn))
|
|
except:
|
|
util.logexc(LOG, ("Failed loading of configuration"
|
|
" from %s"), c_fn)
|
|
|
|
cfgs.extend(self._get_env_configs())
|
|
cfgs.extend(self._get_instance_configs())
|
|
cfgs.extend(self._get_datasource_configs())
|
|
if self._base_cfg:
|
|
cfgs.append(self._base_cfg)
|
|
return util.mergemanydict(cfgs)
|
|
|
|
@property
|
|
def cfg(self):
|
|
# None check to avoid empty case causing re-reading
|
|
if self._cfg is None:
|
|
self._cfg = self._read_cfg()
|
|
return self._cfg
|
|
|
|
|
|
class ContentHandlers(object):
|
|
|
|
def __init__(self):
|
|
self.registered = {}
|
|
|
|
def __contains__(self, item):
|
|
return self.is_registered(item)
|
|
|
|
def __getitem__(self, key):
|
|
return self._get_handler(key)
|
|
|
|
def is_registered(self, content_type):
|
|
return content_type in self.registered
|
|
|
|
def register(self, mod):
|
|
types = set()
|
|
for t in mod.list_types():
|
|
self.registered[t] = mod
|
|
types.add(t)
|
|
return types
|
|
|
|
def _get_handler(self, content_type):
|
|
return self.registered[content_type]
|
|
|
|
def items(self):
|
|
return self.registered.items()
|
|
|
|
def iteritems(self):
|
|
return self.registered.iteritems()
|
|
|
|
def register_defaults(self, defs):
|
|
registered = set()
|
|
for mod in defs:
|
|
for t in mod.list_types():
|
|
if not self.is_registered(t):
|
|
self.registered[t] = mod
|
|
registered.add(t)
|
|
return registered
|
|
|
|
|
|
class Paths(object):
|
|
def __init__(self, path_cfgs, ds=None):
|
|
self.cfgs = path_cfgs
|
|
# Populate all the initial paths
|
|
self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud')
|
|
self.instance_link = os.path.join(self.cloud_dir, 'instance')
|
|
self.boot_finished = os.path.join(self.instance_link, "boot-finished")
|
|
self.upstart_conf_d = path_cfgs.get('upstart_dir')
|
|
self.seed_dir = os.path.join(self.cloud_dir, 'seed')
|
|
# This one isn't joined, since it should just be read-only
|
|
template_dir = path_cfgs.get('templates_dir', '/etc/cloud/templates/')
|
|
self.template_tpl = os.path.join(template_dir, '%s.tmpl')
|
|
self.lookups = {
|
|
"handlers": "handlers",
|
|
"scripts": "scripts",
|
|
"sem": "sem",
|
|
"boothooks": "boothooks",
|
|
"userdata_raw": "user-data.txt",
|
|
"userdata": "user-data.txt.i",
|
|
"obj_pkl": "obj.pkl",
|
|
"cloud_config": "cloud-config.txt",
|
|
"data": "data",
|
|
}
|
|
# Set when a datasource becomes active
|
|
self.datasource = ds
|
|
|
|
# get_ipath_cur: get the current instance path for an item
|
|
def get_ipath_cur(self, name=None):
|
|
ipath = self.instance_link
|
|
add_on = self.lookups.get(name)
|
|
if add_on:
|
|
ipath = os.path.join(ipath, add_on)
|
|
return ipath
|
|
|
|
# get_cpath : get the "clouddir" (/var/lib/cloud/<name>)
|
|
# for a name in dirmap
|
|
def get_cpath(self, name=None):
|
|
cpath = self.cloud_dir
|
|
add_on = self.lookups.get(name)
|
|
if add_on:
|
|
cpath = os.path.join(cpath, add_on)
|
|
return cpath
|
|
|
|
# _get_ipath : get the instance path for a name in pathmap
|
|
# (/var/lib/cloud/instances/<instance>/<name>)
|
|
def _get_ipath(self, name=None):
|
|
if not self.datasource:
|
|
return None
|
|
iid = self.datasource.get_instance_id()
|
|
if iid is None:
|
|
return None
|
|
ipath = os.path.join(self.cloud_dir, 'instances', str(iid))
|
|
add_on = self.lookups.get(name)
|
|
if add_on:
|
|
ipath = os.path.join(ipath, add_on)
|
|
return ipath
|
|
|
|
# get_ipath : get the instance path for a name in pathmap
|
|
# (/var/lib/cloud/instances/<instance>/<name>)
|
|
# returns None + warns if no active datasource....
|
|
def get_ipath(self, name=None):
|
|
ipath = self._get_ipath(name)
|
|
if not ipath:
|
|
LOG.warn(("No per instance data available, "
|
|
"is there an datasource/iid set?"))
|
|
return None
|
|
else:
|
|
return ipath
|
|
|
|
|
|
# This config parser will not throw when sections don't exist
|
|
# and you are setting values on those sections which is useful
|
|
# when writing to new options that may not have corresponding
|
|
# sections. Also it can default other values when doing gets
|
|
# so that if those sections/options do not exist you will
|
|
# get a default instead of an error. Another useful case where
|
|
# you can avoid catching exceptions that you typically don't
|
|
# care about...
|
|
|
|
class DefaultingConfigParser(RawConfigParser):
|
|
DEF_INT = 0
|
|
DEF_FLOAT = 0.0
|
|
DEF_BOOLEAN = False
|
|
DEF_BASE = None
|
|
|
|
def get(self, section, option):
|
|
value = self.DEF_BASE
|
|
try:
|
|
value = RawConfigParser.get(self, section, option)
|
|
except NoSectionError:
|
|
pass
|
|
except NoOptionError:
|
|
pass
|
|
return value
|
|
|
|
def set(self, section, option, value=None):
|
|
if not self.has_section(section) and section.lower() != 'default':
|
|
self.add_section(section)
|
|
RawConfigParser.set(self, section, option, value)
|
|
|
|
def remove_option(self, section, option):
|
|
if self.has_option(section, option):
|
|
RawConfigParser.remove_option(self, section, option)
|
|
|
|
def getboolean(self, section, option):
|
|
if not self.has_option(section, option):
|
|
return self.DEF_BOOLEAN
|
|
return RawConfigParser.getboolean(self, section, option)
|
|
|
|
def getfloat(self, section, option):
|
|
if not self.has_option(section, option):
|
|
return self.DEF_FLOAT
|
|
return RawConfigParser.getfloat(self, section, option)
|
|
|
|
def getint(self, section, option):
|
|
if not self.has_option(section, option):
|
|
return self.DEF_INT
|
|
return RawConfigParser.getint(self, section, option)
|
|
|
|
def stringify(self, header=None):
|
|
contents = ''
|
|
with io.BytesIO() as outputstream:
|
|
self.write(outputstream)
|
|
outputstream.flush()
|
|
contents = outputstream.getvalue()
|
|
if header:
|
|
contents = "\n".join([header, contents])
|
|
return contents
|