From 42faf0279f5217690239113a20629d17ddc26756 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 17:27:56 -0700 Subject: [PATCH 01/22] Increase the robustness of the chef module Add the following adjustments to the chef template and module - Make it so that the chef directories can be provided (defaults to the existing directories) - Make the params much more configurable, and if a parameter is provided in the chef configuration it will override existing template parameters. - Make the template skip lines if the values are None in the configuration so that template lines can be removed if/when this is desirable. - Allow the firstboot json path to be configurable (defaults to the existing location). - Adds a basic set of tests to ensure that good things are happening. --- cloudinit/config/cc_chef.py | 95 +++++++++--- cloudinit/util.py | 135 ++++-------------- templates/chef_client.rb.tmpl | 52 +++++-- .../test_handler/test_handler_chef.py | 84 +++++++++++ 4 files changed, 229 insertions(+), 137 deletions(-) create mode 100644 tests/unittests/test_handler/test_handler_chef.py diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 806deed9..691a51bc 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from datetime import datetime + import json import os @@ -38,6 +40,61 @@ CHEF_DIRS = [ OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" +CHEF_RB_TPL_DEFAULTS = { + # These are ruby symbols... + 'ssl_verify_mode': ':verify_none', + 'log_level': ':info', + # These are not symbols... + 'log_location': '/var/log/chef/client.log', + 'validation_key': "/etc/chef/validation.pem", + 'client_key': "/etc/chef/client.pem", + 'json_attribs': "/etc/chef/firstboot.json", + 'file_cache_path': "/var/cache/chef", + 'file_backup_path': "/var/backups/chef", + 'pid_file': "/var/run/chef/client.pid", + 'show_time': True, +} +CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) +CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) +CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) +CHEF_RB_TPL_KEYS.extend([ + 'server_url', + 'node_name', + 'environment', + 'validation_name', +]) +CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) +CHEF_RB_PATH = '/etc/chef/client.rb' +CHEF_FB_PATH = '/etc/chef/firstboot.json' + + +def get_template_params(iid, chef_cfg, log): + params = CHEF_RB_TPL_DEFAULTS.copy() + params.update({ + 'server_url': chef_cfg['server_url'], + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + '_default'), + 'validation_name': chef_cfg['validation_name'], + }) + # Allow users to overwrite any of the keys they want (if they so choose), + # when a value is None, then the value will be set to None and no boolean + # or string version will be populated... + for (k, v) in chef_cfg.items(): + if k not in CHEF_RB_TPL_KEYS: + log.debug("Skipping unknown chef template key '%s'", k) + continue + if v is None: + params[k] = None + else: + # This will make the value a boolean or string... + if k in CHEF_RB_TPL_BOOL_KEYS: + params[k] = util.get_cfg_option_bool(chef_cfg, k) + else: + params[k] = util.get_cfg_option_str(chef_cfg, k) + params['generated_on'] = datetime.now().isoformat() + return params + def handle(name, cfg, cloud, log, _args): @@ -49,7 +106,7 @@ def handle(name, cfg, cloud, log, _args): chef_cfg = cfg['chef'] # Ensure the chef directories we use exist - for d in CHEF_DIRS: + for d in chef_cfg.get('directories', CHEF_DIRS): util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' @@ -64,26 +121,26 @@ def handle(name, cfg, cloud, log, _args): template_fn = cloud.get_template_filename('chef_client.rb') if template_fn: iid = str(cloud.datasource.get_instance_id()) - params = { - 'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name'] - } - templater.render_to_file(template_fn, '/etc/chef/client.rb', params) + params = get_template_params(iid, chef_cfg, log) + templater.render_to_file(template_fn, CHEF_RB_PATH, params) else: - log.warn("No template found, not rendering to /etc/chef/client.rb") + log.warn("No template found, not rendering to %s", + CHEF_RB_PATH) - # set the firstboot json - initial_json = {} - if 'run_list' in chef_cfg: - initial_json['run_list'] = chef_cfg['run_list'] - if 'initial_attributes' in chef_cfg: - initial_attributes = chef_cfg['initial_attributes'] - for k in list(initial_attributes.keys()): - initial_json[k] = initial_attributes[k] - util.write_file('/etc/chef/firstboot.json', json.dumps(initial_json)) + # Set the firstboot json + fb_filename = util.get_cfg_option_str(chef_cfg, 'firstboot_path', + default=CHEF_FB_PATH) + if not fb_filename: + log.info("First boot path empty, not writing first boot json file") + else: + initial_json = {} + if 'run_list' in chef_cfg: + initial_json['run_list'] = chef_cfg['run_list'] + if 'initial_attributes' in chef_cfg: + initial_attributes = chef_cfg['initial_attributes'] + for k in list(initial_attributes.keys()): + initial_json[k] = initial_attributes[k] + util.write_file(fb_filename, json.dumps(initial_json)) # If chef is not installed, we install chef based on 'install_type' if (not os.path.isfile('/usr/bin/chef-client') or diff --git a/cloudinit/util.py b/cloudinit/util.py index f236d0bf..bdb0f268 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -191,11 +191,11 @@ def ExtendedTemporaryFile(**kwargs): return fh -def fork_cb(child_cb, *args, **kwargs): +def fork_cb(child_cb, *args): fid = os.fork() if fid == 0: try: - child_cb(*args, **kwargs) + child_cb(*args) os._exit(0) except: logexc(LOG, "Failed forking and calling callback %s", @@ -1297,7 +1297,7 @@ def unmounter(umount): yield umount finally: if umount: - umount_cmd = ["umount", umount] + umount_cmd = ["umount", '-l', umount] subp(umount_cmd) @@ -1346,70 +1346,37 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): Mount the device, call method 'callback' passing the directory in which it was mounted, then unmount. Return whatever 'callback' returned. If data != None, also pass data to callback. - - mtype is a filesystem type. it may be a list, string (a single fsname) - or a list of fsnames. """ - - if isinstance(mtype, str): - mtypes = [mtype] - elif isinstance(mtype, (list, tuple)): - mtypes = list(mtype) - elif mtype is None: - mtypes = None - - # clean up 'mtype' input a bit based on platform. - platsys = platform.system().lower() - if platsys == "linux": - if mtypes is None: - mtypes = ["auto"] - elif platsys.endswith("bsd"): - if mtypes is None: - mtypes = ['ufs', 'cd9660', 'vfat'] - for index, mtype in enumerate(mtypes): - if mtype == "iso9660": - mtypes[index] = "cd9660" - else: - # we cannot do a smart "auto", so just call 'mount' once with no -t - mtypes = [''] - mounted = mounts() with tempdir() as tmpd: umount = False if device in mounted: mountpoint = mounted[device]['mountpoint'] else: - for mtype in mtypes: - mountpoint = None - try: - mountcmd = ['mount'] - mountopts = [] - if rw: - mountopts.append('rw') - else: - mountopts.append('ro') - if sync: - # This seems like the safe approach to do - # (ie where this is on by default) - mountopts.append("sync") - if mountopts: - mountcmd.extend(["-o", ",".join(mountopts)]) - if mtype: - mountcmd.extend(['-t', mtype]) - mountcmd.append(device) - mountcmd.append(tmpd) - subp(mountcmd) - umount = tmpd # This forces it to be unmounted (when set) - mountpoint = tmpd - break - except (IOError, OSError) as exc: - LOG.debug("Failed mount of '%s' as '%s': %s", - device, mtype, exc) - pass - if not mountpoint: - raise MountFailedError("Failed mounting %s to %s due to: %s" % + try: + mountcmd = ['mount'] + mountopts = [] + if rw: + mountopts.append('rw') + else: + mountopts.append('ro') + if sync: + # This seems like the safe approach to do + # (ie where this is on by default) + mountopts.append("sync") + if mountopts: + mountcmd.extend(["-o", ",".join(mountopts)]) + if mtype: + mountcmd.extend(['-t', mtype]) + mountcmd.append(device) + mountcmd.append(tmpd) + subp(mountcmd) + umount = tmpd # This forces it to be unmounted (when set) + mountpoint = tmpd + except (IOError, OSError) as exc: + raise MountFailedError(("Failed mounting %s " + "to %s due to: %s") % (device, tmpd, exc)) - # Be nice and ensure it ends with a slash if not mountpoint.endswith("/"): mountpoint += "/" @@ -1957,53 +1924,3 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): raise ValueError("Missing required files: %s", ','.join(missing)) return ret - - -def read_meminfo(meminfo="/proc/meminfo", raw=False): - # read a /proc/meminfo style file and return - # a dict with 'total', 'free', and 'available' - mpliers = {'kB': 2**10, 'mB': 2 ** 20, 'B': 1, 'gB': 2 ** 30} - kmap = {'MemTotal:': 'total', 'MemFree:': 'free', - 'MemAvailable:': 'available'} - ret = {} - for line in load_file(meminfo).splitlines(): - try: - key, value, unit = line.split() - except ValueError: - key, value = line.split() - unit = 'B' - if raw: - ret[key] = int(value) * mpliers[unit] - elif key in kmap: - ret[kmap[key]] = int(value) * mpliers[unit] - - return ret - - -def human2bytes(size): - """Convert human string or integer to size in bytes - 10M => 10485760 - .5G => 536870912 - """ - size_in = size - if size.endswith("B"): - size = size[:-1] - - mpliers = {'B': 1, 'K': 2 ** 10, 'M': 2 ** 20, 'G': 2 ** 30, 'T': 2 ** 40} - - num = size - mplier = 'B' - for m in mpliers: - if size.endswith(m): - mplier = m - num = size[0:-len(m)] - - try: - num = float(num) - except ValueError: - raise ValueError("'%s' is not valid input." % size_in) - - if num < 0: - raise ValueError("'%s': cannot be negative" % size_in) - - return int(num * mpliers[mplier]) diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index 538850ca..7b9e6298 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -9,17 +9,51 @@ you need to add the following to config: validation_name: XYZ server_url: XYZ -#} -log_level :info -log_location "/var/log/chef/client.log" -ssl_verify_mode :verify_none + +{# +The reason these are not in quotes is because they are ruby +symbols that will be placed inside here, and not actual strings... +#} +# This is a generated file, created on {{generated_on}}. +{% if log_level %} +log_level {{log_level}} +{% endif %} +{% if ssl_verify_mode %} +ssl_verify_mode {{ssl_verify_mode}} +{% endif %} +{% if log_location %} +log_location "{{log_location}}" +{% endif %} +{% if validation_name %} validation_client_name "{{validation_name}}" -validation_key "/etc/chef/validation.pem" -client_key "/etc/chef/client.pem" +{% endif %} +{% if validation_key %} +validation_key "{{validation_key}}" +{% endif %} +{% if client_key %} +client_key "{{client_key}}" +{% endif %} +{% if server_url %} chef_server_url "{{server_url}}" +{% endif %} +{% if environment %} environment "{{environment}}" +{% endif %} +{% if node_name %} node_name "{{node_name}}" -json_attribs "/etc/chef/firstboot.json" -file_cache_path "/var/cache/chef" -file_backup_path "/var/backups/chef" -pid_file "/var/run/chef/client.pid" +{% endif %} +{% if json_attribs %} +json_attribs "{{json_attribs}}" +{% endif %} +{% if file_cache_path %} +file_cache_path "{{file_cache_path}}" +{% endif %} +{% if file_backup_path %} +file_backup_path "{{file_backup_path}}" +{% endif %} +{% if pid_file %} +pid_file "{{pid_file}}" +{% endif %} +{% if show_time %} Chef::Log::Formatter.show_time = true +{% endif %} diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py new file mode 100644 index 00000000..5562d18a --- /dev/null +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -0,0 +1,84 @@ +import os +import json + +from cloudinit.config import cc_chef + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import helpers +from cloudinit import util +from cloudinit.sources import DataSourceNone + +from .. import helpers as t_help + +import logging + +LOG = logging.getLogger(__name__) + + +class TestChef(t_help.FilesystemMockingTestCase): + def setUp(self): + super(TestChef, self).setUp() + self.tmp = self.makeDir(prefix="unittest_") + + def fetch_cloud(self, distro_kind): + cls = distros.fetch(distro_kind) + paths = helpers.Paths({}) + distro = cls(distro_kind, {}, paths) + ds = DataSourceNone.DataSourceNone({}, distro, paths, None) + return cloud.Cloud(ds, paths, {}, distro, None) + + def test_no_config(self): + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + cfg = {} + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + for d in cc_chef.CHEF_DIRS: + self.assertFalse(os.path.isdir(d)) + + def test_basic_config(self): + tpl_file = util.load_file('templates/chef_client.rb.tmpl') + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + util.write_file('/etc/cloud/templates/chef_client.rb.tmpl', tpl_file) + cfg = { + 'chef': { + 'server_url': 'localhost', + 'validation_name': 'bob', + }, + } + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + for d in cc_chef.CHEF_DIRS: + self.assertTrue(os.path.isdir(d)) + c = util.load_file(cc_chef.CHEF_RB_PATH) + for k, v in cfg['chef'].items(): + self.assertIn(v, c) + for k, v in cc_chef.CHEF_RB_TPL_DEFAULTS.items(): + if isinstance(v, basestring): + self.assertIn(v, c) + c = util.load_file(cc_chef.CHEF_FB_PATH) + self.assertEqual({}, json.loads(c)) + + def test_firstboot_json(self): + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + cfg = { + 'chef': { + 'server_url': 'localhost', + 'validation_name': 'bob', + 'run_list': ['a', 'b', 'c'], + 'initial_attributes': { + 'c': 'd', + } + }, + } + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + c = util.load_file(cc_chef.CHEF_FB_PATH) + self.assertEqual( + { + 'run_list': ['a', 'b', 'c'], + 'c': 'd', + }, json.loads(c)) From d991a9f26e93e8541fa4b7a8ac30a77b67ddb744 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 17:34:32 -0700 Subject: [PATCH 02/22] Undo changes to the util file, not sure why that happened... --- cloudinit/util.py | 136 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 110 insertions(+), 26 deletions(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index bdb0f268..58f7455c 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -191,11 +191,11 @@ def ExtendedTemporaryFile(**kwargs): return fh -def fork_cb(child_cb, *args): +def fork_cb(child_cb, *args, **kwargs): fid = os.fork() if fid == 0: try: - child_cb(*args) + child_cb(*args, **kwargs) os._exit(0) except: logexc(LOG, "Failed forking and calling callback %s", @@ -1297,7 +1297,7 @@ def unmounter(umount): yield umount finally: if umount: - umount_cmd = ["umount", '-l', umount] + umount_cmd = ["umount", umount] subp(umount_cmd) @@ -1346,37 +1346,70 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None, sync=True): Mount the device, call method 'callback' passing the directory in which it was mounted, then unmount. Return whatever 'callback' returned. If data != None, also pass data to callback. + + mtype is a filesystem type. it may be a list, string (a single fsname) + or a list of fsnames. """ + + if isinstance(mtype, str): + mtypes = [mtype] + elif isinstance(mtype, (list, tuple)): + mtypes = list(mtype) + elif mtype is None: + mtypes = None + + # clean up 'mtype' input a bit based on platform. + platsys = platform.system().lower() + if platsys == "linux": + if mtypes is None: + mtypes = ["auto"] + elif platsys.endswith("bsd"): + if mtypes is None: + mtypes = ['ufs', 'cd9660', 'vfat'] + for index, mtype in enumerate(mtypes): + if mtype == "iso9660": + mtypes[index] = "cd9660" + else: + # we cannot do a smart "auto", so just call 'mount' once with no -t + mtypes = [''] + mounted = mounts() with tempdir() as tmpd: umount = False if device in mounted: mountpoint = mounted[device]['mountpoint'] else: - try: - mountcmd = ['mount'] - mountopts = [] - if rw: - mountopts.append('rw') - else: - mountopts.append('ro') - if sync: - # This seems like the safe approach to do - # (ie where this is on by default) - mountopts.append("sync") - if mountopts: - mountcmd.extend(["-o", ",".join(mountopts)]) - if mtype: - mountcmd.extend(['-t', mtype]) - mountcmd.append(device) - mountcmd.append(tmpd) - subp(mountcmd) - umount = tmpd # This forces it to be unmounted (when set) - mountpoint = tmpd - except (IOError, OSError) as exc: - raise MountFailedError(("Failed mounting %s " - "to %s due to: %s") % + for mtype in mtypes: + mountpoint = None + try: + mountcmd = ['mount'] + mountopts = [] + if rw: + mountopts.append('rw') + else: + mountopts.append('ro') + if sync: + # This seems like the safe approach to do + # (ie where this is on by default) + mountopts.append("sync") + if mountopts: + mountcmd.extend(["-o", ",".join(mountopts)]) + if mtype: + mountcmd.extend(['-t', mtype]) + mountcmd.append(device) + mountcmd.append(tmpd) + subp(mountcmd) + umount = tmpd # This forces it to be unmounted (when set) + mountpoint = tmpd + break + except (IOError, OSError) as exc: + LOG.debug("Failed mount of '%s' as '%s': %s", + device, mtype, exc) + pass + if not mountpoint: + raise MountFailedError("Failed mounting %s to %s due to: %s" % (device, tmpd, exc)) + # Be nice and ensure it ends with a slash if not mountpoint.endswith("/"): mountpoint += "/" @@ -1924,3 +1957,54 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep): raise ValueError("Missing required files: %s", ','.join(missing)) return ret + + +def read_meminfo(meminfo="/proc/meminfo", raw=False): + # read a /proc/meminfo style file and return + # a dict with 'total', 'free', and 'available' + mpliers = {'kB': 2**10, 'mB': 2 ** 20, 'B': 1, 'gB': 2 ** 30} + kmap = {'MemTotal:': 'total', 'MemFree:': 'free', + 'MemAvailable:': 'available'} + ret = {} + for line in load_file(meminfo).splitlines(): + try: + key, value, unit = line.split() + except ValueError: + key, value = line.split() + unit = 'B' + if raw: + ret[key] = int(value) * mpliers[unit] + elif key in kmap: + ret[kmap[key]] = int(value) * mpliers[unit] + + return ret + + +def human2bytes(size): + """Convert human string or integer to size in bytes + 10M => 10485760 + .5G => 536870912 + """ + size_in = size + if size.endswith("B"): + size = size[:-1] + + mpliers = {'B': 1, 'K': 2 ** 10, 'M': 2 ** 20, 'G': 2 ** 30, 'T': 2 ** 40} + + num = size + mplier = 'B' + for m in mpliers: + if size.endswith(m): + mplier = m + num = size[0:-len(m)] + + try: + num = float(num) + except ValueError: + raise ValueError("'%s' is not valid input." % size_in) + + if num < 0: + raise ValueError("'%s': cannot be negative" % size_in) + + return int(num * mpliers[mplier]) + From 6fd4b89088d7412ed31eb4e5b5a4b3103f071d85 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 17:40:10 -0700 Subject: [PATCH 03/22] Fix newline added at end of file --- cloudinit/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudinit/util.py b/cloudinit/util.py index 58f7455c..f236d0bf 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2007,4 +2007,3 @@ def human2bytes(size): raise ValueError("'%s': cannot be negative" % size_in) return int(num * mpliers[mplier]) - From c67916d0917d4d19eaccd4743aab49cb6de0f39c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 17:44:57 -0700 Subject: [PATCH 04/22] Add a few template delete tests --- .../test_handler/test_handler_chef.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index 5562d18a..de7ff2da 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -82,3 +82,22 @@ class TestChef(t_help.FilesystemMockingTestCase): 'run_list': ['a', 'b', 'c'], 'c': 'd', }, json.loads(c)) + + def test_template_deletes(self): + tpl_file = util.load_file('templates/chef_client.rb.tmpl') + self.patchUtils(self.tmp) + self.patchOS(self.tmp) + + util.write_file('/etc/cloud/templates/chef_client.rb.tmpl', tpl_file) + cfg = { + 'chef': { + 'server_url': 'localhost', + 'validation_name': 'bob', + 'json_attribs': None, + 'show_time': None, + }, + } + cc_chef.handle('chef', cfg, self.fetch_cloud('ubuntu'), LOG, []) + c = util.load_file(cc_chef.CHEF_RB_PATH) + self.assertNotIn('json_attribs', c) + self.assertNotIn('Formatter.show_time', c) From 806e6520ff79b829dc2ee09725201af9942b0470 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 10 Oct 2014 19:09:27 -0700 Subject: [PATCH 05/22] Move the installation code to its own function --- cloudinit/config/cc_chef.py | 72 ++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 691a51bc..971b4fce 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -142,36 +142,42 @@ def handle(name, cfg, cloud, log, _args): initial_json[k] = initial_attributes[k] util.write_file(fb_filename, json.dumps(initial_json)) - # If chef is not installed, we install chef based on 'install_type' - if (not os.path.isfile('/usr/bin/chef-client') or - util.get_cfg_option_bool(chef_cfg, - 'force_install', default=False)): + # Try to install chef, if its not already installed... + install_chef(cloud, chef_cfg, log) - install_type = util.get_cfg_option_str(chef_cfg, 'install_type', - 'packages') - if install_type == "gems": - # this will install and run the chef-client from gems - chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) - ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', - RUBY_VERSION_DEFAULT) - install_chef_from_gems(cloud.distro, ruby_version, chef_version) - # and finally, run chef-client - log.debug('Running chef-client') - util.subp(['/usr/bin/chef-client', - '-d', '-i', '1800', '-s', '20'], capture=False) - elif install_type == 'packages': - # this will install and run the chef-client from packages - cloud.distro.install_packages(('chef',)) - elif install_type == 'omnibus': - url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) - content = url_helper.readurl(url=url, retries=5) - with util.tempdir() as tmpd: - # use tmpd over tmpfile to avoid 'Text file busy' on execute - tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, str(content), mode=0700) - util.subp([tmpf], capture=False) - else: - log.warn("Unknown chef install type %s", install_type) + +def install_chef(cloud, chef_cfg, log): + # If chef is not installed, we install chef based on 'install_type' + if os.path.isfile('/usr/bin/chef-client'): + return + if not util.get_cfg_option_bool(chef_cfg, 'force_install', default=False): + return + install_type = util.get_cfg_option_str(chef_cfg, 'install_type', + 'packages') + if install_type == "gems": + # This will install and run the chef-client from gems + chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) + ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', + RUBY_VERSION_DEFAULT) + install_chef_from_gems(cloud.distro, ruby_version, chef_version) + # And finally, run chef-client + log.debug('Running chef-client') + util.subp(['/usr/bin/chef-client', + '-d', '-i', '1800', '-s', '20'], capture=False) + elif install_type == 'packages': + # This will install and run the chef-client from packages + cloud.distro.install_packages(('chef',)) + elif install_type == 'omnibus': + # This will install as a omnibus unified package + url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) + content = url_helper.readurl(url=url, retries=5) + with util.tempdir() as tmpd: + # Use tmpdir over tmpfile to avoid 'text file busy' on execute + tmpf = "%s/chef-omnibus-install" % tmpd + util.write_file(tmpf, str(content), mode=0700) + util.subp([tmpf], capture=False) + else: + log.warn("Unknown chef install type '%s'", install_type) def get_ruby_packages(version): @@ -190,9 +196,9 @@ def install_chef_from_gems(ruby_version, chef_version, distro): util.sym_link('/usr/bin/ruby%s' % ruby_version, '/usr/bin/ruby') if chef_version: util.subp(['/usr/bin/gem', 'install', 'chef', - '-v %s' % chef_version, '--no-ri', - '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) + '-v %s' % chef_version, '--no-ri', + '--no-rdoc', '--bindir', '/usr/bin', '-q'], capture=False) else: util.subp(['/usr/bin/gem', 'install', 'chef', - '--no-ri', '--no-rdoc', '--bindir', - '/usr/bin', '-q'], capture=False) + '--no-ri', '--no-rdoc', '--bindir', + '/usr/bin', '-q'], capture=False) From d40f2a152a1725bce4e7de49d84b2f7f00bb1497 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 16:37:30 -0700 Subject: [PATCH 06/22] Some more reworkings - Make a helper function to tell if already installed. - Have the install routine not run chef after installed but have it instead return a result to tell the caller to run the chef program once completed. --- cloudinit/config/cc_chef.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 971b4fce..fb825404 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -19,7 +19,6 @@ # along with this program. If not, see . from datetime import datetime - import json import os @@ -66,6 +65,16 @@ CHEF_RB_TPL_KEYS.extend([ CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' CHEF_FB_PATH = '/etc/chef/firstboot.json' +CHEF_EXEC_PATH = '/usr/bin/chef-client' +CHEF_EXEC_CMD = tuple([CHEF_EXEC_PATH, '-d', '-i', '1800', '-s', '20']) + + +def is_installed(): + if not os.path.isfile(CHEF_EXEC_PATH): + return False + if not os.access(CHEF_EXEC_PATH, os.X_OK): + return False + return True def get_template_params(iid, chef_cfg, log): @@ -106,7 +115,7 @@ def handle(name, cfg, cloud, log, _args): chef_cfg = cfg['chef'] # Ensure the chef directories we use exist - for d in chef_cfg.get('directories', CHEF_DIRS): + for d in list(chef_cfg.get('directories', CHEF_DIRS)): util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' @@ -143,27 +152,27 @@ def handle(name, cfg, cloud, log, _args): util.write_file(fb_filename, json.dumps(initial_json)) # Try to install chef, if its not already installed... - install_chef(cloud, chef_cfg, log) + force_install = util.get_cfg_option_bool(chef_cfg, + 'force_install', default=False) + if not is_installed() or force_install: + run = install_chef(cloud, chef_cfg, log) + if run: + log.debug('Running chef-client') + util.subp(CHEF_EXEC_CMD, capture=False) def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' - if os.path.isfile('/usr/bin/chef-client'): - return - if not util.get_cfg_option_bool(chef_cfg, 'force_install', default=False): - return install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') + run_after = False if install_type == "gems": # This will install and run the chef-client from gems chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', RUBY_VERSION_DEFAULT) install_chef_from_gems(cloud.distro, ruby_version, chef_version) - # And finally, run chef-client - log.debug('Running chef-client') - util.subp(['/usr/bin/chef-client', - '-d', '-i', '1800', '-s', '20'], capture=False) + run_after = True elif install_type == 'packages': # This will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) @@ -178,6 +187,7 @@ def install_chef(cloud, chef_cfg, log): util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) + return run_after def get_ruby_packages(version): From 4b53ce5e10f26474d41333dd9f950b788638c79b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 16:59:50 -0700 Subject: [PATCH 07/22] More adjustments - Use the generated_by() utility function to give the ruby template a better header comment - Set special parameters after selecting the basic chef parameters. --- cloudinit/config/cc_chef.py | 19 ++++++++++-------- templates/chef_client.rb.tmpl | 3 +-- .../test_handler/test_handler_chef.py | 20 ++++++++++++++++++- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index fb825404..999b658d 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -79,13 +79,6 @@ def is_installed(): def get_template_params(iid, chef_cfg, log): params = CHEF_RB_TPL_DEFAULTS.copy() - params.update({ - 'server_url': chef_cfg['server_url'], - 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', iid), - 'environment': util.get_cfg_option_str(chef_cfg, 'environment', - '_default'), - 'validation_name': chef_cfg['validation_name'], - }) # Allow users to overwrite any of the keys they want (if they so choose), # when a value is None, then the value will be set to None and no boolean # or string version will be populated... @@ -101,7 +94,17 @@ def get_template_params(iid, chef_cfg, log): params[k] = util.get_cfg_option_bool(chef_cfg, k) else: params[k] = util.get_cfg_option_str(chef_cfg, k) - params['generated_on'] = datetime.now().isoformat() + # These ones are overwritten to be exact values... + params.update({ + 'generated_by': util.make_header(), + 'server_url': util.get_cfg_option_str(chef_cfg, 'server_url'), + 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', + default=iid), + 'environment': util.get_cfg_option_str(chef_cfg, 'environment', + default='_default'), + 'validation_name': util.get_cfg_option_str(chef_cfg, + 'validation_name'), + }) return params diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl index 7b9e6298..c4069d22 100644 --- a/templates/chef_client.rb.tmpl +++ b/templates/chef_client.rb.tmpl @@ -9,12 +9,11 @@ you need to add the following to config: validation_name: XYZ server_url: XYZ -#} - +{{generated_by}} {# The reason these are not in quotes is because they are ruby symbols that will be placed inside here, and not actual strings... #} -# This is a generated file, created on {{generated_on}}. {% if log_level %} log_level {{log_level}} {% endif %} diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index de7ff2da..ef1aa208 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -1,5 +1,5 @@ -import os import json +import os from cloudinit.config import cc_chef @@ -38,6 +38,24 @@ class TestChef(t_help.FilesystemMockingTestCase): self.assertFalse(os.path.isdir(d)) def test_basic_config(self): + # This should create a file of the format... + """ + # Created by cloud-init v. 0.7.6 on Sat, 11 Oct 2014 23:57:21 +0000 + log_level :info + ssl_verify_mode :verify_none + log_location "/var/log/chef/client.log" + validation_client_name "bob" + validation_key "/etc/chef/validation.pem" + client_key "/etc/chef/client.pem" + chef_server_url "localhost" + environment "_default" + node_name "iid-datasource-none" + json_attribs "/etc/chef/firstboot.json" + file_cache_path "/var/cache/chef" + file_backup_path "/var/backups/chef" + pid_file "/var/run/chef/client.pid" + Chef::Log::Formatter.show_time = true + """ tpl_file = util.load_file('templates/chef_client.rb.tmpl') self.patchUtils(self.tmp) self.patchOS(self.tmp) From cbb24ba6904e8305e4d96404fb98fbb5d1f93df7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:18:45 -0700 Subject: [PATCH 08/22] Allow for the running after install and run arguments to be configured Instead of only running the client when installed from gems, allow it to be ran from other install modes as well (if configured) and allow the arguments that are passed to the client when ran to be altered (if so desired). --- cloudinit/config/cc_chef.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 999b658d..e503371d 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,7 +18,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from datetime import datetime import json import os @@ -66,7 +65,7 @@ CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_EXEC_PATH = '/usr/bin/chef-client' -CHEF_EXEC_CMD = tuple([CHEF_EXEC_PATH, '-d', '-i', '1800', '-s', '20']) +CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) def is_installed(): @@ -158,24 +157,42 @@ def handle(name, cfg, cloud, log, _args): force_install = util.get_cfg_option_bool(chef_cfg, 'force_install', default=False) if not is_installed() or force_install: - run = install_chef(cloud, chef_cfg, log) - if run: + run_after = install_chef(cloud, chef_cfg, log) + if run_after: log.debug('Running chef-client') - util.subp(CHEF_EXEC_CMD, capture=False) + cmd = [CHEF_EXEC_PATH] + if 'exec_arguments' in chef_cfg: + cmd_args = chef_cfg['exec_arguments'] + if isinstance(cmd_args, (list, tuple)): + cmd.extend(cmd_args) + elif isinstance(cmd_args, (str, basestring)): + cmd.append(cmd_args) + else: + log.warn("Unknown type %s provided for chef" + " 'exec_arguments' expected list, tuple," + " or string", type(cmd_args)) + cmd.extend(CHEF_EXEC_DEF_ARGS) + else: + cmd.extend(CHEF_EXEC_DEF_ARGS) + util.subp(cmd, capture=False) def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') - run_after = False + run_after = util.get_cfg_option_bool(chef_cfg, 'exec_after_install', + default=False) if install_type == "gems": # This will install and run the chef-client from gems chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', RUBY_VERSION_DEFAULT) install_chef_from_gems(cloud.distro, ruby_version, chef_version) - run_after = True + # Retain backwards compat, but preferring True instead of False + # when not provided/overriden... + run_after = util.get_cfg_option_bool(chef_cfg, 'exec_after_install', + default=True) elif install_type == 'packages': # This will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) @@ -190,6 +207,7 @@ def install_chef(cloud, chef_cfg, log): util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) + run_after = False return run_after From 9691d100bc61513ff8ae90032b491192fd50e328 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:23:20 -0700 Subject: [PATCH 09/22] Allow the omnibus url fetching retries to be configurable --- cloudinit/config/cc_chef.py | 6 +++++- cloudinit/util.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index e503371d..205f4b49 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -37,6 +37,7 @@ CHEF_DIRS = [ ] OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" +OMNIBUS_URL_RETRIES = 5 CHEF_RB_TPL_DEFAULTS = { # These are ruby symbols... @@ -199,7 +200,10 @@ def install_chef(cloud, chef_cfg, log): elif install_type == 'omnibus': # This will install as a omnibus unified package url = util.get_cfg_option_str(chef_cfg, "omnibus_url", OMNIBUS_URL) - content = url_helper.readurl(url=url, retries=5) + retries = max(0, util.get_cfg_option_int(chef_cfg, + "omnibus_url_retries", + default=OMNIBUS_URL_RETRIES)) + content = url_helper.readurl(url=url, retries=retries) with util.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd diff --git a/cloudinit/util.py b/cloudinit/util.py index f236d0bf..71221e09 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -399,6 +399,10 @@ def get_cfg_option_str(yobj, key, default=None): return val +def get_cfg_option_int(yobj, key, default=0): + return int(get_cfg_option_str(yobj, key, default=default)) + + def system_info(): return { 'platform': platform.platform(), From f0b23510571c982f3622b96accc7fa4c8c1859f8 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:47:06 -0700 Subject: [PATCH 10/22] Use the util function to get the chef base directories --- cloudinit/config/cc_chef.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 205f4b49..fb8d7641 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -118,7 +118,10 @@ def handle(name, cfg, cloud, log, _args): chef_cfg = cfg['chef'] # Ensure the chef directories we use exist - for d in list(chef_cfg.get('directories', CHEF_DIRS)): + chef_dirs = util.get_cfg_option_list(chef_cfg, 'directories') + if not chef_dirs: + chef_dirs = list(CHEF_DIRS) + for d in chef_dirs: util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' From 455008dec432b67a34977d9aa9b9ebfa3c482de7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:51:33 -0700 Subject: [PATCH 11/22] Always ensure we create the /etc/chef dir --- cloudinit/config/cc_chef.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index fb8d7641..ff08050d 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,6 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import itertools import json import os @@ -35,6 +36,9 @@ CHEF_DIRS = [ '/var/backups/chef', '/var/run/chef', ] +REQUIRED_CHEF_DIRS = [ + '/etc/chef', +] OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" OMNIBUS_URL_RETRIES = 5 @@ -121,7 +125,7 @@ def handle(name, cfg, cloud, log, _args): chef_dirs = util.get_cfg_option_list(chef_cfg, 'directories') if not chef_dirs: chef_dirs = list(CHEF_DIRS) - for d in chef_dirs: + for d in itertools.chain(chef_dirs, REQUIRED_CHEF_DIRS): util.ensure_dir(d) # Set the validation key based on the presence of either 'validation_key' From 8dc69bec02edaf71386dfe5d89d43d624ecd9c1b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 18:54:14 -0700 Subject: [PATCH 12/22] Move the chef running to its own helper function --- cloudinit/config/cc_chef.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index ff08050d..f501f1f7 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -167,22 +167,26 @@ def handle(name, cfg, cloud, log, _args): if not is_installed() or force_install: run_after = install_chef(cloud, chef_cfg, log) if run_after: - log.debug('Running chef-client') - cmd = [CHEF_EXEC_PATH] - if 'exec_arguments' in chef_cfg: - cmd_args = chef_cfg['exec_arguments'] - if isinstance(cmd_args, (list, tuple)): - cmd.extend(cmd_args) - elif isinstance(cmd_args, (str, basestring)): - cmd.append(cmd_args) - else: - log.warn("Unknown type %s provided for chef" - " 'exec_arguments' expected list, tuple," - " or string", type(cmd_args)) - cmd.extend(CHEF_EXEC_DEF_ARGS) - else: - cmd.extend(CHEF_EXEC_DEF_ARGS) - util.subp(cmd, capture=False) + run_chef(chef_cfg, log) + + +def run_chef(chef_cfg, log): + log.debug('Running chef-client') + cmd = [CHEF_EXEC_PATH] + if 'exec_arguments' in chef_cfg: + cmd_args = chef_cfg['exec_arguments'] + if isinstance(cmd_args, (list, tuple)): + cmd.extend(cmd_args) + elif isinstance(cmd_args, (str, basestring)): + cmd.append(cmd_args) + else: + log.warn("Unknown type %s provided for chef" + " 'exec_arguments' expected list, tuple," + " or string", type(cmd_args)) + cmd.extend(CHEF_EXEC_DEF_ARGS) + else: + cmd.extend(CHEF_EXEC_DEF_ARGS) + util.subp(cmd, capture=False) def install_chef(cloud, chef_cfg, log): From 5cea0883800a5726bb1108142ceeeb7b3e2ca2b2 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 19:26:49 -0700 Subject: [PATCH 13/22] Ensure that any template paths have associated directories When the template provides a path, make sure that before the template is written that the path that is now in the template has the associated directory created (if not already created). --- cloudinit/config/cc_chef.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index f501f1f7..28562caf 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -58,8 +58,18 @@ CHEF_RB_TPL_DEFAULTS = { 'show_time': True, } CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) +CHEF_RB_PATH_KEYS = frozenset([ + 'log_location', + 'validation_key', + 'client_key', + 'file_cache_path', + 'json_attribs', + 'file_cache_path', + 'pid_file', +]) CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) +CHEF_RB_TPL_KEYS.extend(CHEF_RB_PATH_KEYS) CHEF_RB_TPL_KEYS.extend([ 'server_url', 'node_name', @@ -109,7 +119,11 @@ def get_template_params(iid, chef_cfg, log): 'validation_name': util.get_cfg_option_str(chef_cfg, 'validation_name'), }) - return params + paths = set() + for (k, v) in params.items(): + if k in CHEF_RB_PATH_KEYS and v: + paths.add(os.path.dirname(v)) + return params, paths def handle(name, cfg, cloud, log, _args): @@ -140,7 +154,9 @@ def handle(name, cfg, cloud, log, _args): template_fn = cloud.get_template_filename('chef_client.rb') if template_fn: iid = str(cloud.datasource.get_instance_id()) - params = get_template_params(iid, chef_cfg, log) + params, paths = get_template_params(iid, chef_cfg, log) + for d in paths: + util.ensure_dir(d) templater.render_to_file(template_fn, CHEF_RB_PATH, params) else: log.warn("No template found, not rendering to %s", From bdf76a6d0398a1e45fa8a61b55be0d26895d5940 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 19:30:12 -0700 Subject: [PATCH 14/22] Have the caller find the param paths instead of the param creator --- cloudinit/config/cc_chef.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 28562caf..66187659 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -119,11 +119,7 @@ def get_template_params(iid, chef_cfg, log): 'validation_name': util.get_cfg_option_str(chef_cfg, 'validation_name'), }) - paths = set() - for (k, v) in params.items(): - if k in CHEF_RB_PATH_KEYS and v: - paths.add(os.path.dirname(v)) - return params, paths + return params def handle(name, cfg, cloud, log, _args): @@ -154,9 +150,12 @@ def handle(name, cfg, cloud, log, _args): template_fn = cloud.get_template_filename('chef_client.rb') if template_fn: iid = str(cloud.datasource.get_instance_id()) - params, paths = get_template_params(iid, chef_cfg, log) - for d in paths: - util.ensure_dir(d) + params = get_template_params(iid, chef_cfg, log) + param_paths = set() + for (k, v) in params.items(): + if k in CHEF_RB_PATH_KEYS and v: + param_paths.add(os.path.dirname(v)) + util.ensure_dirs(param_paths) templater.render_to_file(template_fn, CHEF_RB_PATH, params) else: log.warn("No template found, not rendering to %s", From b7ffdccb7c062f3fe579f765a114a8a1e830c7eb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 19:39:36 -0700 Subject: [PATCH 15/22] Follow the same constant variable naming scheme for the path tpl keys --- cloudinit/config/cc_chef.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 66187659..e9a37652 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -58,7 +58,7 @@ CHEF_RB_TPL_DEFAULTS = { 'show_time': True, } CHEF_RB_TPL_BOOL_KEYS = frozenset(['show_time']) -CHEF_RB_PATH_KEYS = frozenset([ +CHEF_RB_TPL_PATH_KEYS = frozenset([ 'log_location', 'validation_key', 'client_key', @@ -69,7 +69,7 @@ CHEF_RB_PATH_KEYS = frozenset([ ]) CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys()) CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS) -CHEF_RB_TPL_KEYS.extend(CHEF_RB_PATH_KEYS) +CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_PATH_KEYS) CHEF_RB_TPL_KEYS.extend([ 'server_url', 'node_name', @@ -153,7 +153,7 @@ def handle(name, cfg, cloud, log, _args): params = get_template_params(iid, chef_cfg, log) param_paths = set() for (k, v) in params.items(): - if k in CHEF_RB_PATH_KEYS and v: + if k in CHEF_RB_TPL_PATH_KEYS and v: param_paths.add(os.path.dirname(v)) util.ensure_dirs(param_paths) templater.render_to_file(template_fn, CHEF_RB_PATH, params) From f84ae5485f472712264bdd05e948a0ed8ddd28c9 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 21:29:01 -0700 Subject: [PATCH 16/22] Allow running even if installed Standardize on using the chef_cfg key 'exec' which can be used when installing to tell the caller to run the chef client or can also be used if the client is already installed and its requested to be ran. To retain existing behavior 'exec' does not by default assume to be true, unless explicitly provided or a gems mode install is requested. --- cloudinit/config/cc_chef.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index e9a37652..29238861 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -180,9 +180,13 @@ def handle(name, cfg, cloud, log, _args): force_install = util.get_cfg_option_bool(chef_cfg, 'force_install', default=False) if not is_installed() or force_install: - run_after = install_chef(cloud, chef_cfg, log) - if run_after: - run_chef(chef_cfg, log) + run = install_chef(cloud, chef_cfg, log) + elif is_installed(): + run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) + else: + run = False + if run: + run_chef(chef_cfg, log) def run_chef(chef_cfg, log): @@ -208,18 +212,16 @@ def install_chef(cloud, chef_cfg, log): # If chef is not installed, we install chef based on 'install_type' install_type = util.get_cfg_option_str(chef_cfg, 'install_type', 'packages') - run_after = util.get_cfg_option_bool(chef_cfg, 'exec_after_install', - default=False) + run = util.get_cfg_option_bool(chef_cfg, 'exec', default=False) if install_type == "gems": # This will install and run the chef-client from gems chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', RUBY_VERSION_DEFAULT) install_chef_from_gems(cloud.distro, ruby_version, chef_version) - # Retain backwards compat, but preferring True instead of False + # Retain backwards compat, by preferring True instead of False # when not provided/overriden... - run_after = util.get_cfg_option_bool(chef_cfg, 'exec_after_install', - default=True) + run = util.get_cfg_option_bool(chef_cfg, 'exec', default=True) elif install_type == 'packages': # This will install and run the chef-client from packages cloud.distro.install_packages(('chef',)) @@ -237,8 +239,8 @@ def install_chef(cloud, chef_cfg, log): util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) - run_after = False - return run_after + run = False + return run def get_ruby_packages(version): From ec38a6adb4510944badc99810350e56f81b43fc0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 11 Oct 2014 21:36:34 -0700 Subject: [PATCH 17/22] Add a comment explaining the param path logic --- cloudinit/config/cc_chef.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 29238861..687be69f 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -151,6 +151,9 @@ def handle(name, cfg, cloud, log, _args): if template_fn: iid = str(cloud.datasource.get_instance_id()) params = get_template_params(iid, chef_cfg, log) + # Do a best effort attempt to ensure that the template values that + # are associated with paths have there parent directory created + # before they are used by the chef-client itself. param_paths = set() for (k, v) in params.items(): if k in CHEF_RB_TPL_PATH_KEYS and v: From fc4f2f37d221316e7f6135dfc01899b1f72e07f3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sun, 12 Oct 2014 10:12:00 -0700 Subject: [PATCH 18/22] Retain the old behavior for mandatory keys The keys 'server_url' and 'validation_name' were previously mandatory, we should retain that behavior for now. --- cloudinit/config/cc_chef.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 687be69f..1e44ec72 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -111,13 +111,13 @@ def get_template_params(iid, chef_cfg, log): # These ones are overwritten to be exact values... params.update({ 'generated_by': util.make_header(), - 'server_url': util.get_cfg_option_str(chef_cfg, 'server_url'), 'node_name': util.get_cfg_option_str(chef_cfg, 'node_name', default=iid), 'environment': util.get_cfg_option_str(chef_cfg, 'environment', default='_default'), - 'validation_name': util.get_cfg_option_str(chef_cfg, - 'validation_name'), + # These two are mandatory... + 'server_url': chef_cfg['server_url'], + 'validation_name': chef_cfg['validation_name'], }) return params From 6d20769ff85c63660292763512d43d5ed6e5e913 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 13 Oct 2014 18:29:23 -0700 Subject: [PATCH 19/22] Add a post-run method that can be used to delete validation.pem files For those who run chef in non-daemon mode, they would like to delete the validation.pem file if chef finishes as expected to remove that file from existing in an easy to read manner. --- cloudinit/config/cc_chef.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 1e44ec72..4350a353 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -78,6 +78,7 @@ CHEF_RB_TPL_KEYS.extend([ ]) CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' +CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_EXEC_PATH = '/usr/bin/chef-client' CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) @@ -91,6 +92,14 @@ def is_installed(): return True +def post_run_chef(chef_cfg, log): + delete_pem = util.get_cfg_option_bool(chef_cfg, + 'delete_validation_post_exec', + default=False) + if delete_pem and os.path.isfile(CHEF_VALIDATION_PEM_PATH): + os.unlink(CHEF_VALIDATION_PEM_PATH) + + def get_template_params(iid, chef_cfg, log): params = CHEF_RB_TPL_DEFAULTS.copy() # Allow users to overwrite any of the keys they want (if they so choose), @@ -143,7 +152,7 @@ def handle(name, cfg, cloud, log, _args): # takes precedence for key in ('validation_key', 'validation_cert'): if key in chef_cfg and chef_cfg[key]: - util.write_file('/etc/chef/validation.pem', chef_cfg[key]) + util.write_file(CHEF_VALIDATION_PEM_PATH, chef_cfg[key]) break # Create the chef config from template @@ -190,6 +199,7 @@ def handle(name, cfg, cloud, log, _args): run = False if run: run_chef(chef_cfg, log) + post_run_chef(chef_cfg, log) def run_chef(chef_cfg, log): From e8f9ab7c0edf24ab3b40716729645fcccddbd838 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 13 Oct 2014 18:51:19 -0700 Subject: [PATCH 20/22] Use the key contants in the default key => value set --- cloudinit/config/cc_chef.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 4350a353..f6f07bce 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -43,15 +43,17 @@ REQUIRED_CHEF_DIRS = [ OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" OMNIBUS_URL_RETRIES = 5 +CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' +CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_RB_TPL_DEFAULTS = { # These are ruby symbols... 'ssl_verify_mode': ':verify_none', 'log_level': ':info', # These are not symbols... 'log_location': '/var/log/chef/client.log', - 'validation_key': "/etc/chef/validation.pem", + 'validation_key': CHEF_VALIDATION_PEM_PATH, 'client_key': "/etc/chef/client.pem", - 'json_attribs': "/etc/chef/firstboot.json", + 'json_attribs': CHEF_FB_PATH, 'file_cache_path': "/var/cache/chef", 'file_backup_path': "/var/backups/chef", 'pid_file': "/var/run/chef/client.pid", @@ -78,8 +80,6 @@ CHEF_RB_TPL_KEYS.extend([ ]) CHEF_RB_TPL_KEYS = frozenset(CHEF_RB_TPL_KEYS) CHEF_RB_PATH = '/etc/chef/client.rb' -CHEF_VALIDATION_PEM_PATH = '/etc/chef/validation.pem' -CHEF_FB_PATH = '/etc/chef/firstboot.json' CHEF_EXEC_PATH = '/usr/bin/chef-client' CHEF_EXEC_DEF_ARGS = tuple(['-d', '-i', '1800', '-s', '20']) From 3d299e114e800bb67b915bdf2b225391499db13a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 15 Oct 2014 11:40:37 -0700 Subject: [PATCH 21/22] Prefer immutable structures --- cloudinit/config/cc_chef.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index f6f07bce..aa82cb0a 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -28,17 +28,17 @@ from cloudinit import util RUBY_VERSION_DEFAULT = "1.8" -CHEF_DIRS = [ +CHEF_DIRS = tuple([ '/etc/chef', '/var/log/chef', '/var/lib/chef', '/var/cache/chef', '/var/backups/chef', '/var/run/chef', -] -REQUIRED_CHEF_DIRS = [ +]) +REQUIRED_CHEF_DIRS = tuple([ '/etc/chef', -] +]) OMNIBUS_URL = "https://www.opscode.com/chef/install.sh" OMNIBUS_URL_RETRIES = 5 From 60f2d2698f6a519c04d062a3b28a2f62b1e88185 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 21 Nov 2014 17:12:56 -0800 Subject: [PATCH 22/22] Update chef module docstring to reflect the new style --- cloudinit/config/cc_chef.py | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index aa82cb0a..b30d7361 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -18,6 +18,56 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +""" +**Summary:** module that configures, starts and installs chef. + +**Description:** This module enables chef to be installed (from packages or +from gems, or from omnibus). Before this occurs chef configurations are +written to disk (validation.pem, client.pem, firstboot.json, client.rb), +and needed chef folders/directories are created (/etc/chef and /var/log/chef +and so-on). Then once installing proceeds correctly if configured chef will +be started (in daemon mode or in non-daemon mode) and then once that has +finished (if ran in non-daemon mode this will be when chef finishes +converging, if ran in daemon mode then no further actions are possible since +chef will have forked into its own process) then a post run function can +run that can do finishing activities (such as removing the validation pem +file). + +It can be configured with the following option structure:: + + chef: + directories: (defaulting to /etc/chef, /var/log/chef, /var/lib/chef, + /var/cache/chef, /var/backups/chef, /var/run/chef) + validation_key or validation_cert: (optional string to be written to + /etc/chef/validation.pem) + firstboot_path: (path to write run_list and initial_attributes keys that + should also be present in this configuration, defaults + to /etc/chef/firstboot.json) + exec: boolean to run or not run chef (defaults to false, unless + a gem installed is requested + where this will then default + to true) + + chef.rb template keys (if falsey, then will be skipped and not + written to /etc/chef/client.rb) + + chef: + client_key: + environment: + file_backup_path: + file_cache_path: + json_attribs: + log_level: + log_location: + node_name: + pid_file: + server_url: + show_time: + ssl_verify_mode: + validation_key: + validation_name: +""" + import itertools import json import os