From b2c32b15057b51852349a7f6491fd4f491ea7cf3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 16 Oct 2012 15:15:48 -0700 Subject: [PATCH 1/8] Add in a configuration module that can write out the yum.repo format for those that want to hook into different repos for installing. --- cloudinit/config/cc_yum_add_repo.py | 93 +++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 cloudinit/config/cc_yum_add_repo.py diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py new file mode 100644 index 00000000..41dc72a0 --- /dev/null +++ b/cloudinit/config/cc_yum_add_repo.py @@ -0,0 +1,93 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this + +import os + +from cloudinit import templater +from cloudinit import util + +import configobj + + +def _canonicalize_id(repo_id): + repo_id = repo_id.lower().replace("-", "_") + repo_id = repo_id.replace(" ", "_") + return repo_id + + +## TODO(harlowja): move to distro? +# See man yum.conf +def _format_repository_config(repo_id, repo_config): + to_be = configobj.ConfigObj() + to_be[repo_id] = {} + # Do basic translation of + for (k, v) in repo_config.items(): + if isinstance(v, bool): + if v: + v = '1' + else: + v = '0' + elif isinstance(v, (tuple, list)): + v = "\n ".join(v) + # For now assume that peopel using this know + # the format of yum and don't verify further + to_be[repo_id][k] = v + lines = to_be.write() + return "\n".join(lines) + + +def handle(name, cfg, cloud, log, _args): + repos = cfg.get('yum_repos') + if not repos: + log.debug(("Skipping module named %s," + " no 'yum_repos' configuration found"), name) + return + repo_base_path = util.get_cfg_option_str(cfg, 'yum_repo_dir', + '/etc/yum.repos.d/') + repo_locations = {} + repo_configs = {} + for (repo_id, repo_config) in repos.items(): + canon_repo_id = _canonicalize_id(repo_id) + repo_fn_pth = os.path.join(repo_base_path, "%s.repo" % (canon_repo_id)) + if os.path.exists(repo_fn_pth): + log.info("Skipping repo %s, file %s already exists!", + repo_id, repo_fn_pth) + continue + elif canon_repo_id in repo_locations: + log.info("Skipping repo %s, file %s already pending!", + repo_id, repo_fn_pth) + continue + if not repo_config: + repo_config = {} + # Do some basic sanity checks/cleaning + n_repo_config = {} + for (k, v) in repo_config.items(): + k = k.lower().strip().replace("-", "_") + if k: + n_repo_config[k] = v + repo_config = n_repo_config + if not 'baseurl' in repo_config: + log.warn("Repository %s does not contain a baseurl address", repo_id) + else: + repo_configs[canon_repo_id] = repo_config + repo_locations[canon_repo_id] = repo_fn_pth + for (c_repo_id, path) in repo_locations.items(): + repo_blob = _format_repository_config(c_repo_id, repo_configs.get(c_repo_id)) + util.write_file(path, repo_blob) From d4e95c0fd3b8812443e881e25f557ec9a9140d0b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 16 Oct 2012 15:17:20 -0700 Subject: [PATCH 2/8] Remove some pylint buggies. --- cloudinit/config/cc_yum_add_repo.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 41dc72a0..10c423b5 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -20,7 +20,6 @@ import os -from cloudinit import templater from cloudinit import util import configobj @@ -53,7 +52,7 @@ def _format_repository_config(repo_id, repo_config): return "\n".join(lines) -def handle(name, cfg, cloud, log, _args): +def handle(name, cfg, _cloud, log, _args): repos = cfg.get('yum_repos') if not repos: log.debug(("Skipping module named %s," @@ -84,10 +83,12 @@ def handle(name, cfg, cloud, log, _args): n_repo_config[k] = v repo_config = n_repo_config if not 'baseurl' in repo_config: - log.warn("Repository %s does not contain a baseurl address", repo_id) + log.warn("Repository %s does not contain a baseurl address", + repo_id) else: repo_configs[canon_repo_id] = repo_config repo_locations[canon_repo_id] = repo_fn_pth for (c_repo_id, path) in repo_locations.items(): - repo_blob = _format_repository_config(c_repo_id, repo_configs.get(c_repo_id)) + repo_blob = _format_repository_config(c_repo_id, + repo_configs.get(c_repo_id)) util.write_file(path, repo_blob) From b23df6b38b4822b1c2f1c536725a2bcbdfc56775 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 16 Oct 2012 15:19:14 -0700 Subject: [PATCH 3/8] Fix copyright. --- cloudinit/config/cc_yum_add_repo.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 10c423b5..f0487773 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -1,10 +1,8 @@ # vi: ts=4 expandtab # -# Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. # -# Author: Scott Moser -# Author: Juerg Haefliger +# Author: Joshua Harlow # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -16,7 +14,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this +# along with this program. If not, see . import os From 9f75338bab5124b7ed916e3da1f2fecb08382a1c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 16 Oct 2012 15:24:15 -0700 Subject: [PATCH 4/8] Update comments as to why format is the way it is. --- cloudinit/config/cc_yum_add_repo.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index f0487773..fda3236f 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -34,14 +34,17 @@ def _canonicalize_id(repo_id): def _format_repository_config(repo_id, repo_config): to_be = configobj.ConfigObj() to_be[repo_id] = {} - # Do basic translation of + # Do basic translation of the items -> values for (k, v) in repo_config.items(): if isinstance(v, bool): + # Seems like yum prefers 1/0 if v: v = '1' else: v = '0' elif isinstance(v, (tuple, list)): + # Can handle 'lists' in certain cases + # See: http://bit.ly/Qqrf1t v = "\n ".join(v) # For now assume that peopel using this know # the format of yum and don't verify further From d40f27cdbd869927271540c306eb153e779de578 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 16 Oct 2012 21:20:19 -0700 Subject: [PATCH 5/8] Add in a created by cloud-init header. --- cloudinit/config/cc_yum_add_repo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index fda3236f..ee117bcd 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -50,6 +50,7 @@ def _format_repository_config(repo_id, repo_config): # the format of yum and don't verify further to_be[repo_id][k] = v lines = to_be.write() + lines.insert(0, util.make_header()) return "\n".join(lines) From c75870956e801cb83be71973d416c16858a66b15 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 17 Oct 2012 09:32:35 -0700 Subject: [PATCH 6/8] Make the yum value 'string' translation a function. --- cloudinit/config/cc_yum_add_repo.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index ee117bcd..123f7827 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -29,6 +29,19 @@ def _canonicalize_id(repo_id): return repo_id +def _format_repo_value(val): + if isinstance(val, (bool)): + # Seems like yum prefers 1/0 + return str(int(val)) + if isinstance(val, (list, tuple)): + # Can handle 'lists' in certain cases + # See: http://bit.ly/Qqrf1t + return "\n ".join([_format_repo_value(v) for v in val]) + if not isinstance(val, (basestring, str)): + return str(val) + return val + + ## TODO(harlowja): move to distro? # See man yum.conf def _format_repository_config(repo_id, repo_config): @@ -36,19 +49,9 @@ def _format_repository_config(repo_id, repo_config): to_be[repo_id] = {} # Do basic translation of the items -> values for (k, v) in repo_config.items(): - if isinstance(v, bool): - # Seems like yum prefers 1/0 - if v: - v = '1' - else: - v = '0' - elif isinstance(v, (tuple, list)): - # Can handle 'lists' in certain cases - # See: http://bit.ly/Qqrf1t - v = "\n ".join(v) - # For now assume that peopel using this know - # the format of yum and don't verify further - to_be[repo_id][k] = v + # For now assume that people using this know + # the format of yum and don't verify keys/values further + to_be[repo_id][k] = _format_repo_value(v) lines = to_be.write() lines.insert(0, util.make_header()) return "\n".join(lines) From 612303361f8d6e27b8828468e2168e1eb53be99b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 17 Oct 2012 09:42:22 -0700 Subject: [PATCH 7/8] More cleanups, fix header function. --- cloudinit/config/cc_yum_add_repo.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/cloudinit/config/cc_yum_add_repo.py b/cloudinit/config/cc_yum_add_repo.py index 123f7827..5c273825 100644 --- a/cloudinit/config/cc_yum_add_repo.py +++ b/cloudinit/config/cc_yum_add_repo.py @@ -53,7 +53,7 @@ def _format_repository_config(repo_id, repo_config): # the format of yum and don't verify keys/values further to_be[repo_id][k] = _format_repo_value(v) lines = to_be.write() - lines.insert(0, util.make_header()) + lines.insert(0, "# Created by cloud-init on %s" % (util.time_rfc2822())) return "\n".join(lines) @@ -87,12 +87,19 @@ def handle(name, cfg, _cloud, log, _args): if k: n_repo_config[k] = v repo_config = n_repo_config - if not 'baseurl' in repo_config: - log.warn("Repository %s does not contain a baseurl address", - repo_id) - else: + missing_required = 0 + for req_field in ['baseurl']: + if not req_field in repo_config: + log.warn(("Repository %s does not contain a %s" + " configuration 'required' entry"), + repo_id, req_field) + missing_required += 1 + if not missing_required: repo_configs[canon_repo_id] = repo_config repo_locations[canon_repo_id] = repo_fn_pth + else: + log.warn("Repository %s is missing %s required fields, skipping!", + repo_id, missing_required) for (c_repo_id, path) in repo_locations.items(): repo_blob = _format_repository_config(c_repo_id, repo_configs.get(c_repo_id)) From 4efa9ddb09df20cd52af5a696e7436f75c673952 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sat, 10 Nov 2012 22:24:01 -0500 Subject: [PATCH 8/8] revert unrelated whitespace changes that creeped in. --- cloudinit/distros/__init__.py | 38 ++++++++++---------- cloudinit/sources/DataSourceAltCloud.py | 2 +- cloudinit/util.py | 3 +- tests/unittests/test_runs/test_simple_run.py | 6 ++-- tools/hacking.py | 4 +-- 5 files changed, 27 insertions(+), 26 deletions(-) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 8a98e334..d2cb0a8b 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -187,23 +187,23 @@ class Distro(object): # inputs. If something goes wrong, we can end up with a system # that nobody can login to. adduser_opts = { - "gecos": '--comment', - "homedir": '--home', - "primary_group": '--gid', - "groups": '--groups', - "passwd": '--password', - "shell": '--shell', - "expiredate": '--expiredate', - "inactive": '--inactive', - "selinux_user": '--selinux-user', - } + "gecos": '--comment', + "homedir": '--home', + "primary_group": '--gid', + "groups": '--groups', + "passwd": '--password', + "shell": '--shell', + "expiredate": '--expiredate', + "inactive": '--inactive', + "selinux_user": '--selinux-user', + } adduser_opts_flags = { - "no_user_group": '--no-user-group', - "system": '--system', - "no_log_init": '--no-log-init', - "no_create_home": "-M", - } + "no_user_group": '--no-user-group', + "system": '--system', + "no_log_init": '--no-log-init', + "no_create_home": "-M", + } # Now check the value and create the command for option in kwargs: @@ -320,9 +320,11 @@ class Distro(object): raise e util.ensure_dir(path, 0750) - def write_sudo_rules(self, user, rules, sudo_file=None): - if not sudo_file: - sudo_file = "/etc/sudoers.d/90-cloud-init-users" + def write_sudo_rules(self, + user, + rules, + sudo_file="/etc/sudoers.d/90-cloud-init-users", + ): content_header = "# user rules for %s" % user content = "%s\n%s %s\n\n" % (content_header, user, rules) diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 9812bdcb..d7e1204f 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -47,7 +47,7 @@ META_DATA_NOT_SUPPORTED = { 'instance-id': 455, 'local-hostname': 'localhost', 'placement': {}, -} + } def read_user_data_callback(mount_dir): diff --git a/cloudinit/util.py b/cloudinit/util.py index 4f5b15ee..7890a3d6 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1193,7 +1193,8 @@ def yaml_dumps(obj): indent=4, explicit_start=True, explicit_end=True, - default_flow_style=False) + default_flow_style=False, + ) return formatted diff --git a/tests/unittests/test_runs/test_simple_run.py b/tests/unittests/test_runs/test_simple_run.py index 60ef812a..22d6cf2c 100644 --- a/tests/unittests/test_runs/test_simple_run.py +++ b/tests/unittests/test_runs/test_simple_run.py @@ -37,13 +37,11 @@ class TestSimpleRun(helpers.FilesystemMockingTestCase): self.replicateTestRoot('simple_ubuntu', new_root) cfg = { 'datasource_list': ['None'], - 'write_files': [ - { + 'write_files': [{ 'path': '/etc/blah.ini', 'content': 'blah', 'permissions': 0755, - }, - ], + }], 'cloud_init_modules': ['write-files'], } cloud_cfg = util.yaml_dumps(cfg) diff --git a/tools/hacking.py b/tools/hacking.py index 26a07c53..11163df3 100755 --- a/tools/hacking.py +++ b/tools/hacking.py @@ -66,8 +66,8 @@ def cloud_import_alphabetical(physical_line, line_number, lines): # handle import x # use .lower since capitalization shouldn't dictate order split_line = import_normalize(physical_line.strip()).lower().split() - split_previous = import_normalize(lines[line_number - 2]) - split_previous = split_previous.strip().lower().split() + split_previous = import_normalize(lines[line_number - 2] + ).strip().lower().split() # with or without "as y" length = [2, 4] if (len(split_line) in length and len(split_previous) in length and