From b17bbcdef9acfaf6e2b47671c9982d3378045961 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sun, 22 Nov 2015 13:16:44 -0500 Subject: [PATCH] Add support for secure.yaml file for auth info Almost nothing in clouds.yaml is secret, but the file has to be treated as if it were because of the passwords or other secrets contained in it. This makes it difficult to put clouds.yaml into a public or broadly accessible config repository. Add support for having a second optional file, secure.yaml, which can contain any value you can put in clouds.yaml and which will be overlayed on top of clouds.yaml values. Most people probably do not need this, but for folks with complex cloud configs with teams of people working on them, this reduces the amount of things that have to be managed by the privileged system. Change-Id: I631d826588b0a0b1f36244caa7982dd42d9eb498 --- README.rst | 28 ++++++++++++++++++++++ os_client_config/config.py | 33 +++++++++++++++++++++++++- os_client_config/tests/base.py | 12 +++++++++- os_client_config/tests/test_config.py | 21 ++++++++++------ os_client_config/tests/test_environ.py | 14 +++++++---- 5 files changed, 95 insertions(+), 13 deletions(-) diff --git a/README.rst b/README.rst index f16bbc0..0bdc182 100644 --- a/README.rst +++ b/README.rst @@ -145,6 +145,34 @@ as a result of a chosen plugin need to go into the auth dict. For password auth, this includes `auth_url`, `username` and `password` as well as anything related to domains, projects and trusts. +Splitting Secrets +----------------- + +In some scenarios, such as configuragtion managment controlled environments, +it might be eaiser to have secrets in one file and non-secrets in another. +This is fully supported via an optional file `secure.yaml` which follows all +the same location rules as `clouds.yaml`. It can contain anything you put +in `clouds.yaml` and will take precedence over anything in the `clouds.yaml` +file. + +:: + + # clouds.yaml + clouds: + internap: + profile: internap + auth: + username: api-55f9a00fb2619 + project_name: inap-17037 + regions: + - ams01 + - nyj01 + # secure.yaml + clouds: + internap: + auth: + password: XXXXXXXXXXXXXXXXX + SSL Settings ------------ diff --git a/os_client_config/config.py b/os_client_config/config.py index 5f7c402..c12b25a 100644 --- a/os_client_config/config.py +++ b/os_client_config/config.py @@ -51,6 +51,11 @@ CONFIG_FILES = [ for d in CONFIG_SEARCH_PATH for s in YAML_SUFFIXES + JSON_SUFFIXES ] +SECURE_FILES = [ + os.path.join(d, 'secure' + s) + for d in CONFIG_SEARCH_PATH + for s in YAML_SUFFIXES + JSON_SUFFIXES +] VENDOR_FILES = [ os.path.join(d, 'clouds-public' + s) for d in CONFIG_SEARCH_PATH @@ -102,6 +107,20 @@ def _get_os_environ(envvar_prefix=None): return ret +def _merge_clouds(old_dict, new_dict): + """Like dict.update, except handling nested dicts.""" + ret = old_dict.copy() + for (k, v) in new_dict.items(): + if isinstance(v, dict): + if k in ret: + ret[k] = _merge_clouds(ret[k], v) + else: + ret[k] = v.copy() + else: + ret[k] = v + return ret + + def _auth_update(old_dict, new_dict): """Like dict.update, except handling the nested dict called auth.""" for (k, v) in new_dict.items(): @@ -119,20 +138,29 @@ class OpenStackConfig(object): def __init__(self, config_files=None, vendor_files=None, override_defaults=None, force_ipv4=None, - envvar_prefix=None): + envvar_prefix=None, secure_files=None): self._config_files = config_files or CONFIG_FILES + self._secure_files = secure_files or SECURE_FILES self._vendor_files = vendor_files or VENDOR_FILES config_file_override = os.environ.pop('OS_CLIENT_CONFIG_FILE', None) if config_file_override: self._config_files.insert(0, config_file_override) + secure_file_override = os.environ.pop('OS_CLIENT_SECURE_FILE', None) + if secure_file_override: + self._secure_files.insert(0, secure_file_override) + self.defaults = defaults.get_defaults() if override_defaults: self.defaults.update(override_defaults) # First, use a config file if it exists where expected self.config_filename, self.cloud_config = self._load_config_file() + _, secure_config = self._load_secure_file() + if secure_config: + self.cloud_config = _merge_clouds( + self.cloud_config, secure_config) if not self.cloud_config: self.cloud_config = {'clouds': {}} @@ -220,6 +248,9 @@ class OpenStackConfig(object): def _load_config_file(self): return self._load_yaml_json_file(self._config_files) + def _load_secure_file(self): + return self._load_yaml_json_file(self._secure_files) + def _load_vendor_file(self): return self._load_yaml_json_file(self._vendor_files) diff --git a/os_client_config/tests/base.py b/os_client_config/tests/base.py index 3d94e25..33a868d 100644 --- a/os_client_config/tests/base.py +++ b/os_client_config/tests/base.py @@ -64,7 +64,6 @@ USER_CONF = { 'auth': { 'auth_url': 'http://example.com/v2', 'username': 'testuser', - 'password': 'testpass', 'project_name': 'testproject', }, 'region-name': 'test-region', @@ -112,6 +111,15 @@ USER_CONF = { } }, } +SECURE_CONF = { + 'clouds': { + '_test_cloud_no_vendor': { + 'auth': { + 'password': 'testpass', + }, + } + } +} NO_CONF = { 'cache': {'max_age': 1}, } @@ -135,6 +143,7 @@ class TestCase(base.BaseTestCase): tdir = self.useFixture(fixtures.TempDir()) conf['cache']['path'] = tdir.path self.cloud_yaml = _write_yaml(conf) + self.secure_yaml = _write_yaml(SECURE_CONF) self.vendor_yaml = _write_yaml(VENDOR_CONF) self.no_yaml = _write_yaml(NO_CONF) @@ -155,6 +164,7 @@ class TestCase(base.BaseTestCase): self.assertIsNone(cc.cloud) self.assertIn('username', cc.auth) self.assertEqual('testuser', cc.auth['username']) + self.assertEqual('testpass', cc.auth['password']) self.assertFalse(cc.config['image_api_use_tasks']) self.assertTrue('project_name' in cc.auth or 'project_id' in cc.auth) if 'project_name' in cc.auth: diff --git a/os_client_config/tests/test_config.py b/os_client_config/tests/test_config.py index d225b7c..aff8c6d 100644 --- a/os_client_config/tests/test_config.py +++ b/os_client_config/tests/test_config.py @@ -30,7 +30,8 @@ class TestConfig(base.TestCase): def test_get_all_clouds(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) + vendor_files=[self.vendor_yaml], + secure_files=[self.no_yaml]) clouds = c.get_all_clouds() # We add one by hand because the regions cloud is going to exist # twice since it has two regions in it @@ -74,7 +75,8 @@ class TestConfig(base.TestCase): def test_get_one_cloud_with_config_files(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) + vendor_files=[self.vendor_yaml], + secure_files=[self.secure_yaml]) self.assertIsInstance(c.cloud_config, dict) self.assertIn('cache', c.cloud_config) self.assertIsInstance(c.cloud_config['cache'], dict) @@ -129,7 +131,8 @@ class TestConfig(base.TestCase): def test_fallthrough(self): c = config.OpenStackConfig(config_files=[self.no_yaml], - vendor_files=[self.no_yaml]) + vendor_files=[self.no_yaml], + secure_files=[self.no_yaml]) for k in os.environ.keys(): if k.startswith('OS_'): self.useFixture(fixtures.EnvironmentVariable(k)) @@ -137,7 +140,8 @@ class TestConfig(base.TestCase): def test_prefer_ipv6_true(self): c = config.OpenStackConfig(config_files=[self.no_yaml], - vendor_files=[self.no_yaml]) + vendor_files=[self.no_yaml], + secure_files=[self.no_yaml]) cc = c.get_one_cloud(cloud='defaults', validate=False) self.assertTrue(cc.prefer_ipv6) @@ -155,7 +159,8 @@ class TestConfig(base.TestCase): def test_force_ipv4_false(self): c = config.OpenStackConfig(config_files=[self.no_yaml], - vendor_files=[self.no_yaml]) + vendor_files=[self.no_yaml], + secure_files=[self.no_yaml]) cc = c.get_one_cloud(cloud='defaults', validate=False) self.assertFalse(cc.force_ipv4) @@ -166,7 +171,8 @@ class TestConfig(base.TestCase): self.assertEqual('testpass', cc.auth['password']) def test_get_cloud_names(self): - c = config.OpenStackConfig(config_files=[self.cloud_yaml]) + c = config.OpenStackConfig(config_files=[self.cloud_yaml], + secure_files=[self.no_yaml]) self.assertEqual( ['_test-cloud-domain-id_', '_test-cloud-int-project_', @@ -177,7 +183,8 @@ class TestConfig(base.TestCase): ], sorted(c.get_cloud_names())) c = config.OpenStackConfig(config_files=[self.no_yaml], - vendor_files=[self.no_yaml]) + vendor_files=[self.no_yaml], + secure_files=[self.no_yaml]) for k in os.environ.keys(): if k.startswith('OS_'): self.useFixture(fixtures.EnvironmentVariable(k)) diff --git a/os_client_config/tests/test_environ.py b/os_client_config/tests/test_environ.py index 0ff800f..b75db1c 100644 --- a/os_client_config/tests/test_environ.py +++ b/os_client_config/tests/test_environ.py @@ -29,6 +29,8 @@ class TestEnviron(base.TestCase): fixtures.EnvironmentVariable('OS_AUTH_URL', 'https://example.com')) self.useFixture( fixtures.EnvironmentVariable('OS_USERNAME', 'testuser')) + self.useFixture( + fixtures.EnvironmentVariable('OS_PASSWORD', 'testpass')) self.useFixture( fixtures.EnvironmentVariable('OS_PROJECT_NAME', 'testproject')) self.useFixture( @@ -57,13 +59,15 @@ class TestEnviron(base.TestCase): self.useFixture( fixtures.EnvironmentVariable('OS_PREFER_IPV6', 'false')) c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) + vendor_files=[self.vendor_yaml], + secure_files=[self.secure_yaml]) cc = c.get_one_cloud('_test-cloud_') self.assertFalse(cc.prefer_ipv6) def test_environ_exists(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) + vendor_files=[self.vendor_yaml], + secure_files=[self.secure_yaml]) cc = c.get_one_cloud('envvars') self._assert_cloud_details(cc) self.assertNotIn('auth_url', cc.config) @@ -78,7 +82,8 @@ class TestEnviron(base.TestCase): def test_environ_prefix(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], vendor_files=[self.vendor_yaml], - envvar_prefix='NOVA_') + envvar_prefix='NOVA_', + secure_files=[self.secure_yaml]) cc = c.get_one_cloud('envvars') self._assert_cloud_details(cc) self.assertNotIn('auth_url', cc.config) @@ -92,7 +97,8 @@ class TestEnviron(base.TestCase): def test_get_one_cloud_with_config_files(self): c = config.OpenStackConfig(config_files=[self.cloud_yaml], - vendor_files=[self.vendor_yaml]) + vendor_files=[self.vendor_yaml], + secure_files=[self.secure_yaml]) self.assertIsInstance(c.cloud_config, dict) self.assertIn('cache', c.cloud_config) self.assertIsInstance(c.cloud_config['cache'], dict)