Use json for in-tree cloud data

In preparation for sharing the default and vendor data with other
projects, potentially even non-python ones, move the data into json
format, which is slighly less exciting to read, but has more widespread
standard library support. The user-facing config file will still be in
yaml format, because that's easier on the eyes and it's expected to be
read and edited by humans.

Continue to accept yaml everywhere, because an end user may have dropped
a yaml config file into a dir somewhere, and that's fine.

Change-Id: I269d31e61da433ac20abb39acdde0f9f9fe12837
This commit is contained in:
Monty Taylor 2015-06-06 09:40:03 -04:00
parent 48c0668b15
commit 796bfad22d
44 changed files with 661 additions and 174 deletions

@ -13,6 +13,7 @@
# under the License.
import json
import os
import warnings
@ -44,15 +45,16 @@ CONFIG_SEARCH_PATH = [
SITE_CONFIG_HOME, UNIX_SITE_CONFIG_HOME
]
YAML_SUFFIXES = ('.yaml', '.yml')
JSON_SUFFIXES = ('.json',)
CONFIG_FILES = [
os.path.join(d, 'clouds' + s)
for d in CONFIG_SEARCH_PATH
for s in YAML_SUFFIXES
for s in YAML_SUFFIXES + JSON_SUFFIXES
]
VENDOR_FILES = [
os.path.join(d, 'clouds-public' + s)
for d in CONFIG_SEARCH_PATH
for s in YAML_SUFFIXES
for s in YAML_SUFFIXES + JSON_SUFFIXES
]
BOOL_KEYS = ('insecure', 'cache')
@ -212,16 +214,19 @@ class OpenStackConfig(object):
'expiration', self._cache_expiration)
def _load_config_file(self):
return self._load_yaml_file(self._config_files)
return self._load_yaml_json_file(self._config_files)
def _load_vendor_file(self):
return self._load_yaml_file(self._vendor_files)
return self._load_yaml_json_file(self._vendor_files)
def _load_yaml_file(self, filelist):
def _load_yaml_json_file(self, filelist):
for path in filelist:
if os.path.exists(path):
with open(path, 'r') as f:
return path, yaml.safe_load(f)
if path.endswith('json'):
return path, json.load(f)
else:
return path, yaml.safe_load(f)
return (None, None)
def _normalize_keys(self, config):

@ -0,0 +1,19 @@
{
"auth_type": "password",
"baremetal_api_version": "1",
"compute_api_version": "2",
"database_api_version": "1.0",
"disable_vendor_agent": {},
"dns_api_version": "2",
"interface": "public",
"floating_ip_source": "neutron",
"identity_api_version": "2.0",
"image_api_use_tasks": false,
"image_api_version": "2",
"image_format": "qcow2",
"network_api_version": "2",
"object_api_version": "1",
"orchestration_api_version": "1",
"secgroup_source": "neutron",
"volume_api_version": "1"
}

@ -12,12 +12,11 @@
# License for the specific language governing permissions and limitations
# under the License.
import json
import os
import yaml
_yaml_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'defaults.yaml')
_json_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'defaults.json')
_defaults = None
@ -34,8 +33,8 @@ def get_defaults():
cert=None,
key=None,
)
with open(_yaml_path, 'r') as yaml_file:
updates = yaml.load(yaml_file.read())
with open(_json_path, 'r') as json_file:
updates = json.load(json_file)
if updates is not None:
_defaults.update(updates)

@ -1,17 +0,0 @@
auth_type: password
baremetal_api_version: '1'
compute_api_version: '2'
database_api_version: '1.0'
disable_vendor_agent: {}
dns_api_version: '2'
interface: public
floating_ip_source: neutron
identity_api_version: '2.0'
image_api_use_tasks: false
image_api_version: '2'
image_format: qcow2
network_api_version: '2'
object_api_version: '1'
orchestration_api_version: '1'
secgroup_source: neutron
volume_api_version: '1'

@ -0,0 +1,121 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"id": "https://git.openstack.org/cgit/openstack/cloud-data/plain/schema.json#",
"type": "object",
"properties": {
"auth_type": {
"name": "Auth Type",
"description": "Name of authentication plugin to be used",
"default": "password",
"type": "string"
},
"disable_vendor_agent": {
"name": "Disable Vendor Agent Properties",
"description": "Image properties required to disable vendor agent",
"type": "object",
"properties": {}
},
"floating_ip_source": {
"name": "Floating IP Source",
"description": "Which service provides Floating IPs",
"enum": [ "neutron", "nova", "None" ],
"default": "neutron"
},
"image_api_use_tasks": {
"name": "Image Task API",
"description": "Does the cloud require the Image Task API",
"default": false,
"type": "boolean"
},
"image_format": {
"name": "Image Format",
"description": "Format for uploaded Images",
"default": "qcow2",
"type": "string"
},
"interface": {
"name": "API Interface",
"description": "Which API Interface should connections hit",
"default": "public",
"enum": [ "public", "internal", "admin" ]
},
"secgroup_source": {
"name": "Security Group Source",
"description": "Which service provides security groups",
"default": "neutron",
"enum": [ "neutron", "nova", "None" ]
},
"baremetal_api_version": {
"name": "Baremetal API Service Type",
"description": "Baremetal API Service Type",
"default": "1",
"type": "string"
},
"compute_api_version": {
"name": "Compute API Version",
"description": "Compute API Version",
"default": "2",
"type": "string"
},
"database_api_version": {
"name": "Database API Version",
"description": "Database API Version",
"default": "1.0",
"type": "string"
},
"dns_api_version": {
"name": "DNS API Version",
"description": "DNS API Version",
"default": "2",
"type": "string"
},
"identity_api_version": {
"name": "Identity API Version",
"description": "Identity API Version",
"default": "2",
"type": "string"
},
"image_api_version": {
"name": "Image API Version",
"description": "Image API Version",
"default": "1",
"type": "string"
},
"network_api_version": {
"name": "Network API Version",
"description": "Network API Version",
"default": "2",
"type": "string"
},
"object_api_version": {
"name": "Object Storage API Version",
"description": "Object Storage API Version",
"default": "1",
"type": "string"
},
"volume_api_version": {
"name": "Volume API Version",
"description": "Volume API Version",
"default": "2",
"type": "string"
}
},
"required": [
"auth_type",
"baremetal_api_version",
"compute_api_version",
"database_api_version",
"disable_vendor_agent",
"dns_api_version",
"floating_ip_source",
"identity_api_version",
"image_api_use_tasks",
"image_api_version",
"image_format",
"interface",
"network_api_version",
"object_api_version",
"secgroup_source",
"volume_api_version"
]
}

@ -0,0 +1,62 @@
# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import glob
import json
import os
import jsonschema
from testtools import content
from os_client_config import defaults
from os_client_config.tests import base
class TestConfig(base.TestCase):
def json_diagnostics(self, exc_info):
self.addDetail('filename', content.text_content(self.filename))
for error in sorted(self.validator.iter_errors(self.json_data)):
self.addDetail('jsonschema', content.text_content(str(error)))
def test_defaults_valid_json(self):
_schema_path = os.path.join(
os.path.dirname(os.path.realpath(defaults.__file__)),
'schema.json')
schema = json.load(open(_schema_path, 'r'))
self.validator = jsonschema.Draft4Validator(schema)
self.addOnException(self.json_diagnostics)
self.filename = os.path.join(
os.path.dirname(os.path.realpath(defaults.__file__)),
'defaults.json')
self.json_data = json.load(open(self.filename, 'r'))
self.assertTrue(self.validator.is_valid(self.json_data))
def test_vendors_valid_json(self):
_schema_path = os.path.join(
os.path.dirname(os.path.realpath(defaults.__file__)),
'vendor-schema.json')
schema = json.load(open(_schema_path, 'r'))
self.validator = jsonschema.Draft4Validator(schema)
self.addOnException(self.json_diagnostics)
_vendors_path = os.path.join(
os.path.dirname(os.path.realpath(defaults.__file__)),
'vendors')
for self.filename in glob.glob(os.path.join(_vendors_path, '*.json')):
self.json_data = json.load(open(self.filename, 'r'))
self.assertTrue(self.validator.is_valid(self.json_data))

@ -0,0 +1,206 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"id": "https://git.openstack.org/cgit/openstack/cloud-data/plain/vendor-schema.json#",
"type": "object",
"properties": {
"name": {
"type": "string"
},
"profile": {
"type": "object",
"properties": {
"auth": {
"type": "object",
"properties": {
"auth_url": {
"name": "Auth URL",
"description": "URL of the primary Keystone endpoint",
"type": "string"
}
}
},
"auth_type": {
"name": "Auth Type",
"description": "Name of authentication plugin to be used",
"default": "password",
"type": "string"
},
"disable_vendor_agent": {
"name": "Disable Vendor Agent Properties",
"description": "Image properties required to disable vendor agent",
"type": "object",
"properties": {}
},
"floating_ip_source": {
"name": "Floating IP Source",
"description": "Which service provides Floating IPs",
"enum": [ "neutron", "nova", "None" ],
"default": "neutron"
},
"image_api_use_tasks": {
"name": "Image Task API",
"description": "Does the cloud require the Image Task API",
"default": false,
"type": "boolean"
},
"image_format": {
"name": "Image Format",
"description": "Format for uploaded Images",
"default": "qcow2",
"type": "string"
},
"interface": {
"name": "API Interface",
"description": "Which API Interface should connections hit",
"default": "public",
"enum": [ "public", "internal", "admin" ]
},
"secgroup_source": {
"name": "Security Group Source",
"description": "Which service provides security groups",
"enum": [ "neutron", "nova", "None" ],
"default": "neutron"
},
"compute_api_service_name": {
"name": "Compute API Service Name",
"description": "Compute API Service Name",
"type": "string"
},
"database_api_service_name": {
"name": "Database API Service Name",
"description": "Database API Service Name",
"type": "string"
},
"dns_api_service_name": {
"name": "DNS API Service Name",
"description": "DNS API Service Name",
"type": "string"
},
"identity_api_service_name": {
"name": "Identity API Service Name",
"description": "Identity API Service Name",
"type": "string"
},
"image_api_service_name": {
"name": "Image API Service Name",
"description": "Image API Service Name",
"type": "string"
},
"volume_api_service_name": {
"name": "Volume API Service Name",
"description": "Volume API Service Name",
"type": "string"
},
"network_api_service_name": {
"name": "Network API Service Name",
"description": "Network API Service Name",
"type": "string"
},
"object_api_service_name": {
"name": "Object Storage API Service Name",
"description": "Object Storage API Service Name",
"type": "string"
},
"baremetal_api_service_name": {
"name": "Baremetal API Service Name",
"description": "Baremetal API Service Name",
"type": "string"
},
"compute_api_service_type": {
"name": "Compute API Service Type",
"description": "Compute API Service Type",
"type": "string"
},
"database_api_service_type": {
"name": "Database API Service Type",
"description": "Database API Service Type",
"type": "string"
},
"dns_api_service_type": {
"name": "DNS API Service Type",
"description": "DNS API Service Type",
"type": "string"
},
"identity_api_service_type": {
"name": "Identity API Service Type",
"description": "Identity API Service Type",
"type": "string"
},
"image_api_service_type": {
"name": "Image API Service Type",
"description": "Image API Service Type",
"type": "string"
},
"volume_api_service_type": {
"name": "Volume API Service Type",
"description": "Volume API Service Type",
"type": "string"
},
"network_api_service_type": {
"name": "Network API Service Type",
"description": "Network API Service Type",
"type": "string"
},
"object_api_service_type": {
"name": "Object Storage API Service Type",
"description": "Object Storage API Service Type",
"type": "string"
},
"baremetal_api_version": {
"name": "Baremetal API Service Type",
"description": "Baremetal API Service Type",
"type": "string"
},
"compute_api_version": {
"name": "Compute API Version",
"description": "Compute API Version",
"type": "string"
},
"database_api_version": {
"name": "Database API Version",
"description": "Database API Version",
"type": "string"
},
"dns_api_version": {
"name": "DNS API Version",
"description": "DNS API Version",
"type": "string"
},
"identity_api_version": {
"name": "Identity API Version",
"description": "Identity API Version",
"type": "string"
},
"image_api_version": {
"name": "Image API Version",
"description": "Image API Version",
"type": "string"
},
"volume_api_version": {
"name": "Volume API Version",
"description": "Volume API Version",
"type": "string"
},
"network_api_version": {
"name": "Network API Version",
"description": "Network API Version",
"type": "string"
},
"object_api_version": {
"name": "Object Storage API Version",
"description": "Object Storage API Version",
"type": "string"
},
"baremetal_api_version": {
"name": "Baremetal API Version",
"description": "Baremetal API Version",
"type": "string"
}
}
}
},
"required": [
"name",
"profile"
]
}

@ -13,6 +13,7 @@
# under the License.
import glob
import json
import os
import yaml
@ -27,6 +28,10 @@ def get_profile(profile_name):
_vendor_defaults = {}
for vendor in glob.glob(os.path.join(_vendors_path, '*.yaml')):
with open(vendor, 'r') as f:
vendor_data = yaml.load(f)
vendor_data = yaml.safe_load(f)
_vendor_defaults[vendor_data['name']] = vendor_data['profile']
for vendor in glob.glob(os.path.join(_vendors_path, '*.json')):
with open(vendor, 'r') as f:
vendor_data = json.load(f)
_vendor_defaults[vendor_data['name']] = vendor_data['profile']
return _vendor_defaults.get(profile_name)

9
os_client_config/vendors/auro.json vendored Normal file

@ -0,0 +1,9 @@
{
"name": "auro",
"profile": {
"auth": {
"auth_url": "https://api.van1.auro.io:5000/v2.0"
},
"region_name": "van1"
}
}

@ -1,5 +0,0 @@
name: auro
profile:
auth:
auth_url: https://api.van1.auro.io:5000/v2.0
region_name: van1

6
os_client_config/vendors/bluebox.json vendored Normal file

@ -0,0 +1,6 @@
{
"name": "bluebox",
"profile": {
"region_name": "RegionOne"
}
}

14
os_client_config/vendors/catalyst.json vendored Normal file

@ -0,0 +1,14 @@
{
"name": "catalyst",
"profile": {
"auth": {
"auth_url": "https://api.cloud.catalyst.net.nz:5000/v2.0"
},
"regions": [
"nz-por-1",
"nz_wlg_2"
],
"image_api_version": "1",
"image_format": "raw"
}
}

@ -1,9 +0,0 @@
name: catalyst
profile:
auth:
auth_url: https://api.cloud.catalyst.net.nz:5000/v2.0
regions:
- nz-por-1
- nz_wlg_2
image_api_version: '1'
image_format: raw

14
os_client_config/vendors/citycloud.json vendored Normal file

@ -0,0 +1,14 @@
{
"name": "citycloud",
"profile": {
"auth": {
"auth_url": "https://identity1.citycloud.com:5000/v3/"
},
"regions": [
"Lon1",
"Sto2",
"Kna1"
],
"identity_api_version": "3"
}
}

@ -1,9 +0,0 @@
name: citycloud
profile:
auth:
auth_url: https://identity1.citycloud.com:5000/v3/
regions:
- Lon1
- Sto2
- Kna1
identity_api_version: '3'

13
os_client_config/vendors/conoha.json vendored Normal file

@ -0,0 +1,13 @@
{
"name": "conoha",
"profile": {
"auth": {
"auth_url": "https://identity.{region_name}.conoha.io/v2.0"
},
"regions": [
"sin1",
"lon1",
"tyo1"
]
}
}

@ -1,8 +0,0 @@
name: conoha
profile:
auth:
auth_url: https://identity.{region_name}.conoha.io/v2.0
regions:
- sin1
- lon1
- tyo1

@ -0,0 +1,10 @@
{
"name": "datacentred",
"profile": {
"auth": {
"auth_url": "https://compute.datacentred.io:5000/v2.0"
},
"region-name": "sal01",
"image_api_version": "1"
}
}

@ -1,6 +0,0 @@
name: datacentred
profile:
auth:
auth_url: https://compute.datacentred.io:5000/v2.0
region-name: sal01
image_api_version: '1'

10
os_client_config/vendors/dreamhost.json vendored Normal file

@ -0,0 +1,10 @@
{
"name": "dreamhost",
"profile": {
"auth": {
"auth_url": "https://keystone.dream.io/v2.0"
},
"region_name": "RegionOne",
"image_format": "raw"
}
}

@ -1,6 +0,0 @@
name: dreamhost
profile:
auth:
auth_url: https://keystone.dream.io/v2.0
region_name: RegionOne
image_format: raw

9
os_client_config/vendors/elastx.json vendored Normal file

@ -0,0 +1,9 @@
{
"name": "elastx",
"profile": {
"auth": {
"auth_url": "https://ops.elastx.net:5000/v2.0"
},
"region_name": "regionOne"
}
}

@ -1,5 +0,0 @@
name: elastx
profile:
auth:
auth_url: https://ops.elastx.net:5000/v2.0
region_name: regionOne

@ -0,0 +1,13 @@
{
"name": "entercloudsuite",
"profile": {
"auth": {
"auth_url": "https://api.entercloudsuite.com/v2.0"
},
"regions": [
"it-mil1",
"nl-ams1",
"de-fra1"
]
}
}

@ -1,8 +0,0 @@
name: entercloudsuite
profile:
auth:
auth_url: https://api.entercloudsuite.com/v2.0
regions:
- it-mil1
- nl-ams1
- de-fra1

14
os_client_config/vendors/hp.json vendored Normal file

@ -0,0 +1,14 @@
{
"name": "hp",
"profile": {
"auth": {
"auth_url": "https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0"
},
"regions": [
"region-a.geo-1",
"region-b.geo-1"
],
"dns_service_type": "hpext:dns",
"image_api_version": "1"
}
}

@ -1,9 +0,0 @@
name: hp
profile:
auth:
auth_url: https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0
regions:
- region-a.geo-1
- region-b.geo-1
dns_service_type: hpext:dns
image_api_version: '1'

15
os_client_config/vendors/internap.json vendored Normal file

@ -0,0 +1,15 @@
{
"name": "internap",
"profile": {
"auth": {
"auth_url": "https://identity.api.cloud.iweb.com/v2.0"
},
"regions": [
"ams01",
"da01",
"nyj01"
],
"image_api_version": "1",
"floating_ip_source": "None"
}
}

@ -1,10 +0,0 @@
name: internap
profile:
auth:
auth_url: https://identity.api.cloud.iweb.com/v2.0
regions:
- ams01
- da01
- nyj01
image_api_version: '1'
floating_ip_source: None

14
os_client_config/vendors/ovh.json vendored Normal file

@ -0,0 +1,14 @@
{
"name": "ovh",
"profile": {
"auth": {
"auth_url": "https://auth.cloud.ovh.net/v2.0"
},
"regions": [
"GRA1",
"SBG1"
],
"image_format": "raw",
"floating_ip_source": "None"
}
}

@ -1,9 +0,0 @@
name: ovh
profile:
auth:
auth_url: https://auth.cloud.ovh.net/v2.0
regions:
- GRA1
- SBG1
image_format: raw
floating_ip_source: None

27
os_client_config/vendors/rackspace.json vendored Normal file

@ -0,0 +1,27 @@
{
"name": "rackspace",
"profile": {
"auth": {
"auth_url": "https://identity.api.rackspacecloud.com/v2.0/"
},
"regions": [
"DFW",
"HKG",
"IAD",
"ORD",
"SYD",
"LON"
],
"database_service_type": "rax:database",
"compute_service_name": "cloudServersOpenStack",
"image_api_use_tasks": true,
"image_format": "vhd",
"floating_ip_source": "None",
"secgroup_source": "None",
"disable_vendor_agent": {
"vm_mode": "hvm",
"xenapi_use_agent": false
},
"has_network": false
}
}

@ -1,21 +0,0 @@
name: rackspace
profile:
auth:
auth_url: https://identity.api.rackspacecloud.com/v2.0/
regions:
- DFW
- HKG
- IAD
- ORD
- SYD
- LON
database_service_type: rax:database
compute_service_name: cloudServersOpenStack
image_api_use_tasks: true
image_format: vhd
floating_ip_source: None
secgroup_source: None
disable_vendor_agent:
vm_mode: hvm
xenapi_use_agent: false
has_network: false

14
os_client_config/vendors/runabove.json vendored Normal file

@ -0,0 +1,14 @@
{
"name": "runabove",
"profile": {
"auth": {
"auth_url": "https://auth.runabove.io/v2.0"
},
"regions": [
"BHS-1",
"SBG-1"
],
"image_format": "qcow2",
"floating_ip_source": "None"
}
}

@ -1,9 +0,0 @@
name: runabove
profile:
auth:
auth_url: https://auth.runabove.io/v2.0
regions:
- BHS-1
- SBG-1
image_format: qcow2
floating_ip_source: None

@ -0,0 +1,14 @@
{
"name": "switchengines",
"profile": {
"auth": {
"auth_url": "https://keystone.cloud.switch.ch:5000/v2.0"
},
"regions": [
"LS",
"ZH"
],
"image_api_use_tasks": true,
"image_format": "raw"
}
}

@ -1,9 +0,0 @@
name: switchengines
profile:
auth:
auth_url: https://keystone.cloud.switch.ch:5000/v2.0
regions:
- LS
- ZH
image_api_use_tasks: true
image_format: raw

9
os_client_config/vendors/ultimum.json vendored Normal file

@ -0,0 +1,9 @@
{
"name": "ultimum",
"profile": {
"auth": {
"auth_url": "https://console.ultimum-cloud.com:5000/v2.0"
},
"region-name": "RegionOne"
}
}

@ -1,5 +0,0 @@
name: ultimum
profile:
auth:
auth_url: https://console.ultimum-cloud.com:5000/v2.0
region-name: RegionOne

@ -0,0 +1,15 @@
{
"name": "unitedstack",
"profile": {
"auth": {
"auth_url": "https://identity.api.ustack.com/v3"
},
"regions": [
"bj1",
"gd1"
],
"identity_api_version": "3",
"image_format": "raw",
"floating_ip_source": "None"
}
}

@ -1,10 +0,0 @@
name: unitedstack
profile:
auth:
auth_url: https://identity.api.ustack.com/v3
regions:
- bj1
- gd1
identity_api_version: '3'
image_format: raw
floating_ip_source: None

10
os_client_config/vendors/vexxhost.json vendored Normal file

@ -0,0 +1,10 @@
{
"name": "vexxhost",
"profile": {
"auth": {
"auth_url": "http://auth.api.thenebulacloud.com:5000/v2.0/"
},
"region_name": "ca-ymq-1",
"floating_ip_source": "None"
}
}

@ -1,6 +0,0 @@
name: vexxhost
profile:
auth:
auth_url: http://auth.api.thenebulacloud.com:5000/v2.0/
region_name: ca-ymq-1
floating_ip_source: None

@ -8,6 +8,7 @@ coverage>=3.6
extras
fixtures>=0.3.14
discover
jsonschema>=2.0.0,<3.0.0,!=2.5.0
python-keystoneclient>=1.1.0
python-subunit>=0.0.18
sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3