diff --git a/os_collect_config/collect.py b/os_collect_config/collect.py
index 084b41c..dfbafe1 100644
--- a/os_collect_config/collect.py
+++ b/os_collect_config/collect.py
@@ -31,10 +31,11 @@ from os_collect_config import heat_local
from os_collect_config import keystone
from os_collect_config import local
from os_collect_config.openstack.common import log
+from os_collect_config import request
from os_collect_config import version
from oslo.config import cfg
-DEFAULT_COLLECTORS = ['heat_local', 'ec2', 'cfn', 'heat']
+DEFAULT_COLLECTORS = ['heat_local', 'ec2', 'cfn', 'heat', 'request']
opts = [
cfg.StrOpt('command', short='c',
help='Command to run on metadata changes. If specified,'
@@ -83,7 +84,8 @@ COLLECTORS = {ec2.name: ec2,
cfn.name: cfn,
heat.name: heat,
heat_local.name: heat_local,
- local.name: local}
+ local.name: local,
+ request.name: request}
def setup_conf():
@@ -102,6 +104,9 @@ def setup_conf():
heat_group = cfg.OptGroup(name='heat',
title='Heat Metadata options')
+ request_group = cfg.OptGroup(name='request',
+ title='Request Metadata options')
+
keystone_group = cfg.OptGroup(name='keystone',
title='Keystone auth options')
@@ -110,12 +115,14 @@ def setup_conf():
CONF.register_group(heat_local_group)
CONF.register_group(local_group)
CONF.register_group(heat_group)
+ CONF.register_group(request_group)
CONF.register_group(keystone_group)
CONF.register_cli_opts(ec2.opts, group='ec2')
CONF.register_cli_opts(cfn.opts, group='cfn')
CONF.register_cli_opts(heat_local.opts, group='heat_local')
CONF.register_cli_opts(local.opts, group='local')
CONF.register_cli_opts(heat.opts, group='heat')
+ CONF.register_cli_opts(request.opts, group='request')
CONF.register_cli_opts(keystone.opts, group='keystone')
CONF.register_cli_opts(opts)
diff --git a/os_collect_config/exc.py b/os_collect_config/exc.py
index 74d2d22..78c9f19 100644
--- a/os_collect_config/exc.py
+++ b/os_collect_config/exc.py
@@ -50,5 +50,13 @@ class LocalMetadataNotAvailable(SourceNotAvailable):
"""The local metadata is not available."""
+class RequestMetadataNotAvailable(SourceNotAvailable):
+ """The request metadata is not available."""
+
+
+class RequestMetadataNotConfigured(SourceNotAvailable):
+ """The request metadata is not fully configured."""
+
+
class InvalidArguments(ValueError):
"""Invalid arguments."""
diff --git a/os_collect_config/request.py b/os_collect_config/request.py
new file mode 100644
index 0000000..5910ee4
--- /dev/null
+++ b/os_collect_config/request.py
@@ -0,0 +1,93 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import calendar
+import json
+import time
+
+from oslo.config import cfg
+
+from os_collect_config import common
+from os_collect_config import exc
+from os_collect_config.openstack.common import log
+
+CONF = cfg.CONF
+logger = log.getLogger(__name__)
+
+opts = [
+ cfg.StrOpt('metadata-url',
+ help='URL to query for metadata'),
+]
+name = 'request'
+
+
+class Collector(object):
+ def __init__(self, requests_impl=common.requests):
+ self._requests_impl = requests_impl
+ self._session = requests_impl.Session()
+ self.last_modified = None
+
+ def check_fetch_content(self, headers):
+ '''Raises RequestMetadataNotAvailable if metadata should not be
+ fetched.
+ '''
+
+ # no last-modified header, so fetch
+ lm = headers.get('last-modified')
+ if not lm:
+ return
+
+ last_modified = calendar.timegm(
+ time.strptime(lm, '%a, %d %b %Y %H:%M:%S %Z'))
+
+ # first run, so fetch
+ if not self.last_modified:
+ return last_modified
+
+ if last_modified < self.last_modified:
+ logger.warn(
+ 'Last-Modified is older than previous collection')
+
+ if last_modified <= self.last_modified:
+ raise exc.RequestMetadataNotAvailable
+ return last_modified
+
+ def collect(self):
+ if CONF.request.metadata_url is None:
+ logger.warn('No metadata_url configured.')
+ raise exc.RequestMetadataNotConfigured
+ url = CONF.request.metadata_url
+ final_content = {}
+
+ try:
+ head = self._session.head(url)
+ last_modified = self.check_fetch_content(head.headers)
+
+ content = self._session.get(url)
+ content.raise_for_status()
+ self.last_modified = last_modified
+
+ except self._requests_impl.exceptions.RequestException as e:
+ logger.warn(e)
+ raise exc.RequestMetadataNotAvailable
+ try:
+ value = json.loads(content.text)
+ except ValueError as e:
+ logger.warn(
+ 'Failed to parse as json. (%s)' % e)
+ raise exc.RequestMetadataNotAvailable
+ final_content.update(value)
+
+ return [('request', final_content)]
diff --git a/os_collect_config/tests/test_collect.py b/os_collect_config/tests/test_collect.py
index 09dbc43..1feab93 100644
--- a/os_collect_config/tests/test_collect.py
+++ b/os_collect_config/tests/test_collect.py
@@ -35,6 +35,7 @@ from os_collect_config.tests import test_cfn
from os_collect_config.tests import test_ec2
from os_collect_config.tests import test_heat
from os_collect_config.tests import test_heat_local
+from os_collect_config.tests import test_request
def _setup_local_metadata(test_case):
@@ -63,7 +64,8 @@ class TestCollect(testtools.TestCase):
'heat': {
'keystoneclient': test_heat.FakeKeystoneClient(self),
'heatclient': test_heat.FakeHeatClient(self)
- }
+ },
+ 'request': {'requests_impl': test_request.FakeRequests},
}
return collect.__main__(args=fake_args,
collector_kwargs_map=collector_kwargs_map)
@@ -338,6 +340,7 @@ class TestCollectAll(testtools.TestCase):
cfg.CONF.heat.project_id = '9f6b09df-4d7f-4a33-8ec3-9924d8f46f10'
cfg.CONF.heat.stack_id = 'a/c482680f-7238-403d-8f76-36acf0c8e0aa'
cfg.CONF.heat.resource_name = 'server'
+ cfg.CONF.request.metadata_url = 'http://127.0.0.1:8000/my_metadata/'
@mock.patch.object(ks_discover.Discover, '__init__')
@mock.patch.object(ks_discover.Discover, 'url_for')
@@ -352,7 +355,8 @@ class TestCollectAll(testtools.TestCase):
'heat': {
'keystoneclient': test_heat.FakeKeystoneClient(self),
'heatclient': test_heat.FakeHeatClient(self)
- }
+ },
+ 'request': {'requests_impl': test_request.FakeRequests},
}
if collectors is None:
collectors = cfg.CONF.collectors
@@ -366,7 +370,8 @@ class TestCollectAll(testtools.TestCase):
(changed_keys, paths) = self._call_collect_all(
store=True, collector_kwargs_map=collector_kwargs_map)
if expected_changed is None:
- expected_changed = set(['heat_local', 'cfn', 'ec2', 'heat'])
+ expected_changed = set(
+ ['heat_local', 'cfn', 'ec2', 'heat', 'request'])
self.assertEqual(expected_changed, changed_keys)
self.assertThat(paths, matchers.IsInstance(list))
for path in paths:
@@ -384,10 +389,11 @@ class TestCollectAll(testtools.TestCase):
'heat': {
'keystoneclient': test_heat.FakeKeystoneClient(self),
'heatclient': test_heat.FakeHeatClient(self)
- }
+ },
+ 'request': {'requests_impl': test_request.FakeRequests},
}
expected_changed = set((
- 'heat_local', 'ec2', 'cfn', 'heat',
+ 'heat_local', 'ec2', 'cfn', 'heat', 'request',
'dep-name1', 'dep-name2', 'dep-name3'))
self._test_collect_all_store(collector_kwargs_map=soft_config_map,
expected_changed=expected_changed)
@@ -422,7 +428,8 @@ class TestCollectAll(testtools.TestCase):
'heat': {
'keystoneclient': test_heat.FakeKeystoneClient(self),
'heatclient': test_heat.FakeHeatClient(self)
- }
+ },
+ 'request': {'requests_impl': test_request.FakeRequests},
}
(changed_keys, paths) = self._call_collect_all(
store=True, collector_kwargs_map=soft_config_map)
diff --git a/os_collect_config/tests/test_request.py b/os_collect_config/tests/test_request.py
new file mode 100644
index 0000000..a48f969
--- /dev/null
+++ b/os_collect_config/tests/test_request.py
@@ -0,0 +1,148 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import calendar
+import json
+import time
+
+import fixtures
+from oslo.config import cfg
+import requests
+import testtools
+from testtools import matchers
+
+from os_collect_config import collect
+from os_collect_config import exc
+from os_collect_config import request
+
+
+META_DATA = {u'int1': 1,
+ u'strfoo': u'foo',
+ u'map_ab': {
+ u'a': 'apple',
+ u'b': 'banana',
+ }}
+
+
+class FakeResponse(dict):
+ def __init__(self, text, headers=None):
+ self.text = text
+ self.headers = headers
+
+ def raise_for_status(self):
+ pass
+
+
+class FakeRequests(object):
+ exceptions = requests.exceptions
+
+ class Session(object):
+ def get(self, url):
+ return FakeResponse(json.dumps(META_DATA))
+
+ def head(self, url):
+ return FakeResponse('', headers={
+ 'last-modified': time.strftime(
+ "%a, %d %b %Y %H:%M:%S %Z", time.gmtime())})
+
+
+class FakeFailRequests(object):
+ exceptions = requests.exceptions
+
+ class Session(object):
+ def get(self, url):
+ raise requests.exceptions.HTTPError(403, 'Forbidden')
+
+ def head(self, url):
+ raise requests.exceptions.HTTPError(403, 'Forbidden')
+
+
+class TestRequestBase(testtools.TestCase):
+ def setUp(self):
+ super(TestRequestBase, self).setUp()
+ self.log = self.useFixture(fixtures.FakeLogger())
+ collect.setup_conf()
+ cfg.CONF.request.metadata_url = 'http://127.0.0.1:8000/my_metadata'
+
+
+class TestRequest(TestRequestBase):
+
+ def test_collect_request(self):
+ req_collect = request.Collector(requests_impl=FakeRequests)
+ self.assertIsNone(req_collect.last_modified)
+ req_md = req_collect.collect()
+ self.assertIsNotNone(req_collect.last_modified)
+ self.assertThat(req_md, matchers.IsInstance(list))
+ self.assertEqual('request', req_md[0][0])
+ req_md = req_md[0][1]
+
+ for k in ('int1', 'strfoo', 'map_ab'):
+ self.assertIn(k, req_md)
+ self.assertEqual(req_md[k], META_DATA[k])
+
+ self.assertEqual('', self.log.output)
+
+ def test_collect_request_fail(self):
+ req_collect = request.Collector(requests_impl=FakeFailRequests)
+ self.assertRaises(exc.RequestMetadataNotAvailable, req_collect.collect)
+ self.assertIn('Forbidden', self.log.output)
+
+ def test_collect_request_no_metadata_url(self):
+ cfg.CONF.request.metadata_url = None
+ req_collect = request.Collector(requests_impl=FakeRequests)
+ self.assertRaises(exc.RequestMetadataNotConfigured,
+ req_collect.collect)
+ self.assertIn('No metadata_url configured', self.log.output)
+
+ def test_check_fetch_content(self):
+ req_collect = request.Collector()
+
+ now_secs = calendar.timegm(time.gmtime())
+ now_str = time.strftime("%a, %d %b %Y %H:%M:%S %Z",
+ time.gmtime(now_secs))
+
+ future_secs = calendar.timegm(time.gmtime()) + 10
+ future_str = time.strftime("%a, %d %b %Y %H:%M:%S %Z",
+ time.gmtime(future_secs))
+
+ past_secs = calendar.timegm(time.gmtime()) - 10
+ past_str = time.strftime("%a, %d %b %Y %H:%M:%S %Z",
+ time.gmtime(past_secs))
+
+ self.assertIsNone(req_collect.last_modified)
+
+ # first run always collects
+ self.assertEqual(
+ now_secs,
+ req_collect.check_fetch_content({'last-modified': now_str}))
+
+ # second run unmodified, does not collect
+ req_collect.last_modified = now_secs
+ self.assertRaises(exc.RequestMetadataNotAvailable,
+ req_collect.check_fetch_content,
+ {'last-modified': now_str})
+
+ # run with later date, collects
+ self.assertEqual(
+ future_secs,
+ req_collect.check_fetch_content({'last-modified': future_str}))
+
+ # run with earlier date, does not collect
+ self.assertRaises(exc.RequestMetadataNotAvailable,
+ req_collect.check_fetch_content,
+ {'last-modified': past_str})
+
+ # run no last-modified header, collects
+ self.assertIsNone(req_collect.check_fetch_content({}))