openstack/ceilometermiddleware
Code Issues Proposed changes

retrieve project id to ignore from keystone

Browse Source 
We currently allows only project uuid, but this is a pain for deployer.
Also the default is a project name which doesn't work...

This change queries keystone to retrieve project ids when the
ignore_projects list are names.

By default, if auth_type is not set, we keep the previous behavior to
not query keystone.

Change-Id: I270d080d3e65eb6b0cd823498a4dd37389c49221
This commit is contained in:
Mehdi Abaakouk 2017-05-13 13:37:33 +02:00
parent 0cea730b10
commit e2bf485044
5 changed files with 639 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
ceilometermiddleware/swift.py
View File

@ -36,7 +36,7 @@ before "proxy-server" and add the following filter in the file:
# set topic
topic = notifications
# skip metering of requests from listed project ids
ignore_projects = <proj_uuid>, <proj_uuid2>
ignore_projects = <proj_uuid>, <proj_uuid2>, <proj_name>
# Whether to send events to messaging driver in a background thread
nonblocking_notify = False
# Queue size for sending notifications in background thread (0=unlimited).
@ -44,11 +44,27 @@ before "proxy-server" and add the following filter in the file:
send_queue_size = 1000
# Logging level control
log_level = WARNING
# All keystoneauth1 options can be set to query project name for
# ignore_projects option, here is just a example:
auth_type = password
auth_url = https://[::1]:5000
project_name = services
project_domain_name = Default
username = user
user_domain_name = Default
password = a_big_secret
interface = public
"""
import datetime
import functools
import logging
from keystoneauth1 import exceptions as ksa_exc
from keystoneauth1.loading import adapter as ksa_adapter
from keystoneauth1.loading import base as ksa_base
from keystoneauth1.loading import session as ksa_session
from keystoneclient.v3 import client as ks_client
from oslo_config import cfg
import oslo_messaging
from oslo_utils import strutils
@ -62,9 +78,19 @@ import six.moves.queue as queue
import six.moves.urllib.parse as urlparse
import threading
LOG = logging.getLogger(__name__)
def list_from_csv(comma_separated_str):
if comma_separated_str:
return list(
filter(lambda x: x,
map(lambda x: x.strip(),
comma_separated_str.split(','))))
return []
def _log_and_ignore_error(fn):
@functools.wraps(fn)
def wrapper(*args, **kwargs):
@ -106,17 +132,30 @@ class InputProxy(object):
return line
class KeystoneClientLoader(ksa_adapter.Adapter):
"""Keystone client adapter loader.
Keystone client and Keystoneauth1 adapter take exactly the same options, so
it's safe to create a keystone client with keystoneauth adapter options.
"""
@property
def plugin_class(self):
return ks_client.Client
class Swift(object):
"""Swift middleware used for counting requests."""
event_queue = None
threadLock = threading.Lock()
DEFAULT_IGNORE_PROJECT_NAMES = ['services']
def __init__(self, app, conf):
self._app = app
self.ignore_projects = [
proj.strip() for proj in
conf.get('ignore_projects', 'gnocchi').split(',')]
self.ignore_projects = self._get_ignore_projects(conf)
oslo_messaging.set_transport_defaults(conf.get('control_exchange',
'swift'))
@ -156,6 +195,54 @@ class Swift(object):
self.start_sender_thread()
Swift.threadLock.release()
def _get_ignore_projects(self, conf):
if 'auth_type' not in conf:
LOG.info("'auth_type' is not set assuming ignore_projects are "
"only project uuid.")
return list_from_csv(conf.get('ignore_projects'))
if 'ignore_projects' in conf:
ignore_projects = list_from_csv(conf.get('ignore_projects'))
else:
ignore_projects = self.DEFAULT_IGNORE_PROJECT_NAMES
if not ignore_projects:
return []
def opt_getter(opt):
# TODO(sileht): This method does not support deprecated opt names
val = conf.get(opt.name)
if val is None:
val = conf.get(opt.dest)
return val
auth_type = conf.get('auth_type')
plugin = ksa_base.get_plugin_loader(auth_type)
auth = plugin.load_from_options_getter(opt_getter)
session = ksa_session.Session().load_from_options_getter(
opt_getter, auth=auth)
client = KeystoneClientLoader().load_from_options_getter(
opt_getter, session=session)
projects = []
for name_or_id in ignore_projects:
projects.extend(self._get_keystone_projects(client, name_or_id))
return projects
@staticmethod
def _get_keystone_projects(client, name_or_id):
try:
return [client.projects.get(name_or_id)]
except ksa_exc.NotFound:
pass
if isinstance(name_or_id, six.binary_type):
name_or_id = name_or_id.decode('utf-8', 'strict')
projects = client.projects.list(name=name_or_id)
if not projects:
LOG.warning("fail to find project '%s' in keystone", name_or_id)
return [p.id for p in projects]
def __call__(self, env, start_response):
start_response_args = [None]
input_proxy = InputProxy(env['wsgi.input'])

514
ceilometermiddleware/tests/data/list_projects.yaml Normal file
View File

@ -0,0 +1,514 @@
http_interactions:
- recorded_at: '2017-05-15T07:49:52'
request:
body:
encoding: utf-8
string: |-
{
"auth": {
"tenantName": "dummy",
"passwordCredentials": {
"username": "dummy",
"password": "********"
}
}
}
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '107'
Content-Type:
- application/json
User-Agent:
- run.py keystoneauth1/2.20.0 python-requests/2.14.2 CPython/2.7.13
method: POST
uri: https://[::1]:5000/v2.0/tokens
response:
body:
encoding: null
string: |-
{
"access": {
"serviceCatalog": [
{
"type": "compute",
"endpoints_links": [],
"name": "nova",
"endpoints": [
{
"internalURL": "https://[::1]:8774/v2.1",
"adminURL": "https://[::1]:8774/v2.1",
"id": "1e879ab434b54b8abfd275feeb2ef9f3",
"region": "RegionOne",
"publicURL": "https://[::1]:8774/v2.1"
}
]
},
{
"type": "network",
"endpoints_links": [],
"name": "neutron",
"endpoints": [
{
"internalURL": "http://[::1]:9696",
"adminURL": "http://[::1]:9696",
"id": "83fcb786f646437f9a61cef72a9e43d7",
"region": "RegionOne",
"publicURL": "http://[::1]:9696"
}
]
},
{
"type": "volumev2",
"endpoints_links": [],
"name": "cinderv2",
"endpoints": [
{
"internalURL": "https://[::1]:8776/v2/ed980105f9d047e2bee738b3f261f126",
"adminURL": "https://[::1]:8776/v2/ed980105f9d047e2bee738b3f261f126",
"id": "973ef665c2ea4ec3b5c3d48932fad7a4",
"region": "RegionOne",
"publicURL": "https://[::1]:8776/v2/ed980105f9d047e2bee738b3f261f126"
}
]
},
{
"type": "volumev3",
"endpoints_links": [],
"name": "cinderv3",
"endpoints": [
{
"internalURL": "https://[::1]:8776/v3/ed980105f9d047e2bee738b3f261f126",
"adminURL": "https://[::1]:8776/v3/ed980105f9d047e2bee738b3f261f126",
"id": "0e80fe643d4d44729db99d0a5c882d1b",
"region": "RegionOne",
"publicURL": "https://[::1]:8776/v3/ed980105f9d047e2bee738b3f261f126"
}
]
},
{
"type": "image",
"endpoints_links": [],
"name": "glance",
"endpoints": [
{
"internalURL": "http://[::1]:9292",
"adminURL": "http://[::1]:9292",
"id": "7aad24b660a94254adc3546e4de4d668",
"region": "RegionOne",
"publicURL": "http://[::1]:9292"
}
]
},
{
"type": "volume",
"endpoints_links": [],
"name": "cinder",
"endpoints": [
{
"internalURL": "https://[::1]:8776/v1/ed980105f9d047e2bee738b3f261f126",
"adminURL": "https://[::1]:8776/v1/ed980105f9d047e2bee738b3f261f126",
"id": "8191ee00b695483796a9531bca70279b",
"region": "RegionOne",
"publicURL": "https://[::1]:8776/v1/ed980105f9d047e2bee738b3f261f126"
}
]
},
{
"type": "identity",
"endpoints_links": [],
"name": "keystone",
"endpoints": [
{
"internalURL": "https://[::1]:5000",
"adminURL": "https://[::1]:35357",
"id": "24ab268f1a7b47d4af493c4c74cd6130",
"region": "RegionOne",
"publicURL": "https://[::1]:5000"
}
]
}
],
"user": {
"username": "dummy",
"roles_links": [],
"id": "f18b121edda04346b86610fa23983a0e",
"roles": [
{
"name": "admin"
}
],
"name": "dummy"
},
"token": {
"issued_at": "2017-05-15T07:49:52.000000Z",
"tenant": {
"enabled": true,
"id": "ed980105f9d047e2bee738b3f261f126",
"name": "dummy",
"description": "admin tenant"
},
"audit_ids": [
"VzK7yoNFT0qlUWg5KhDuMQ"
],
"expires": "9999-12-31T23:59:59Z",
"id": "gAAAAABZGV2gZwV0SlycA_OIohX7kRAmTp84SnzsAYD5Uhey7RTzCf0NvxNRxLk5RsjRSMncSdro7eWgtMrSblZJCPl485IvHANL3E3gsxFJP9TjebqDiF4DtGhQmc4mHPB3kOBNzg3v2FrHB2hK77Cc4M7V1Pm_-nWBHxYxVNopVhrd80Y4-2c"
},
"metadata": {
"is_admin": 0,
"roles": [
"d3b61a4656d64cbbbdb0f13690e2ffe4"
]
}
}
}
headers:
Connection:
- Keep-Alive
Content-Length:
- '3183'
Content-Type:
- application/json
Date:
- Mon, 15 May 2017 07:49:51 GMT
Keep-Alive:
- timeout=3, max=100
Server:
- Apache/2.4.18 (Ubuntu)
Strict-Transport-Security:
- max-age=15768000
Vary:
- X-Auth-Token
X-Distribution:
- Ubuntu
x-openstack-request-id:
- req-84cb5714-49dc-4bab-93ba-2b66ba566c30
status:
code: 200
message: OK
url: https://[::1]:5000/v2.0/tokens
- recorded_at: '2017-05-15T07:49:53'
request:
body:
encoding: utf-8
string: ''
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- run.py keystoneauth1/2.20.0 python-requests/2.14.2 CPython/2.7.13
method: GET
uri: https://[::1]:35357/
response:
body:
encoding: null
string: |-
{
"versions": {
"values": [
{
"status": "stable",
"updated": "2016-10-06T00:00:00Z",
"id": "v3.7",
"links": [
{
"rel": "self",
"href": "https://[::1]:35357/v3/"
}
],
"media-types": [
{
"type": "application/vnd.openstack.identity-v3+json",
"base": "application/json"
}
]
},
{
"status": "deprecated",
"updated": "2016-08-04T00:00:00Z",
"id": "v2.0",
"links": [
{
"rel": "self",
"href": "https://[::1]:35357/v2.0/"
},
{
"type": "text/html",
"rel": "describedby",
"href": "http://docs.openstack.org/"
}
],
"media-types": [
{
"type": "application/vnd.openstack.identity-v2.0+json",
"base": "application/json"
}
]
}
]
}
}
headers:
Connection:
- Keep-Alive
Content-Length:
- '627'
Content-Type:
- application/json
Date:
- Mon, 15 May 2017 07:49:52 GMT
Keep-Alive:
- timeout=3, max=100
Server:
- Apache/2.4.18 (Ubuntu)
Strict-Transport-Security:
- max-age=15768000
Vary:
- X-Auth-Token
X-Distribution:
- Ubuntu
status:
code: 300
message: Multiple Choices
url: https://[::1]:35357/
- recorded_at: '2017-05-15T07:49:53'
request:
body:
encoding: utf-8
string: ''
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- python-keystoneclient
X-Auth-Token:
- gAAAAABZGV2gZwV0SlycA_OIohX7kRAmTp84SnzsAYD5Uhey7RTzCf0NvxNRxLk5RsjRSMncSdro7eWgtMrSblZJCPl485IvHANL3E3gsxFJP9TjebqDiF4DtGhQmc4mHPB3kOBNzg3v2FrHB2hK77Cc4M7V1Pm_-nWBHxYxVNopVhrd80Y4-2c
method: GET
uri: https://[::1]:35357/v3/projects/services
response:
body:
encoding: null
string: |-
{
"error": {
"code": 404,
"title": "Not Found",
"message": "Could not find project: services"
}
}
headers:
Connection:
- Keep-Alive
Content-Length:
- '93'
Content-Type:
- application/json
Date:
- Mon, 15 May 2017 07:49:53 GMT
Keep-Alive:
- timeout=3, max=99
Server:
- Apache/2.4.18 (Ubuntu)
Strict-Transport-Security:
- max-age=15768000
Vary:
- X-Auth-Token
X-Distribution:
- Ubuntu
x-openstack-request-id:
- req-6107025c-e09e-437a-90c2-61a559154d32
status:
code: 404
message: Not Found
url: https://[::1]:35357/v3/projects/services
- recorded_at: '2017-05-15T07:49:53'
request:
body:
encoding: utf-8
string: ''
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- python-keystoneclient
X-Auth-Token:
- gAAAAABZGV2gZwV0SlycA_OIohX7kRAmTp84SnzsAYD5Uhey7RTzCf0NvxNRxLk5RsjRSMncSdro7eWgtMrSblZJCPl485IvHANL3E3gsxFJP9TjebqDiF4DtGhQmc4mHPB3kOBNzg3v2FrHB2hK77Cc4M7V1Pm_-nWBHxYxVNopVhrd80Y4-2c
method: GET
uri: https://[::1]:35357/v3/projects?name=services
response:
body:
encoding: null
string: |-
{
"projects": [
{
"enabled": true,
"id": "147cc0a9263c4964926f3ee7b6ba3685",
"domain_id": "default",
"parent_id": "default",
"is_domain": false,
"name": "services",
"links": {
"self": "https://[::1]:5000/v3/projects/147cc0a9263c4964926f3ee7b6ba3685"
},
"description": "Tenant for the openstack services"
}
],
"links": {
"self": "https://[::1]:5000/v3/projects?name=services",
"next": null,
"previous": null
}
}
headers:
Connection:
- Keep-Alive
Content-Length:
- '440'
Content-Type:
- application/json
Date:
- Mon, 15 May 2017 07:49:53 GMT
Keep-Alive:
- timeout=3, max=98
Server:
- Apache/2.4.18 (Ubuntu)
Strict-Transport-Security:
- max-age=15768000
Vary:
- X-Auth-Token
X-Distribution:
- Ubuntu
x-openstack-request-id:
- req-1915b2be-f116-4831-a7c3-5ba0a32d416f
status:
code: 200
message: OK
url: https://[::1]:35357/v3/projects?name=services
- recorded_at: '2017-05-15T07:49:53'
request:
body:
encoding: utf-8
string: ''
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- python-keystoneclient
X-Auth-Token:
- gAAAAABZGV2gZwV0SlycA_OIohX7kRAmTp84SnzsAYD5Uhey7RTzCf0NvxNRxLk5RsjRSMncSdro7eWgtMrSblZJCPl485IvHANL3E3gsxFJP9TjebqDiF4DtGhQmc4mHPB3kOBNzg3v2FrHB2hK77Cc4M7V1Pm_-nWBHxYxVNopVhrd80Y4-2c
method: GET
uri: https://[::1]:35357/v3/projects/gnocchi
response:
body:
encoding: null
string: |-
{
"error": {
"code": 404,
"title": "Not Found",
"message": "Could not find project: gnocchi"
}
}
headers:
Connection:
- Keep-Alive
Content-Length:
- '92'
Content-Type:
- application/json
Date:
- Mon, 15 May 2017 07:49:53 GMT
Keep-Alive:
- timeout=3, max=97
Server:
- Apache/2.4.18 (Ubuntu)
Strict-Transport-Security:
- max-age=15768000
Vary:
- X-Auth-Token
X-Distribution:
- Ubuntu
x-openstack-request-id:
- req-b23e72d3-742e-4e10-b9a7-d1161f1eeab4
status:
code: 404
message: Not Found
url: https://[::1]:35357/v3/projects/gnocchi
- recorded_at: '2017-05-15T07:49:53'
request:
body:
encoding: utf-8
string: ''
headers:
Accept:
- application/json
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- python-keystoneclient
X-Auth-Token:
- gAAAAABZGV2gZwV0SlycA_OIohX7kRAmTp84SnzsAYD5Uhey7RTzCf0NvxNRxLk5RsjRSMncSdro7eWgtMrSblZJCPl485IvHANL3E3gsxFJP9TjebqDiF4DtGhQmc4mHPB3kOBNzg3v2FrHB2hK77Cc4M7V1Pm_-nWBHxYxVNopVhrd80Y4-2c
method: GET
uri: https://[::1]:35357/v3/projects?name=gnocchi
response:
body:
encoding: null
string: |-
{
"projects": [],
"links": {
"self": "https://[::1]:5000/v3/projects?name=gnocchi",
"next": null,
"previous": null
}
}
headers:
Connection:
- Keep-Alive
Content-Length:
- '134'
Content-Type:
- application/json
Date:
- Mon, 15 May 2017 07:49:53 GMT
Keep-Alive:
- timeout=3, max=96
Server:
- Apache/2.4.18 (Ubuntu)
Strict-Transport-Security:
- max-age=15768000
Vary:
- X-Auth-Token
X-Distribution:
- Ubuntu
x-openstack-request-id:
- req-fdeed726-18a4-4e73-bf8d-d24a5b56246e
status:
code: 200
message: OK
url: https://[::1]:35357/v3/projects?name=gnocchi
recorded_with: betamax/0.8.0

32
ceilometermiddleware/tests/test_swift.py
View File

@ -12,13 +12,15 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import threading
import mock
from oslo_config import cfg
import six
from ceilometermiddleware import swift
from ceilometermiddleware.tests import base as tests_base
import threading
from keystoneauth1.fixture import keystoneauth_betamax as betamax
class FakeApp(object):
@ -429,3 +431,31 @@ class TestSwift(tests_base.TestCase):
with mock.patch('oslo_messaging.Notifier.info') as notify:
list(app(req.environ, self.start_response))
self.assertFalse(notify.called)
def test_ignore_projects_without_keystone(self):
app = swift.Swift(FakeApp(), {
'ignore_projects': 'cf0356aaac7c42bba5a744339a6169fa,'
'18157dd635bb413c9e27686fee93c583',
})
self.assertEqual(["cf0356aaac7c42bba5a744339a6169fa",
"18157dd635bb413c9e27686fee93c583"],
app.ignore_projects)
@mock.patch.object(swift.LOG, 'warning')
def test_ignore_projects_with_keystone(self, warning):
self.useFixture(betamax.BetamaxFixture(
cassette_name='list_projects',
cassette_library_dir='ceilometermiddleware/tests/data',
))
app = swift.Swift(FakeApp(), {
'auth_type': 'v2password',
'auth_url': 'https://[::1]:5000/v2.0',
'username': 'admin',
'tenant_name': 'admin',
'password': 'secret',
'ignore_projects': 'services,gnocchi',
})
self.assertEqual(["147cc0a9263c4964926f3ee7b6ba3685"],
app.ignore_projects)
warning.assert_called_once_with(
"fail to find project '%s' in keystone", "gnocchi")

2
requirements.txt
View File

@ -8,3 +8,5 @@ oslo.utils
pbr>=1.6 # Apache-2.0
pycadf!=2.0.0,>=1.1.0 # Apache-2.0
six>=1.9.0 # MIT
keystoneauth1>=2.18.0 # Apache-2.0
python-keystoneclient>=3.8.0 # Apache-2.0

2
test-requirements.txt
View File

@ -10,4 +10,4 @@ oslotest>=1.10.0 # Apache-2.0
testrepository>=0.0.18 # Apache-2.0/BSD
mock>=1.2 # BSD
reno>=0.1.1 # Apache-2.0
betamax>=0.7.0 # Apache-2.0