From 41b840a6b6f17afc8f6b9a28c5fa223f7fc24fc5 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Fri, 29 Mar 2013 18:03:51 +0100 Subject: [PATCH 01/13] Add a swift middleware Last-Modified. --- middlewares/last_modified.py | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 middlewares/last_modified.py diff --git a/middlewares/last_modified.py b/middlewares/last_modified.py new file mode 100644 index 0000000..a038052 --- /dev/null +++ b/middlewares/last_modified.py @@ -0,0 +1,68 @@ +# Copyright (c) 2013 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from swift.common.utils import get_logger +from swift.common.swob import Request, wsgify + + +class LastModified(object): + """ + """ + + def __init__(self, app, conf): + self.app = app + self.conf = conf + self.logger = get_logger(self.conf, log_route='last_modified') + + def update_last_modified_meta(self, req, env): + vrs, account, container, obj = req.split_path(1, 4, True) + if obj: + env['PATH_INFO'] = env['PATH_INFO'].split('/%s' % obj)[0] + env['REQUEST_METHOD'] = 'POST' + headers = {'X-Container-Meta-Last-Modified': str(time.time())} + set_meta_req = Request.blank(env['PATH_INFO'], + headers=headers, + environ=env) + return set_meta_req.get_response(self.app) + + def req_passthrough(self, req): + return req.get_response(self.app) + + @wsgify + def __call__(self, req): + vrs, account, container, obj = req.split_path(1, 4, True) + if req.method in ('POST', 'PUT') and container or \ + req.method == 'DELETE' and obj: + new_env = req.environ.copy() + user_resp = self.req_passthrough(req) + if user_resp.status_int // 100 == 2: + # Update Container Meta Last-Modified in case of + # successful request + update_resp = self.update_last_modified_meta(req, + new_env) + if update_resp.status_int // 100 != 2: + self.logger.info('Unable to update Meta Last-Modified') + return user_resp + return self.app + + +def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + + def last_modified_filter(app): + return LastModified(app, conf) + return last_modified_filter From c33804586eaf8016c3c082a9ddffb46bbab15ee6 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Tue, 2 Apr 2013 13:47:20 +0200 Subject: [PATCH 02/13] Fix wrong licence header. --- middlewares/last_modified.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/middlewares/last_modified.py b/middlewares/last_modified.py index a038052..498275a 100644 --- a/middlewares/last_modified.py +++ b/middlewares/last_modified.py @@ -1,17 +1,20 @@ -# Copyright (c) 2013 OpenStack, LLC. +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2013 eNovance SAS # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Author: Fabien Boucher # -# http://www.apache.org/licenses/LICENSE-2.0 +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. import time from swift.common.utils import get_logger From 139bde7d8052706a66623731e745f637e7ed497f Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Tue, 2 Apr 2013 14:03:12 +0200 Subject: [PATCH 03/13] Use is_success facility from common/http.py --- middlewares/last_modified.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/middlewares/last_modified.py b/middlewares/last_modified.py index 498275a..b8f9b7e 100644 --- a/middlewares/last_modified.py +++ b/middlewares/last_modified.py @@ -18,6 +18,7 @@ import time from swift.common.utils import get_logger +from swift.common.http import is_success from swift.common.swob import Request, wsgify @@ -51,12 +52,12 @@ class LastModified(object): req.method == 'DELETE' and obj: new_env = req.environ.copy() user_resp = self.req_passthrough(req) - if user_resp.status_int // 100 == 2: + if is_success(user_resp.status_int): # Update Container Meta Last-Modified in case of # successful request update_resp = self.update_last_modified_meta(req, new_env) - if update_resp.status_int // 100 != 2: + if is_success(update_resp.status_int): self.logger.info('Unable to update Meta Last-Modified') return user_resp return self.app From a3069306d00da1665e90a8c331799d895c71f11b Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Tue, 2 Apr 2013 14:20:55 +0200 Subject: [PATCH 04/13] Fix PEP8 not compliant --- middlewares/last_modified.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/middlewares/last_modified.py b/middlewares/last_modified.py index b8f9b7e..ed8af31 100644 --- a/middlewares/last_modified.py +++ b/middlewares/last_modified.py @@ -48,8 +48,8 @@ class LastModified(object): @wsgify def __call__(self, req): vrs, account, container, obj = req.split_path(1, 4, True) - if req.method in ('POST', 'PUT') and container or \ - req.method == 'DELETE' and obj: + if (req.method in ('POST', 'PUT') and + container or req.method == 'DELETE' and obj): new_env = req.environ.copy() user_resp = self.req_passthrough(req) if is_success(user_resp.status_int): From de83a597814fe4de4bc759d1830df68911be79b8 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Tue, 2 Apr 2013 14:26:19 +0200 Subject: [PATCH 05/13] Use lambda function to prettify middleware creation --- middlewares/last_modified.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/middlewares/last_modified.py b/middlewares/last_modified.py index ed8af31..ea32928 100644 --- a/middlewares/last_modified.py +++ b/middlewares/last_modified.py @@ -22,7 +22,7 @@ from swift.common.http import is_success from swift.common.swob import Request, wsgify -class LastModified(object): +class LastModifiedMiddleware(object): """ """ @@ -67,6 +67,4 @@ def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) - def last_modified_filter(app): - return LastModified(app, conf) - return last_modified_filter + return lambda app: LastModifiedMiddleware(app, conf) From da28663f0e5b1f3878af1bdf316366b5b5443268 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Tue, 2 Apr 2013 17:04:56 +0200 Subject: [PATCH 06/13] Make use of make_pre_authed_request Use make_pre_authed_request instead of Request.Blank and set environnement correctly. --- middlewares/last_modified.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/middlewares/last_modified.py b/middlewares/last_modified.py index ea32928..2d35b0a 100644 --- a/middlewares/last_modified.py +++ b/middlewares/last_modified.py @@ -19,7 +19,8 @@ import time from swift.common.utils import get_logger from swift.common.http import is_success -from swift.common.swob import Request, wsgify +from swift.common.swob import wsgify +from swift.common.wsgi import make_pre_authed_request class LastModifiedMiddleware(object): @@ -34,12 +35,14 @@ class LastModifiedMiddleware(object): def update_last_modified_meta(self, req, env): vrs, account, container, obj = req.split_path(1, 4, True) if obj: - env['PATH_INFO'] = env['PATH_INFO'].split('/%s' % obj)[0] - env['REQUEST_METHOD'] = 'POST' + path = env['PATH_INFO'].split('/%s' % obj)[0] headers = {'X-Container-Meta-Last-Modified': str(time.time())} - set_meta_req = Request.blank(env['PATH_INFO'], - headers=headers, - environ=env) + set_meta_req = make_pre_authed_request(env, + method='POST', + path=path, + headers=headers, + environ=env, + swift_source='lm') return set_meta_req.get_response(self.app) def req_passthrough(self, req): From b59438c28f2d9fd589e5371c47803aaf68c46213 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Tue, 2 Apr 2013 17:42:56 +0200 Subject: [PATCH 07/13] Add comments about middleware aim and usage --- middlewares/last_modified.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/middlewares/last_modified.py b/middlewares/last_modified.py index 2d35b0a..c16d22b 100644 --- a/middlewares/last_modified.py +++ b/middlewares/last_modified.py @@ -25,6 +25,23 @@ from swift.common.wsgi import make_pre_authed_request class LastModifiedMiddleware(object): """ + LastModified is a middleware that add a meta to a container + when that container and/or objects in it are modified. The meta + data will contains the epoch timestamp. This middleware aims + to be used with the synchronizer. It limits the tree parsing + by giving a way to know a container has been modified since the + last container synchronization. + + Actions that lead to the container meta modification : + - POST/PUT on container + - POST/PUT/DELETE on object in it + + The following shows an example of proxy-server.conf: + [pipeline:main] + pipeline = catch_errors cache tempauth last-modified proxy-server + + [filter:last-modified] + use = egg:swift#last_modified """ def __init__(self, app, conf): From 1039bcdf0c70ed6dbb68452287fc76cbcc4093c8 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Tue, 2 Apr 2013 22:48:02 +0200 Subject: [PATCH 08/13] Fix stupid errors untested previous commit --- middlewares/last_modified.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/middlewares/last_modified.py b/middlewares/last_modified.py index c16d22b..c165cd6 100644 --- a/middlewares/last_modified.py +++ b/middlewares/last_modified.py @@ -51,14 +51,14 @@ class LastModifiedMiddleware(object): def update_last_modified_meta(self, req, env): vrs, account, container, obj = req.split_path(1, 4, True) + path = env['PATH_INFO'] if obj: - path = env['PATH_INFO'].split('/%s' % obj)[0] + path = path.split('/%s' % obj)[0] headers = {'X-Container-Meta-Last-Modified': str(time.time())} set_meta_req = make_pre_authed_request(env, method='POST', path=path, headers=headers, - environ=env, swift_source='lm') return set_meta_req.get_response(self.app) From f8593c20aa3349deb0f5c389838ac57841c1be2a Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Tue, 2 Apr 2013 22:52:06 +0200 Subject: [PATCH 09/13] Add a configuration statement to configure key name --- middlewares/last_modified.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/middlewares/last_modified.py b/middlewares/last_modified.py index c165cd6..ffc01da 100644 --- a/middlewares/last_modified.py +++ b/middlewares/last_modified.py @@ -42,19 +42,22 @@ class LastModifiedMiddleware(object): [filter:last-modified] use = egg:swift#last_modified + key_name = Last-Modified """ def __init__(self, app, conf): self.app = app self.conf = conf self.logger = get_logger(self.conf, log_route='last_modified') + self.key_name = conf.get('key_name', 'Last-Modified').replace(' ', '-') def update_last_modified_meta(self, req, env): vrs, account, container, obj = req.split_path(1, 4, True) path = env['PATH_INFO'] if obj: path = path.split('/%s' % obj)[0] - headers = {'X-Container-Meta-Last-Modified': str(time.time())} + metakey = 'X-Container-Meta-%s' % self.key_name + headers = {metakey: str(time.time())} set_meta_req = make_pre_authed_request(env, method='POST', path=path, From cf77fbab8308fee91db4f9a0b1ab5dbcc17a6971 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Wed, 3 Apr 2013 10:12:18 +0200 Subject: [PATCH 10/13] Keep it simple Do not check original status response before updating container meta. --- middlewares/last_modified.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/middlewares/last_modified.py b/middlewares/last_modified.py index ffc01da..2bb03fe 100644 --- a/middlewares/last_modified.py +++ b/middlewares/last_modified.py @@ -18,7 +18,6 @@ import time from swift.common.utils import get_logger -from swift.common.http import is_success from swift.common.swob import wsgify from swift.common.wsgi import make_pre_authed_request @@ -74,14 +73,10 @@ class LastModifiedMiddleware(object): if (req.method in ('POST', 'PUT') and container or req.method == 'DELETE' and obj): new_env = req.environ.copy() + # Keep it simple and do not check original request + # response status before updating the container meta user_resp = self.req_passthrough(req) - if is_success(user_resp.status_int): - # Update Container Meta Last-Modified in case of - # successful request - update_resp = self.update_last_modified_meta(req, - new_env) - if is_success(update_resp.status_int): - self.logger.info('Unable to update Meta Last-Modified') + self.update_last_modified_meta(req, new_env) return user_resp return self.app From 678c82e7ca247627463aed7679db34c19caf5a0d Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Wed, 3 Apr 2013 12:06:05 +0200 Subject: [PATCH 11/13] Let the orginal request to pass over the pipeline. --- middlewares/last_modified.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/middlewares/last_modified.py b/middlewares/last_modified.py index 2bb03fe..4f75856 100644 --- a/middlewares/last_modified.py +++ b/middlewares/last_modified.py @@ -17,9 +17,10 @@ # under the License. import time + from swift.common.utils import get_logger -from swift.common.swob import wsgify from swift.common.wsgi import make_pre_authed_request +from swift.common.swob import wsgify class LastModifiedMiddleware(object): @@ -41,6 +42,7 @@ class LastModifiedMiddleware(object): [filter:last-modified] use = egg:swift#last_modified + # will show as X-Container-Meta-${key_name} for the container's header. key_name = Last-Modified """ @@ -48,7 +50,8 @@ class LastModifiedMiddleware(object): self.app = app self.conf = conf self.logger = get_logger(self.conf, log_route='last_modified') - self.key_name = conf.get('key_name', 'Last-Modified').replace(' ', '-') + self.key_name = conf.get('key_name', + 'Last-Modified').strip().replace(' ', '-') def update_last_modified_meta(self, req, env): vrs, account, container, obj = req.split_path(1, 4, True) @@ -62,10 +65,7 @@ class LastModifiedMiddleware(object): path=path, headers=headers, swift_source='lm') - return set_meta_req.get_response(self.app) - - def req_passthrough(self, req): - return req.get_response(self.app) + set_meta_req.get_response(self.app) @wsgify def __call__(self, req): @@ -73,11 +73,7 @@ class LastModifiedMiddleware(object): if (req.method in ('POST', 'PUT') and container or req.method == 'DELETE' and obj): new_env = req.environ.copy() - # Keep it simple and do not check original request - # response status before updating the container meta - user_resp = self.req_passthrough(req) self.update_last_modified_meta(req, new_env) - return user_resp return self.app From 823c7c9b050876619a601c74b0c6995d53debf71 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Fri, 5 Apr 2013 10:34:32 +0200 Subject: [PATCH 12/13] Set swift version to master tarball as dependency --- tools/pip-requires | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pip-requires b/tools/pip-requires index 3814adb..fe2547f 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -1,5 +1,5 @@ simplejson -swift +http://tarballs.openstack.org/swift/swift-master.tar.gz#egg=swift python-swiftclient webob python-dateutil From 920dc639cf68ceef87caf4f3ba16ec573fca4970 Mon Sep 17 00:00:00 2001 From: Fabien Boucher Date: Fri, 5 Apr 2013 10:36:24 +0200 Subject: [PATCH 13/13] Add unittest for last modified middleware --- middlewares/__init__.py | 0 tests/test_middleware_lm.py | 156 ++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 middlewares/__init__.py create mode 100644 tests/test_middleware_lm.py diff --git a/middlewares/__init__.py b/middlewares/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_middleware_lm.py b/tests/test_middleware_lm.py new file mode 100644 index 0000000..769f402 --- /dev/null +++ b/tests/test_middleware_lm.py @@ -0,0 +1,156 @@ +# -*- encoding: utf-8 -*- + +# Copyright 2013 eNovance. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Author : "Fabien Boucher " +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +from middlewares import last_modified as middleware +from swift.common.swob import Response, Request + + +class FakeApp(object): + def __init__(self, status_headers_body=None): + self.status_headers_body = status_headers_body + if not self.status_headers_body: + self.status_headers_body = ('204 No Content', {}, '') + + def __call__(self, env, start_response): + status, headers, body = self.status_headers_body + return Response(status=status, headers=headers, + body=body)(env, start_response) + + +class FakeRequest(object): + def get_response(self, app): + pass + + +class TestLastModifiedMiddleware(unittest.TestCase): + + def _make_request(self, path, **kwargs): + req = Request.blank("/v1/AUTH_account/%s" % path, **kwargs) + return req + + def setUp(self): + self.conf = {'key_name': 'Last-Modified'} + self.test_default = middleware.filter_factory(self.conf)(FakeApp()) + + def test_denied_method_conf(self): + app = FakeApp() + test = middleware.filter_factory({})(app) + self.assertEquals(test.key_name, 'Last-Modified') + test = middleware.filter_factory({'key_name': "Last Modified"})(app) + self.assertEquals(test.key_name, 'Last-Modified') + test = middleware.filter_factory({'key_name': "Custom Key"})(app) + self.assertEquals(test.key_name, 'Custom-Key') + + def test_PUT_on_container(self): + self.called = False + + def make_pre_authed_request(*args, **kargs): + self.called = True + return FakeRequest() + + middleware.make_pre_authed_request = make_pre_authed_request + req = self._make_request('cont', + environ={'REQUEST_METHOD': 'PUT'}) + req.get_response(self.test_default) + self.assertEqual(self.called, True) + + def test_POST_on_container(self): + self.called = False + + def make_pre_authed_request(*args, **kargs): + self.called = True + return FakeRequest() + + middleware.make_pre_authed_request = make_pre_authed_request + req = self._make_request('cont', + environ={'REQUEST_METHOD': 'POST'}) + req.get_response(self.test_default) + self.assertEqual(self.called, True) + + def test_DELETE_on_container(self): + self.called = False + + def make_pre_authed_request(*args, **kargs): + self.called = True + return FakeRequest() + + middleware.make_pre_authed_request = make_pre_authed_request + req = self._make_request('cont', + environ={'REQUEST_METHOD': 'DELETE'}) + req.get_response(self.test_default) + self.assertEqual(self.called, False) + + def test_GET_on_container_and_object(self): + self.called = False + + def make_pre_authed_request(*args, **kargs): + self.called = True + return FakeRequest() + + middleware.make_pre_authed_request = make_pre_authed_request + req = self._make_request('cont', + environ={'REQUEST_METHOD': 'GET'}) + req.get_response(self.test_default) + self.assertEqual(self.called, False) + self.called = False + req = self._make_request('cont/obj', + environ={'REQUEST_METHOD': 'GET'}) + req.get_response(self.test_default) + self.assertEqual(self.called, False) + + def test_POST_on_object(self): + self.called = False + + def make_pre_authed_request(*args, **kargs): + self.called = True + return FakeRequest() + + middleware.make_pre_authed_request = make_pre_authed_request + req = self._make_request('cont/obj', + environ={'REQUEST_METHOD': 'POST'}) + req.get_response(self.test_default) + self.assertEqual(self.called, True) + + def test_PUT_on_object(self): + self.called = False + + def make_pre_authed_request(*args, **kargs): + self.called = True + return FakeRequest() + + middleware.make_pre_authed_request = make_pre_authed_request + req = self._make_request('cont/obj', + environ={'REQUEST_METHOD': 'PUT'}) + req.get_response(self.test_default) + self.assertEqual(self.called, True) + + def test_DELETE_on_object(self): + self.called = False + + def make_pre_authed_request(*args, **kargs): + self.called = True + return FakeRequest() + + middleware.make_pre_authed_request = make_pre_authed_request + req = self._make_request('cont/obj', + environ={'REQUEST_METHOD': 'DELETE'}) + req.get_response(self.test_default) + self.assertEqual(self.called, True)