From 02374dbf195880c853c022758344764394fc9296 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Mon, 7 May 2012 06:22:56 +0900 Subject: [PATCH] initial commit swift3 will be developed separately from swift. This is the version included in the commit 5dcc9083a76baaddd318303a0541c560d97b33d4 of swift. Signed-off-by: FUJITA Tomonori --- swift3/swift3.py | 480 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 480 insertions(+) create mode 100644 swift3/swift3.py diff --git a/swift3/swift3.py b/swift3/swift3.py new file mode 100644 index 00000000..e3390985 --- /dev/null +++ b/swift3/swift3.py @@ -0,0 +1,480 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The swift3 middleware will emulate the S3 REST api on top of swift. + +The following opperations are currently supported: + + * GET Service + * DELETE Bucket + * GET Bucket (List Objects) + * PUT Bucket + * DELETE Object + * GET Object + * HEAD Object + * PUT Object + * PUT Object (Copy) + +To add this middleware to your configuration, add the swift3 middleware +in front of the auth middleware, and before any other middleware that +look at swift requests (like rate limiting). + +To set up your client, the access key will be the concatenation of the +account and user strings that should look like test:tester, and the +secret access key is the account password. The host should also point +to the swift storage hostname. It also will have to use the old style +calling format, and not the hostname based container format. + +An example client using the python boto library might look like the +following for an SAIO setup:: + + connection = boto.s3.Connection( + aws_access_key_id='test:tester', + aws_secret_access_key='testing', + port=8080, + host='127.0.0.1', + is_secure=False, + calling_format=boto.s3.connection.OrdinaryCallingFormat()) +""" + +from urllib import unquote, quote +import base64 +from xml.sax.saxutils import escape as xml_escape +import urlparse + +from webob import Request, Response +from simplejson import loads + +from swift.common.utils import split_path +from swift.common.wsgi import WSGIContext +from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \ + HTTP_NO_CONTENT, HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, \ + HTTP_NOT_FOUND, HTTP_CONFLICT, is_success + + +MAX_BUCKET_LISTING = 1000 + + +def get_err_response(code): + """ + Given an HTTP response code, create a properly formatted xml error response + + :param code: error code + :returns: webob.response object + """ + error_table = { + 'AccessDenied': + (HTTP_FORBIDDEN, 'Access denied'), + 'BucketAlreadyExists': + (HTTP_CONFLICT, 'The requested bucket name is not available'), + 'BucketNotEmpty': + (HTTP_CONFLICT, 'The bucket you tried to delete is not empty'), + 'InvalidArgument': + (HTTP_BAD_REQUEST, 'Invalid Argument'), + 'InvalidBucketName': + (HTTP_BAD_REQUEST, 'The specified bucket is not valid'), + 'InvalidURI': + (HTTP_BAD_REQUEST, 'Could not parse the specified URI'), + 'NoSuchBucket': + (HTTP_NOT_FOUND, 'The specified bucket does not exist'), + 'SignatureDoesNotMatch': + (HTTP_FORBIDDEN, 'The calculated request signature does not '\ + 'match your provided one'), + 'NoSuchKey': + (HTTP_NOT_FOUND, 'The resource you requested does not exist')} + + resp = Response(content_type='text/xml') + resp.status = error_table[code][0] + resp.body = error_table[code][1] + resp.body = '\r\n\r\n ' \ + '%s\r\n %s\r\n\r\n' \ + % (code, error_table[code][1]) + return resp + + +def get_acl(account_name): + body = ('' + '' + '%s' + '' + '' + '' + '' + '%s' + '' + 'FULL_CONTROL' + '' + '' + '' % + (account_name, account_name)) + return Response(body=body, content_type="text/plain") + + +def canonical_string(req): + """ + Canonicalize a request to a token that can be signed. + """ + amz_headers = {} + + buf = "%s\n%s\n%s\n" % (req.method, req.headers.get('Content-MD5', ''), + req.headers.get('Content-Type') or '') + + for amz_header in sorted((key.lower() for key in req.headers + if key.lower().startswith('x-amz-'))): + amz_headers[amz_header] = req.headers[amz_header] + + if 'x-amz-date' in amz_headers: + buf += "\n" + elif 'Date' in req.headers: + buf += "%s\n" % req.headers['Date'] + + for k in sorted(key.lower() for key in amz_headers): + buf += "%s:%s\n" % (k, amz_headers[k]) + + path = req.path_qs + if '?' in path: + path, args = path.split('?', 1) + for key in urlparse.parse_qs(args, keep_blank_values=True): + if key in ('acl', 'logging', 'torrent', 'location', + 'requestPayment'): + return "%s%s?%s" % (buf, path, key) + return buf + path + + +class ServiceController(WSGIContext): + """ + Handles account level requests. + """ + def __init__(self, env, app, account_name, token, **kwargs): + WSGIContext.__init__(self, app) + env['HTTP_X_AUTH_TOKEN'] = token + env['PATH_INFO'] = '/v1/%s' % account_name + + def GET(self, env, start_response): + """ + Handle GET Service request + """ + env['QUERY_STRING'] = 'format=json' + body_iter = self._app_call(env) + status = self._get_status_int() + + if status != HTTP_OK: + if status == HTTP_UNAUTHORIZED: + return get_err_response('AccessDenied') + else: + return get_err_response('InvalidURI') + + containers = loads(''.join(list(body_iter))) + # we don't keep the creation time of a backet (s3cmd doesn't + # work without that) so we use something bogus. + body = '' \ + '' \ + '%s' \ + '' \ + % ("".join(['%s' \ + '2009-02-03T16:45:09.000Z' % + xml_escape(i['name']) for i in containers])) + resp = Response(status=HTTP_OK, content_type='application/xml', + body=body) + return resp + + +class BucketController(WSGIContext): + """ + Handles bucket request. + """ + def __init__(self, env, app, account_name, token, container_name, + **kwargs): + WSGIContext.__init__(self, app) + self.container_name = unquote(container_name) + self.account_name = unquote(account_name) + env['HTTP_X_AUTH_TOKEN'] = token + env['PATH_INFO'] = '/v1/%s/%s' % (account_name, container_name) + + def GET(self, env, start_response): + """ + Handle GET Bucket (List Objects) request + """ + if 'QUERY_STRING' in env: + args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1)) + else: + args = {} + max_keys = min(int(args.get('max-keys', MAX_BUCKET_LISTING)), + MAX_BUCKET_LISTING) + env['QUERY_STRING'] = 'format=json&limit=%s' % (max_keys + 1) + if 'marker' in args: + env['QUERY_STRING'] += '&marker=%s' % quote(args['marker']) + if 'prefix' in args: + env['QUERY_STRING'] += '&prefix=%s' % quote(args['prefix']) + if 'delimiter' in args: + env['QUERY_STRING'] += '&delimiter=%s' % quote(args['delimiter']) + body_iter = self._app_call(env) + status = self._get_status_int() + + if status != HTTP_OK: + if status == HTTP_UNAUTHORIZED: + return get_err_response('AccessDenied') + elif status == HTTP_NOT_FOUND: + return get_err_response('NoSuchBucket') + else: + return get_err_response('InvalidURI') + + if 'acl' in args: + return get_acl(self.account_name) + + objects = loads(''.join(list(body_iter))) + body = ('' + '' + '%s' + '%s' + '%s' + '%s' + '%s' + '%s' + '%s' + '%s' + '' % + ( + xml_escape(args.get('prefix', '')), + xml_escape(args.get('marker', '')), + xml_escape(args.get('delimiter', '')), + 'true' if len(objects) == (max_keys + 1) else 'false', + max_keys, + xml_escape(self.container_name), + "".join(['%s%sZ%s%sSTA'\ + 'NDARD' % + (xml_escape(i['name']), i['last_modified'], i['hash'], + i['bytes']) + for i in objects[:max_keys] if 'subdir' not in i]), + "".join(['%s' + % xml_escape(i['subdir']) + for i in objects[:max_keys] if 'subdir' in i]))) + return Response(body=body, content_type='application/xml') + + def PUT(self, env, start_response): + """ + Handle PUT Bucket request + """ + body_iter = self._app_call(env) + status = self._get_status_int() + + if status != HTTP_CREATED: + if status == HTTP_UNAUTHORIZED: + return get_err_response('AccessDenied') + elif status == HTTP_ACCEPTED: + return get_err_response('BucketAlreadyExists') + else: + return get_err_response('InvalidURI') + + resp = Response() + resp.headers.add('Location', self.container_name) + resp.status = HTTP_OK + return resp + + def DELETE(self, env, start_response): + """ + Handle DELETE Bucket request + """ + body_iter = self._app_call(env) + status = self._get_status_int() + + if status != HTTP_NO_CONTENT: + if status == HTTP_UNAUTHORIZED: + return get_err_response('AccessDenied') + elif status == HTTP_NOT_FOUND: + return get_err_response('NoSuchBucket') + elif status == HTTP_CONFLICT: + return get_err_response('BucketNotEmpty') + else: + return get_err_response('InvalidURI') + + resp = Response() + resp.status = HTTP_NO_CONTENT + return resp + + +class ObjectController(WSGIContext): + """ + Handles requests on objects + """ + def __init__(self, env, app, account_name, token, container_name, + object_name, **kwargs): + WSGIContext.__init__(self, app) + self.account_name = unquote(account_name) + self.container_name = unquote(container_name) + env['HTTP_X_AUTH_TOKEN'] = token + env['PATH_INFO'] = '/v1/%s/%s/%s' % (account_name, container_name, + object_name) + + def GETorHEAD(self, env, start_response): + app_iter = self._app_call(env) + status = self._get_status_int() + headers = dict(self._response_headers) + + if is_success(status): + if 'QUERY_STRING' in env: + args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1)) + else: + args = {} + if 'acl' in args: + return get_acl(self.account_name) + + new_hdrs = {} + for key, val in headers.iteritems(): + _key = key.lower() + if _key.startswith('x-object-meta-'): + new_hdrs['x-amz-meta-' + key[14:]] = val + elif _key in ('content-length', 'content-type', + 'content-range', 'content-encoding', + 'etag', 'last-modified'): + new_hdrs[key] = val + return Response(status=status, headers=new_hdrs, app_iter=app_iter) + elif status == HTTP_UNAUTHORIZED: + return get_err_response('AccessDenied') + elif status == HTTP_NOT_FOUND: + return get_err_response('NoSuchKey') + else: + return get_err_response('InvalidURI') + + def HEAD(self, env, start_response): + """ + Handle HEAD Object request + """ + return self.GETorHEAD(env, start_response) + + def GET(self, env, start_response): + """ + Handle GET Object request + """ + return self.GETorHEAD(env, start_response) + + def PUT(self, env, start_response): + """ + Handle PUT Object and PUT Object (Copy) request + """ + for key, value in env.items(): + if key.startswith('HTTP_X_AMZ_META_'): + del env[key] + env['HTTP_X_OBJECT_META_' + key[16:]] = value + elif key == 'HTTP_CONTENT_MD5': + env['HTTP_ETAG'] = value.decode('base64').encode('hex') + elif key == 'HTTP_X_AMZ_COPY_SOURCE': + env['HTTP_X_COPY_FROM'] = value + + body_iter = self._app_call(env) + status = self._get_status_int() + + if status != HTTP_CREATED: + if status == HTTP_UNAUTHORIZED: + return get_err_response('AccessDenied') + elif status == HTTP_NOT_FOUND: + return get_err_response('NoSuchBucket') + else: + return get_err_response('InvalidURI') + + if 'HTTP_X_COPY_FROM' in env: + body = '' \ + '"%s"' \ + '' % self._response_header_value('etag') + return Response(status=HTTP_OK, body=body) + + return Response(status=200, etag=self._response_header_value('etag')) + + def DELETE(self, env, start_response): + """ + Handle DELETE Object request + """ + body_iter = self._app_call(env) + status = self._get_status_int() + + if status != HTTP_NO_CONTENT: + if status == HTTP_UNAUTHORIZED: + return get_err_response('AccessDenied') + elif status == HTTP_NOT_FOUND: + return get_err_response('NoSuchKey') + else: + return get_err_response('InvalidURI') + + resp = Response() + resp.status = HTTP_NO_CONTENT + return resp + + +class Swift3Middleware(object): + """Swift3 S3 compatibility midleware""" + def __init__(self, app, conf, *args, **kwargs): + self.app = app + + def get_controller(self, path): + container, obj = split_path(path, 0, 2, True) + d = dict(container_name=container, object_name=obj) + + if container and obj: + return ObjectController, d + elif container: + return BucketController, d + return ServiceController, d + + def __call__(self, env, start_response): + req = Request(env) + + if 'AWSAccessKeyId' in req.GET: + try: + req.headers['Date'] = req.GET['Expires'] + req.headers['Authorization'] = \ + 'AWS %(AWSAccessKeyId)s:%(Signature)s' % req.GET + except KeyError: + return get_err_response('InvalidArgument')(env, start_response) + + if not 'Authorization' in req.headers: + return self.app(env, start_response) + + try: + account, signature = \ + req.headers['Authorization'].split(' ')[-1].rsplit(':', 1) + except Exception: + return get_err_response('InvalidArgument')(env, start_response) + + try: + controller, path_parts = self.get_controller(req.path) + except ValueError: + return get_err_response('InvalidURI')(env, start_response) + + token = base64.urlsafe_b64encode(canonical_string(req)) + + controller = controller(env, self.app, account, token, **path_parts) + + if hasattr(controller, req.method): + res = getattr(controller, req.method)(env, start_response) + else: + return get_err_response('InvalidURI')(env, start_response) + + return res(env, start_response) + + +def filter_factory(global_conf, **local_conf): + """Standard filter factory to use the middleware with paste.deploy""" + conf = global_conf.copy() + conf.update(local_conf) + + def swift3_filter(app): + return Swift3Middleware(app, conf) + + return swift3_filter