initial commit

swift3 will be developed separately from swift. This is the version
included in the commit 5dcc9083a76baaddd318303a0541c560d97b33d4 of
swift.

Signed-off-by: FUJITA Tomonori <fujita.tomonori@lab.ntt.co.jp>

swift3/swift3.py

@@ -0,0 +1,480 @@
+# Copyright (c) 2010 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+The swift3 middleware will emulate the S3 REST api on top of swift.
+
+The following opperations are currently supported:
+
+    * GET Service
+    * DELETE Bucket
+    * GET Bucket (List Objects)
+    * PUT Bucket
+    * DELETE Object
+    * GET Object
+    * HEAD Object
+    * PUT Object
+    * PUT Object (Copy)
+
+To add this middleware to your configuration, add the swift3 middleware
+in front of the auth middleware, and before any other middleware that
+look at swift requests (like rate limiting).
+
+To set up your client, the access key will be the concatenation of the
+account and user strings that should look like test:tester, and the
+secret access key is the account password.  The host should also point
+to the swift storage hostname.  It also will have to use the old style
+calling format, and not the hostname based container format.
+
+An example client using the python boto library might look like the
+following for an SAIO setup::
+
+    connection = boto.s3.Connection(
+        aws_access_key_id='test:tester',
+        aws_secret_access_key='testing',
+        port=8080,
+        host='127.0.0.1',
+        is_secure=False,
+        calling_format=boto.s3.connection.OrdinaryCallingFormat())
+"""
+
+from urllib import unquote, quote
+import base64
+from xml.sax.saxutils import escape as xml_escape
+import urlparse
+
+from webob import Request, Response
+from simplejson import loads
+
+from swift.common.utils import split_path
+from swift.common.wsgi import WSGIContext
+from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \
+    HTTP_NO_CONTENT, HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, \
+    HTTP_NOT_FOUND, HTTP_CONFLICT, is_success
+
+
+MAX_BUCKET_LISTING = 1000
+
+
+def get_err_response(code):
+    """
+    Given an HTTP response code, create a properly formatted xml error response
+
+    :param code: error code
+    :returns: webob.response object
+    """
+    error_table = {
+        'AccessDenied':
+            (HTTP_FORBIDDEN, 'Access denied'),
+        'BucketAlreadyExists':
+            (HTTP_CONFLICT, 'The requested bucket name is not available'),
+        'BucketNotEmpty':
+            (HTTP_CONFLICT, 'The bucket you tried to delete is not empty'),
+        'InvalidArgument':
+            (HTTP_BAD_REQUEST, 'Invalid Argument'),
+        'InvalidBucketName':
+            (HTTP_BAD_REQUEST, 'The specified bucket is not valid'),
+        'InvalidURI':
+            (HTTP_BAD_REQUEST, 'Could not parse the specified URI'),
+        'NoSuchBucket':
+            (HTTP_NOT_FOUND, 'The specified bucket does not exist'),
+        'SignatureDoesNotMatch':
+            (HTTP_FORBIDDEN, 'The calculated request signature does not '\
+            'match your provided one'),
+        'NoSuchKey':
+            (HTTP_NOT_FOUND, 'The resource you requested does not exist')}
+
+    resp = Response(content_type='text/xml')
+    resp.status = error_table[code][0]
+    resp.body = error_table[code][1]
+    resp.body = '<?xml version="1.0" encoding="UTF-8"?>\r\n<Error>\r\n  ' \
+                '<Code>%s</Code>\r\n  <Message>%s</Message>\r\n</Error>\r\n' \
+                 % (code, error_table[code][1])
+    return resp
+
+
+def get_acl(account_name):
+    body = ('<AccessControlPolicy>'
+            '<Owner>'
+            '<ID>%s</ID>'
+            '</Owner>'
+            '<AccessControlList>'
+            '<Grant>'
+            '<Grantee xmlns:xsi="http://www.w3.org/2001/'\
+            'XMLSchema-instance" xsi:type="CanonicalUser">'
+            '<ID>%s</ID>'
+            '</Grantee>'
+            '<Permission>FULL_CONTROL</Permission>'
+            '</Grant>'
+            '</AccessControlList>'
+            '</AccessControlPolicy>' %
+            (account_name, account_name))
+    return Response(body=body, content_type="text/plain")
+
+
+def canonical_string(req):
+    """
+    Canonicalize a request to a token that can be signed.
+    """
+    amz_headers = {}
+
+    buf = "%s\n%s\n%s\n" % (req.method, req.headers.get('Content-MD5', ''),
+            req.headers.get('Content-Type') or '')
+
+    for amz_header in sorted((key.lower() for key in req.headers
+                              if key.lower().startswith('x-amz-'))):
+        amz_headers[amz_header] = req.headers[amz_header]
+
+    if 'x-amz-date' in amz_headers:
+        buf += "\n"
+    elif 'Date' in req.headers:
+        buf += "%s\n" % req.headers['Date']
+
+    for k in sorted(key.lower() for key in amz_headers):
+        buf += "%s:%s\n" % (k, amz_headers[k])
+
+    path = req.path_qs
+    if '?' in path:
+        path, args = path.split('?', 1)
+        for key in urlparse.parse_qs(args, keep_blank_values=True):
+            if key in ('acl', 'logging', 'torrent', 'location',
+                       'requestPayment'):
+                return "%s%s?%s" % (buf, path, key)
+    return buf + path
+
+
+class ServiceController(WSGIContext):
+    """
+    Handles account level requests.
+    """
+    def __init__(self, env, app, account_name, token, **kwargs):
+        WSGIContext.__init__(self, app)
+        env['HTTP_X_AUTH_TOKEN'] = token
+        env['PATH_INFO'] = '/v1/%s' % account_name
+
+    def GET(self, env, start_response):
+        """
+        Handle GET Service request
+        """
+        env['QUERY_STRING'] = 'format=json'
+        body_iter = self._app_call(env)
+        status = self._get_status_int()
+
+        if status != HTTP_OK:
+            if status == HTTP_UNAUTHORIZED:
+                return get_err_response('AccessDenied')
+            else:
+                return get_err_response('InvalidURI')
+
+        containers = loads(''.join(list(body_iter)))
+        # we don't keep the creation time of a backet (s3cmd doesn't
+        # work without that) so we use something bogus.
+        body = '<?xml version="1.0" encoding="UTF-8"?>' \
+            '<ListAllMyBucketsResult ' \
+              'xmlns="http://doc.s3.amazonaws.com/2006-03-01">' \
+            '<Buckets>%s</Buckets>' \
+            '</ListAllMyBucketsResult>' \
+            % ("".join(['<Bucket><Name>%s</Name><CreationDate>' \
+                         '2009-02-03T16:45:09.000Z</CreationDate></Bucket>' %
+                         xml_escape(i['name']) for i in containers]))
+        resp = Response(status=HTTP_OK, content_type='application/xml',
+            body=body)
+        return resp
+
+
+class BucketController(WSGIContext):
+    """
+    Handles bucket request.
+    """
+    def __init__(self, env, app, account_name, token, container_name,
+                    **kwargs):
+        WSGIContext.__init__(self, app)
+        self.container_name = unquote(container_name)
+        self.account_name = unquote(account_name)
+        env['HTTP_X_AUTH_TOKEN'] = token
+        env['PATH_INFO'] = '/v1/%s/%s' % (account_name, container_name)
+
+    def GET(self, env, start_response):
+        """
+        Handle GET Bucket (List Objects) request
+        """
+        if 'QUERY_STRING' in env:
+            args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1))
+        else:
+            args = {}
+        max_keys = min(int(args.get('max-keys', MAX_BUCKET_LISTING)),
+                        MAX_BUCKET_LISTING)
+        env['QUERY_STRING'] = 'format=json&limit=%s' % (max_keys + 1)
+        if 'marker' in args:
+            env['QUERY_STRING'] += '&marker=%s' % quote(args['marker'])
+        if 'prefix' in args:
+            env['QUERY_STRING'] += '&prefix=%s' % quote(args['prefix'])
+        if 'delimiter' in args:
+            env['QUERY_STRING'] += '&delimiter=%s' % quote(args['delimiter'])
+        body_iter = self._app_call(env)
+        status = self._get_status_int()
+
+        if status != HTTP_OK:
+            if status == HTTP_UNAUTHORIZED:
+                return get_err_response('AccessDenied')
+            elif status == HTTP_NOT_FOUND:
+                return get_err_response('NoSuchBucket')
+            else:
+                return get_err_response('InvalidURI')
+
+        if 'acl' in args:
+            return get_acl(self.account_name)
+
+        objects = loads(''.join(list(body_iter)))
+        body = ('<?xml version="1.0" encoding="UTF-8"?>'
+            '<ListBucketResult '
+                'xmlns="http://s3.amazonaws.com/doc/2006-03-01">'
+            '<Prefix>%s</Prefix>'
+            '<Marker>%s</Marker>'
+            '<Delimiter>%s</Delimiter>'
+            '<IsTruncated>%s</IsTruncated>'
+            '<MaxKeys>%s</MaxKeys>'
+            '<Name>%s</Name>'
+            '%s'
+            '%s'
+            '</ListBucketResult>' %
+            (
+                xml_escape(args.get('prefix', '')),
+                xml_escape(args.get('marker', '')),
+                xml_escape(args.get('delimiter', '')),
+                'true' if len(objects) == (max_keys + 1) else 'false',
+                max_keys,
+                xml_escape(self.container_name),
+                "".join(['<Contents><Key>%s</Key><LastModified>%sZ</LastModif'\
+                        'ied><ETag>%s</ETag><Size>%s</Size><StorageClass>STA'\
+                        'NDARD</StorageClass></Contents>' %
+                        (xml_escape(i['name']), i['last_modified'], i['hash'],
+                           i['bytes'])
+                           for i in objects[:max_keys] if 'subdir' not in i]),
+                "".join(['<CommonPrefixes><Prefix>%s</Prefix></CommonPrefixes>'
+                         % xml_escape(i['subdir'])
+                         for i in objects[:max_keys] if 'subdir' in i])))
+        return Response(body=body, content_type='application/xml')
+
+    def PUT(self, env, start_response):
+        """
+        Handle PUT Bucket request
+        """
+        body_iter = self._app_call(env)
+        status = self._get_status_int()
+
+        if status != HTTP_CREATED:
+            if status == HTTP_UNAUTHORIZED:
+                return get_err_response('AccessDenied')
+            elif status == HTTP_ACCEPTED:
+                return get_err_response('BucketAlreadyExists')
+            else:
+                return get_err_response('InvalidURI')
+
+        resp = Response()
+        resp.headers.add('Location', self.container_name)
+        resp.status = HTTP_OK
+        return resp
+
+    def DELETE(self, env, start_response):
+        """
+        Handle DELETE Bucket request
+        """
+        body_iter = self._app_call(env)
+        status = self._get_status_int()
+
+        if status != HTTP_NO_CONTENT:
+            if status == HTTP_UNAUTHORIZED:
+                return get_err_response('AccessDenied')
+            elif status == HTTP_NOT_FOUND:
+                return get_err_response('NoSuchBucket')
+            elif status == HTTP_CONFLICT:
+                return get_err_response('BucketNotEmpty')
+            else:
+                return get_err_response('InvalidURI')
+
+        resp = Response()
+        resp.status = HTTP_NO_CONTENT
+        return resp
+
+
+class ObjectController(WSGIContext):
+    """
+    Handles requests on objects
+    """
+    def __init__(self, env, app, account_name, token, container_name,
+                    object_name, **kwargs):
+        WSGIContext.__init__(self, app)
+        self.account_name = unquote(account_name)
+        self.container_name = unquote(container_name)
+        env['HTTP_X_AUTH_TOKEN'] = token
+        env['PATH_INFO'] = '/v1/%s/%s/%s' % (account_name, container_name,
+                                             object_name)
+
+    def GETorHEAD(self, env, start_response):
+        app_iter = self._app_call(env)
+        status = self._get_status_int()
+        headers = dict(self._response_headers)
+
+        if is_success(status):
+            if 'QUERY_STRING' in env:
+                args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1))
+            else:
+                args = {}
+            if 'acl' in args:
+                return get_acl(self.account_name)
+
+            new_hdrs = {}
+            for key, val in headers.iteritems():
+                _key = key.lower()
+                if _key.startswith('x-object-meta-'):
+                    new_hdrs['x-amz-meta-' + key[14:]] = val
+                elif _key in ('content-length', 'content-type',
+                              'content-range', 'content-encoding',
+                              'etag', 'last-modified'):
+                    new_hdrs[key] = val
+            return Response(status=status, headers=new_hdrs, app_iter=app_iter)
+        elif status == HTTP_UNAUTHORIZED:
+            return get_err_response('AccessDenied')
+        elif status == HTTP_NOT_FOUND:
+            return get_err_response('NoSuchKey')
+        else:
+            return get_err_response('InvalidURI')
+
+    def HEAD(self, env, start_response):
+        """
+        Handle HEAD Object request
+        """
+        return self.GETorHEAD(env, start_response)
+
+    def GET(self, env, start_response):
+        """
+        Handle GET Object request
+        """
+        return self.GETorHEAD(env, start_response)
+
+    def PUT(self, env, start_response):
+        """
+        Handle PUT Object and PUT Object (Copy) request
+        """
+        for key, value in env.items():
+            if key.startswith('HTTP_X_AMZ_META_'):
+                del env[key]
+                env['HTTP_X_OBJECT_META_' + key[16:]] = value
+            elif key == 'HTTP_CONTENT_MD5':
+                env['HTTP_ETAG'] = value.decode('base64').encode('hex')
+            elif key == 'HTTP_X_AMZ_COPY_SOURCE':
+                env['HTTP_X_COPY_FROM'] = value
+
+        body_iter = self._app_call(env)
+        status = self._get_status_int()
+
+        if status != HTTP_CREATED:
+            if status == HTTP_UNAUTHORIZED:
+                return get_err_response('AccessDenied')
+            elif status == HTTP_NOT_FOUND:
+                return get_err_response('NoSuchBucket')
+            else:
+                return get_err_response('InvalidURI')
+
+        if 'HTTP_X_COPY_FROM' in env:
+            body = '<CopyObjectResult>' \
+                   '<ETag>"%s"</ETag>' \
+                   '</CopyObjectResult>' % self._response_header_value('etag')
+            return Response(status=HTTP_OK, body=body)
+
+        return Response(status=200, etag=self._response_header_value('etag'))
+
+    def DELETE(self, env, start_response):
+        """
+        Handle DELETE Object request
+        """
+        body_iter = self._app_call(env)
+        status = self._get_status_int()
+
+        if status != HTTP_NO_CONTENT:
+            if status == HTTP_UNAUTHORIZED:
+                return get_err_response('AccessDenied')
+            elif status == HTTP_NOT_FOUND:
+                return get_err_response('NoSuchKey')
+            else:
+                return get_err_response('InvalidURI')
+
+        resp = Response()
+        resp.status = HTTP_NO_CONTENT
+        return resp
+
+
+class Swift3Middleware(object):
+    """Swift3 S3 compatibility midleware"""
+    def __init__(self, app, conf, *args, **kwargs):
+        self.app = app
+
+    def get_controller(self, path):
+        container, obj = split_path(path, 0, 2, True)
+        d = dict(container_name=container, object_name=obj)
+
+        if container and obj:
+            return ObjectController, d
+        elif container:
+            return BucketController, d
+        return ServiceController, d
+
+    def __call__(self, env, start_response):
+        req = Request(env)
+
+        if 'AWSAccessKeyId' in req.GET:
+            try:
+                req.headers['Date'] = req.GET['Expires']
+                req.headers['Authorization'] = \
+                    'AWS %(AWSAccessKeyId)s:%(Signature)s' % req.GET
+            except KeyError:
+                return get_err_response('InvalidArgument')(env, start_response)
+
+        if not 'Authorization' in req.headers:
+            return self.app(env, start_response)
+
+        try:
+            account, signature = \
+                req.headers['Authorization'].split(' ')[-1].rsplit(':', 1)
+        except Exception:
+            return get_err_response('InvalidArgument')(env, start_response)
+
+        try:
+            controller, path_parts = self.get_controller(req.path)
+        except ValueError:
+            return get_err_response('InvalidURI')(env, start_response)
+
+        token = base64.urlsafe_b64encode(canonical_string(req))
+
+        controller = controller(env, self.app, account, token, **path_parts)
+
+        if hasattr(controller, req.method):
+            res = getattr(controller, req.method)(env, start_response)
+        else:
+            return get_err_response('InvalidURI')(env, start_response)
+
+        return res(env, start_response)
+
+
+def filter_factory(global_conf, **local_conf):
+    """Standard filter factory to use the middleware with paste.deploy"""
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def swift3_filter(app):
+        return Swift3Middleware(app, conf)
+
+    return swift3_filter