x/swift3
Code Issues Proposed changes

Make multipart uploads compatible with ProxyFS

Browse Source 
ProxyFS is a soon-to-be-released project that provides a filesystem on
top of Swift, then also tries to provide the Swift API on top of that
filesystem. However, there are some places where
Swift-on-filesystem-on-Swift works differently from plain Swift, and
this is one of them.

When we make a multipart upload, it looks like this:

/v1/a/c+segments/obj/uploadId
/v1/a/c+segments/obj/uploadId/1
/v1/a/c+segments/obj/uploadId/2
...

The problem is that .../obj/uploadId is a file, and in
Swift-on-filesystem-on-Swift, that means you can't make another file
.../obj/uploadId/1, just like you can't on your local filesystem.

However, if you create .../obj/uploadId with a Content-Type of
application/directory and a Content-Length of 0, then it *is* a
directory, and the rest of the upload can proceed.

Swift3 is currently using Content-Type on the upload marker to store
the user-supplied Content-Type. This commit moves that data into
sysmeta in the handler for Initiate Multipart Upload, and then looks
in both new and old locations in the handler for Finalize Multipart
Upload.

Change-Id: If64c914b6d9ace7700ca77eead3ef66a771cd92e
This commit is contained in:
Samuel Merritt 2017-09-01 16:53:59 -07:00
parent c416369d38
commit 93b97c924c
3 changed files with 106 additions and 5 deletions

31
swift3/controllers/multi_upload.py

@ -59,7 +59,8 @@ from swift3.response import InvalidArgument, ErrorResponse, MalformedXML, \
InvalidRequest, HTTPOk, HTTPNoContent, NoSuchKey, NoSuchUpload, \
NoSuchBucket
from swift3.exception import BadSwiftRequest
from swift3.utils import LOGGER, unique_id, MULTIUPLOAD_SUFFIX, S3Timestamp
from swift3.utils import LOGGER, unique_id, MULTIUPLOAD_SUFFIX, S3Timestamp, \
sysmeta_header
from swift3.etree import Element, SubElement, fromstring, tostring, \
XMLSyntaxError, DocumentInvalid
from swift3.cfg import CONF
@ -333,6 +334,16 @@ class UploadsController(Controller):
upload_id = unique_id()
container = req.container_name + MULTIUPLOAD_SUFFIX
content_type = req.headers.get('Content-Type')
if content_type:
req.headers[sysmeta_header('object', 'has-content-type')] = 'yes'
req.headers[
sysmeta_header('object', 'content-type')] = content_type
else:
req.headers[sysmeta_header('object', 'has-content-type')] = 'no'
req.headers['Content-Type'] = 'application/directory'
try:
req.get_response(self.app, 'PUT', container, '')
except BucketAlreadyExists:
@ -511,8 +522,22 @@ class UploadController(Controller):
_key = key.lower()
if _key.startswith('x-amz-meta-'):
headers['x-object-meta-' + _key[11:]] = val
elif _key == 'content-type':
headers['Content-Type'] = val
hct_header = sysmeta_header('object', 'has-content-type')
if resp.sysmeta_headers.get(hct_header) == 'yes':
content_type = resp.sysmeta_headers.get(
sysmeta_header('object', 'content-type'))
elif hct_header in resp.sysmeta_headers:
# has-content-type is present but false, so no content type was
# set on initial upload. In that case, we won't set one on our
# PUT request. Swift will end up guessing one based on the
# object name.
content_type = None
else:
content_type = resp.headers.get('Content-Type')
if content_type:
headers['Content-Type'] = content_type
# Query for the objects in the segments area to make sure it completed
query = {

6
swift3/test/unit/helpers.py

@ -151,5 +151,11 @@ class FakeSwift(object):
self._responses[(method, path)] = (response_class, headers, body)
def register_unconditionally(self, method, path, response_class, headers,
body):
# register() keeps old sysmeta around, but
# register_unconditionally() keeps nothing.
self._responses[(method, path)] = (response_class, headers, body)
def clear_calls(self):
del self._calls[:]

74
swift3/test/unit/test_multi_upload.py

@ -85,8 +85,12 @@ class TestSwift3MultiUpload(Swift3TestCase):
self.swift.register('GET', segment_bucket, swob.HTTPOk, {},
object_list)
self.swift.register('HEAD', segment_bucket + '/object/X',
swob.HTTPOk, {'x-object-meta-foo': 'bar',
'content-type': 'baz/quux'}, None)
swob.HTTPOk,
{'x-object-meta-foo': 'bar',
'content-type': 'application/directory',
'x-object-sysmeta-swift3-has-content-type': 'yes',
'x-object-sysmeta-swift3-content-type':
'baz/quux'}, None)
self.swift.register('PUT', segment_bucket + '/object/X',
swob.HTTPCreated, {}, None)
self.swift.register('DELETE', segment_bucket + '/object/X',
@ -549,6 +553,35 @@ class TestSwift3MultiUpload(Swift3TestCase):
@s3acl(s3acl_only=True)
@patch('swift3.controllers.multi_upload.unique_id', lambda: 'X')
def test_object_multipart_upload_initiate_s3acl(self):
req = Request.blank('/bucket/object?uploads',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization':
'AWS test:tester:hmac',
'Date': self.get_date_header(),
'x-amz-acl': 'public-read',
'x-amz-meta-foo': 'bar',
'Content-Type': 'cat/picture'})
status, headers, body = self.call_swift3(req)
fromstring(body, 'InitiateMultipartUploadResult')
self.assertEqual(status.split()[0], '200')
_, _, req_headers = self.swift.calls_with_headers[-1]
self.assertEqual(req_headers.get('X-Object-Meta-Foo'), 'bar')
self.assertEqual(req_headers.get(
'X-Object-Sysmeta-Swift3-Has-Content-Type'), 'yes')
self.assertEqual(req_headers.get(
'X-Object-Sysmeta-Swift3-Content-Type'), 'cat/picture')
tmpacl_header = req_headers.get(sysmeta_header('object', 'tmpacl'))
self.assertTrue(tmpacl_header)
acl_header = encode_acl('object',
ACLPublicRead(Owner('test:tester',
'test:tester')))
self.assertEqual(acl_header.get(sysmeta_header('object', 'acl')),
tmpacl_header)
@s3acl(s3acl_only=True)
@patch('swift3.controllers.multi_upload.unique_id', lambda: 'X')
def test_object_multipart_upload_initiate_no_content_type(self):
req = Request.blank('/bucket/object?uploads',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization':
@ -562,6 +595,8 @@ class TestSwift3MultiUpload(Swift3TestCase):
_, _, req_headers = self.swift.calls_with_headers[-1]
self.assertEqual(req_headers.get('X-Object-Meta-Foo'), 'bar')
self.assertEqual(req_headers.get(
'X-Object-Sysmeta-Swift3-Has-Content-Type'), 'no')
tmpacl_header = req_headers.get(sysmeta_header('object', 'tmpacl'))
self.assertTrue(tmpacl_header)
acl_header = encode_acl('object',
@ -638,6 +673,41 @@ class TestSwift3MultiUpload(Swift3TestCase):
self.assertEqual(headers.get('X-Object-Meta-Foo'), 'bar')
self.assertEqual(headers.get('Content-Type'), 'baz/quux')
def test_object_multipart_upload_complete_old_content_type(self):
self.swift.register_unconditionally(
'HEAD', '/v1/AUTH_test/bucket+segments/object/X',
swob.HTTPOk, {"Content-Type": "thingy/dingy"}, None)
req = Request.blank('/bucket/object?uploadId=X',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(), },
body=xml)
status, headers, body = self.call_swift3(req)
fromstring(body, 'CompleteMultipartUploadResult')
self.assertEqual(status.split()[0], '200')
_, _, headers = self.swift.calls_with_headers[-2]
self.assertEqual(headers.get('Content-Type'), 'thingy/dingy')
def test_object_multipart_upload_complete_no_content_type(self):
self.swift.register_unconditionally(
'HEAD', '/v1/AUTH_test/bucket+segments/object/X',
swob.HTTPOk, {"X-Object-Sysmeta-Swift3-Has-Content-Type": "no"},
None)
req = Request.blank('/bucket/object?uploadId=X',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header(), },
body=xml)
status, headers, body = self.call_swift3(req)
fromstring(body, 'CompleteMultipartUploadResult')
self.assertEqual(status.split()[0], '200')
_, _, headers = self.swift.calls_with_headers[-2]
self.assertNotIn('Content-Type', headers)
def test_object_multipart_upload_complete_weird_host_name(self):
# This happens via boto signature v4
req = Request.blank('/bucket/object?uploadId=X',