x/swiftonfile
Code Issues Proposed changes
swiftonfile/swift/1.4.8/swift.diff
Peter Portante 3ad78098b5 object-storage: Refactor code to reduce Swift diffs carried 
The upstream Swift code base contains the following commit which adds the
ability to modify constraint values using the swift.conf file:

 a2ac5efaa6

These changes rely on the above commit being back-ported to Swift 1.4.8 (see
fc2421b040).

As a result, a good number of differences we carry can be removed, since the
provided swift.conf file now contains the values we need.

Along with these changes, we add middleware to get constraints loaded properly
(via monkey patching) until subclassing is implemented in a future commit. We
further simplify the diffs to a minimal set by storing the timestamp in file
system metadata and moving new files to the plugins/middleware directory.

Note that the original "gluster" middleware was in the swift.diff file, and
has been renamed to "glusterfs" and moved to the new plugins/middleware
directory. The "gluster" middleware is now a temporary way to get the
constraints properly loaded (both of these middleware pieces should disappear
in future commits when we refactor further to subclass the Swift objects
instead of patching them).

Change-Id: I9dc00d6b6cdd64e277896d75c2fb06431c3e69cb
BUG: 862052
Signed-off-by: Peter Portante <peter.portante@redhat.com>
Reviewed-on: http://review.gluster.org/4093
Tested-by: Peter Portante <pportant@redhat.com>
Reviewed-by: Anand Avati <avati@redhat.com>
2013-04-29 16:35:55 -04:00

310 lines
14 KiB
Diff
Raw Blame History

diff --git a/setup.py b/setup.py
index d195d34..ab236ee 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,6 @@
#!/usr/bin/python
# Copyright (c) 2010-2012 OpenStack, LLC.
+# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -94,6 +95,8 @@ setup(
'tempurl=swift.common.middleware.tempurl:filter_factory',
'formpost=swift.common.middleware.formpost:filter_factory',
'name_check=swift.common.middleware.name_check:filter_factory',
+ 'gluster=swift.plugins.middleware.gluster:filter_factory',
+ 'glusterfs=swift.plugins.middleware.glusterfs:filter_factory',
],
},
)
diff --git a/swift/account/server.py b/swift/account/server.py
index 800b3c0..77f9879 100644
--- a/swift/account/server.py
+++ b/swift/account/server.py
@@ -1,4 +1,5 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
+# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -35,6 +36,9 @@ from swift.common.utils import get_logger, get_param, hash_path, \
from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \
check_mount, check_float, check_utf8
from swift.common.db_replicator import ReplicatorRpc
+from swift.plugins.utils import plugin_enabled
+if plugin_enabled():
+ from swift.plugins.DiskDir import DiskAccount
DATADIR = 'accounts'
@@ -52,8 +56,12 @@ class AccountController(object):
self.mount_check, logger=self.logger)
self.auto_create_account_prefix = \
conf.get('auto_create_account_prefix') or '.'
+ self.fs_object = None
def _get_account_broker(self, drive, part, account):
+ if self.fs_object:
+ return DiskAccount(self.root, account, self.fs_object);
+
hsh = hash_path(account)
db_dir = storage_directory(DATADIR, part, hsh)
db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
@@ -153,6 +161,9 @@ class AccountController(object):
broker.stale_reads_ok = True
if broker.is_deleted():
return HTTPNotFound(request=req)
+ if self.fs_object and not self.fs_object.object_only:
+ broker.list_containers_iter(None, None,None,
+ None, None)
info = broker.get_info()
headers = {
'X-Account-Container-Count': info['container_count'],
@@ -305,8 +316,17 @@ class AccountController(object):
broker.update_metadata(metadata)
return HTTPNoContent(request=req)
+ def plugin(self, env):
+ if env.get('Gluster_enabled', False):
+ self.fs_object = env.get('fs_object')
+ self.root = env.get('root')
+ self.mount_check = False
+ else:
+ self.fs_object = None
+
def __call__(self, env, start_response):
start_time = time.time()
+ self.plugin(env)
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
diff --git a/swift/container/server.py b/swift/container/server.py
index 8a18cfd..952b8cd 100644
--- a/swift/container/server.py
+++ b/swift/container/server.py
@@ -1,4 +1,5 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
+# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -37,6 +38,9 @@ from swift.common.constraints import CONTAINER_LISTING_LIMIT, \
from swift.common.bufferedhttp import http_connect
from swift.common.exceptions import ConnectionTimeout
from swift.common.db_replicator import ReplicatorRpc
+from swift.plugins.utils import plugin_enabled
+if plugin_enabled():
+ from swift.plugins.DiskDir import DiskDir
DATADIR = 'containers'
@@ -62,6 +66,7 @@ class ContainerController(object):
ContainerBroker, self.mount_check, logger=self.logger)
self.auto_create_account_prefix = \
conf.get('auto_create_account_prefix') or '.'
+ self.fs_object = None
def _get_container_broker(self, drive, part, account, container):
"""
@@ -73,6 +78,10 @@ class ContainerController(object):
:param container: container name
:returns: ContainerBroker object
"""
+ if self.fs_object:
+ return DiskDir(self.root, drive, part, account,
+ container, self.logger,
+ fs_object = self.fs_object)
hsh = hash_path(account, container)
db_dir = storage_directory(DATADIR, part, hsh)
db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
@@ -245,6 +254,9 @@ class ContainerController(object):
broker.stale_reads_ok = True
if broker.is_deleted():
return HTTPNotFound(request=req)
+ if self.fs_object and not self.fs_object.object_only:
+ broker.list_objects_iter(None, None, None, None,
+ None, None)
info = broker.get_info()
headers = {
'X-Container-Object-Count': info['object_count'],
@@ -427,8 +439,19 @@ class ContainerController(object):
broker.update_metadata(metadata)
return HTTPNoContent(request=req)
+ def plugin(self, env):
+ if env.get('Gluster_enabled', False):
+ self.fs_object = env.get('fs_object')
+ if not self.fs_object:
+ raise NoneTypeError
+ self.root = env.get('root')
+ self.mount_check = False
+ else:
+ self.fs_object = None
+
def __call__(self, env, start_response):
start_time = time.time()
+ self.plugin(env)
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
diff --git a/swift/obj/server.py b/swift/obj/server.py
index 9cca16b..82eaa40 100644
--- a/swift/obj/server.py
+++ b/swift/obj/server.py
@@ -1,4 +1,5 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
+# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -45,6 +46,10 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileError, \
DiskFileNotExist
from swift.obj.replicator import tpooled_get_hashes, invalidate_hash, \
quarantine_renamer
+from swift.plugins.utils import plugin_enabled
+if plugin_enabled():
+ from swift.plugins.utils import X_TYPE, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, \
+ OBJECT, DIR_TYPE, FILE_TYPE
DATADIR = 'objects'
@@ -340,6 +345,9 @@ class DiskFile(object):
raise DiskFileNotExist('Data File does not exist.')
+if plugin_enabled():
+ from swift.plugins.DiskFile import Gluster_DiskFile
+
class ObjectController(object):
"""Implements the WSGI application for the Swift Object Server."""
@@ -377,6 +385,17 @@ class ObjectController(object):
'expiring_objects'
self.expiring_objects_container_divisor = \
int(conf.get('expiring_objects_container_divisor') or 86400)
+ self.fs_object = None
+
+ def get_DiskFile_obj(self, path, device, partition, account, container, obj,
+ logger, keep_data_fp=False, disk_chunk_size=65536):
+ if self.fs_object:
+ return Gluster_DiskFile(path, device, partition, account, container,
+ obj, logger, keep_data_fp,
+ disk_chunk_size, fs_object = self.fs_object);
+ else:
+ return DiskFile(path, device, partition, account, container,
+ obj, logger, keep_data_fp, disk_chunk_size)
def async_update(self, op, account, container, obj, host, partition,
contdevice, headers_out, objdevice):
@@ -493,7 +512,7 @@ class ObjectController(object):
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):
return Response(status='507 %s is not mounted' % device)
- file = DiskFile(self.devices, device, partition, account, container,
+ file = self.get_DiskFile_obj(self.devices, device, partition, account, container,
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
if 'X-Delete-At' in file.metadata and \
@@ -548,7 +567,7 @@ class ObjectController(object):
if new_delete_at and new_delete_at < time.time():
return HTTPBadRequest(body='X-Delete-At in past', request=request,
content_type='text/plain')
- file = DiskFile(self.devices, device, partition, account, container,
+ file = self.get_DiskFile_obj(self.devices, device, partition, account, container,
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
orig_timestamp = file.metadata.get('X-Timestamp')
upload_expiration = time.time() + self.max_upload_time
@@ -580,12 +599,26 @@ class ObjectController(object):
if 'etag' in request.headers and \
request.headers['etag'].lower() != etag:
return HTTPUnprocessableEntity(request=request)
- metadata = {
- 'X-Timestamp': request.headers['x-timestamp'],
- 'Content-Type': request.headers['content-type'],
- 'ETag': etag,
- 'Content-Length': str(os.fstat(fd).st_size),
- }
+ content_type = request.headers['content-type']
+ if self.fs_object and not content_type:
+ content_type = FILE_TYPE
+ if not self.fs_object:
+ metadata = {
+ 'X-Timestamp': request.headers['x-timestamp'],
+ 'Content-Type': content_type,
+ 'ETag': etag,
+ 'Content-Length': str(os.fstat(fd).st_size),
+ }
+ else:
+ x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE
+ metadata = {
+ 'X-Timestamp': request.headers['x-timestamp'],
+ 'Content-Type': content_type,
+ 'ETag': etag,
+ 'Content-Length': str(os.fstat(fd).st_size),
+ X_TYPE: OBJECT,
+ X_OBJECT_TYPE: x_object_type,
+ }
metadata.update(val for val in request.headers.iteritems()
if val[0].lower().startswith('x-object-meta-') and
len(val[0]) > 14)
@@ -612,7 +645,7 @@ class ObjectController(object):
'x-timestamp': file.metadata['X-Timestamp'],
'x-etag': file.metadata['ETag'],
'x-trans-id': request.headers.get('x-trans-id', '-')},
- device)
+ (self.fs_object and account) or device)
resp = HTTPCreated(request=request, etag=etag)
return resp
@@ -626,9 +659,9 @@ class ObjectController(object):
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):
return Response(status='507 %s is not mounted' % device)
- file = DiskFile(self.devices, device, partition, account, container,
- obj, self.logger, keep_data_fp=True,
- disk_chunk_size=self.disk_chunk_size)
+ file = self.get_DiskFile_obj(self.devices, device, partition, account, container,
+ obj, self.logger, keep_data_fp=True,
+ disk_chunk_size=self.disk_chunk_size)
if file.is_deleted() or ('X-Delete-At' in file.metadata and
int(file.metadata['X-Delete-At']) <= time.time()):
if request.headers.get('if-match') == '*':
@@ -702,7 +735,7 @@ class ObjectController(object):
return resp
if self.mount_check and not check_mount(self.devices, device):
return Response(status='507 %s is not mounted' % device)
- file = DiskFile(self.devices, device, partition, account, container,
+ file = self.get_DiskFile_obj(self.devices, device, partition, account, container,
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
if file.is_deleted() or ('X-Delete-At' in file.metadata and
int(file.metadata['X-Delete-At']) <= time.time()):
@@ -744,7 +777,7 @@ class ObjectController(object):
if self.mount_check and not check_mount(self.devices, device):
return Response(status='507 %s is not mounted' % device)
response_class = HTTPNoContent
- file = DiskFile(self.devices, device, partition, account, container,
+ file = self.get_DiskFile_obj(self.devices, device, partition, account, container,
obj, self.logger, disk_chunk_size=self.disk_chunk_size)
if 'x-if-delete-at' in request.headers and \
int(request.headers['x-if-delete-at']) != \
@@ -797,9 +830,18 @@ class ObjectController(object):
raise hashes
return Response(body=pickle.dumps(hashes))
+ def plugin(self, env):
+ if env.get('Gluster_enabled', False):
+ self.fs_object = env.get('fs_object')
+ self.devices = env.get('root')
+ self.mount_check = False
+ else:
+ self.fs_object = None
+
def __call__(self, env, start_response):
"""WSGI Application entry point for the Swift Object Server."""
start_time = time.time()
+ self.plugin(env)
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
View Git Blame Copy Permalink