e625cc40ff
This patch also contains fixes to bugs * 811501 * 812498 * 821310 Also, removed the default set of users in the proxy-server.conf file. Change-Id: Ief83905d10ff7bf7c43685ada4d7f05959cee9d1 BUG: 821310 Signed-off-by: Mohammed Junaid <junaid@redhat.com> Reviewed-on: http://review.gluster.com/3390 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Vijay Bellur <vijay@gluster.com>
485 lines
17 KiB
Python
485 lines
17 KiB
Python
# Copyright (c) 2011 Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
|
|
from swift.plugins.utils import clean_metadata, dir_empty, rmdirs, mkdirs, \
|
|
validate_account, validate_container, check_valid_account, is_marker, \
|
|
get_container_details, get_account_details, create_container_metadata, \
|
|
create_account_metadata, DEFAULT_GID, DEFAULT_UID, get_account_details, \
|
|
validate_object, create_object_metadata, read_metadata, write_metadata
|
|
|
|
from swift.common.constraints import CONTAINER_LISTING_LIMIT, \
|
|
check_mount
|
|
|
|
from swift.plugins.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP,\
|
|
X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \
|
|
X_CONTAINER_COUNT, CONTAINER
|
|
|
|
from swift import plugins
|
|
def strip_obj_storage_path(path, string='/mnt/gluster-object'):
|
|
"""
|
|
strip /mnt/gluster-object
|
|
"""
|
|
return path.replace(string, '').strip('/')
|
|
|
|
DATADIR = 'containers'
|
|
|
|
|
|
class DiskCommon(object):
|
|
def is_deleted(self):
|
|
return not os.path.exists(self.datadir)
|
|
|
|
def filter_prefix(self, objects, prefix):
|
|
"""
|
|
Accept sorted list.
|
|
"""
|
|
found = 0
|
|
filtered_objs = []
|
|
for object_name in objects:
|
|
if object_name.startswith(prefix):
|
|
filtered_objs.append(object_name)
|
|
found = 1
|
|
else:
|
|
if found:
|
|
break
|
|
return filtered_objs
|
|
|
|
def filter_delimiter(self, objects, delimiter, prefix):
|
|
"""
|
|
Accept sorted list.
|
|
Objects should start with prefix.
|
|
"""
|
|
filtered_objs=[]
|
|
for object_name in objects:
|
|
tmp_obj = object_name.replace(prefix, '', 1)
|
|
sufix = tmp_obj.split(delimiter, 1)
|
|
new_obj = prefix + sufix[0]
|
|
if new_obj and new_obj not in filtered_objs:
|
|
filtered_objs.append(new_obj)
|
|
|
|
return filtered_objs
|
|
|
|
def filter_marker(self, objects, marker):
|
|
"""
|
|
TODO: We can traverse in reverse order to optimize.
|
|
Accept sorted list.
|
|
"""
|
|
filtered_objs=[]
|
|
found = 0
|
|
if objects[-1] < marker:
|
|
return filtered_objs
|
|
for object_name in objects:
|
|
if object_name > marker:
|
|
filtered_objs.append(object_name)
|
|
|
|
return filtered_objs
|
|
|
|
def filter_end_marker(self, objects, end_marker):
|
|
"""
|
|
Accept sorted list.
|
|
"""
|
|
filtered_objs=[]
|
|
for object_name in objects:
|
|
if object_name < end_marker:
|
|
filtered_objs.append(object_name)
|
|
else:
|
|
break
|
|
|
|
return filtered_objs
|
|
|
|
def filter_limit(self, objects, limit):
|
|
filtered_objs=[]
|
|
for i in range(0, limit):
|
|
filtered_objs.append(objects[i])
|
|
|
|
return filtered_objs
|
|
|
|
def update_account(self, metadata):
|
|
acc_path = self.datadir
|
|
write_metadata(acc_path, metadata)
|
|
self.metadata = metadata
|
|
|
|
class DiskDir(DiskCommon):
|
|
"""
|
|
Manage object files on disk.
|
|
|
|
:param path: path to devices on the node
|
|
:param device: device name
|
|
:param partition: partition on the device the object lives in
|
|
:param account: account name for the object
|
|
:param container: container name for the object
|
|
:param obj: object name for the object
|
|
:param keep_data_fp: if True, don't close the fp, otherwise close it
|
|
:param disk_chunk_Size: size of chunks on file reads
|
|
"""
|
|
|
|
def __init__(self, path, device, partition, account, container, logger,
|
|
uid=DEFAULT_UID, gid=DEFAULT_GID, fs_object=None):
|
|
self.root = path
|
|
device = account
|
|
if container:
|
|
self.name = container
|
|
else:
|
|
self.name = None
|
|
if self.name:
|
|
self.datadir = os.path.join(path, account, self.name)
|
|
else:
|
|
self.datadir = os.path.join(path, device)
|
|
self.account = account
|
|
self.device_path = os.path.join(path, device)
|
|
if not check_mount(path, device):
|
|
check_valid_account(account, fs_object)
|
|
self.logger = logger
|
|
self.metadata = {}
|
|
self.uid = int(uid)
|
|
self.gid = int(gid)
|
|
# Create a dummy db_file in /etc/swift
|
|
self.db_file = '/etc/swift/db_file.db'
|
|
if not os.path.exists(self.db_file):
|
|
file(self.db_file, 'w+')
|
|
self.dir_exists = os.path.exists(self.datadir)
|
|
if self.dir_exists:
|
|
try:
|
|
self.metadata = read_metadata(self.datadir)
|
|
except EOFError:
|
|
create_container_metadata(self.datadir)
|
|
else:
|
|
return
|
|
if container:
|
|
if not self.metadata:
|
|
create_container_metadata(self.datadir)
|
|
self.metadata = read_metadata(self.datadir)
|
|
else:
|
|
if not validate_container(self.metadata):
|
|
create_container_metadata(self.datadir)
|
|
self.metadata = read_metadata(self.datadir)
|
|
else:
|
|
if not self.metadata:
|
|
create_account_metadata(self.datadir)
|
|
self.metadata = read_metadata(self.datadir)
|
|
else:
|
|
if not validate_account(self.metadata):
|
|
create_account_metadata(self.datadir)
|
|
self.metadata = read_metadata(self.datadir)
|
|
|
|
def empty(self):
|
|
return dir_empty(self.datadir)
|
|
|
|
def delete(self):
|
|
if self.empty():
|
|
#For delete account.
|
|
if os.path.ismount(self.datadir):
|
|
clean_metadata(self.datadir)
|
|
else:
|
|
rmdirs(self.datadir)
|
|
self.dir_exists = False
|
|
|
|
|
|
def put_metadata(self, metadata):
|
|
"""
|
|
Write metadata to directory/container.
|
|
"""
|
|
write_metadata(self.datadir, metadata)
|
|
self.metadata = metadata
|
|
|
|
def put(self, metadata):
|
|
"""
|
|
Create and write metatdata to directory/container.
|
|
:param metadata: Metadata to write.
|
|
"""
|
|
if not self.dir_exists:
|
|
mkdirs(self.datadir)
|
|
|
|
os.chown(self.datadir, self.uid, self.gid)
|
|
write_metadata(self.datadir, metadata)
|
|
self.metadata = metadata
|
|
self.dir_exists = True
|
|
|
|
def put_obj(self, content_length, timestamp):
|
|
self.metadata[X_OBJECTS_COUNT] = int(self.metadata[X_OBJECTS_COUNT]) + 1
|
|
self.metadata[X_PUT_TIMESTAMP] = timestamp
|
|
self.metadata[X_BYTES_USED] = int(self.metadata[X_BYTES_USED]) + int(content_length)
|
|
#TODO: define update_metadata instad of writing whole metadata again.
|
|
self.put_metadata(self.metadata)
|
|
|
|
def delete_obj(self, content_length):
|
|
self.metadata[X_OBJECTS_COUNT] = int(self.metadata[X_OBJECTS_COUNT]) - 1
|
|
self.metadata[X_BYTES_USED] = int(self.metadata[X_BYTES_USED]) - int(content_length)
|
|
self.put_metadata(self.metadata)
|
|
|
|
def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used):
|
|
"""
|
|
For account server.
|
|
"""
|
|
self.metadata[X_OBJECTS_COUNT] = 0
|
|
self.metadata[X_BYTES_USED] = 0
|
|
self.metadata[X_CONTAINER_COUNT] = int(self.metadata[X_CONTAINER_COUNT]) + 1
|
|
self.metadata[X_PUT_TIMESTAMP] = 1
|
|
self.put_metadata(self.metadata)
|
|
|
|
def delete_container(self, object_count, bytes_used):
|
|
"""
|
|
For account server.
|
|
"""
|
|
self.metadata[X_OBJECTS_COUNT] = 0
|
|
self.metadata[X_BYTES_USED] = 0
|
|
self.metadata[X_CONTAINER_COUNT] = int(self.metadata[X_CONTAINER_COUNT]) - 1
|
|
self.put_metadata(self.metadata)
|
|
|
|
def unlink(self):
|
|
"""
|
|
Remove directory/container if empty.
|
|
"""
|
|
if dir_empty(self.datadir):
|
|
rmdirs(self.datadir)
|
|
|
|
def list_objects_iter(self, limit, marker, end_marker,
|
|
prefix, delimiter, path):
|
|
"""
|
|
Returns tuple of name, created_at, size, content_type, etag.
|
|
"""
|
|
if path:
|
|
prefix = path = path.rstrip('/') + '/'
|
|
delimiter = '/'
|
|
if delimiter and not prefix:
|
|
prefix = ''
|
|
|
|
objects = []
|
|
object_count = 0
|
|
bytes_used = 0
|
|
container_list = []
|
|
|
|
objects, object_count, bytes_used = get_container_details(self.datadir)
|
|
|
|
if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \
|
|
int(self.metadata[X_BYTES_USED]) != bytes_used:
|
|
self.metadata[X_OBJECTS_COUNT] = object_count
|
|
self.metadata[X_BYTES_USED] = bytes_used
|
|
self.update_container(self.metadata)
|
|
|
|
if objects:
|
|
objects.sort()
|
|
|
|
if objects and prefix:
|
|
objects = self.filter_prefix(objects, prefix)
|
|
|
|
if objects and delimiter:
|
|
objects = self.filter_delimiter(objects, delimiter, prefix)
|
|
|
|
if objects and marker:
|
|
objects = self.filter_marker(objects, marker)
|
|
|
|
if objects and end_marker:
|
|
objects = self.filter_end_marker(objects, end_marker)
|
|
|
|
if objects and limit:
|
|
if len(objects) > limit:
|
|
objects = self.filter_limit(objects, limit)
|
|
|
|
if objects:
|
|
for obj in objects:
|
|
list_item = []
|
|
list_item.append(obj)
|
|
metadata = read_metadata(self.datadir + '/' + obj)
|
|
if not metadata or not validate_object(metadata):
|
|
metadata = create_object_metadata(self.datadir + '/' + obj)
|
|
if metadata:
|
|
list_item.append(metadata[X_TIMESTAMP])
|
|
list_item.append(int(metadata[X_CONTENT_LENGTH]))
|
|
list_item.append(metadata[X_CONTENT_TYPE])
|
|
list_item.append(metadata[X_ETAG])
|
|
container_list.append(list_item)
|
|
|
|
return container_list
|
|
|
|
def update_container(self, metadata):
|
|
cont_path = self.datadir
|
|
write_metadata(cont_path, metadata)
|
|
self.metadata = metadata
|
|
|
|
def update_object_count(self):
|
|
objects = []
|
|
object_count = 0
|
|
bytes_used = 0
|
|
objects, object_count, bytes_used = get_container_details(self.datadir)
|
|
|
|
|
|
if int(self.metadata[X_OBJECTS_COUNT]) != object_count or \
|
|
int(self.metadata[X_BYTES_USED]) != bytes_used:
|
|
self.metadata[X_OBJECTS_COUNT] = object_count
|
|
self.metadata[X_BYTES_USED] = bytes_used
|
|
self.update_container(self.metadata)
|
|
|
|
def update_container_count(self):
|
|
containers = []
|
|
container_count = 0
|
|
|
|
containers, container_count = get_account_details(self.datadir)
|
|
|
|
if int(self.metadata[X_CONTAINER_COUNT]) != container_count:
|
|
self.metadata[X_CONTAINER_COUNT] = container_count
|
|
self.update_account(self.metadata)
|
|
|
|
def get_info(self, include_metadata=False):
|
|
"""
|
|
Get global data for the container.
|
|
:returns: dict with keys: account, container, created_at,
|
|
put_timestamp, delete_timestamp, object_count, bytes_used,
|
|
reported_put_timestamp, reported_delete_timestamp,
|
|
reported_object_count, reported_bytes_used, hash, id,
|
|
x_container_sync_point1, and x_container_sync_point2.
|
|
If include_metadata is set, metadata is included as a key
|
|
pointing to a dict of tuples of the metadata
|
|
"""
|
|
# TODO: delete_timestamp, reported_put_timestamp
|
|
# reported_delete_timestamp, reported_object_count,
|
|
# reported_bytes_used, created_at
|
|
|
|
metadata = {}
|
|
if os.path.exists(self.datadir):
|
|
metadata = read_metadata(self.datadir)
|
|
|
|
data = {'account' : self.account, 'container' : self.name,
|
|
'object_count' : metadata.get(X_OBJECTS_COUNT, '0'),
|
|
'bytes_used' : metadata.get(X_BYTES_USED, '0'),
|
|
'hash': '', 'id' : '', 'created_at' : '1',
|
|
'put_timestamp' : metadata.get(X_PUT_TIMESTAMP, '0'),
|
|
'delete_timestamp' : '1',
|
|
'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1',
|
|
'reported_object_count' : '1', 'reported_bytes_used' : '1'}
|
|
if include_metadata:
|
|
data['metadata'] = metadata
|
|
return data
|
|
|
|
def put_object(self, name, timestamp, size, content_type,
|
|
etag, deleted=0):
|
|
# TODO: Implement the specifics of this func.
|
|
pass
|
|
|
|
def initialize(self, timestamp):
|
|
pass
|
|
|
|
def update_put_timestamp(self, timestamp):
|
|
"""
|
|
Create the container if it doesn't exist and update the timestamp
|
|
"""
|
|
if not os.path.exists(self.datadir):
|
|
self.put(self.metadata)
|
|
|
|
def delete_object(self, name, timestamp):
|
|
# TODO: Implement the delete object
|
|
pass
|
|
|
|
def delete_db(self, timestamp):
|
|
"""
|
|
Delete the container
|
|
"""
|
|
self.unlink()
|
|
|
|
def update_metadata(self, metadata):
|
|
self.metadata.update(metadata)
|
|
write_metadata(self.datadir, self.metadata)
|
|
|
|
|
|
class DiskAccount(DiskDir):
|
|
def __init__(self, root, account, fs_object = None):
|
|
self.root = root
|
|
self.account = account
|
|
self.datadir = os.path.join(self.root, self.account)
|
|
if not check_mount(root, account):
|
|
check_valid_account(account, fs_object)
|
|
self.metadata = read_metadata(self.datadir)
|
|
if not self.metadata or not validate_account(self.metadata):
|
|
self.metadata = create_account_metadata(self.datadir)
|
|
|
|
def list_containers_iter(self, limit, marker, end_marker,
|
|
prefix, delimiter):
|
|
"""
|
|
Return tuple of name, object_count, bytes_used, 0(is_subdir).
|
|
Used by account server.
|
|
"""
|
|
if delimiter and not prefix:
|
|
prefix = ''
|
|
containers = []
|
|
container_count = 0
|
|
account_list = []
|
|
|
|
containers, container_count = get_account_details(self.datadir)
|
|
|
|
if int(self.metadata[X_CONTAINER_COUNT]) != container_count:
|
|
self.metadata[X_CONTAINER_COUNT] = container_count
|
|
self.update_account(self.metadata)
|
|
|
|
if containers:
|
|
containers.sort()
|
|
|
|
if containers and prefix:
|
|
containers = self.filter_prefix(containers, prefix)
|
|
|
|
if containers and delimiter:
|
|
containers = self.filter_delimiter(containers, delimiter, prefix)
|
|
|
|
if containers and marker:
|
|
containers = self.filter_marker(containers, marker)
|
|
|
|
if containers and end_marker:
|
|
containers = self.filter_end_marker(containers, end_marker)
|
|
|
|
if containers and limit:
|
|
if len(containers) > limit:
|
|
containers = self.filter_limit(containers, limit)
|
|
|
|
if containers:
|
|
for cont in containers:
|
|
list_item = []
|
|
metadata = None
|
|
list_item.append(cont)
|
|
metadata = read_metadata(self.datadir + '/' + cont)
|
|
if not metadata or not validate_container(metadata):
|
|
metadata = create_container_metadata(self.datadir + '/' + cont)
|
|
|
|
if metadata:
|
|
list_item.append(metadata[X_OBJECTS_COUNT])
|
|
list_item.append(metadata[X_BYTES_USED])
|
|
list_item.append(0)
|
|
account_list.append(list_item)
|
|
|
|
return account_list
|
|
|
|
def get_info(self, include_metadata=False):
|
|
"""
|
|
Get global data for the account.
|
|
:returns: dict with keys: account, created_at, put_timestamp,
|
|
delete_timestamp, container_count, object_count,
|
|
bytes_used, hash, id
|
|
"""
|
|
metadata = {}
|
|
if (os.path.exists(self.datadir)):
|
|
metadata = read_metadata(self.datadir)
|
|
if not metadata:
|
|
metadata = create_account_metadata(self.datadir)
|
|
|
|
data = {'account' : self.account, 'created_at' : '1',
|
|
'put_timestamp' : '1', 'delete_timestamp' : '1',
|
|
'container_count' : metadata.get(X_CONTAINER_COUNT, 0),
|
|
'object_count' : metadata.get(X_OBJECTS_COUNT, 0),
|
|
'bytes_used' : metadata.get(X_BYTES_USED, 0),
|
|
'hash' : '', 'id' : ''}
|
|
|
|
if include_metadata:
|
|
data['metadata'] = metadata
|
|
return data
|