Support Storage Policy
* The file path on mountpoint was /container/obj and has been changed to /account/container/obj. This structure is not created on account and container creation but rather during the first object PUT to that container. * Rename OnDiskManager class to DiskFileManager * Remove overridden container_update() method and use Swift's default implementation that updates container DB. * Add support for async_dir in storage policies - failed database updates are stored in these directories to be later picked up by object-updater daemon. This change reintroduced write_pickle and pickle_async_update methods which are slightly modified versions of Swift's methods of the same name. * Updated tox.ini dependency on Swift to point to feature/ec branch of Swift's github repo. Signed-off-by: Prashanth Pai <ppai@redhat.com>
This commit is contained in:
Prashanth Pai
parent
ba39ade716
commit
3f4d0f98de
@ -16,6 +16,7 @@
|
||||
import os
|
||||
import stat
|
||||
import errno
|
||||
import random
|
||||
import logging
|
||||
from hashlib import md5
|
||||
from eventlet import sleep
|
||||
@ -25,7 +26,7 @@ from swift.common.exceptions import DiskFileNoSpace
|
||||
from gluster.swift.common.fs_utils import do_getctime, do_getmtime, do_stat, \
|
||||
do_listdir, do_walk, do_rmdir, do_log_rl, get_filename_from_fd, do_open, \
|
||||
do_isdir, do_getsize, do_getxattr, do_setxattr, do_removexattr, do_read, \
|
||||
do_close, do_dup, do_lseek, do_fstat
|
||||
do_close, do_dup, do_lseek, do_fstat, do_fsync, do_rename
|
||||
from gluster.swift.common import Glusterfs
|
||||
|
||||
X_CONTENT_TYPE = 'Content-Type'
|
||||
@ -530,3 +531,41 @@ def rmobjdir(dir_path):
|
||||
raise
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
|
||||
"""
|
||||
Ensure that a pickle file gets written to disk. The file is first written
|
||||
to a tmp file location in the destination directory path, ensured it is
|
||||
synced to disk, then moved to its final destination name.
|
||||
|
||||
This version takes advantage of Gluster's dot-prefix-dot-suffix naming
|
||||
where the a file named ".thefile.name.9a7aasv" is hashed to the same
|
||||
Gluster node as "thefile.name". This ensures the renaming of a temp file
|
||||
once written does not move it to another Gluster node.
|
||||
|
||||
:param obj: python object to be pickled
|
||||
:param dest: path of final destination file
|
||||
:param tmp: path to tmp to use, defaults to None (ignored)
|
||||
:param pickle_protocol: protocol to pickle the obj with, defaults to 0
|
||||
"""
|
||||
dirname = os.path.dirname(dest)
|
||||
# Create destination directory
|
||||
try:
|
||||
os.makedirs(dirname)
|
||||
except OSError as err:
|
||||
if err.errno != errno.EEXIST:
|
||||
raise
|
||||
basename = os.path.basename(dest)
|
||||
tmpname = '.' + basename + '.' + \
|
||||
md5(basename + str(random.random())).hexdigest()
|
||||
tmppath = os.path.join(dirname, tmpname)
|
||||
with open(tmppath, 'wb') as fo:
|
||||
pickle.dump(obj, fo, pickle_protocol)
|
||||
# TODO: This flush() method call turns into a flush() system call
|
||||
# We'll need to wrap this as well, but we would do this by writing
|
||||
# a context manager for our own open() method which returns an object
|
||||
# in fo which makes the gluster API call.
|
||||
fo.flush()
|
||||
do_fsync(fo)
|
||||
do_rename(tmppath, dest)
|
||||
|
||||
@ -31,7 +31,8 @@ from greenlet import getcurrent
|
||||
from contextlib import contextmanager
|
||||
from gluster.swift.common.exceptions import AlreadyExistsAsFile, \
|
||||
AlreadyExistsAsDir
|
||||
from swift.common.utils import TRUE_VALUES, ThreadPool, config_true_value
|
||||
from swift.common.utils import TRUE_VALUES, ThreadPool, config_true_value, \
|
||||
hash_path, normalize_timestamp
|
||||
from swift.common.exceptions import DiskFileNotExist, DiskFileError, \
|
||||
DiskFileNoSpace, DiskFileDeviceUnavailable, DiskFileNotOpen, \
|
||||
DiskFileExpired
|
||||
@ -44,12 +45,14 @@ from gluster.swift.common.fs_utils import do_fstat, do_open, do_close, \
|
||||
do_fadvise64, do_rename, do_fdatasync, do_lseek, do_mkdir
|
||||
from gluster.swift.common.utils import read_metadata, write_metadata, \
|
||||
validate_object, create_object_metadata, rmobjdir, dir_is_object, \
|
||||
get_object_metadata
|
||||
get_object_metadata, write_pickle
|
||||
from gluster.swift.common.utils import X_CONTENT_TYPE, \
|
||||
X_TIMESTAMP, X_TYPE, X_OBJECT_TYPE, FILE, OBJECT, DIR_TYPE, \
|
||||
FILE_TYPE, DEFAULT_UID, DEFAULT_GID, DIR_NON_OBJECT, DIR_OBJECT, \
|
||||
X_ETAG, X_CONTENT_LENGTH
|
||||
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
|
||||
from swift.common.storage_policy import get_policy_string
|
||||
from functools import partial
|
||||
|
||||
# FIXME: Hopefully we'll be able to move to Python 2.7+ where O_CLOEXEC will
|
||||
# be back ported. See http://www.python.org/dev/peps/pep-0433/
|
||||
@ -67,6 +70,9 @@ MAX_OPEN_ATTEMPTS = 10
|
||||
_cur_pid = str(os.getpid())
|
||||
_cur_host = str(gethostname())
|
||||
|
||||
ASYNCDIR_BASE = 'async_pending'
|
||||
get_async_dir = partial(get_policy_string, ASYNCDIR_BASE)
|
||||
|
||||
|
||||
def _random_sleep():
|
||||
sleep(random.uniform(0.5, 0.15))
|
||||
@ -223,7 +229,7 @@ def _adjust_metadata(metadata):
|
||||
return metadata
|
||||
|
||||
|
||||
class OnDiskManager(object):
|
||||
class DiskFileManager(object):
|
||||
"""
|
||||
Management class for devices, providing common place for shared parameters
|
||||
and methods not provided by the DiskFile class (which primarily services
|
||||
@ -254,6 +260,15 @@ class OnDiskManager(object):
|
||||
self.threadpools = defaultdict(
|
||||
lambda: ThreadPool(nthreads=threads_per_disk))
|
||||
|
||||
def construct_dev_path(self, device):
|
||||
"""
|
||||
Construct the path to a device without checking if it is mounted.
|
||||
|
||||
:param device: name of target device
|
||||
:returns: full path to the device
|
||||
"""
|
||||
return os.path.join(self.devices, device)
|
||||
|
||||
def _get_dev_path(self, device):
|
||||
"""
|
||||
Return the path to a device, checking to see that it is a proper mount
|
||||
@ -270,13 +285,26 @@ class OnDiskManager(object):
|
||||
return dev_path
|
||||
|
||||
def get_diskfile(self, device, account, container, obj,
|
||||
**kwargs):
|
||||
policy_idx=0, **kwargs):
|
||||
dev_path = self._get_dev_path(device)
|
||||
if not dev_path:
|
||||
raise DiskFileDeviceUnavailable()
|
||||
return DiskFile(self, dev_path, self.threadpools[device],
|
||||
account, container, obj, **kwargs)
|
||||
|
||||
def pickle_async_update(self, device, account, container, obj, data,
|
||||
timestamp, policy_idx):
|
||||
device_path = self.construct_dev_path(device)
|
||||
async_dir = os.path.join(device_path, get_async_dir(policy_idx))
|
||||
ohash = hash_path(account, container, obj)
|
||||
self.threadpools[device].run_in_thread(
|
||||
write_pickle,
|
||||
data,
|
||||
os.path.join(async_dir, ohash[-3:], ohash + '-' +
|
||||
normalize_timestamp(timestamp)),
|
||||
os.path.join(device_path, 'tmp'))
|
||||
self.logger.increment('async_pendings')
|
||||
|
||||
|
||||
class DiskFileWriter(object):
|
||||
"""
|
||||
@ -607,7 +635,15 @@ class DiskFile(object):
|
||||
# Don't store a value for data_file until we know it exists.
|
||||
self._data_file = None
|
||||
|
||||
self._container_path = os.path.join(self._device_path, container)
|
||||
if not hasattr(self._mgr, 'reseller_prefix'):
|
||||
self._mgr.reseller_prefix = 'AUTH_'
|
||||
if account.startswith(self._mgr.reseller_prefix):
|
||||
account = account[len(self._mgr.reseller_prefix):]
|
||||
self._account = account
|
||||
self._container = container
|
||||
|
||||
self._container_path = \
|
||||
os.path.join(self._device_path, self._account, self._container)
|
||||
obj = obj.strip(os.path.sep)
|
||||
obj_path, self._obj = os.path.split(obj)
|
||||
if obj_path:
|
||||
@ -862,6 +898,13 @@ class DiskFile(object):
|
||||
:raises AlreadyExistsAsFile: if path or part of a path is not a \
|
||||
directory
|
||||
"""
|
||||
# Create /account/container directory structure on mount point root
|
||||
try:
|
||||
os.makedirs(self._container_path)
|
||||
except OSError as err:
|
||||
if err.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
data_file = os.path.join(self._put_datadir, self._obj)
|
||||
|
||||
# Assume the full directory path exists to the file already, and
|
||||
|
||||
@ -26,7 +26,7 @@ from swift.common.request_helpers import split_and_validate_path
|
||||
|
||||
from swift.obj import server
|
||||
|
||||
from gluster.swift.obj.diskfile import OnDiskManager
|
||||
from gluster.swift.obj.diskfile import DiskFileManager
|
||||
|
||||
|
||||
class ObjectController(server.ObjectController):
|
||||
@ -46,11 +46,12 @@ class ObjectController(server.ObjectController):
|
||||
"""
|
||||
# Common on-disk hierarchy shared across account, container and object
|
||||
# servers.
|
||||
self._ondisk_mgr = OnDiskManager(conf, self.logger)
|
||||
self.swift_dir = conf.get('swift_dir', '/etc/swift')
|
||||
self._diskfile_mgr = DiskFileManager(conf, self.logger)
|
||||
self._diskfile_mgr.reseller_prefix = \
|
||||
conf.get('reseller_prefix', 'AUTH_').strip()
|
||||
|
||||
def get_diskfile(self, device, partition, account, container, obj,
|
||||
**kwargs):
|
||||
policy_idx=0, **kwargs):
|
||||
"""
|
||||
Utility method for instantiating a DiskFile object supporting a given
|
||||
REST API.
|
||||
@ -59,17 +60,8 @@ class ObjectController(server.ObjectController):
|
||||
DiskFile class would simply over-ride this method to provide that
|
||||
behavior.
|
||||
"""
|
||||
return self._ondisk_mgr.get_diskfile(device, account, container, obj,
|
||||
**kwargs)
|
||||
|
||||
def container_update(self, *args, **kwargs):
|
||||
"""
|
||||
Update the container when objects are updated.
|
||||
|
||||
For Gluster, this is just a no-op, since a container is just the
|
||||
directory holding all the objects (sub-directory hierarchy of files).
|
||||
"""
|
||||
return
|
||||
return self._diskfile_mgr.get_diskfile(device, account, container, obj,
|
||||
**kwargs)
|
||||
|
||||
@public
|
||||
@timing_stats()
|
||||
|
||||
@ -36,7 +36,7 @@ from gluster.swift.common.exceptions import GlusterFileSystemOSError
|
||||
import gluster.swift.common.utils
|
||||
from gluster.swift.common.utils import normalize_timestamp
|
||||
import gluster.swift.obj.diskfile
|
||||
from gluster.swift.obj.diskfile import DiskFileWriter, DiskFile, OnDiskManager
|
||||
from gluster.swift.obj.diskfile import DiskFileWriter, DiskFile, DiskFileManager
|
||||
from gluster.swift.common.utils import DEFAULT_UID, DEFAULT_GID, X_TYPE, \
|
||||
X_OBJECT_TYPE, DIR_OBJECT
|
||||
|
||||
@ -136,7 +136,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
self.td = tempfile.mkdtemp()
|
||||
self.conf = dict(devices=self.td, mb_per_sync=2,
|
||||
keep_cache_size=(1024 * 1024), mount_check=False)
|
||||
self.mgr = OnDiskManager(self.conf, self.lg)
|
||||
self.mgr = DiskFileManager(self.conf, self.lg)
|
||||
|
||||
def tearDown(self):
|
||||
tpool.execute = self._orig_tpool_exc
|
||||
@ -161,9 +161,9 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert gdf._gid == DEFAULT_GID
|
||||
assert gdf._obj == "z"
|
||||
assert gdf._obj_path == ""
|
||||
assert gdf._datadir == os.path.join(self.td, "vol0", "bar"), gdf._datadir
|
||||
assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar"), gdf._datadir
|
||||
assert gdf._datadir == gdf._put_datadir
|
||||
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
|
||||
assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
|
||||
assert gdf._is_dir is False
|
||||
assert gdf._logger == self.lg
|
||||
assert gdf._fd is None
|
||||
@ -172,10 +172,10 @@ class TestDiskFile(unittest.TestCase):
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "/b/a/z/")
|
||||
assert gdf._obj == "z"
|
||||
assert gdf._obj_path == "b/a"
|
||||
assert gdf._datadir == os.path.join(self.td, "vol0", "bar", "b", "a"), gdf._datadir
|
||||
assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "b", "a"), gdf._datadir
|
||||
|
||||
def test_open_no_metadata(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_file = os.path.join(the_path, "z")
|
||||
os.makedirs(the_path)
|
||||
with open(the_file, "wb") as fd:
|
||||
@ -215,7 +215,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
self.assertTrue(mock_close.called)
|
||||
|
||||
def test_open_existing_metadata(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_file = os.path.join(the_path, "z")
|
||||
os.makedirs(the_path)
|
||||
with open(the_file, "wb") as fd:
|
||||
@ -243,7 +243,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert gdf._metadata == exp_md, "%r != %r" % (gdf._metadata, exp_md)
|
||||
|
||||
def test_open_invalid_existing_metadata(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_file = os.path.join(the_path, "z")
|
||||
os.makedirs(the_path)
|
||||
with open(the_file, "wb") as fd:
|
||||
@ -263,7 +263,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert gdf._metadata != inv_md
|
||||
|
||||
def test_open_isdir(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_dir = os.path.join(the_path, "d")
|
||||
os.makedirs(the_dir)
|
||||
ini_md = {
|
||||
@ -287,7 +287,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
|
||||
def _create_and_get_diskfile(self, dev, par, acc, con, obj, fsize=256):
|
||||
# FIXME: assumes account === volume
|
||||
the_path = os.path.join(self.td, dev, con)
|
||||
the_path = os.path.join(self.td, dev, acc, con)
|
||||
the_file = os.path.join(the_path, obj)
|
||||
base_obj = os.path.basename(the_file)
|
||||
base_dir = os.path.dirname(the_file)
|
||||
@ -312,7 +312,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
|
||||
with gdf.open():
|
||||
assert gdf._fd is not None
|
||||
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
|
||||
assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
|
||||
reader = gdf.reader()
|
||||
assert reader._fd is not None
|
||||
fd[0] = reader._fd
|
||||
@ -324,7 +324,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
def test_reader_disk_chunk_size(self):
|
||||
conf = dict(disk_chunk_size=64)
|
||||
conf.update(self.conf)
|
||||
self.mgr = OnDiskManager(conf, self.lg)
|
||||
self.mgr = DiskFileManager(conf, self.lg)
|
||||
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
|
||||
with gdf.open():
|
||||
reader = gdf.reader()
|
||||
@ -365,7 +365,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z", fsize=1024*1024*2)
|
||||
with gdf.open():
|
||||
assert gdf._fd is not None
|
||||
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
|
||||
assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
|
||||
reader = gdf.reader()
|
||||
assert reader._fd is not None
|
||||
fd[0] = reader._fd
|
||||
@ -380,7 +380,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
called[0] = True
|
||||
os.close(fd)
|
||||
|
||||
the_cont = os.path.join(self.td, "vol0", "bar")
|
||||
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
os.makedirs(os.path.join(the_cont, "dir"))
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
|
||||
with gdf.open():
|
||||
@ -396,7 +396,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
reader.close()
|
||||
|
||||
def test_create_dir_object_no_md(self):
|
||||
the_cont = os.path.join(self.td, "vol0", "bar")
|
||||
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_dir = "dir"
|
||||
os.makedirs(the_cont)
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar",
|
||||
@ -409,7 +409,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert _mapit(full_dir_path) not in _metadata
|
||||
|
||||
def test_create_dir_object_with_md(self):
|
||||
the_cont = os.path.join(self.td, "vol0", "bar")
|
||||
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_dir = "dir"
|
||||
os.makedirs(the_cont)
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar",
|
||||
@ -468,7 +468,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
self.assertFalse(_mapit(the_dir) in _metadata)
|
||||
|
||||
def test_write_metadata(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_dir = os.path.join(the_path, "z")
|
||||
os.makedirs(the_dir)
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
|
||||
@ -481,7 +481,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
self.assertTrue(fmd['Content-Type'], md['Content-Type'])
|
||||
|
||||
def test_add_metadata_to_existing_file(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_file = os.path.join(the_path, "z")
|
||||
os.makedirs(the_path)
|
||||
with open(the_file, "wb") as fd:
|
||||
@ -509,7 +509,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
self.assertFalse('a' in on_disk_md)
|
||||
|
||||
def test_add_md_to_existing_file_with_md_in_gdf(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_file = os.path.join(the_path, "z")
|
||||
os.makedirs(the_path)
|
||||
with open(the_file, "wb") as fd:
|
||||
@ -537,7 +537,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
self.assertFalse('a' in on_disk_md)
|
||||
|
||||
def test_add_metadata_to_existing_dir(self):
|
||||
the_cont = os.path.join(self.td, "vol0", "bar")
|
||||
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_dir = os.path.join(the_cont, "dir")
|
||||
os.makedirs(the_dir)
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
|
||||
@ -571,7 +571,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
|
||||
|
||||
def test_write_metadata_w_meta_file(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_file = os.path.join(the_path, "z")
|
||||
os.makedirs(the_path)
|
||||
with open(the_file, "wb") as fd:
|
||||
@ -583,7 +583,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert _metadata[_mapit(the_file)] == newmd
|
||||
|
||||
def test_write_metadata_w_meta_file_no_content_type(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_file = os.path.join(the_path, "z")
|
||||
os.makedirs(the_path)
|
||||
with open(the_file, "wb") as fd:
|
||||
@ -596,7 +596,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert _metadata[_mapit(the_file)] == newmd
|
||||
|
||||
def test_write_metadata_w_meta_dir(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_dir = os.path.join(the_path, "dir")
|
||||
os.makedirs(the_dir)
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
|
||||
@ -606,7 +606,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert _metadata[_mapit(the_dir)] == newmd
|
||||
|
||||
def test_write_metadata_w_marker_dir(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_dir = os.path.join(the_path, "dir")
|
||||
os.makedirs(the_dir)
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
|
||||
@ -616,7 +616,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert _metadata[_mapit(the_dir)] == newmd
|
||||
|
||||
def test_put_w_marker_dir_create(self):
|
||||
the_cont = os.path.join(self.td, "vol0", "bar")
|
||||
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_dir = os.path.join(the_cont, "dir")
|
||||
os.makedirs(the_cont)
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
|
||||
@ -633,7 +633,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert _metadata[_mapit(the_dir)][X_OBJECT_TYPE] == DIR_OBJECT
|
||||
|
||||
def test_put_is_dir(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_dir = os.path.join(the_path, "dir")
|
||||
os.makedirs(the_dir)
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
|
||||
@ -664,14 +664,14 @@ class TestDiskFile(unittest.TestCase):
|
||||
origfmd, _metadata[_mapit(the_dir)])
|
||||
|
||||
def test_put(self):
|
||||
the_cont = os.path.join(self.td, "vol0", "bar")
|
||||
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
os.makedirs(the_cont)
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
|
||||
assert gdf._obj == "z"
|
||||
assert gdf._obj_path == ""
|
||||
assert gdf._container_path == os.path.join(self.td, "vol0", "bar")
|
||||
assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
assert gdf._datadir == the_cont
|
||||
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
|
||||
assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
|
||||
|
||||
body = '1234\n'
|
||||
etag = md5()
|
||||
@ -694,14 +694,14 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert not os.path.exists(tmppath)
|
||||
|
||||
def test_put_ENOSPC(self):
|
||||
the_cont = os.path.join(self.td, "vol0", "bar")
|
||||
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
os.makedirs(the_cont)
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
|
||||
assert gdf._obj == "z"
|
||||
assert gdf._obj_path == ""
|
||||
assert gdf._container_path == os.path.join(self.td, "vol0", "bar")
|
||||
assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
assert gdf._datadir == the_cont
|
||||
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
|
||||
assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
|
||||
|
||||
body = '1234\n'
|
||||
etag = md5()
|
||||
@ -729,14 +729,14 @@ class TestDiskFile(unittest.TestCase):
|
||||
self.fail("Expected exception DiskFileNoSpace")
|
||||
|
||||
def test_put_rename_ENOENT(self):
|
||||
the_cont = os.path.join(self.td, "vol0", "bar")
|
||||
the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
os.makedirs(the_cont)
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
|
||||
assert gdf._obj == "z"
|
||||
assert gdf._obj_path == ""
|
||||
assert gdf._container_path == os.path.join(self.td, "vol0", "bar")
|
||||
assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
assert gdf._datadir == the_cont
|
||||
assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z")
|
||||
assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
|
||||
|
||||
body = '1234\n'
|
||||
etag = md5()
|
||||
@ -775,10 +775,10 @@ class TestDiskFile(unittest.TestCase):
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", the_file)
|
||||
assert gdf._obj == "z"
|
||||
assert gdf._obj_path == the_obj_path
|
||||
assert gdf._container_path == os.path.join(self.td, "vol0", "bar")
|
||||
assert gdf._datadir == os.path.join(self.td, "vol0", "bar", "b", "a")
|
||||
assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "b", "a")
|
||||
assert gdf._data_file == os.path.join(
|
||||
self.td, "vol0", "bar", "b", "a", "z")
|
||||
self.td, "vol0", "ufo47", "bar", "b", "a", "z")
|
||||
|
||||
body = '1234\n'
|
||||
etag = md5()
|
||||
@ -801,7 +801,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert not os.path.exists(tmppath)
|
||||
|
||||
def test_delete(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_file = os.path.join(the_path, "z")
|
||||
os.makedirs(the_path)
|
||||
with open(the_file, "wb") as fd:
|
||||
@ -816,7 +816,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert not os.path.exists(os.path.join(gdf._datadir, gdf._obj))
|
||||
|
||||
def test_delete_same_timestamp(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_file = os.path.join(the_path, "z")
|
||||
os.makedirs(the_path)
|
||||
with open(the_file, "wb") as fd:
|
||||
@ -831,7 +831,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert os.path.exists(os.path.join(gdf._datadir, gdf._obj))
|
||||
|
||||
def test_delete_file_not_found(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_file = os.path.join(the_path, "z")
|
||||
os.makedirs(the_path)
|
||||
with open(the_file, "wb") as fd:
|
||||
@ -850,7 +850,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert not os.path.exists(os.path.join(gdf._datadir, gdf._obj))
|
||||
|
||||
def test_delete_file_unlink_error(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_file = os.path.join(the_path, "z")
|
||||
os.makedirs(the_path)
|
||||
with open(the_file, "wb") as fd:
|
||||
@ -884,7 +884,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
assert os.path.exists(os.path.join(gdf._datadir, gdf._obj))
|
||||
|
||||
def test_delete_is_dir(self):
|
||||
the_path = os.path.join(self.td, "vol0", "bar")
|
||||
the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
|
||||
the_dir = os.path.join(the_path, "d")
|
||||
os.makedirs(the_dir)
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
|
||||
@ -899,7 +899,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
saved_tmppath = ''
|
||||
saved_fd = None
|
||||
with gdf.create() as dw:
|
||||
assert gdf._datadir == os.path.join(self.td, "vol0", "bar", "dir")
|
||||
assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "dir")
|
||||
assert os.path.isdir(gdf._datadir)
|
||||
saved_tmppath = dw._tmppath
|
||||
assert os.path.dirname(saved_tmppath) == gdf._datadir
|
||||
@ -922,7 +922,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
|
||||
saved_tmppath = ''
|
||||
with gdf.create() as dw:
|
||||
assert gdf._datadir == os.path.join(self.td, "vol0", "bar", "dir")
|
||||
assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "dir")
|
||||
assert os.path.isdir(gdf._datadir)
|
||||
saved_tmppath = dw._tmppath
|
||||
assert os.path.dirname(saved_tmppath) == gdf._datadir
|
||||
@ -937,7 +937,7 @@ class TestDiskFile(unittest.TestCase):
|
||||
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
|
||||
saved_tmppath = ''
|
||||
with gdf.create() as dw:
|
||||
assert gdf._datadir == os.path.join(self.td, "vol0", "bar", "dir")
|
||||
assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "dir")
|
||||
assert os.path.isdir(gdf._datadir)
|
||||
saved_tmppath = dw._tmppath
|
||||
assert os.path.dirname(saved_tmppath) == gdf._datadir
|
||||
|
||||
4
tox.ini
4
tox.ini
@ -18,7 +18,9 @@ setenv = VIRTUAL_ENV={envdir}
|
||||
NOSE_COVER_BRANCHES=1
|
||||
NOSE_COVER_PACKAGE=gluster
|
||||
deps =
|
||||
https://launchpad.net/swift/icehouse/1.13.1/+download/swift-1.13.1.tar.gz
|
||||
# GitHub's .zip URL won't work! pip supports installing from git repos.
|
||||
# https://pip.pypa.io/en/latest/reference/pip_install.html#git
|
||||
git+https://github.com/openstack/swift.git@feature/ec
|
||||
-r{toxinidir}/requirements.txt
|
||||
-r{toxinidir}/test-requirements.txt
|
||||
changedir = {toxinidir}/test/unit
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user