diff --git a/gluster/swift/common/utils.py b/gluster/swift/common/utils.py index 145add3..2012b78 100644 --- a/gluster/swift/common/utils.py +++ b/gluster/swift/common/utils.py @@ -16,6 +16,7 @@ import os import stat import errno +import random import logging from hashlib import md5 from eventlet import sleep @@ -25,7 +26,7 @@ from swift.common.exceptions import DiskFileNoSpace from gluster.swift.common.fs_utils import do_getctime, do_getmtime, do_stat, \ do_listdir, do_walk, do_rmdir, do_log_rl, get_filename_from_fd, do_open, \ do_isdir, do_getsize, do_getxattr, do_setxattr, do_removexattr, do_read, \ - do_close, do_dup, do_lseek, do_fstat + do_close, do_dup, do_lseek, do_fstat, do_fsync, do_rename from gluster.swift.common import Glusterfs X_CONTENT_TYPE = 'Content-Type' @@ -530,3 +531,41 @@ def rmobjdir(dir_path): raise else: return True + + +def write_pickle(obj, dest, tmp=None, pickle_protocol=0): + """ + Ensure that a pickle file gets written to disk. The file is first written + to a tmp file location in the destination directory path, ensured it is + synced to disk, then moved to its final destination name. + + This version takes advantage of Gluster's dot-prefix-dot-suffix naming + where the a file named ".thefile.name.9a7aasv" is hashed to the same + Gluster node as "thefile.name". This ensures the renaming of a temp file + once written does not move it to another Gluster node. + + :param obj: python object to be pickled + :param dest: path of final destination file + :param tmp: path to tmp to use, defaults to None (ignored) + :param pickle_protocol: protocol to pickle the obj with, defaults to 0 + """ + dirname = os.path.dirname(dest) + # Create destination directory + try: + os.makedirs(dirname) + except OSError as err: + if err.errno != errno.EEXIST: + raise + basename = os.path.basename(dest) + tmpname = '.' + basename + '.' + \ + md5(basename + str(random.random())).hexdigest() + tmppath = os.path.join(dirname, tmpname) + with open(tmppath, 'wb') as fo: + pickle.dump(obj, fo, pickle_protocol) + # TODO: This flush() method call turns into a flush() system call + # We'll need to wrap this as well, but we would do this by writing + # a context manager for our own open() method which returns an object + # in fo which makes the gluster API call. + fo.flush() + do_fsync(fo) + do_rename(tmppath, dest) diff --git a/gluster/swift/obj/diskfile.py b/gluster/swift/obj/diskfile.py index 3223dcf..332ec83 100644 --- a/gluster/swift/obj/diskfile.py +++ b/gluster/swift/obj/diskfile.py @@ -31,7 +31,8 @@ from greenlet import getcurrent from contextlib import contextmanager from gluster.swift.common.exceptions import AlreadyExistsAsFile, \ AlreadyExistsAsDir -from swift.common.utils import TRUE_VALUES, ThreadPool, config_true_value +from swift.common.utils import TRUE_VALUES, ThreadPool, config_true_value, \ + hash_path, normalize_timestamp from swift.common.exceptions import DiskFileNotExist, DiskFileError, \ DiskFileNoSpace, DiskFileDeviceUnavailable, DiskFileNotOpen, \ DiskFileExpired @@ -44,12 +45,14 @@ from gluster.swift.common.fs_utils import do_fstat, do_open, do_close, \ do_fadvise64, do_rename, do_fdatasync, do_lseek, do_mkdir from gluster.swift.common.utils import read_metadata, write_metadata, \ validate_object, create_object_metadata, rmobjdir, dir_is_object, \ - get_object_metadata + get_object_metadata, write_pickle from gluster.swift.common.utils import X_CONTENT_TYPE, \ X_TIMESTAMP, X_TYPE, X_OBJECT_TYPE, FILE, OBJECT, DIR_TYPE, \ FILE_TYPE, DEFAULT_UID, DEFAULT_GID, DIR_NON_OBJECT, DIR_OBJECT, \ X_ETAG, X_CONTENT_LENGTH from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from swift.common.storage_policy import get_policy_string +from functools import partial # FIXME: Hopefully we'll be able to move to Python 2.7+ where O_CLOEXEC will # be back ported. See http://www.python.org/dev/peps/pep-0433/ @@ -67,6 +70,9 @@ MAX_OPEN_ATTEMPTS = 10 _cur_pid = str(os.getpid()) _cur_host = str(gethostname()) +ASYNCDIR_BASE = 'async_pending' +get_async_dir = partial(get_policy_string, ASYNCDIR_BASE) + def _random_sleep(): sleep(random.uniform(0.5, 0.15)) @@ -223,7 +229,7 @@ def _adjust_metadata(metadata): return metadata -class OnDiskManager(object): +class DiskFileManager(object): """ Management class for devices, providing common place for shared parameters and methods not provided by the DiskFile class (which primarily services @@ -254,6 +260,15 @@ class OnDiskManager(object): self.threadpools = defaultdict( lambda: ThreadPool(nthreads=threads_per_disk)) + def construct_dev_path(self, device): + """ + Construct the path to a device without checking if it is mounted. + + :param device: name of target device + :returns: full path to the device + """ + return os.path.join(self.devices, device) + def _get_dev_path(self, device): """ Return the path to a device, checking to see that it is a proper mount @@ -270,13 +285,26 @@ class OnDiskManager(object): return dev_path def get_diskfile(self, device, account, container, obj, - **kwargs): + policy_idx=0, **kwargs): dev_path = self._get_dev_path(device) if not dev_path: raise DiskFileDeviceUnavailable() return DiskFile(self, dev_path, self.threadpools[device], account, container, obj, **kwargs) + def pickle_async_update(self, device, account, container, obj, data, + timestamp, policy_idx): + device_path = self.construct_dev_path(device) + async_dir = os.path.join(device_path, get_async_dir(policy_idx)) + ohash = hash_path(account, container, obj) + self.threadpools[device].run_in_thread( + write_pickle, + data, + os.path.join(async_dir, ohash[-3:], ohash + '-' + + normalize_timestamp(timestamp)), + os.path.join(device_path, 'tmp')) + self.logger.increment('async_pendings') + class DiskFileWriter(object): """ @@ -607,7 +635,15 @@ class DiskFile(object): # Don't store a value for data_file until we know it exists. self._data_file = None - self._container_path = os.path.join(self._device_path, container) + if not hasattr(self._mgr, 'reseller_prefix'): + self._mgr.reseller_prefix = 'AUTH_' + if account.startswith(self._mgr.reseller_prefix): + account = account[len(self._mgr.reseller_prefix):] + self._account = account + self._container = container + + self._container_path = \ + os.path.join(self._device_path, self._account, self._container) obj = obj.strip(os.path.sep) obj_path, self._obj = os.path.split(obj) if obj_path: @@ -862,6 +898,13 @@ class DiskFile(object): :raises AlreadyExistsAsFile: if path or part of a path is not a \ directory """ + # Create /account/container directory structure on mount point root + try: + os.makedirs(self._container_path) + except OSError as err: + if err.errno != errno.EEXIST: + raise + data_file = os.path.join(self._put_datadir, self._obj) # Assume the full directory path exists to the file already, and diff --git a/gluster/swift/obj/server.py b/gluster/swift/obj/server.py index c9af6b5..1fdeeb1 100644 --- a/gluster/swift/obj/server.py +++ b/gluster/swift/obj/server.py @@ -26,7 +26,7 @@ from swift.common.request_helpers import split_and_validate_path from swift.obj import server -from gluster.swift.obj.diskfile import OnDiskManager +from gluster.swift.obj.diskfile import DiskFileManager class ObjectController(server.ObjectController): @@ -46,11 +46,12 @@ class ObjectController(server.ObjectController): """ # Common on-disk hierarchy shared across account, container and object # servers. - self._ondisk_mgr = OnDiskManager(conf, self.logger) - self.swift_dir = conf.get('swift_dir', '/etc/swift') + self._diskfile_mgr = DiskFileManager(conf, self.logger) + self._diskfile_mgr.reseller_prefix = \ + conf.get('reseller_prefix', 'AUTH_').strip() def get_diskfile(self, device, partition, account, container, obj, - **kwargs): + policy_idx=0, **kwargs): """ Utility method for instantiating a DiskFile object supporting a given REST API. @@ -59,17 +60,8 @@ class ObjectController(server.ObjectController): DiskFile class would simply over-ride this method to provide that behavior. """ - return self._ondisk_mgr.get_diskfile(device, account, container, obj, - **kwargs) - - def container_update(self, *args, **kwargs): - """ - Update the container when objects are updated. - - For Gluster, this is just a no-op, since a container is just the - directory holding all the objects (sub-directory hierarchy of files). - """ - return + return self._diskfile_mgr.get_diskfile(device, account, container, obj, + **kwargs) @public @timing_stats() diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py index ec813c7..c6b8586 100644 --- a/test/unit/obj/test_diskfile.py +++ b/test/unit/obj/test_diskfile.py @@ -36,7 +36,7 @@ from gluster.swift.common.exceptions import GlusterFileSystemOSError import gluster.swift.common.utils from gluster.swift.common.utils import normalize_timestamp import gluster.swift.obj.diskfile -from gluster.swift.obj.diskfile import DiskFileWriter, DiskFile, OnDiskManager +from gluster.swift.obj.diskfile import DiskFileWriter, DiskFile, DiskFileManager from gluster.swift.common.utils import DEFAULT_UID, DEFAULT_GID, X_TYPE, \ X_OBJECT_TYPE, DIR_OBJECT @@ -136,7 +136,7 @@ class TestDiskFile(unittest.TestCase): self.td = tempfile.mkdtemp() self.conf = dict(devices=self.td, mb_per_sync=2, keep_cache_size=(1024 * 1024), mount_check=False) - self.mgr = OnDiskManager(self.conf, self.lg) + self.mgr = DiskFileManager(self.conf, self.lg) def tearDown(self): tpool.execute = self._orig_tpool_exc @@ -161,9 +161,9 @@ class TestDiskFile(unittest.TestCase): assert gdf._gid == DEFAULT_GID assert gdf._obj == "z" assert gdf._obj_path == "" - assert gdf._datadir == os.path.join(self.td, "vol0", "bar"), gdf._datadir + assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar"), gdf._datadir assert gdf._datadir == gdf._put_datadir - assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z") + assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z") assert gdf._is_dir is False assert gdf._logger == self.lg assert gdf._fd is None @@ -172,10 +172,10 @@ class TestDiskFile(unittest.TestCase): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "/b/a/z/") assert gdf._obj == "z" assert gdf._obj_path == "b/a" - assert gdf._datadir == os.path.join(self.td, "vol0", "bar", "b", "a"), gdf._datadir + assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "b", "a"), gdf._datadir def test_open_no_metadata(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: @@ -215,7 +215,7 @@ class TestDiskFile(unittest.TestCase): self.assertTrue(mock_close.called) def test_open_existing_metadata(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: @@ -243,7 +243,7 @@ class TestDiskFile(unittest.TestCase): assert gdf._metadata == exp_md, "%r != %r" % (gdf._metadata, exp_md) def test_open_invalid_existing_metadata(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: @@ -263,7 +263,7 @@ class TestDiskFile(unittest.TestCase): assert gdf._metadata != inv_md def test_open_isdir(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_dir = os.path.join(the_path, "d") os.makedirs(the_dir) ini_md = { @@ -287,7 +287,7 @@ class TestDiskFile(unittest.TestCase): def _create_and_get_diskfile(self, dev, par, acc, con, obj, fsize=256): # FIXME: assumes account === volume - the_path = os.path.join(self.td, dev, con) + the_path = os.path.join(self.td, dev, acc, con) the_file = os.path.join(the_path, obj) base_obj = os.path.basename(the_file) base_dir = os.path.dirname(the_file) @@ -312,7 +312,7 @@ class TestDiskFile(unittest.TestCase): gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z") with gdf.open(): assert gdf._fd is not None - assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z") + assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z") reader = gdf.reader() assert reader._fd is not None fd[0] = reader._fd @@ -324,7 +324,7 @@ class TestDiskFile(unittest.TestCase): def test_reader_disk_chunk_size(self): conf = dict(disk_chunk_size=64) conf.update(self.conf) - self.mgr = OnDiskManager(conf, self.lg) + self.mgr = DiskFileManager(conf, self.lg) gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z") with gdf.open(): reader = gdf.reader() @@ -365,7 +365,7 @@ class TestDiskFile(unittest.TestCase): gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z", fsize=1024*1024*2) with gdf.open(): assert gdf._fd is not None - assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z") + assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z") reader = gdf.reader() assert reader._fd is not None fd[0] = reader._fd @@ -380,7 +380,7 @@ class TestDiskFile(unittest.TestCase): called[0] = True os.close(fd) - the_cont = os.path.join(self.td, "vol0", "bar") + the_cont = os.path.join(self.td, "vol0", "ufo47", "bar") os.makedirs(os.path.join(the_cont, "dir")) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir") with gdf.open(): @@ -396,7 +396,7 @@ class TestDiskFile(unittest.TestCase): reader.close() def test_create_dir_object_no_md(self): - the_cont = os.path.join(self.td, "vol0", "bar") + the_cont = os.path.join(self.td, "vol0", "ufo47", "bar") the_dir = "dir" os.makedirs(the_cont) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", @@ -409,7 +409,7 @@ class TestDiskFile(unittest.TestCase): assert _mapit(full_dir_path) not in _metadata def test_create_dir_object_with_md(self): - the_cont = os.path.join(self.td, "vol0", "bar") + the_cont = os.path.join(self.td, "vol0", "ufo47", "bar") the_dir = "dir" os.makedirs(the_cont) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", @@ -468,7 +468,7 @@ class TestDiskFile(unittest.TestCase): self.assertFalse(_mapit(the_dir) in _metadata) def test_write_metadata(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_dir = os.path.join(the_path, "z") os.makedirs(the_dir) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") @@ -481,7 +481,7 @@ class TestDiskFile(unittest.TestCase): self.assertTrue(fmd['Content-Type'], md['Content-Type']) def test_add_metadata_to_existing_file(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: @@ -509,7 +509,7 @@ class TestDiskFile(unittest.TestCase): self.assertFalse('a' in on_disk_md) def test_add_md_to_existing_file_with_md_in_gdf(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: @@ -537,7 +537,7 @@ class TestDiskFile(unittest.TestCase): self.assertFalse('a' in on_disk_md) def test_add_metadata_to_existing_dir(self): - the_cont = os.path.join(self.td, "vol0", "bar") + the_cont = os.path.join(self.td, "vol0", "ufo47", "bar") the_dir = os.path.join(the_cont, "dir") os.makedirs(the_dir) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir") @@ -571,7 +571,7 @@ class TestDiskFile(unittest.TestCase): def test_write_metadata_w_meta_file(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: @@ -583,7 +583,7 @@ class TestDiskFile(unittest.TestCase): assert _metadata[_mapit(the_file)] == newmd def test_write_metadata_w_meta_file_no_content_type(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: @@ -596,7 +596,7 @@ class TestDiskFile(unittest.TestCase): assert _metadata[_mapit(the_file)] == newmd def test_write_metadata_w_meta_dir(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_dir = os.path.join(the_path, "dir") os.makedirs(the_dir) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir") @@ -606,7 +606,7 @@ class TestDiskFile(unittest.TestCase): assert _metadata[_mapit(the_dir)] == newmd def test_write_metadata_w_marker_dir(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_dir = os.path.join(the_path, "dir") os.makedirs(the_dir) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir") @@ -616,7 +616,7 @@ class TestDiskFile(unittest.TestCase): assert _metadata[_mapit(the_dir)] == newmd def test_put_w_marker_dir_create(self): - the_cont = os.path.join(self.td, "vol0", "bar") + the_cont = os.path.join(self.td, "vol0", "ufo47", "bar") the_dir = os.path.join(the_cont, "dir") os.makedirs(the_cont) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir") @@ -633,7 +633,7 @@ class TestDiskFile(unittest.TestCase): assert _metadata[_mapit(the_dir)][X_OBJECT_TYPE] == DIR_OBJECT def test_put_is_dir(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_dir = os.path.join(the_path, "dir") os.makedirs(the_dir) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir") @@ -664,14 +664,14 @@ class TestDiskFile(unittest.TestCase): origfmd, _metadata[_mapit(the_dir)]) def test_put(self): - the_cont = os.path.join(self.td, "vol0", "bar") + the_cont = os.path.join(self.td, "vol0", "ufo47", "bar") os.makedirs(the_cont) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._obj_path == "" - assert gdf._container_path == os.path.join(self.td, "vol0", "bar") + assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar") assert gdf._datadir == the_cont - assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z") + assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z") body = '1234\n' etag = md5() @@ -694,14 +694,14 @@ class TestDiskFile(unittest.TestCase): assert not os.path.exists(tmppath) def test_put_ENOSPC(self): - the_cont = os.path.join(self.td, "vol0", "bar") + the_cont = os.path.join(self.td, "vol0", "ufo47", "bar") os.makedirs(the_cont) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._obj_path == "" - assert gdf._container_path == os.path.join(self.td, "vol0", "bar") + assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar") assert gdf._datadir == the_cont - assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z") + assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z") body = '1234\n' etag = md5() @@ -729,14 +729,14 @@ class TestDiskFile(unittest.TestCase): self.fail("Expected exception DiskFileNoSpace") def test_put_rename_ENOENT(self): - the_cont = os.path.join(self.td, "vol0", "bar") + the_cont = os.path.join(self.td, "vol0", "ufo47", "bar") os.makedirs(the_cont) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") assert gdf._obj == "z" assert gdf._obj_path == "" - assert gdf._container_path == os.path.join(self.td, "vol0", "bar") + assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar") assert gdf._datadir == the_cont - assert gdf._data_file == os.path.join(self.td, "vol0", "bar", "z") + assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z") body = '1234\n' etag = md5() @@ -775,10 +775,10 @@ class TestDiskFile(unittest.TestCase): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", the_file) assert gdf._obj == "z" assert gdf._obj_path == the_obj_path - assert gdf._container_path == os.path.join(self.td, "vol0", "bar") - assert gdf._datadir == os.path.join(self.td, "vol0", "bar", "b", "a") + assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar") + assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "b", "a") assert gdf._data_file == os.path.join( - self.td, "vol0", "bar", "b", "a", "z") + self.td, "vol0", "ufo47", "bar", "b", "a", "z") body = '1234\n' etag = md5() @@ -801,7 +801,7 @@ class TestDiskFile(unittest.TestCase): assert not os.path.exists(tmppath) def test_delete(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: @@ -816,7 +816,7 @@ class TestDiskFile(unittest.TestCase): assert not os.path.exists(os.path.join(gdf._datadir, gdf._obj)) def test_delete_same_timestamp(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: @@ -831,7 +831,7 @@ class TestDiskFile(unittest.TestCase): assert os.path.exists(os.path.join(gdf._datadir, gdf._obj)) def test_delete_file_not_found(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: @@ -850,7 +850,7 @@ class TestDiskFile(unittest.TestCase): assert not os.path.exists(os.path.join(gdf._datadir, gdf._obj)) def test_delete_file_unlink_error(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_file = os.path.join(the_path, "z") os.makedirs(the_path) with open(the_file, "wb") as fd: @@ -884,7 +884,7 @@ class TestDiskFile(unittest.TestCase): assert os.path.exists(os.path.join(gdf._datadir, gdf._obj)) def test_delete_is_dir(self): - the_path = os.path.join(self.td, "vol0", "bar") + the_path = os.path.join(self.td, "vol0", "ufo47", "bar") the_dir = os.path.join(the_path, "d") os.makedirs(the_dir) gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d") @@ -899,7 +899,7 @@ class TestDiskFile(unittest.TestCase): saved_tmppath = '' saved_fd = None with gdf.create() as dw: - assert gdf._datadir == os.path.join(self.td, "vol0", "bar", "dir") + assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "dir") assert os.path.isdir(gdf._datadir) saved_tmppath = dw._tmppath assert os.path.dirname(saved_tmppath) == gdf._datadir @@ -922,7 +922,7 @@ class TestDiskFile(unittest.TestCase): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z") saved_tmppath = '' with gdf.create() as dw: - assert gdf._datadir == os.path.join(self.td, "vol0", "bar", "dir") + assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "dir") assert os.path.isdir(gdf._datadir) saved_tmppath = dw._tmppath assert os.path.dirname(saved_tmppath) == gdf._datadir @@ -937,7 +937,7 @@ class TestDiskFile(unittest.TestCase): gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z") saved_tmppath = '' with gdf.create() as dw: - assert gdf._datadir == os.path.join(self.td, "vol0", "bar", "dir") + assert gdf._datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "dir") assert os.path.isdir(gdf._datadir) saved_tmppath = dw._tmppath assert os.path.dirname(saved_tmppath) == gdf._datadir diff --git a/tox.ini b/tox.ini index 82bc69c..75fbf84 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,9 @@ setenv = VIRTUAL_ENV={envdir} NOSE_COVER_BRANCHES=1 NOSE_COVER_PACKAGE=gluster deps = - https://launchpad.net/swift/icehouse/1.13.1/+download/swift-1.13.1.tar.gz +# GitHub's .zip URL won't work! pip supports installing from git repos. +# https://pip.pypa.io/en/latest/reference/pip_install.html#git + git+https://github.com/openstack/swift.git@feature/ec -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt changedir = {toxinidir}/test/unit