x/swiftonhpss
Code Issues Proposed changes

object-storage: Initial unittest of DiskFile class

Browse Source 
If we had this ahead of time, we could have avoided the errors that
were encountered leading to the fix-account-mapping fix (see
http://review.gluster.org/4222).

This represents 100% coverage of the DiskFile module, but the coverage
report says otherwise, unfortunately. That is because the put() method
invokes eventlets during the test run, and coverage is not able to
accurately track the coverage lines properly. If one comments out the
"tpool.execute()" line in DiskFile.put() the coverage report then
reports 100% for the DiskFile module.

Additionally, we changed DiskFile.put() in four ways that should not
change its behavior:

  1. Comments were added to explain various code paths and mark
     potential issues / fixes

  2. It no longer returns a boolean value, matching the behavior of
     swift.obj.server.DiskFile.put()

  3. It no longer logs a message when it detects a directory that
     already exists, instead is raises an exception

     We believe this is okay because we cannot find a code path that
     would lead to his condition. As a result, it makes it easier to
     test all the code paths in that routine.

  4. It no longer logs a message when create_dir_object() fails, since
     create_dir_object() raises an exception on failure only

     We also modified create_dir_object() to not return a boolean as a
     result of the above behavior.

Note that by implementing these tests up to this point we found three
code paths that would have failed if encountered due to missing
imports. We also made changes to the DiskFile module to make it a bit
easier to test, also eliminating an extra stat system call when
deleting directory objects.

Change-Id: I3286de83c1ec7c5e8d6cab9354e1c4397cee7497
BUG: 870589
Signed-off-by: Peter Portante <peter.portante@redhat.com>
Reviewed-on: http://review.gluster.org/4284
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Mohammed Junaid <junaid@redhat.com>
This commit is contained in:
Peter Portante 2012-12-07 16:39:37 -05:00
parent 56f3291567
commit 4183990ef1
5 changed files with 1081 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
ufo/gluster/swift/common/DiskFile.py
View File

@ -14,10 +14,12 @@
# limitations under the License.
import os
import errno
from eventlet import tpool
from tempfile import mkstemp
from contextlib import contextmanager
from swift.common.utils import normalize_timestamp, renamer
from swift.common.exceptions import DiskFileNotExist
from gluster.swift.common.utils import mkdirs, rmdirs, validate_object, \
create_object_metadata, do_open, do_close, do_unlink, do_chown, \
do_stat, do_listdir, read_metadata, write_metadata
@ -37,6 +39,10 @@ KEEP_CACHE_SIZE = (5 * 1024 * 1024)
DISALLOWED_HEADERS = set('content-length content-type deleted etag'.split())
class AlreadyExistsAsDir(Exception):
pass
class Gluster_DiskFile(DiskFile):
"""
Manage object files on disk.
@ -142,7 +148,6 @@ class Gluster_DiskFile(DiskFile):
mkdirs(dir_path)
do_chown(dir_path, self.uid, self.gid)
create_object_metadata(dir_path)
return True
def put_metadata(self, metadata):
obj_path = self.datadir + '/' + self.obj
@ -162,7 +167,7 @@ class Gluster_DiskFile(DiskFile):
"""
if extension == '.ts':
# TombStone marker (deleted)
return True
return
# Fix up the metadata to ensure it has a proper value for the
# Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE
@ -170,6 +175,10 @@ class Gluster_DiskFile(DiskFile):
content_type = metadata['Content-Type']
if not content_type:
# FIXME: How can this be some object that evaluates to False?
#
# FIXME: If the file exists, we would already know it is a
# directory.
metadata['Content-Type'] = FILE_TYPE
x_object_type = FILE
else:
@ -178,23 +187,31 @@ class Gluster_DiskFile(DiskFile):
metadata[X_OBJECT_TYPE] = x_object_type
if extension == '.meta':
# Metadata recorded separately from the file
# Metadata recorded separately from the file, we just update the
# metadata for the file.
#
# FIXME: If the file does not exist, this call will fail.
self.put_metadata(metadata)
return True
return
# Our caller will use '.data' here; we just ignore it since we map the
# URL directly to the file system.
extension = ''
if metadata[X_OBJECT_TYPE] == MARKER_DIR:
# FIXME: If we know it already exists, why call
# create_dir_object()?
self.create_dir_object(os.path.join(self.datadir, self.obj))
self.put_metadata(metadata)
self.data_file = self.datadir + '/' + self.obj
return True
return
# Check if directory already exists.
if self.is_dir:
self.logger.error('Directory already exists %s/%s' % \
(self.datadir , self.obj))
return False
# FIXME: How can we have a directory and it not be marked as a
# MARKER_DIR (see above)?
raise AlreadyExistsAsDir('File object already exists ' \
'as a directory: %s/%s' % (self.datadir , self.obj))
timestamp = normalize_timestamp(metadata[X_TIMESTAMP])
write_metadata(tmppath, metadata)
@ -203,18 +220,15 @@ class Gluster_DiskFile(DiskFile):
tpool.execute(os.fsync, fd)
if self.obj_path:
dir_objs = self.obj_path.split('/')
assert len(dir_objs) >= 1
tmp_path = ''
if len(dir_objs):
for dir_name in dir_objs:
if tmp_path:
tmp_path = tmp_path + '/' + dir_name
else:
tmp_path = dir_name
if not self.create_dir_object(os.path.join(self.container_path,
tmp_path)):
self.logger.error("Failed in subdir %s",\
os.path.join(self.container_path,tmp_path))
return False
for dir_name in dir_objs:
if tmp_path:
tmp_path = tmp_path + '/' + dir_name
else:
tmp_path = dir_name
self.create_dir_object(
os.path.join(self.container_path, tmp_path))
renamer(tmppath, os.path.join(self.datadir,
self.obj + extension))
@ -222,7 +236,7 @@ class Gluster_DiskFile(DiskFile):
self.uid, self.gid)
self.metadata = metadata
self.data_file = self.datadir + '/' + self.obj + extension
return True
return
def unlinkold(self, timestamp):
"""
@ -231,33 +245,19 @@ class Gluster_DiskFile(DiskFile):
:param timestamp: timestamp to compare with each file
"""
if self.metadata and self.metadata['X-Timestamp'] != timestamp:
self.unlink()
def unlink(self):
"""
Remove the file.
"""
#Marker dir.
if self.is_dir:
rmdirs(os.path.join(self.datadir, self.obj))
if not os.path.isdir(os.path.join(self.datadir, self.obj)):
self.metadata = {}
self.data_file = None
else:
logging.error('Unable to delete dir %s' % os.path.join(self.datadir, self.obj))
if not self.metadata or self.metadata['X-Timestamp'] >= timestamp:
return
for fname in do_listdir(self.datadir):
if fname == self.obj:
try:
if self.is_dir:
# Marker directory object
if not rmdirs(os.path.join(self.datadir, self.obj)):
logging.error('Unable to delete dir %s' % os.path.join(self.datadir, self.obj))
return
else:
# File object
for fname in do_listdir(self.datadir):
if fname == self.obj:
do_unlink(os.path.join(self.datadir, fname))
except OSError, err:
if err.errno != errno.ENOENT:
raise
#Remove entire path for object.
#remove_dir_path(self.obj_path, self.container_path)
self.metadata = {}
self.data_file = None
@ -286,7 +286,7 @@ class Gluster_DiskFile(DiskFile):
self.update_object(self.metadata)
return file_size
except OSError, err:
except OSError as err:
if err.errno != errno.ENOENT:
raise
raise DiskFileNotExist('Data File does not exist.')

13
ufo/gluster/swift/common/fs_utils.py
View File

@ -101,7 +101,10 @@ def do_rmdir(path):
logging.exception("Rmdir failed on %s err: %s", path, str(err))
if err.errno != errno.ENOENT:
raise
return True
res = False
else:
res = True
return res
def do_rename(old_path, new_path):
try:
@ -149,8 +152,8 @@ def dir_empty(path):
return True
def rmdirs(path):
if os.path.isdir(path) and dir_empty(path):
do_rmdir(path)
else:
logging.error("rmdirs failed dir may not be empty or not valid dir")
if not os.path.isdir(path) or not dir_empty(path):
logging.error("rmdirs failed: %s may not be empty or not valid dir", path)
return False
return do_rmdir(path)

49
ufo/test/__init__.py
View File

@ -0,0 +1,49 @@
# See http://code.google.com/p/python-nose/issues/detail?id=373
# The code below enables nosetests to work with i18n _() blocks
import __builtin__
import sys
import os
from ConfigParser import MissingSectionHeaderError
from StringIO import StringIO
from swift.common.utils import readconf
setattr(__builtin__, '_', lambda x: x)
# Work around what seems to be a Python bug.
# c.f. https://bugs.launchpad.net/swift/+bug/820185.
import logging
logging.raiseExceptions = False
def get_config(section_name=None, defaults=None):
"""
Attempt to get a test config dictionary.
:param section_name: the section to read (all sections if not defined)
:param defaults: an optional dictionary namespace of defaults
"""
config_file = os.environ.get('SWIFT_TEST_CONFIG_FILE',
'/etc/swift/test.conf')
config = {}
if defaults is not None:
config.update(defaults)
try:
config = readconf(config_file, section_name)
except SystemExit:
if not os.path.exists(config_file):
print >>sys.stderr, \
'Unable to read test config %s - file not found' \
% config_file
elif not os.access(config_file, os.R_OK):
print >>sys.stderr, \
'Unable to read test config %s - permission denied' \
% config_file
else:
print >>sys.stderr, \
'Unable to read test config %s - section %s not found' \
% (config_file, section_name)
return config

95
ufo/test/unit/__init__.py
View File

@ -0,0 +1,95 @@
""" Gluster Swift Unit Tests """
import logging
from collections import defaultdict
from test import get_config
from swift.common.utils import TRUE_VALUES
class NullLoggingHandler(logging.Handler):
def emit(self, record):
pass
class FakeLogger(object):
# a thread safe logger
def __init__(self, *args, **kwargs):
self._clear()
self.level = logging.NOTSET
if 'facility' in kwargs:
self.facility = kwargs['facility']
def _clear(self):
self.log_dict = defaultdict(list)
def _store_in(store_name):
def stub_fn(self, *args, **kwargs):
self.log_dict[store_name].append((args, kwargs))
return stub_fn
error = _store_in('error')
info = _store_in('info')
warning = _store_in('warning')
debug = _store_in('debug')
def exception(self, *args, **kwargs):
self.log_dict['exception'].append((args, kwargs, str(exc_info()[1])))
# mock out the StatsD logging methods:
increment = _store_in('increment')
decrement = _store_in('decrement')
timing = _store_in('timing')
timing_since = _store_in('timing_since')
update_stats = _store_in('update_stats')
set_statsd_prefix = _store_in('set_statsd_prefix')
def setFormatter(self, obj):
self.formatter = obj
def close(self):
self._clear()
def set_name(self, name):
# don't touch _handlers
self._name = name
def acquire(self):
pass
def release(self):
pass
def createLock(self):
pass
def emit(self, record):
pass
def handle(self, record):
pass
def flush(self):
pass
def handleError(self, record):
pass
original_syslog_handler = logging.handlers.SysLogHandler
def fake_syslog_handler():
for attr in dir(original_syslog_handler):
if attr.startswith('LOG'):
setattr(FakeLogger, attr,
copy.copy(getattr(logging.handlers.SysLogHandler, attr)))
FakeLogger.priority_map = \
copy.deepcopy(logging.handlers.SysLogHandler.priority_map)
logging.handlers.SysLogHandler = FakeLogger
if get_config('unit_test').get('fake_syslog', 'False').lower() in TRUE_VALUES:
fake_syslog_handler()

884
ufo/test/unit/common/test_diskfile.py Normal file
View File

@ -0,0 +1,884 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Tests for gluster.swift.common.DiskFile """
import os
import stat
import errno
import unittest
import tempfile
import shutil
from hashlib import md5
from swift.common.utils import normalize_timestamp
from swift.common.exceptions import DiskFileNotExist
import gluster.swift.common.DiskFile
import gluster.swift.common.utils
from gluster.swift.common.DiskFile import Gluster_DiskFile, \
AlreadyExistsAsDir
from gluster.swift.common.utils import DEFAULT_UID, DEFAULT_GID, X_TYPE, \
X_OBJECT_TYPE
from test_utils import _initxattr, _destroyxattr
from test.unit import FakeLogger
_metadata = {}
def _mock_read_metadata(filename):
if filename in _metadata:
md = _metadata[filename]
else:
md = {}
return md
def _mock_write_metadata(filename, metadata):
_metadata[filename] = metadata
def _mock_clear_metadata():
_metadata = {}
class MockException(Exception):
pass
def _mock_rmdirs(p):
raise MockException("gluster.swift.common.DiskFile.rmdirs() called")
def _mock_do_listdir(p):
raise MockException("gluster.swift.common.DiskFile.do_listdir() called")
def _mock_do_unlink(f):
ose = OSError()
ose.errno = errno.ENOENT
raise ose
class MockRenamerCalled(Exception):
pass
def _mock_renamer(a, b):
raise MockRenamerCalled()
class TestDiskFile(unittest.TestCase):
""" Tests for gluster.swift.common.DiskFile """
def setUp(self):
self.lg = FakeLogger()
_initxattr()
_mock_clear_metadata()
self._saved_df_wm = gluster.swift.common.DiskFile.write_metadata
self._saved_df_rm = gluster.swift.common.DiskFile.read_metadata
gluster.swift.common.DiskFile.write_metadata = _mock_write_metadata
gluster.swift.common.DiskFile.read_metadata = _mock_read_metadata
self._saved_ut_wm = gluster.swift.common.utils.write_metadata
self._saved_ut_rm = gluster.swift.common.utils.read_metadata
gluster.swift.common.utils.write_metadata = _mock_write_metadata
gluster.swift.common.utils.read_metadata = _mock_read_metadata
def tearDown(self):
self.lg = None
_destroyxattr()
gluster.swift.common.DiskFile.write_metadata = self._saved_df_wm
gluster.swift.common.DiskFile.read_metadata = self._saved_df_rm
gluster.swift.common.utils.write_metadata = self._saved_ut_wm
gluster.swift.common.utils.read_metadata = self._saved_ut_rm
def test_constructor_no_slash(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.obj == "z"
assert gdf.obj_path == ""
assert gdf.name == "bar"
assert gdf.datadir == "/tmp/foo/vol0/bar"
assert gdf.device_path == "/tmp/foo/vol0"
assert gdf.container_path == "/tmp/foo/vol0/bar"
assert gdf.tmpdir == "/tmp/foo/vol0/tmp"
assert gdf.disk_chunk_size == 65536
assert gdf.logger == self.lg
assert gdf.uid == DEFAULT_UID
assert gdf.gid == DEFAULT_GID
assert gdf.metadata == {}
assert gdf.data_file == None
assert gdf.fp == None
assert gdf.iter_etag == None
assert not gdf.started_at_0
assert not gdf.read_to_eof
assert gdf.quarantined_dir == None
assert not gdf.keep_cache
assert not gdf.is_dir
assert gdf.is_valid
def test_constructor_leadtrail_slash(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"/b/a/z/", self.lg)
assert gdf.obj == "z"
assert gdf.obj_path == "b/a"
assert gdf.name == "bar/b/a"
assert gdf.datadir == "/tmp/foo/vol0/bar/b/a"
assert gdf.device_path == "/tmp/foo/vol0"
assert gdf.tmpdir == "/tmp/foo/vol0/tmp"
def test_constructor_no_metadata(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
stats = os.stat(the_file)
ts = normalize_timestamp(stats.st_ctime)
etag = md5()
etag.update("1234")
etag = etag.hexdigest()
exp_md = {
'Content-Length': 4,
'ETag': etag,
'X-Timestamp': ts,
'Content-Type': 'application/octet-stream'}
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.obj == "z"
assert gdf.data_file == the_file
assert not gdf.is_dir
assert gdf.fp is None
assert gdf.metadata == exp_md
finally:
shutil.rmtree(td)
def test_constructor_existing_metadata(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[the_file] = ini_md
exp_md = ini_md.copy()
del exp_md['X-Type']
del exp_md['X-Object-Type']
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.obj == "z"
assert gdf.data_file == the_file
assert not gdf.is_dir
assert gdf.fp is None
assert gdf.metadata == exp_md
finally:
shutil.rmtree(td)
def test_constructor_invalid_existing_metadata(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
inv_md = {
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[the_file] = inv_md
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.obj == "z"
assert gdf.data_file == the_file
assert not gdf.is_dir
assert gdf.fp is None
assert gdf.metadata != inv_md
finally:
shutil.rmtree(td)
def test_constructor_isdir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
try:
os.makedirs(the_dir)
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'dir',
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[the_dir] = ini_md
exp_md = ini_md.copy()
del exp_md['X-Type']
del exp_md['X-Object-Type']
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"d", self.lg, keep_data_fp=True)
assert gdf.obj == "d"
assert gdf.data_file == the_dir
assert gdf.is_dir
assert gdf.fp is None
assert gdf.metadata == exp_md
finally:
shutil.rmtree(td)
def test_constructor_keep_data_fp(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg, keep_data_fp=True)
assert gdf.obj == "z"
assert gdf.data_file == the_file
assert not gdf.is_dir
assert gdf.fp is not None
finally:
shutil.rmtree(td)
def test_constructor_chunk_size(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg, disk_chunk_size=8192)
assert gdf.disk_chunk_size == 8192
def test_close(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
# Should be a no-op, as by default is_dir is False, but fp is None
gdf.close()
gdf.is_dir = True
gdf.fp = "123"
# Should still be a no-op as is_dir is True (marker directory)
gdf.close()
assert gdf.fp == "123"
gdf.is_dir = False
saved_dc = gluster.swift.common.DiskFile.do_close
self.called = False
def our_do_close(fp):
self.called = True
gluster.swift.common.DiskFile.do_close = our_do_close
try:
gdf.close()
assert self.called
assert gdf.fp is None
finally:
gluster.swift.common.DiskFile.do_close = saved_dc
def test_is_deleted(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.is_deleted()
gdf.data_file = "/tmp/foo/bar"
assert not gdf.is_deleted()
def test_create_dir_object(self):
td = tempfile.mkdtemp()
the_dir = os.path.join(td, "vol0", "bar", "dir")
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
# Not created, dir object path is different, just checking
assert gdf.obj == "z"
gdf.create_dir_object(the_dir)
assert os.path.isdir(the_dir)
assert the_dir in _metadata
finally:
shutil.rmtree(td)
def test_create_dir_object_exists(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
os.makedirs(the_path)
with open(the_dir, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
# Not created, dir object path is different, just checking
assert gdf.obj == "z"
def _mock_do_chown(p, u, g):
assert u == DEFAULT_UID
assert g == DEFAULT_GID
dc = gluster.swift.common.DiskFile.do_chown
gluster.swift.common.DiskFile.do_chown = _mock_do_chown
try:
gdf.create_dir_object(the_dir)
finally:
gluster.swift.common.DiskFile.do_chown = dc
assert os.path.isdir(the_dir)
assert the_dir in _metadata
finally:
shutil.rmtree(td)
def test_put_metadata(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
md = { 'a': 'b' }
gdf.put_metadata(md)
assert gdf.metadata == md
assert _metadata[os.path.join(the_dir, "z")] == md
finally:
shutil.rmtree(td)
def test_put_w_tombstone(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.metadata == {}
gdf.put(None, '', {'x': '1'}, extension='.ts')
assert gdf.metadata == {}
def test_put_w_meta_file(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
newmd = gdf.metadata.copy()
newmd['X-Object-Meta-test'] = '1234'
with gdf.mkstemp() as (fd, tmppath):
gdf.put(fd, tmppath, newmd, extension='.meta')
assert gdf.metadata == newmd
assert _metadata[the_file] == newmd
finally:
shutil.rmtree(td)
def test_put_w_meta_file_no_content_type(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
newmd = gdf.metadata.copy()
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
with gdf.mkstemp() as (fd, tmppath):
gdf.put(fd, tmppath, newmd, extension='.meta')
assert gdf.metadata == newmd
assert _metadata[the_file] == newmd
finally:
shutil.rmtree(td)
def test_put_w_meta_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir", self.lg)
newmd = gdf.metadata.copy()
newmd['X-Object-Meta-test'] = '1234'
gdf.put(None, None, newmd, extension='.meta')
assert gdf.metadata == newmd
assert _metadata[the_dir] == newmd
finally:
shutil.rmtree(td)
def test_put_w_marker_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir", self.lg)
newmd = gdf.metadata.copy()
newmd['X-Object-Meta-test'] = '1234'
gdf.put(None, None, newmd, extension='.data')
assert gdf.metadata == newmd
assert _metadata[the_dir] == newmd
finally:
shutil.rmtree(td)
def test_put_is_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir", self.lg)
origmd = gdf.metadata.copy()
origfmd = _metadata[the_dir]
newmd = gdf.metadata.copy()
# FIXME: This is a hack to get to the code-path; it is not clear
# how this can happen normally.
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
try:
gdf.put(None, None, newmd, extension='.data')
except AlreadyExistsAsDir:
pass
else:
self.fail("Expected to encounter 'already-exists-as-dir' exception")
assert gdf.metadata == origmd
assert _metadata[the_dir] == origfmd
finally:
shutil.rmtree(td)
def test_put(self):
td = tempfile.mkdtemp()
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.obj == "z"
assert gdf.obj_path == ""
assert gdf.name == "bar"
assert gdf.datadir == os.path.join(td, "vol0", "bar")
assert gdf.data_file is None
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
with gdf.mkstemp() as (fd, tmppath):
os.write(fd, body)
gdf.put(fd, tmppath, metadata)
assert gdf.data_file == os.path.join(td, "vol0", "bar", "z")
assert os.path.exists(gdf.data_file)
assert not os.path.exists(tmppath)
finally:
shutil.rmtree(td)
def test_put_obj_path(self):
the_obj_path = os.path.join("b", "a")
the_file = os.path.join(the_obj_path, "z")
td = tempfile.mkdtemp()
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
the_file, self.lg)
assert gdf.obj == "z"
assert gdf.obj_path == the_obj_path
assert gdf.name == os.path.join("bar", "b", "a")
assert gdf.datadir == os.path.join(td, "vol0", "bar", "b", "a")
assert gdf.data_file is None
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
with gdf.mkstemp() as (fd, tmppath):
os.write(fd, body)
gdf.put(fd, tmppath, metadata)
assert gdf.data_file == os.path.join(td, "vol0", "bar", "b", "a", "z")
assert os.path.exists(gdf.data_file)
assert not os.path.exists(tmppath)
finally:
shutil.rmtree(td)
def test_unlinkold_no_metadata(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.metadata == {}
_saved_rmdirs = gluster.swift.common.DiskFile.rmdirs
_saved_do_listdir = gluster.swift.common.DiskFile.do_listdir
gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs
gluster.swift.common.DiskFile.do_listdir = _mock_do_listdir
try:
gdf.unlinkold(None)
except MockException as exp:
self.fail(str(exp))
finally:
gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs
gluster.swift.common.DiskFile.do_listdir = _saved_do_listdir
def test_unlinkold_same_timestamp(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.metadata == {}
gdf.metadata['X-Timestamp'] = 1
_saved_rmdirs = gluster.swift.common.DiskFile.rmdirs
_saved_do_listdir = gluster.swift.common.DiskFile.do_listdir
gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs
gluster.swift.common.DiskFile.do_listdir = _mock_do_listdir
try:
gdf.unlinkold(1)
except MockException as exp:
self.fail(str(exp))
finally:
gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs
gluster.swift.common.DiskFile.do_listdir = _saved_do_listdir
def test_unlinkold_file(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.obj == "z"
assert gdf.data_file == the_file
assert not gdf.is_dir
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
assert os.path.isdir(gdf.datadir)
assert not os.path.exists(os.path.join(gdf.datadir, gdf.obj))
finally:
shutil.rmtree(td)
def test_unlinkold_file_not_found(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.obj == "z"
assert gdf.data_file == the_file
assert not gdf.is_dir
# Handle the case the file is not in the directory listing.
os.unlink(the_file)
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
assert os.path.isdir(gdf.datadir)
assert not os.path.exists(os.path.join(gdf.datadir, gdf.obj))
finally:
shutil.rmtree(td)
def test_unlinkold_file_unlink_error(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.obj == "z"
assert gdf.data_file == the_file
assert not gdf.is_dir
later = float(gdf.metadata['X-Timestamp']) + 1
stats = os.stat(the_path)
os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR))
# Handle the case do_unlink() raises an OSError
try:
gdf.unlinkold(normalize_timestamp(later))
except OSError as e:
assert e.errno != errno.ENOENT
else:
self.fail("Excepted an OSError when unlinking file")
finally:
os.chmod(the_path, stats.st_mode)
assert os.path.isdir(gdf.datadir)
assert os.path.exists(os.path.join(gdf.datadir, gdf.obj))
finally:
shutil.rmtree(td)
def test_unlinkold_is_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"d", self.lg, keep_data_fp=True)
assert gdf.data_file == the_dir
assert gdf.is_dir
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
assert os.path.isdir(gdf.datadir)
assert not os.path.exists(os.path.join(gdf.datadir, gdf.obj))
finally:
shutil.rmtree(td)
def test_unlinkold_is_dir_failure(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"d", self.lg, keep_data_fp=True)
assert gdf.data_file == the_dir
assert gdf.is_dir
stats = os.stat(gdf.datadir)
os.chmod(gdf.datadir, 0)
try:
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
finally:
os.chmod(gdf.datadir, stats.st_mode)
assert os.path.isdir(gdf.datadir)
assert os.path.isdir(gdf.data_file)
finally:
shutil.rmtree(td)
def test_get_data_file_size(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.obj == "z"
assert gdf.data_file == the_file
assert not gdf.is_dir
assert 4 == gdf.get_data_file_size()
finally:
shutil.rmtree(td)
def test_get_data_file_size(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.obj == "z"
assert gdf.data_file == the_file
assert not gdf.is_dir
assert 4 == gdf.metadata['Content-Length']
gdf.metadata['Content-Length'] = 3
assert 4 == gdf.get_data_file_size()
assert 4 == gdf.metadata['Content-Length']
finally:
shutil.rmtree(td)
def test_get_data_file_size_dne(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"/b/a/z/", self.lg)
try:
s = gdf.get_data_file_size()
except DiskFileNotExist:
pass
else:
self.fail("Expected DiskFileNotExist exception")
def test_get_data_file_size_dne_os_err(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.obj == "z"
assert gdf.data_file == the_file
assert not gdf.is_dir
gdf.data_file = gdf.data_file + ".dne"
try:
s = gdf.get_data_file_size()
except DiskFileNotExist:
pass
else:
self.fail("Expected DiskFileNotExist exception")
finally:
shutil.rmtree(td)
def test_get_data_file_size_os_err(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.obj == "z"
assert gdf.data_file == the_file
assert not gdf.is_dir
stats = os.stat(the_path)
os.chmod(the_path, 0)
try:
s = gdf.get_data_file_size()
except OSError as err:
assert err.errno != errno.ENOENT
else:
self.fail("Expected OSError exception")
finally:
os.chmod(the_path, stats.st_mode)
finally:
shutil.rmtree(td)
def test_get_data_file_size_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"d", self.lg, keep_data_fp=True)
assert gdf.obj == "d"
assert gdf.data_file == the_dir
assert gdf.is_dir
assert 0 == gdf.get_data_file_size()
finally:
shutil.rmtree(td)
def test_update_object(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
md = { 'a': 'b' }
gdf.update_object(md)
assert gdf.metadata == md
assert _metadata[os.path.join(the_dir, "z")] == md
finally:
shutil.rmtree(td)
def test_filter_metadata(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.metadata == {}
gdf.filter_metadata()
assert gdf.metadata == {}
gdf.metadata[X_TYPE] = 'a'
gdf.metadata[X_OBJECT_TYPE] = 'b'
gdf.metadata['foobar'] = 'c'
gdf.filter_metadata()
assert X_TYPE not in gdf.metadata
assert X_OBJECT_TYPE not in gdf.metadata
assert 'foobar' in gdf.metadata
def test_mkstemp(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
saved_tmppath = ''
with gdf.mkstemp() as (fd, tmppath):
assert gdf.tmpdir == os.path.join(td, "vol0", "tmp")
assert os.path.isdir(gdf.tmpdir)
assert os.path.dirname(tmppath) == gdf.tmpdir
assert os.path.exists(tmppath)
saved_tmppath = tmppath
os.write(fd, "123")
assert not os.path.exists(saved_tmppath)
finally:
shutil.rmtree(td)
def test_mkstemp_err_on_close(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
saved_tmppath = ''
with gdf.mkstemp() as (fd, tmppath):
assert gdf.tmpdir == os.path.join(td, "vol0", "tmp")
assert os.path.isdir(gdf.tmpdir)
assert os.path.dirname(tmppath) == gdf.tmpdir
assert os.path.exists(tmppath)
saved_tmppath = tmppath
os.write(fd, "123")
os.close(fd)
assert not os.path.exists(saved_tmppath)
finally:
shutil.rmtree(td)
def test_mkstemp_err_on_unlink(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
saved_tmppath = ''
with gdf.mkstemp() as (fd, tmppath):
assert gdf.tmpdir == os.path.join(td, "vol0", "tmp")
assert os.path.isdir(gdf.tmpdir)
assert os.path.dirname(tmppath) == gdf.tmpdir
assert os.path.exists(tmppath)
saved_tmppath = tmppath
os.write(fd, "123")
os.unlink(tmppath)
assert not os.path.exists(saved_tmppath)
finally:
shutil.rmtree(td)