swiftonfile/test/unit/obj/test_diskfile.py

# Copyright (c) 2012-2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Tests for swiftonfile.swift.obj.diskfile """

import os
import stat
import errno
import unittest
import tempfile
import shutil
import mock
from eventlet import tpool
from mock import Mock, patch
from hashlib import md5
from copy import deepcopy
from contextlib import nested
from swiftonfile.swift.common.exceptions import AlreadyExistsAsDir, \
    AlreadyExistsAsFile
from swift.common.exceptions import DiskFileNoSpace, DiskFileNotOpen, \
    DiskFileNotExist, DiskFileExpired
from swift.common.utils import ThreadPool

from swiftonfile.swift.common.exceptions import SwiftOnFileSystemOSError
import swiftonfile.swift.common.utils
from swiftonfile.swift.common.utils import normalize_timestamp
import swiftonfile.swift.obj.diskfile
from swiftonfile.swift.obj.diskfile import DiskFileWriter, DiskFileManager
from swiftonfile.swift.common.utils import DEFAULT_UID, DEFAULT_GID, \
    X_OBJECT_TYPE, DIR_OBJECT

from test.unit.common.test_utils import _initxattr, _destroyxattr
from test.unit import FakeLogger

_metadata = {}


def _mapit(filename_or_fd):
    if isinstance(filename_or_fd, int):
        statmeth = os.fstat
    else:
        statmeth = os.lstat
    stats = statmeth(filename_or_fd)
    return stats.st_ino


def _mock_read_metadata(filename_or_fd):
    global _metadata
    ino = _mapit(filename_or_fd)
    if ino in _metadata:
        md = _metadata[ino]
    else:
        md = {}
    return md


def _mock_write_metadata(filename_or_fd, metadata):
    global _metadata
    ino = _mapit(filename_or_fd)
    _metadata[ino] = metadata


def _mock_clear_metadata():
    global _metadata
    _metadata = {}


class MockException(Exception):
    pass


def _mock_rmobjdir(p):
    raise MockException("swiftonfile.swift.obj.diskfile.rmobjdir() called")


def _mock_do_fsync(fd):
    return


def _mock_fallocate(fd, size):
    return


class MockRenamerCalled(Exception):
    pass


def _mock_renamer(a, b):
    raise MockRenamerCalled()

class TestDiskFileWriter(unittest.TestCase):
    """ Tests for swiftonfile.swift.obj.diskfile.DiskFileWriter """

    def test_close(self):
        dw = DiskFileWriter(100, 'a', None)
        mock_close = Mock()
        with patch("swiftonfile.swift.obj.diskfile.do_close", mock_close):
            # It should call do_close
            self.assertEqual(100, dw._fd)
            dw.close()
            self.assertEqual(1, mock_close.call_count)
            self.assertEqual(None, dw._fd)

            # It should not call do_close since it should
            # have made fd equal to None
            dw.close()
            self.assertEqual(None, dw._fd)
            self.assertEqual(1, mock_close.call_count)

class TestDiskFile(unittest.TestCase):
    """ Tests for swiftonfile.swift.obj.diskfile """

    def setUp(self):
        self._orig_tpool_exc = tpool.execute
        tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs)
        self.lg = FakeLogger()
        _initxattr()
        _mock_clear_metadata()
        self._saved_df_wm = swiftonfile.swift.obj.diskfile.write_metadata
        self._saved_df_rm = swiftonfile.swift.obj.diskfile.read_metadata
        swiftonfile.swift.obj.diskfile.write_metadata = _mock_write_metadata
        swiftonfile.swift.obj.diskfile.read_metadata = _mock_read_metadata
        self._saved_ut_wm = swiftonfile.swift.common.utils.write_metadata
        self._saved_ut_rm = swiftonfile.swift.common.utils.read_metadata
        swiftonfile.swift.common.utils.write_metadata = _mock_write_metadata
        swiftonfile.swift.common.utils.read_metadata = _mock_read_metadata
        self._saved_do_fsync = swiftonfile.swift.obj.diskfile.do_fsync
        swiftonfile.swift.obj.diskfile.do_fsync = _mock_do_fsync
        self._saved_fallocate = swiftonfile.swift.obj.diskfile.fallocate
        swiftonfile.swift.obj.diskfile.fallocate = _mock_fallocate
        self.td = tempfile.mkdtemp()
        self.conf = dict(devices=self.td, mb_per_sync=2,
                         keep_cache_size=(1024 * 1024), mount_check=False)
        self.mgr = DiskFileManager(self.conf, self.lg)

    def tearDown(self):
        tpool.execute = self._orig_tpool_exc
        self.lg = None
        _destroyxattr()
        swiftonfile.swift.obj.diskfile.write_metadata = self._saved_df_wm
        swiftonfile.swift.obj.diskfile.read_metadata = self._saved_df_rm
        swiftonfile.swift.common.utils.write_metadata = self._saved_ut_wm
        swiftonfile.swift.common.utils.read_metadata = self._saved_ut_rm
        swiftonfile.swift.obj.diskfile.do_fsync = self._saved_do_fsync
        swiftonfile.swift.obj.diskfile.fallocate = self._saved_fallocate
        shutil.rmtree(self.td)

    def _get_diskfile(self, d, p, a, c, o, **kwargs):
        return self.mgr.get_diskfile(d, p, a, c, o, **kwargs)

    def test_constructor_no_slash(self):
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        assert gdf._mgr is self.mgr
        assert gdf._device_path == os.path.join(self.td, "vol0")
        assert isinstance(gdf._threadpool, ThreadPool)
        assert gdf._uid == DEFAULT_UID
        assert gdf._gid == DEFAULT_GID
        assert gdf._obj == "z"
        assert gdf._obj_path == ""
        assert gdf._put_datadir == os.path.join(self.td, "vol0", "ufo47", "bar"), gdf._put_datadir
        assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
        assert gdf._is_dir is False
        assert gdf._fd is None

    def test_constructor_leadtrail_slash(self):
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "/b/a/z/")
        assert gdf._obj == "z"
        assert gdf._obj_path == "b/a"
        assert gdf._put_datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "b", "a"), gdf._put_datadir

    def test_open_no_metadata(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "wb") as fd:
            fd.write("1234")
        stats = os.stat(the_file)
        ts = normalize_timestamp(stats.st_ctime)
        etag = md5()
        etag.update("1234")
        etag = etag.hexdigest()
        exp_md = {
            'Content-Length': 4,
            'ETag': etag,
            'X-Timestamp': ts,
            'X-Object-PUT-Mtime': normalize_timestamp(stats.st_mtime),
            'Content-Type': 'application/octet-stream'}
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        assert gdf._obj == "z"
        assert gdf._fd is None
        assert gdf._disk_file_open is False
        assert gdf._metadata is None
        assert not gdf._is_dir
        with gdf.open():
            assert gdf._data_file == the_file
            assert not gdf._is_dir
            assert gdf._fd is not None
            assert gdf._metadata == exp_md
            assert gdf._disk_file_open is True
        assert gdf._disk_file_open is False

    def test_read_metadata_optimize_open_close(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "wb") as fd:
            fd.write("1234")
        init_md = {
            'X-Type': 'Object',
            'X-Object-Type': 'file',
            'Content-Length': 4,
            'ETag': md5("1234").hexdigest(),
            'X-Timestamp': normalize_timestamp(os.stat(the_file).st_ctime),
            'Content-Type': 'application/octet-stream'}
        _metadata[_mapit(the_file)] = init_md
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        assert gdf._obj == "z"
        assert gdf._fd is None
        assert gdf._disk_file_open is False
        assert gdf._metadata is None
        assert not gdf._is_dir

        # Case 1
        # Ensure that reading metadata for non-GET requests
        # does not incur opening and closing the file when
        # metadata is NOT stale.
        mock_open = Mock()
        mock_close = Mock()
        with mock.patch("swiftonfile.swift.obj.diskfile.do_open", mock_open):
            with mock.patch("swiftonfile.swift.obj.diskfile.do_close",
                            mock_close):
                md = gdf.read_metadata()
                self.assertEqual(md, init_md)
        self.assertFalse(mock_open.called)
        self.assertFalse(mock_close.called)

        # Case 2
        # Ensure that reading metadata for non-GET requests
        # still opens and reads the file when metadata is stale
        with open(the_file, "a") as fd:
            # Append to the existing file to make the stored metadata
            # invalid/stale.
            fd.write("5678")
        md = gdf.read_metadata()
        # Check that the stale metadata is recalculated to account for
        # change in file content
        self.assertNotEqual(md, init_md)
        self.assertEqual(md['Content-Length'], 8)
        self.assertEqual(md['ETag'], md5("12345678").hexdigest())

    def test_open_and_close(self):
        mock_close = Mock()

        with mock.patch("swiftonfile.swift.obj.diskfile.do_close", mock_close):
            gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47",
                                                "bar", "z")
            with gdf.open():
                assert gdf._fd is not None
            self.assertTrue(mock_close.called)

    def test_open_existing_metadata(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "wb") as fd:
            fd.write("1234")
        ini_md = {
            'X-Type': 'Object',
            'X-Object-Type': 'file',
            'Content-Length': 4,
            'ETag': 'etag',
            'X-Timestamp': 'ts',
            'Content-Type': 'application/loctet-stream'}
        _metadata[_mapit(the_file)] = ini_md
        exp_md = ini_md.copy()
        del exp_md['X-Type']
        del exp_md['X-Object-Type']
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        assert gdf._obj == "z"
        assert gdf._fd is None
        assert gdf._metadata is None
        assert gdf._disk_file_open is False
        assert not gdf._is_dir
        with gdf.open():
            assert not gdf._is_dir
            assert gdf._data_file == the_file
            assert gdf._fd is not None
            assert gdf._metadata == exp_md, "%r != %r" % (gdf._metadata, exp_md)
            assert gdf._disk_file_open is True
        assert gdf._disk_file_open is False

    def test_open_invalid_existing_metadata(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "wb") as fd:
            fd.write("1234")
        inv_md = {
            'Content-Length': 5,
            'ETag': 'etag',
            'X-Timestamp': 'ts',
            'Content-Type': 'application/loctet-stream'}
        _metadata[_mapit(the_file)] = inv_md
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        assert gdf._obj == "z"
        assert not gdf._is_dir
        assert gdf._fd is None
        assert gdf._disk_file_open is False
        with gdf.open():
            assert gdf._data_file == the_file
            assert gdf._metadata != inv_md
            assert gdf._disk_file_open is True
        assert gdf._disk_file_open is False

    def test_open_isdir(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_dir = os.path.join(the_path, "d")
        os.makedirs(the_dir)
        ini_md = {
            'X-Type': 'Object',
            'X-Object-Type': 'dir',
            'Content-Length': 5,
            'ETag': 'etag',
            'X-Timestamp': 'ts',
            'Content-Type': 'application/loctet-stream'}
        _metadata[_mapit(the_dir)] = ini_md
        exp_md = ini_md.copy()
        del exp_md['X-Type']
        del exp_md['X-Object-Type']
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
        assert gdf._obj == "d"
        assert gdf._is_dir is False
        assert gdf._disk_file_open is False
        with gdf.open():
            assert gdf._is_dir
            assert gdf._data_file == the_dir
            assert gdf._disk_file_open is True
        assert gdf._disk_file_open is False

    def _create_and_get_diskfile(self, dev, par, acc, con, obj, fsize=256):
        # FIXME: assumes account === volume
        the_path = os.path.join(self.td, dev, acc, con)
        the_file = os.path.join(the_path, obj)
        base_obj = os.path.basename(the_file)
        base_dir = os.path.dirname(the_file)
        os.makedirs(base_dir)
        with open(the_file, "wb") as fd:
            fd.write("y" * fsize)
        gdf = self._get_diskfile(dev, par, acc, con, obj)
        assert gdf._obj == base_obj
        assert not gdf._is_dir
        assert gdf._fd is None
        return gdf

    def test_reader(self):
        closed = [False]
        fd = [-1]

        def mock_close(*args, **kwargs):
            closed[0] = True
            os.close(fd[0])

        with mock.patch("swiftonfile.swift.obj.diskfile.do_close", mock_close):
            gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
            with gdf.open():
                assert gdf._fd is not None
                assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
                reader = gdf.reader()
            assert reader._fd is not None
            fd[0] = reader._fd
            chunks = [ck for ck in reader]
            assert reader._fd is None
            assert closed[0]
            assert len(chunks) == 1, repr(chunks)

    def test_reader_disk_chunk_size(self):
        conf = dict(disk_chunk_size=64)
        conf.update(self.conf)
        self.mgr = DiskFileManager(conf, self.lg)
        gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        with gdf.open():
            reader = gdf.reader()
        try:
            assert reader._disk_chunk_size == 64
            chunks = [ck for ck in reader]
        finally:
            reader.close()
        assert len(chunks) == 4, repr(chunks)
        for chunk in chunks:
            assert len(chunk) == 64, repr(chunks)

    def test_reader_iter_hook(self):
        called = [0]

        def mock_sleep(*args, **kwargs):
            called[0] += 1

        gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        with gdf.open():
            reader = gdf.reader(iter_hook=mock_sleep)
        try:
            chunks = [ck for ck in reader]
        finally:
            reader.close()
        assert len(chunks) == 1, repr(chunks)
        assert called[0] == 1, called

    def test_reader_larger_file(self):
        closed = [False]
        fd = [-1]

        def mock_close(*args, **kwargs):
            closed[0] = True
            os.close(fd[0])

        with mock.patch("swiftonfile.swift.obj.diskfile.do_close", mock_close):
            gdf = self._create_and_get_diskfile("vol0", "p57", "ufo47", "bar", "z", fsize=1024*1024*2)
            with gdf.open():
                assert gdf._fd is not None
                assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")
                reader = gdf.reader()
            assert reader._fd is not None
            fd[0] = reader._fd
            chunks = [ck for ck in reader]
            assert reader._fd is None
            assert closed[0]

    def test_reader_dir_object(self):
        called = [False]

        def our_do_close(fd):
            called[0] = True
            os.close(fd)

        the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
        os.makedirs(os.path.join(the_cont, "dir"))
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
        with gdf.open():
            reader = gdf.reader()
        try:
            chunks = [ck for ck in reader]
            assert len(chunks) == 0, repr(chunks)
            with mock.patch("swiftonfile.swift.obj.diskfile.do_close",
                            our_do_close):
                reader.close()
            assert not called[0]
        finally:
            reader.close()

    def test_create_dir_object_no_md(self):
        the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_dir = "dir"
        os.makedirs(the_cont)
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar",
                                 os.path.join(the_dir, "z"))
        # Not created, dir object path is different, just checking
        assert gdf._obj == "z"
        gdf._create_dir_object(the_dir)
        full_dir_path = os.path.join(the_cont, the_dir)
        assert os.path.isdir(full_dir_path)
        assert _mapit(full_dir_path) not in _metadata

    def test_create_dir_object_with_md(self):
        the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_dir = "dir"
        os.makedirs(the_cont)
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar",
                                 os.path.join(the_dir, "z"))
        # Not created, dir object path is different, just checking
        assert gdf._obj == "z"
        dir_md = {'Content-Type': 'application/directory',
                  X_OBJECT_TYPE: DIR_OBJECT}
        gdf._create_dir_object(the_dir, dir_md)
        full_dir_path = os.path.join(the_cont, the_dir)
        assert os.path.isdir(full_dir_path)
        assert _mapit(full_dir_path) in _metadata

    def test_create_dir_object_exists(self):
        the_path = os.path.join(self.td, "vol0", "bar")
        the_dir = os.path.join(the_path, "dir")
        os.makedirs(the_path)
        with open(the_dir, "wb") as fd:
            fd.write("1234")
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
        # Not created, dir object path is different, just checking
        assert gdf._obj == "z"

        def _mock_do_chown(p, u, g):
            assert u == DEFAULT_UID
            assert g == DEFAULT_GID

        dc = swiftonfile.swift.obj.diskfile.do_chown
        swiftonfile.swift.obj.diskfile.do_chown = _mock_do_chown
        self.assertRaises(
            AlreadyExistsAsFile, gdf._create_dir_object, the_dir)
        swiftonfile.swift.obj.diskfile.do_chown = dc
        self.assertFalse(os.path.isdir(the_dir))
        self.assertFalse(_mapit(the_dir) in _metadata)

    def test_create_dir_object_do_stat_failure(self):
        the_path = os.path.join(self.td, "vol0", "bar")
        the_dir = os.path.join(the_path, "dir")
        os.makedirs(the_path)
        with open(the_dir, "wb") as fd:
            fd.write("1234")
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
        # Not created, dir object path is different, just checking
        assert gdf._obj == "z"

        def _mock_do_chown(p, u, g):
            assert u == DEFAULT_UID
            assert g == DEFAULT_GID

        dc = swiftonfile.swift.obj.diskfile.do_chown
        swiftonfile.swift.obj.diskfile.do_chown = _mock_do_chown
        self.assertRaises(
            AlreadyExistsAsFile, gdf._create_dir_object, the_dir)
        swiftonfile.swift.obj.diskfile.do_chown = dc
        self.assertFalse(os.path.isdir(the_dir))
        self.assertFalse(_mapit(the_dir) in _metadata)

    def test_write_metadata(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_dir = os.path.join(the_path, "z")
        os.makedirs(the_dir)
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        md = {'Content-Type': 'application/octet-stream', 'a': 'b'}
        gdf.write_metadata(md.copy())
        fmd = _metadata[_mapit(the_dir)]
        md.update({'X-Object-Type': 'file', 'X-Type': 'Object'})
        self.assertTrue(fmd['a'], md['a'])
        self.assertTrue(fmd['Content-Type'], md['Content-Type'])

    def test_add_metadata_to_existing_file(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "wb") as fd:
            fd.write("1234")
        ini_md = {
            'X-Type': 'Object',
            'X-Object-Type': 'file',
            'Content-Length': 4,
            'ETag': 'etag',
            'X-Timestamp': 'ts',
            'Content-Type': 'application/loctet-stream'}
        _metadata[_mapit(the_file)] = ini_md
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        md = {'Content-Type': 'application/octet-stream', 'a': 'b'}
        gdf.write_metadata(md.copy())
        self.assertTrue(_metadata[_mapit(the_file)]['a'], 'b')
        newmd = {'X-Object-Meta-test':'1234'}
        gdf.write_metadata(newmd.copy())
        on_disk_md = _metadata[_mapit(the_file)]
        self.assertTrue(on_disk_md['Content-Length'], 4)
        self.assertTrue(on_disk_md['X-Object-Meta-test'], '1234')
        self.assertTrue(on_disk_md['X-Type'], 'Object')
        self.assertTrue(on_disk_md['X-Object-Type'], 'file')
        self.assertTrue(on_disk_md['ETag'], 'etag')
        self.assertFalse('a' in on_disk_md)

    def test_add_md_to_existing_file_with_md_in_gdf(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "wb") as fd:
            fd.write("1234")
        ini_md = {
            'X-Type': 'Object',
            'X-Object-Type': 'file',
            'Content-Length': 4,
            'name': 'z',
            'ETag': 'etag',
            'X-Timestamp': 'ts'}
        _metadata[_mapit(the_file)] = ini_md
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")

        # make sure gdf has the _metadata
        gdf.open()
        md = {'a': 'b'}
        gdf.write_metadata(md.copy())
        self.assertTrue(_metadata[_mapit(the_file)]['a'], 'b')
        newmd = {'X-Object-Meta-test':'1234'}
        gdf.write_metadata(newmd.copy())
        on_disk_md = _metadata[_mapit(the_file)]
        self.assertTrue(on_disk_md['Content-Length'], 4)
        self.assertTrue(on_disk_md['X-Object-Meta-test'], '1234')
        self.assertFalse('a' in on_disk_md)

    def test_add_metadata_to_existing_dir(self):
        the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_dir = os.path.join(the_cont, "dir")
        os.makedirs(the_dir)
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
        self.assertEquals(gdf._metadata, None)
        init_md = {
            'X-Type': 'Object',
            'Content-Length': 0,
            'ETag': 'etag',
            'X-Timestamp': 'ts',
            'X-Object-Meta-test':'test',
            'Content-Type': 'application/directory'}
        _metadata[_mapit(the_dir)] = init_md

        md = {'X-Object-Meta-test':'test'}
        gdf.write_metadata(md.copy())
        self.assertEqual(_metadata[_mapit(the_dir)]['X-Object-Meta-test'],
                'test')
        self.assertEqual(_metadata[_mapit(the_dir)]['Content-Type'].lower(),
                'application/directory')

        # set new metadata
        newmd = {'X-Object-Meta-test2':'1234'}
        gdf.write_metadata(newmd.copy())
        self.assertEqual(_metadata[_mapit(the_dir)]['Content-Type'].lower(),
                'application/directory')
        self.assertEqual(_metadata[_mapit(the_dir)]["X-Object-Meta-test2"],
                '1234')
        self.assertEqual(_metadata[_mapit(the_dir)]['X-Object-Type'],
                DIR_OBJECT)
        self.assertFalse('X-Object-Meta-test' in _metadata[_mapit(the_dir)])


    def test_write_metadata_w_meta_file(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "wb") as fd:
            fd.write("1234")
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        newmd = deepcopy(gdf.read_metadata())
        newmd['X-Object-Meta-test'] = '1234'
        gdf.write_metadata(newmd)
        assert _metadata[_mapit(the_file)] == newmd

    def test_write_metadata_w_meta_file_no_content_type(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "wb") as fd:
            fd.write("1234")
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        newmd = deepcopy(gdf.read_metadata())
        newmd['Content-Type'] = ''
        newmd['X-Object-Meta-test'] = '1234'
        gdf.write_metadata(newmd)
        assert _metadata[_mapit(the_file)] == newmd

    def test_write_metadata_w_meta_dir(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_dir = os.path.join(the_path, "dir")
        os.makedirs(the_dir)
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
        newmd = deepcopy(gdf.read_metadata())
        newmd['X-Object-Meta-test'] = '1234'
        gdf.write_metadata(newmd)
        assert _metadata[_mapit(the_dir)] == newmd

    def test_write_metadata_w_marker_dir(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_dir = os.path.join(the_path, "dir")
        os.makedirs(the_dir)
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
        newmd = deepcopy(gdf.read_metadata())
        newmd['X-Object-Meta-test'] = '1234'
        gdf.write_metadata(newmd)
        assert _metadata[_mapit(the_dir)] == newmd

    def test_put_w_marker_dir_create(self):
        the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_dir = os.path.join(the_cont, "dir")
        os.makedirs(the_cont)
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
        assert gdf._metadata is None
        newmd = {
            'ETag': 'etag',
            'X-Timestamp': 'ts',
            'Content-Type': 'application/directory'}
        with gdf.create() as dw:
            dw.put(newmd)
        assert gdf._data_file == the_dir
        for key, val in newmd.items():
            assert _metadata[_mapit(the_dir)][key] == val
        assert _metadata[_mapit(the_dir)][X_OBJECT_TYPE] == DIR_OBJECT

    def test_put_is_dir(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_dir = os.path.join(the_path, "dir")
        os.makedirs(the_dir)
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
        with gdf.open():
            origmd = gdf.get_metadata()
        origfmd = _metadata[_mapit(the_dir)]
        newmd = deepcopy(origmd)
        # FIXME: This is a hack to get to the code-path; it is not clear
        # how this can happen normally.
        newmd['Content-Type'] = ''
        newmd['X-Object-Meta-test'] = '1234'
        with gdf.create() as dw:
            try:
                # FIXME: We should probably be able to detect in .create()
                # when the target file name already exists as a directory to
                # avoid reading the data off the wire only to fail as a
                # directory.
                dw.write('12345\n')
                dw.put(newmd)
            except AlreadyExistsAsDir:
                pass
            else:
                self.fail("Expected to encounter"
                          " 'already-exists-as-dir' exception")
        with gdf.open():
            assert gdf.get_metadata() == origmd
        assert _metadata[_mapit(the_dir)] == origfmd, "was: %r, is: %r" % (
            origfmd, _metadata[_mapit(the_dir)])

    def test_put(self):
        the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
        os.makedirs(the_cont)
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        assert gdf._obj == "z"
        assert gdf._obj_path == ""
        assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar")
        assert gdf._put_datadir == the_cont
        assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")

        body = '1234\n'
        etag = md5()
        etag.update(body)
        etag = etag.hexdigest()
        metadata = {
            'X-Timestamp': '1234',
            'Content-Type': 'file',
            'ETag': etag,
            'Content-Length': '5',
        }

        with gdf.create() as dw:
            assert dw._tmppath is not None
            tmppath = dw._tmppath
            dw.write(body)
            dw.put(metadata)

        assert os.path.exists(gdf._data_file)
        assert not os.path.exists(tmppath)

    def test_put_ENOSPC(self):
        the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
        os.makedirs(the_cont)
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        assert gdf._obj == "z"
        assert gdf._obj_path == ""
        assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar")
        assert gdf._put_datadir == the_cont
        assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")

        body = '1234\n'
        etag = md5()
        etag.update(body)
        etag = etag.hexdigest()
        metadata = {
            'X-Timestamp': '1234',
            'Content-Type': 'file',
            'ETag': etag,
            'Content-Length': '5',
        }

        def mock_open(*args, **kwargs):
            raise OSError(errno.ENOSPC, os.strerror(errno.ENOSPC))

        with mock.patch("os.open", mock_open):
            try:
                with gdf.create() as dw:
                    assert dw._tmppath is not None
                    dw.write(body)
                    dw.put(metadata)
            except DiskFileNoSpace:
                pass
            else:
                self.fail("Expected exception DiskFileNoSpace")

    def test_put_rename_ENOENT(self):
        the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
        os.makedirs(the_cont)
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        assert gdf._obj == "z"
        assert gdf._obj_path == ""
        assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar")
        assert gdf._put_datadir == the_cont
        assert gdf._data_file == os.path.join(self.td, "vol0", "ufo47", "bar", "z")

        body = '1234\n'
        etag = md5()
        etag.update(body)
        etag = etag.hexdigest()
        metadata = {
            'X-Timestamp': '1234',
            'Content-Type': 'file',
            'ETag': etag,
            'Content-Length': '5',
        }

        def mock_sleep(*args, **kwargs):
            # Return without sleep, no need to dely unit tests
            return

        def mock_rename(*args, **kwargs):
            raise OSError(errno.ENOENT, os.strerror(errno.ENOENT))

        with mock.patch("swiftonfile.swift.obj.diskfile.sleep", mock_sleep):
            with mock.patch("os.rename", mock_rename):
                try:
                    with gdf.create() as dw:
                        assert dw._tmppath is not None
                        tmppath = dw._tmppath
                        dw.write(body)
                        dw.put(metadata)
                except SwiftOnFileSystemOSError:
                    pass
                else:
                    self.fail("Expected exception DiskFileError")

    def test_put_obj_path(self):
        the_obj_path = os.path.join("b", "a")
        the_file = os.path.join(the_obj_path, "z")
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", the_file)
        assert gdf._obj == "z"
        assert gdf._obj_path == the_obj_path
        assert gdf._container_path == os.path.join(self.td, "vol0", "ufo47", "bar")
        assert gdf._put_datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "b", "a")
        assert gdf._data_file == os.path.join(
            self.td, "vol0", "ufo47", "bar", "b", "a", "z")

        body = '1234\n'
        etag = md5()
        etag.update(body)
        etag = etag.hexdigest()
        metadata = {
            'X-Timestamp': '1234',
            'Content-Type': 'file',
            'ETag': etag,
            'Content-Length': '5',
        }

        with gdf.create() as dw:
            assert dw._tmppath is not None
            tmppath = dw._tmppath
            dw.write(body)
            dw.put(metadata)

        assert os.path.exists(gdf._data_file)
        assert not os.path.exists(tmppath)

    def test_delete(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "wb") as fd:
            fd.write("1234")
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        assert gdf._obj == "z"
        assert gdf._data_file == the_file
        assert not gdf._is_dir
        later = float(gdf.read_metadata()['X-Timestamp']) + 1
        gdf.delete(normalize_timestamp(later))
        assert os.path.isdir(gdf._put_datadir)
        assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))

    def test_delete_same_timestamp(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "wb") as fd:
            fd.write("1234")
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        assert gdf._obj == "z"
        assert gdf._data_file == the_file
        assert not gdf._is_dir
        now = float(gdf.read_metadata()['X-Timestamp'])
        gdf.delete(normalize_timestamp(now))
        assert os.path.isdir(gdf._put_datadir)
        assert os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))

    def test_delete_file_not_found(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "wb") as fd:
            fd.write("1234")
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        assert gdf._obj == "z"
        assert gdf._data_file == the_file
        assert not gdf._is_dir
        later = float(gdf.read_metadata()['X-Timestamp']) + 1

        # Handle the case the file is not in the directory listing.
        os.unlink(the_file)

        gdf.delete(normalize_timestamp(later))
        assert os.path.isdir(gdf._put_datadir)
        assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))

    def test_delete_file_unlink_error(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "wb") as fd:
            fd.write("1234")
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        assert gdf._obj == "z"
        assert gdf._data_file == the_file
        assert not gdf._is_dir

        later = float(gdf.read_metadata()['X-Timestamp']) + 1

        def _mock_os_unlink_eacces_err(f):
            raise OSError(errno.EACCES, os.strerror(errno.EACCES))

        stats = os.stat(the_path)
        try:
            os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR))

            # Handle the case os_unlink() raises an OSError
            with patch("os.unlink", _mock_os_unlink_eacces_err):
                try:
                    gdf.delete(normalize_timestamp(later))
                except OSError as e:
                    assert e.errno == errno.EACCES
                else:
                    self.fail("Excepted an OSError when unlinking file")
        finally:
            os.chmod(the_path, stats.st_mode)

        assert os.path.isdir(gdf._put_datadir)
        assert os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))

    def test_delete_is_dir(self):
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_dir = os.path.join(the_path, "d")
        os.makedirs(the_dir)
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
        assert gdf._data_file == the_dir
        later = float(gdf.read_metadata()['X-Timestamp']) + 1
        gdf.delete(normalize_timestamp(later))
        assert os.path.isdir(gdf._put_datadir)
        assert not os.path.exists(os.path.join(gdf._put_datadir, gdf._obj))

    def test_create(self):
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
        saved_tmppath = ''
        saved_fd = None
        with gdf.create() as dw:
            assert gdf._put_datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "dir")
            assert os.path.isdir(gdf._put_datadir)
            saved_tmppath = dw._tmppath
            assert os.path.dirname(saved_tmppath) == gdf._put_datadir
            assert os.path.basename(saved_tmppath)[:3] == '.z.'
            assert os.path.exists(saved_tmppath)
            dw.write("123")
            saved_fd = dw._fd
        # At the end of previous with block a close on fd is called.
        # Calling os.close on the same fd will raise an OSError
        # exception and we must catch it.
        try:
            os.close(saved_fd)
        except OSError:
            pass
        else:
            self.fail("Exception expected")
        assert not os.path.exists(saved_tmppath)

    def test_create_err_on_close(self):
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
        saved_tmppath = ''
        with gdf.create() as dw:
            assert gdf._put_datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "dir")
            assert os.path.isdir(gdf._put_datadir)
            saved_tmppath = dw._tmppath
            assert os.path.dirname(saved_tmppath) == gdf._put_datadir
            assert os.path.basename(saved_tmppath)[:3] == '.z.'
            assert os.path.exists(saved_tmppath)
            dw.write("123")
            # Closing the fd prematurely should not raise any exceptions.
            dw.close()
        assert not os.path.exists(saved_tmppath)

    def test_create_err_on_unlink(self):
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
        saved_tmppath = ''
        with gdf.create() as dw:
            assert gdf._put_datadir == os.path.join(self.td, "vol0", "ufo47", "bar", "dir")
            assert os.path.isdir(gdf._put_datadir)
            saved_tmppath = dw._tmppath
            assert os.path.dirname(saved_tmppath) == gdf._put_datadir
            assert os.path.basename(saved_tmppath)[:3] == '.z.'
            assert os.path.exists(saved_tmppath)
            dw.write("123")
            os.unlink(saved_tmppath)
        assert not os.path.exists(saved_tmppath)

    def test_unlink_not_called_after_rename(self):
        the_obj_path = os.path.join("b", "a")
        the_file = os.path.join(the_obj_path, "z")
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", the_file)

        body = '1234\n'
        etag = md5(body).hexdigest()
        metadata = {
            'X-Timestamp': '1234',
            'Content-Type': 'file',
            'ETag': etag,
            'Content-Length': '5',
        }

        _mock_do_unlink = Mock()  # Shouldn't be called
        with patch("swiftonfile.swift.obj.diskfile.do_unlink", _mock_do_unlink):
            with gdf.create() as dw:
                assert dw._tmppath is not None
                tmppath = dw._tmppath
                dw.write(body)
                dw.put(metadata)
                # do_unlink is not called if dw._tmppath is set to None
                assert dw._tmppath is None
        self.assertFalse(_mock_do_unlink.called)

        assert os.path.exists(gdf._data_file)  # Real file exists
        assert not os.path.exists(tmppath)  # Temp file does not exist

    def test_fd_closed_when_diskfile_open_raises_exception_race(self):
        # do_open() succeeds but read_metadata() fails(GlusterFS)
        _m_do_open = Mock(return_value=999)
        _m_do_fstat = Mock(return_value=
                           os.stat_result((33261, 2753735, 2053, 1, 1000,
                                           1000, 6873, 1431415969,
                                           1376895818, 1433139196)))
        _m_rmd = Mock(side_effect=IOError(errno.ENOENT,
                                          os.strerror(errno.ENOENT)))
        _m_do_close = Mock()
        _m_log = Mock()

        with nested(
                patch("swiftonfile.swift.obj.diskfile.do_open", _m_do_open),
                patch("swiftonfile.swift.obj.diskfile.do_fstat", _m_do_fstat),
                patch("swiftonfile.swift.obj.diskfile.read_metadata", _m_rmd),
                patch("swiftonfile.swift.obj.diskfile.do_close", _m_do_close),
                patch("swiftonfile.swift.obj.diskfile.logging.warn", _m_log)):
            gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
            try:
                with gdf.open():
                    pass
            except DiskFileNotExist:
                pass
            else:
                self.fail("Expecting DiskFileNotExist")
            _m_do_fstat.assert_called_once_with(999)
            _m_rmd.assert_called_once_with(999)
            _m_do_close.assert_called_once_with(999)
            self.assertFalse(gdf._fd)
            # Make sure ENOENT failure is logged
            self.assertTrue("failed with ENOENT" in _m_log.call_args[0][0])

    def test_fd_closed_when_diskfile_open_raises_DiskFileExpired(self):
        # A GET/DELETE on an expired object should close fd
        the_path = os.path.join(self.td, "vol0", "ufo47", "bar")
        the_file = os.path.join(the_path, "z")
        os.makedirs(the_path)
        with open(the_file, "w") as fd:
            fd.write("1234")
        md = {
            'X-Type': 'Object',
            'X-Object-Type': 'file',
            'Content-Length': str(os.path.getsize(the_file)),
            'ETag': md5("1234").hexdigest(),
            'X-Timestamp': os.stat(the_file).st_mtime,
            'X-Delete-At': 0,  # This is in the past
            'Content-Type': 'application/octet-stream'}
        _metadata[_mapit(the_file)] = md

        _m_do_close = Mock()

        with patch("swiftonfile.swift.obj.diskfile.do_close", _m_do_close):
            gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
            try:
                with gdf.open():
                    pass
            except DiskFileExpired:
                # Confirm that original exception is re-raised
                pass
            else:
                self.fail("Expecting DiskFileExpired")
            self.assertEqual(_m_do_close.call_count, 1)
            self.assertFalse(gdf._fd)
            # Close the actual fd, as we had mocked do_close
            os.close(_m_do_close.call_args[0][0])

    def make_directory_chown_call(self):
        path = os.path.join(self.td, "a/b/c")
        _m_do_chown = Mock()
        with patch("swiftonfile.swift.obj.diskfile.do_chown", _m_do_chown):
            diskfile.make_directory(path, -1, -1)
        self.assertFalse(_m_do_chown.called)
        self.assertTrue(os.path.isdir(path))

        path = os.path.join(self.td, "d/e/f")
        _m_do_chown.reset_mock()
        with patch("swiftonfile.swift.obj.diskfile.do_chown", _m_do_chown):
            diskfile.make_directory(path, -1, 99)
        self.assertEqual(_m_do_chown.call_count, 3)
        self.assertTrue(os.path.isdir(path))

        path = os.path.join(self.td, "g/h/i")
        _m_do_chown.reset_mock()
        with patch("swiftonfile.swift.obj.diskfile.do_chown", _m_do_chown):
            diskfile.make_directory(path, 99, -1)
        self.assertEqual(_m_do_chown.call_count, 3)
        self.assertTrue(os.path.isdir(path))

    def test_fchown_not_called_on_default_uid_gid_values(self):
        the_cont = os.path.join(self.td, "vol0", "ufo47", "bar")
        os.makedirs(the_cont)
        body = '1234'
        metadata = {
            'X-Timestamp': '1234',
            'Content-Type': 'file',
            'ETag': md5(body).hexdigest(),
            'Content-Length': len(body),
        }

        _m_do_fchown = Mock()
        gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
        with gdf.create() as dw:
            assert dw._tmppath is not None
            tmppath = dw._tmppath
            dw.write(body)
            with patch("swiftonfile.swift.obj.diskfile.do_fchown",
                       _m_do_fchown):
                dw.put(metadata)
        self.assertFalse(_m_do_fchown.called)
        assert os.path.exists(gdf._data_file)
        assert not os.path.exists(tmppath)