Properly handle read_metadata() exceptions

* Gracefully handle ENOENT encountered due to races in deletion.
* Fixed a consumer of read_metadata() that used to catch OSError
  instead of IOError.

Change-Id: I238c05ac8a04b707dd75c454f1722bd788c25339
Signed-off-by: Prashanth Pai <ppai@redhat.com>

gluster/swift/common/DiskDir.py

@@ -26,7 +26,8 @@ from gluster.swift.common.utils import validate_account, validate_container, \
     X_CONTENT_LENGTH, X_TIMESTAMP, X_PUT_TIMESTAMP, X_ETAG, X_OBJECTS_COUNT, \
     X_BYTES_USED, X_CONTAINER_COUNT, DIR_TYPE, rmobjdir, dir_is_object
 from gluster.swift.common import Glusterfs
-from gluster.swift.common.exceptions import FileOrDirNotFoundError
+from gluster.swift.common.exceptions import FileOrDirNotFoundError, \
+    GlusterFileSystemIOError
 
 
 DATADIR = 'containers'
@@ -362,7 +363,15 @@ class DiskDir(DiskCommon):
         count = 0
         for obj in objects:
             obj_path = os.path.join(self.datadir, obj)
-            metadata = read_metadata(obj_path)
+            try:
+                metadata = read_metadata(obj_path)
+            except GlusterFileSystemIOError as err:
+                if err.errno == errno.ENOENT:
+                    # obj might have been deleted by another process
+                    # since the objects list was originally built
+                    continue
+                else:
+                    raise err
             if not metadata or not validate_object(metadata):
                 if delimiter == '/' and obj_path[-1] == delimiter:
                     clean_obj_path = obj_path[:-1]

gluster/swift/common/utils.py

@@ -242,8 +242,16 @@ def _update_list(path, cont_path, src_list, reg_file=True, object_count=0,
         if not reg_file and not Glusterfs._implicit_dir_objects:
             # Now check if this is a dir object or a gratuiously crated
             # directory
-            metadata = \
-                read_metadata(os.path.join(cont_path, obj_path, obj_name))
+            try:
+                metadata = \
+                    read_metadata(os.path.join(cont_path, obj_path, obj_name))
+            except GlusterFileSystemIOError as err:
+                if err.errno == errno.ENOENT:
+                    # object might have been deleted by another process
+                    # since the src_list was originally built
+                    continue
+                else:
+                    raise err
             if not dir_is_object(metadata):
                 continue
 
@@ -494,7 +502,7 @@ def rmobjdir(dir_path):
 
             try:
                 metadata = read_metadata(fullpath)
-            except OSError as err:
+            except GlusterFileSystemIOError as err:
                 if err.errno == errno.ENOENT:
                     # Ignore removal from another entity.
                     continue

test/unit/common/test_utils.py

@@ -27,7 +27,8 @@ import shutil
 from collections import defaultdict
 from mock import patch
 from gluster.swift.common import utils, Glusterfs
-from gluster.swift.common.exceptions import GlusterFileSystemOSError
+from gluster.swift.common.exceptions import GlusterFileSystemOSError,\
+    GlusterFileSystemIOError
 from swift.common.exceptions import DiskFileNoSpace
 
 #
@@ -794,7 +795,8 @@ class TestUtilsDirObjects(unittest.TestCase):
         def _mock_rm(path):
             print "_mock_rm-metadata_enoent(%s)" % path
             shutil.rmtree(path)
-            raise OSError(errno.ENOENT, os.strerror(errno.ENOENT))
+            raise GlusterFileSystemIOError(errno.ENOENT,
+                                           os.strerror(errno.ENOENT))
 
         # Remove the files
         for f in self.files:
@@ -805,8 +807,8 @@ class TestUtilsDirObjects(unittest.TestCase):
         try:
             try:
                 self.assertTrue(utils.rmobjdir(self.rootdir))
-            except OSError:
-                self.fail("Unexpected OSError")
+            except IOError:
+                self.fail("Unexpected IOError")
             else:
                 pass
         finally: