swift: Fix for multiple mounts on the same mount point.

When swift server receives multiple requests on a volume and if the
volume is not mounted before hand, for each request swift tries to
mount the GlusterFS volume on the mount point. This process is racy
and may mount multiple times on the same mount point.

Also added a new option object_only in fs.conf which should be enabled
only if the user requires ReST interface and will have better performance.
By default it is set "off".

Change-Id: Ie1718554c5aaf577e823bbd84da8e78d803e954d
BUG: 821310
Signed-off-by: Mohammed Junaid <junaid@redhat.com>
Reviewed-on: http://review.gluster.com/3477
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vijay@gluster.com>

swift/1.4.8/plugins/Glusterfs.py

@@ -13,10 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import os
+import os, fcntl, time
 from ConfigParser import ConfigParser
 from swift.common.utils import TRUE_VALUES
 from hashlib import md5
+from swift.plugins.utils import mkdirs
 
 class Glusterfs(object):
     def __init__(self):
@@ -27,14 +28,41 @@ class Glusterfs(object):
         self.auth_account = self.fs_conf.get('DEFAULT', 'auth_account', 'auth')
         self.mount_ip = self.fs_conf.get('DEFAULT', 'mount_ip', 'localhost')
         self.remote_cluster = self.fs_conf.get('DEFAULT', 'remote_cluster', False) in TRUE_VALUES
+        self.object_only = self.fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES
+
+    def busy_wait(self, mount_path):
+        # Iterate for definite number of time over a given
+        # interval for successful mount
+        for i in range(0, 5):
+            if os.path.ismount(os.path.join(mount_path)):
+                return True
+            time.sleep(2)
+        return False
 
     def mount(self, account):
-        export = self.get_export_from_account_id(account)
         mount_path = os.path.join(self.mount_path, account)
+        export = self.get_export_from_account_id(account)
+
+        pid_dir  = "/var/lib/glusterd/vols/%s/run/" %export
+        pid_file = os.path.join(pid_dir, 'swift.pid');
+
+        if not os.path.exists(pid_dir):
+            mkdirs(pid_dir)
+
+        fd = os.open(pid_file, os.O_CREAT|os.O_RDWR)
+        with os.fdopen(fd, 'r+b') as f:
+            try:
+                fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB)
+            except:
+                ex = sys.exc_info()[1]
+                if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN):
+                # This means that some other process is mounting the
+                # filesystem, so wait for the mount process to complete
+                    return self.busy_wait(mount_path)
+
             mnt_cmd = 'mount -t glusterfs %s:%s %s' % (self.mount_ip, export, \
                                                        mount_path)
-        if os.system(mnt_cmd) or \
-        not os.path.exists(os.path.join(mount_path)):
+            if os.system(mnt_cmd) or not self.busy_wait(mount_path):
                 raise Exception('Mount failed %s: %s' % (self.name, mnt_cmd))
                 return False
         return True

swift/1.4.8/plugins/conf/fs.conf

@@ -6,3 +6,4 @@ mount_ip = localhost
 #fs server need not be local, remote server can also be used,
 #set remote_cluster=yes for using remote server.
 remote_cluster = no
+object_only = no

swift/1.4.8/plugins/utils.py

@@ -332,7 +332,6 @@ def _check_valid_account(account, fs_object):
 
     if not os.path.isdir(os.path.join(mount_path, account)):
         mkdirs(os.path.join(mount_path, account))
-        fs_object.unmount(os.path.join(mount_path, account))
 
     if fs_object:
         if not fs_object.mount(account):

swift/1.4.8/swift.diff

@@ -18,7 +18,7 @@ index d195d34..b5b5ca2 100644
          },
      )
 diff --git a/swift/account/server.py b/swift/account/server.py
-index 800b3c0..99f5de3 100644
+index 800b3c0..cb17970 100644
 --- a/swift/account/server.py
 +++ b/swift/account/server.py
 @@ -1,4 +1,5 @@
@@ -81,7 +81,7 @@ index 800b3c0..99f5de3 100644
              broker.stale_reads_ok = True
          if broker.is_deleted():
              return HTTPNotFound(request=req)
-+        if self.fs_object:
++        if self.fs_object and not self.fs_object.object_only:
 +            broker.list_containers_iter(None, None,None,
 +                                        None, None)
          info = broker.get_info()
@@ -229,7 +229,7 @@ index 0000000..341285d
 +        return Gluster_plugin(app, conf)
 +    return gluster_filter
 diff --git a/swift/common/utils.py b/swift/common/utils.py
-index 47edce8..afc356c 100644
+index 47edce8..03701ce 100644
 --- a/swift/common/utils.py
 +++ b/swift/common/utils.py
 @@ -1,4 +1,5 @@
@@ -238,7 +238,7 @@ index 47edce8..afc356c 100644
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
-@@ -1138,3 +1139,12 @@ def streq_const_time(s1, s2):
+@@ -1138,3 +1139,11 @@ def streq_const_time(s1, s2):
      for (a, b) in zip(s1, s2):
          result |= ord(a) ^ ord(b)
      return result == 0
@@ -250,9 +250,8 @@ index 47edce8..afc356c 100644
 +        return swift_conf.get('DEFAULT', 'Enable_plugin', 'no') in TRUE_VALUES
 +    except NoOptionError, NoSectionError:
 +        return False
-+
 diff --git a/swift/container/server.py b/swift/container/server.py
-index 8a18cfd..356150f 100644
+index 8a18cfd..93943a3 100644
 --- a/swift/container/server.py
 +++ b/swift/container/server.py
 @@ -1,4 +1,5 @@
@@ -337,7 +336,7 @@ index 8a18cfd..356150f 100644
          if broker.is_deleted():
              return HTTPNotFound(request=req)
 +
-+        if self.fs_object:
++        if self.fs_object and not self.fs_object.object_only:
 +            broker.list_objects_iter(None, None, None, None,
 +                                     None, None)
 +