diff --git a/etc/config-sample.ini b/etc/config-sample.ini index edf1121..6c33afe 100644 --- a/etc/config-sample.ini +++ b/etc/config-sample.ini @@ -9,7 +9,7 @@ keystone_dest_credentials = demo:demo:ADMIN [sync] # This is usually bound to the max open files. -max_gthreads = 500 +max_gthreads = 10 [filler] swift_operator_role = Member diff --git a/sync/accounts.py b/sync/accounts.py index 89873b6..fd23219 100644 --- a/sync/accounts.py +++ b/sync/accounts.py @@ -41,7 +41,7 @@ def sync_an_account(orig_storage_url, None, orig_token, http_conn=orig_storage_cnx, full_listing=True) for container in orig_containers: - print container, + print container dt1 = datetime.datetime.fromtimestamp(time.time()) sync_container(orig_storage_cnx, orig_storage_url, orig_token, dest_storage_cnx, dest_storage_url, dest_token, @@ -51,6 +51,7 @@ def sync_an_account(orig_storage_url, rd = dateutil.relativedelta.relativedelta(dt2, dt1) print "%d hours, %d minutes and %d seconds" % (rd.hours, rd.minutes, rd.seconds) + print def sync_accounts(): diff --git a/sync/containers.py b/sync/containers.py index bf4f2be..f7054e2 100644 --- a/sync/containers.py +++ b/sync/containers.py @@ -1,6 +1,10 @@ +import sys +import os + import swiftclient import eventlet +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from sync.objects import sync_object from common.utils import get_config @@ -34,28 +38,21 @@ def sync_container(orig_storage_cnx, orig_storage_url, None, dest_token, container_name, http_conn=dest_storage_cnx, ) - set1 = set((x['hash'], x['name']) for x in orig_objects) - set2 = set((x['hash'], x['name']) for x in dest_objects) + set1 = set((x['last_modified'], x['name']) for x in orig_objects) + set2 = set((x['last_modified'], x['name']) for x in dest_objects) + diff = set1 - set2 if not diff: return - pool = eventlet.GreenPool() - - count = 0 + pool = eventlet.GreenPool(size=int(MAX_GTHREADS)) pile = eventlet.GreenPile(pool) for obj in diff: + print obj pile.spawn(sync_object, orig_storage_url, orig_token, dest_storage_url, dest_token, container_name, obj) - if count == MAX_GTHREADS: - pool.waitall() - pile = eventlet.GreenPile(pool) - count = 0 - else: - count += 1 - pool.waitall()