stackforge/swiftsync
Code Issues Proposed changes

Change the threadpool is constructed.

Browse Source
This commit is contained in:
Chmouel Boudjnah 2013-03-25 14:16:48 +01:00
parent c0601d7a4e
commit 53f95305e0
3 changed files with 12 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/config-sample.ini
View File

@ -9,7 +9,7 @@ keystone_dest_credentials = demo:demo:ADMIN
[sync] [sync]
# This is usually bound to the max open files. # This is usually bound to the max open files.
max_gthreads = 500 max_gthreads = 10
[filler] [filler]
swift_operator_role = Member swift_operator_role = Member

3
sync/accounts.py
View File

@ -41,7 +41,7 @@ def sync_an_account(orig_storage_url,
None, orig_token, http_conn=orig_storage_cnx, full_listing=True) None, orig_token, http_conn=orig_storage_cnx, full_listing=True)
for container in orig_containers: for container in orig_containers:
print container, print container
dt1 = datetime.datetime.fromtimestamp(time.time()) dt1 = datetime.datetime.fromtimestamp(time.time())
sync_container(orig_storage_cnx, orig_storage_url, orig_token, sync_container(orig_storage_cnx, orig_storage_url, orig_token,
dest_storage_cnx, dest_storage_url, dest_token, dest_storage_cnx, dest_storage_url, dest_token,
@ -51,6 +51,7 @@ def sync_an_account(orig_storage_url,
rd = dateutil.relativedelta.relativedelta(dt2, dt1) rd = dateutil.relativedelta.relativedelta(dt2, dt1)
print "%d hours, %d minutes and %d seconds" % (rd.hours, rd.minutes, print "%d hours, %d minutes and %d seconds" % (rd.hours, rd.minutes,
rd.seconds) rd.seconds)
print
def sync_accounts(): def sync_accounts():

21
sync/containers.py
View File

@ -1,6 +1,10 @@
import sys
import os
import swiftclient import swiftclient
import eventlet import eventlet
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from sync.objects import sync_object from sync.objects import sync_object
from common.utils import get_config from common.utils import get_config
@ -34,28 +38,21 @@ def sync_container(orig_storage_cnx, orig_storage_url,
None, dest_token, container_name, http_conn=dest_storage_cnx, None, dest_token, container_name, http_conn=dest_storage_cnx,
) )
set1 = set((x['hash'], x['name']) for x in orig_objects) set1 = set((x['last_modified'], x['name']) for x in orig_objects)
set2 = set((x['hash'], x['name']) for x in dest_objects) set2 = set((x['last_modified'], x['name']) for x in dest_objects)
diff = set1 - set2 diff = set1 - set2
if not diff: if not diff:
return return
pool = eventlet.GreenPool() pool = eventlet.GreenPool(size=int(MAX_GTHREADS))
count = 0
pile = eventlet.GreenPile(pool) pile = eventlet.GreenPile(pool)
for obj in diff: for obj in diff:
print obj
pile.spawn(sync_object, pile.spawn(sync_object,
orig_storage_url, orig_storage_url,
orig_token, orig_token,
dest_storage_url, dest_storage_url,
dest_token, container_name, dest_token, container_name,
obj) obj)
if count == MAX_GTHREADS:
pool.waitall()
pile = eventlet.GreenPile(pool)
count = 0
else:
count += 1
pool.waitall() pool.waitall()