stackforge/swiftsync
Code Issues Proposed changes

Delete objects.

Browse Source 
- Todo containers next.
This commit is contained in:
Chmouel Boudjnah 2013-03-29 14:34:40 +01:00
parent 5d51e3873d
commit ac48c55687
3 changed files with 49 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
swsync/containers.py
View File

@ -19,7 +19,7 @@ import logging
import swiftclient
import eventlet
from swsync.objects import sync_object
from swsync.objects import sync_object, delete_object
from swsync.utils import get_config
@ -27,7 +27,8 @@ class Containers(object):
"""Containers syncornization."""
def __init__(self):
self.max_gthreads = int(get_config("sync", "max_gthreads"))
self.objects_cls = sync_object
self.sync_object = sync_object
self.delete_object = delete_object
def sync(self, orig_storage_cnx, orig_storage_url,
orig_token, dest_storage_cnx, dest_storage_url, dest_token,
@ -58,18 +59,27 @@ class Containers(object):
set1 = set((x['last_modified'], x['name']) for x in orig_objects)
set2 = set((x['last_modified'], x['name']) for x in dest_objects)
diff = set1 - set2
delete_diff = set2 - set1
if not diff:
if not diff and not delete_diff:
return
pool = eventlet.GreenPool(size=self.max_gthreads)
pile = eventlet.GreenPile(pool)
for obj in diff:
logging.info("sending: %s ts:%s", obj[1], obj[0])
pile.spawn(self.objects_cls,
pile.spawn(self.sync_object,
orig_storage_url,
orig_token,
dest_storage_url,
dest_token, container_name,
obj)
for obj in delete_diff:
logging.info("deleting: %s ts:%s", obj[1], obj[0])
pile.spawn(self.delete_object,
dest_storage_url,
dest_token, container_name,
obj)
pool.waitall()

9
swsync/objects.py
View File

@ -77,6 +77,15 @@ def get_object(storage_url, token,
return (resp_headers, object_body)
def delete_object(dest_storage_url, dest_token,
container_name, object_name_etag):
delete_object = "%s/%s/%s" % (dest_storage_url,
container_name, object_name_etag[1])
delete_headers = {'x-auth-token': dest_token}
swiftclient.delete_object(delete_object, headers=delete_headers)
def sync_object(orig_storage_url, orig_token, dest_storage_url,
dest_token, container_name, object_name_etag):
orig_headers, orig_body = get_object(orig_storage_url,

43
tests/test_containers.py
View File

@ -32,10 +32,10 @@ class TestContainers(test_base.TestCase):
self.tenant_name = 'foo1'
self.tenant_id = TENANTS_LIST[self.tenant_name]['id']
self.orig_storage_url = "%s/AUTH_%s" % (STORAGE_ORIG, self.tenant_id)
self.orig_storage_url = '%s/AUTH_%s' % (STORAGE_ORIG, self.tenant_id)
self.orig_storage_cnx = (urlparse.urlparse(self.orig_storage_url),
None)
self.dest_storage_url = "%s/AUTH_%s" % (STORAGE_DEST, self.tenant_id)
self.dest_storage_url = '%s/AUTH_%s' % (STORAGE_DEST, self.tenant_id)
self.dest_storage_cnx = (urlparse.urlparse(self.dest_storage_url),
None)
@ -46,7 +46,7 @@ class TestContainers(test_base.TestCase):
get_cnt_called.append(args)
def head_container(*args, **kwargs):
raise swiftclient.client.ClientException("Not Here")
raise swiftclient.client.ClientException('Not Here')
def get_container(_, token, name, **kwargs):
for clist in CONTAINERS_LIST:
@ -58,18 +58,26 @@ class TestContainers(test_base.TestCase):
self.stubs.Set(swiftclient, 'head_container', head_container)
self.container_cls.sync(
self.orig_storage_cnx, self.orig_storage_url, "token",
self.dest_storage_cnx, self.dest_storage_url, "token",
"cont1"
self.orig_storage_cnx, self.orig_storage_url, 'token',
self.dest_storage_cnx, self.dest_storage_url, 'token',
'cont1'
)
self.assertEqual(len(get_cnt_called), 1)
def test_dont_sync_dest(self):
def test_delete_dest(self):
# probably need to change that to mox properly
get_cnt_called = []
sync_object_called = []
delete_object_called = []
def delete_object(*args, **kwargs):
delete_object_called.append((args, kwargs))
self.stubs.Set(swsync.objects.swiftclient,
'delete_object', delete_object)
def head_container(*args, **kwargs):
pass
return True
self.stubs.Set(swiftclient, 'head_container', head_container)
def get_container(*args, **kwargs):
# MASTER
@ -89,20 +97,21 @@ class TestContainers(test_base.TestCase):
def sync_object(*args, **kwargs):
sync_object_called.append(args)
self.stubs.Set(swiftclient, 'head_container', head_container)
self.stubs.Set(swiftclient, 'get_container', get_container)
self.container_cls.objects_cls = sync_object
self.container_cls.sync_object = sync_object
self.container_cls.sync(
self.orig_storage_cnx,
self.orig_storage_url,
"token",
'token',
self.dest_storage_cnx,
self.dest_storage_url,
"token",
"cont1")
'token',
'cont1')
self.assertEqual(len(sync_object_called), 0)
self.assertEqual(len(delete_object_called), 1)
def test_sync(self):
get_cnt_called = []
@ -130,15 +139,15 @@ class TestContainers(test_base.TestCase):
self.stubs.Set(swiftclient, 'head_container', head_container)
self.stubs.Set(swiftclient, 'get_container', get_container)
self.container_cls.objects_cls = sync_object
self.container_cls.sync_object = sync_object
self.container_cls.sync(
self.orig_storage_cnx,
self.orig_storage_url,
"token",
'token',
self.dest_storage_cnx,
self.dest_storage_url,
"token",
"cont1")
'token',
'cont1')
self.assertEqual(sync_object_called[0][-1][1], 'NEWOBJ')