sync account.

bin/filler.py

@@ -9,7 +9,6 @@ import sys
 import eventlet
 from keystoneclient.v2_0 import client as ksclient
 
-sys.path.append("../")
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 from common.utils import get_config
 from sync.filler import (load_index, load_containers_index,

bin/sync-accounts.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-import swiftclient
-
-from sync.containers import sync_container
-from utils import get_config, get_auth
-
-orig_auth_url = get_config('auth', 'keystone_origin')
-orig_tenant, orig_user, orig_password = \
-    get_config('auth', 'keystone_origin_credentials').split(':')
-
-dest_auth_url = get_config('auth', 'keystone_dest')
-dest_tenant, dest_user, dest_password = \
-    get_config('auth', 'keystone_dest_credentials').split(':')
-
-orig_storage_url, orig_token = \
-    get_auth(orig_auth_url, orig_tenant, orig_user, orig_password)
-orig_storage_cnx = swiftclient.http_connection(orig_storage_url)
-
-dest_storage_url, dest_token = \
-    get_auth(dest_auth_url, dest_tenant, dest_user, dest_password)
-dest_storage_cnx = swiftclient.http_connection(dest_storage_url)
-
-
-orig_account_stats, orig_containers = swiftclient.get_account(
-    None, orig_token, http_conn=orig_storage_cnx, full_listing=True
-)
-
-for container in orig_containers:
-    print "Synching %s.." % (container)
-    sync_container(orig_storage_cnx, orig_storage_url, orig_token,
-                   dest_storage_cnx, dest_storage_url, dest_token,
-                   container['name'])

bin/syncit.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+import sys
+import os
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from sync.accounts import sync_accounts
+
+
+if __name__ == '__main__':
+    sync_accounts()

etc/config-sample.ini

@@ -3,6 +3,7 @@ keystone_origin = http://vm:5000/v2.0
 keystone_origin_admin_credentials = admin:admin:ADMIN
 
 keystone_dest = http://vm2:5000/v2.0
+
 keystone_origin_demo_credentials = demo:demo:ADMIN
 keystone_dest_credentials = demo:demo:ADMIN
 

sync/accounts.py

@@ -0,0 +1,71 @@
+# -*- encoding: utf-8 -*-
+import os
+import sys
+
+import swiftclient
+from keystoneclient.v2_0 import client as ksclient
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+from common.utils import get_config, get_auth
+from sync.containers import sync_container
+
+
+def get_ks_auth_orig():
+    orig_auth_url = get_config('auth', 'keystone_origin')
+    tenant_name, username, password = \
+        get_config('auth', 'keystone_origin_admin_credentials').split(':')
+
+    return ksclient.Client(auth_url=orig_auth_url,
+                           username=username,
+                           password=password,
+                           tenant_name=tenant_name)
+
+
+def list_accounts(cnx):
+    for x in cnx.tenants.list():
+        yield x
+
+
+def sync_an_account(orig_storage_url,
+                    orig_token,
+                    dest_storage_url,
+                    dest_token):
+
+    orig_storage_cnx = swiftclient.http_connection(orig_storage_url)
+    dest_storage_cnx = swiftclient.http_connection(dest_storage_url)
+
+    orig_account_stats, orig_containers = swiftclient.get_account(
+        None, orig_token, http_conn=orig_storage_cnx, full_listing=True)
+
+    for container in orig_containers:
+        print "Synching %s.." % (container)
+        sync_container(orig_storage_cnx, orig_storage_url, orig_token,
+                       dest_storage_cnx, dest_storage_url, dest_token,
+                       container['name'])
+
+
+def sync_accounts():
+    orig_auth_url = get_config('auth', 'keystone_origin')
+    orig_admin_tenant, orig_admin_user, orig_admin_password = (
+        get_config('auth', 'keystone_origin_admin_credentials').split(':'))
+    oa_st_url, orig_admin_token = (get_auth(orig_auth_url, orig_admin_tenant,
+                                            orig_admin_user,
+                                            orig_admin_password))
+
+    dest_auth_url = get_config('auth', 'keystone_dest')
+    # we assume orig and dest passwd are the same obv synchronized.
+    dst_st_url, dest_admin_token = (get_auth(dest_auth_url, orig_admin_tenant,
+                                    orig_admin_user,
+                                    orig_admin_password))
+
+    bare_oa_st_url = oa_st_url[:oa_st_url.find('AUTH_')] + "AUTH_"
+    bare_dst_st_url = dst_st_url[:dst_st_url.find('AUTH_')] + "AUTH_"
+
+    for x in list_accounts(get_ks_auth_orig()):
+        user_orig_st_url = bare_oa_st_url + x.id
+        user_dst_st_url = bare_dst_st_url + x.id
+
+        sync_an_account(user_orig_st_url,
+                        orig_admin_token,
+                        user_dst_st_url,
+                        dest_admin_token)

sync/containers.py

@@ -1,5 +1,3 @@
-import sys
-
 import swiftclient
 import eventlet
 
@@ -29,7 +27,6 @@ def sync_container(orig_storage_cnx, orig_storage_url,
             dest_token, container_name,
             headers=container_headers)
 
-
     dest_container_stats, dest_objects = swiftclient.get_container(
         None, dest_token, container_name, http_conn=dest_storage_cnx,
     )

sync/objects.py

@@ -5,6 +5,16 @@ from swift.common.http import is_success
 from swift.container.sync import _Iter2FileLikeObject
 from eventlet import Timeout
 import urllib2
+from urllib import quote as urllib_quote
+
+
+def quote(value, safe='/'):
+    """
+    Patched version of urllib.quote that encodes utf-8 strings before quoting
+    """
+    if isinstance(value, unicode):
+        value = value.encode('utf-8')
+    return urllib_quote(value, safe)
 
 
 def get_object(storage_url, token,
@@ -17,6 +27,7 @@ def get_object(storage_url, token,
     x = urllib2.urlparse.urlparse(storage_url)
 
     path = x.path + '/' + container_name + '/' + object_name
+    path = quote(path)
     with Timeout(conn_timeout):
         conn = http_connect_raw(
             x.hostname,
@@ -58,10 +69,13 @@ def sync_object(orig_storage_url, orig_token, dest_storage_url,
                                          container_name,
                                          object_name_etag[1],
                                          )
+    container_name = quote(container_name)
+    object_name = quote(object_name_etag[1])
+
     post_headers = orig_headers
     post_headers['x-auth-token'] = dest_token
     sync_to = dest_storage_url + "/" + container_name
-    swiftclient.put_object(sync_to, name=object_name_etag[1],
+    swiftclient.put_object(sync_to, name=object_name,
                            headers=post_headers,
                            contents=_Iter2FileLikeObject(orig_body))