From 57466ac8ab9fdf26a780fc401381df6f0ae54f91 Mon Sep 17 00:00:00 2001 From: Carlos 'sn1p3r' Martinez Date: Thu, 16 May 2013 15:22:35 -0500 Subject: [PATCH] Subject: Add Licenses. Pep8 fixes. added licenses to tests. added __init__.py's where they were needed. pep8 fixes Change-Id: I5404685c098b75d862dfb9c64482d2685e0b645b --- cloudcafe/common/__init__.py | 1 - cloudcafe/common/tools/__init__.py | 15 + cloudcafe/common/tools/md5hash.py | 41 ++ cloudcafe/common/tools/randomstring.py | 50 +++ cloudcafe/objectstorage/__init__.py | 1 - .../objectstorage_api/__init__.py | 15 + .../objectstorage_api/behaviors.py | 17 + .../objectstorage/objectstorage_api/client.py | 366 +++++++++++++++--- .../objectstorage_api/models/__init__.py | 15 + .../objectstorage_api/models/responses.py | 98 +++++ 10 files changed, 554 insertions(+), 65 deletions(-) create mode 100644 cloudcafe/common/tools/__init__.py create mode 100644 cloudcafe/common/tools/md5hash.py create mode 100644 cloudcafe/common/tools/randomstring.py create mode 100644 cloudcafe/objectstorage/objectstorage_api/__init__.py create mode 100644 cloudcafe/objectstorage/objectstorage_api/models/__init__.py create mode 100644 cloudcafe/objectstorage/objectstorage_api/models/responses.py diff --git a/cloudcafe/common/__init__.py b/cloudcafe/common/__init__.py index dd8b1e4e..59ab77fa 100644 --- a/cloudcafe/common/__init__.py +++ b/cloudcafe/common/__init__.py @@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - diff --git a/cloudcafe/common/tools/__init__.py b/cloudcafe/common/tools/__init__.py new file mode 100644 index 00000000..59ab77fa --- /dev/null +++ b/cloudcafe/common/tools/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2013 Rackspace + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/cloudcafe/common/tools/md5hash.py b/cloudcafe/common/tools/md5hash.py new file mode 100644 index 00000000..3d2ed978 --- /dev/null +++ b/cloudcafe/common/tools/md5hash.py @@ -0,0 +1,41 @@ +""" +Copyright 2013 Rackspace + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import hashlib + + +def get_md5_hash(data, block_size_multiplier=1): + """ + returns an md5 sum. data is a string or file pointer. + block size is 512 (md5 msg length). + """ + hash_ = None + default_block_size = 2 ** 9 + block_size = block_size_multiplier * default_block_size + md5 = hashlib.md5() + + if type(data) is file: + while True: + read_data = data.read(block_size) + if not read_data: + break + md5.update(read_data) + data.close() + else: + md5.update(str(data)) + + hash_ = md5.hexdigest() + + return hash_ diff --git a/cloudcafe/common/tools/randomstring.py b/cloudcafe/common/tools/randomstring.py new file mode 100644 index 00000000..47d3e226 --- /dev/null +++ b/cloudcafe/common/tools/randomstring.py @@ -0,0 +1,50 @@ +""" +Copyright 2013 Rackspace + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from uuid import uuid4 + + +def get_random_string(prefix=None, suffix=None, size=8): + """ + Return exactly size bytes worth of base_text as a string + surrounded by any defined pre or suf-fixes + """ + body = '' + base_text = str(uuid4()).replace('-', '0') + + if size <= 0: + body = '{0}{1}'.format(prefix, suffix) + else: + extra = size % len(base_text) + + if extra == 0: + body = base_text * size + + if extra == size: + body = base_text[:size] + + if (extra > 0) and (extra < size): + temp_len = (size / len(base_text)) + base_one = base_text * temp_len + base_two = base_text[:extra] + body = '{0}{1}'.format(base_one, base_two) + + if prefix is not None: + body = '{0}{1}'.format(str(prefix), str(body)) + + if suffix is not None: + body = '{0}{1}'.format(str(body), str(suffix)) + + return body diff --git a/cloudcafe/objectstorage/__init__.py b/cloudcafe/objectstorage/__init__.py index dd8b1e4e..59ab77fa 100644 --- a/cloudcafe/objectstorage/__init__.py +++ b/cloudcafe/objectstorage/__init__.py @@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - diff --git a/cloudcafe/objectstorage/objectstorage_api/__init__.py b/cloudcafe/objectstorage/objectstorage_api/__init__.py new file mode 100644 index 00000000..59ab77fa --- /dev/null +++ b/cloudcafe/objectstorage/objectstorage_api/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2013 Rackspace + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/cloudcafe/objectstorage/objectstorage_api/behaviors.py b/cloudcafe/objectstorage/objectstorage_api/behaviors.py index dd8b1e4e..c8b7cf5d 100644 --- a/cloudcafe/objectstorage/objectstorage_api/behaviors.py +++ b/cloudcafe/objectstorage/objectstorage_api/behaviors.py @@ -14,3 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. """ +from cafe.engine.behaviors import BaseBehavior, behavior +from cloudcafe.objectstorage.objectstorage_api.config \ + import ObjectStorageAPIConfig +from cloudcafe.objectstorage.objectstorage_api.client \ + import ObjectStorageAPIClient + + +class ObjectStorageAPI_Behaviors(BaseBehavior): + def __init__(self, client=None): + self.client = client + self.cofnig = ObjectStorageAPIConfig() + + @behavior(ObjectStorageAPIClient) + def create_container(self, name=None): + response = self.client.create_container(name) + if not response.ok: + raise Exception('could not create container') diff --git a/cloudcafe/objectstorage/objectstorage_api/client.py b/cloudcafe/objectstorage/objectstorage_api/client.py index 95c2ec14..c30b515b 100644 --- a/cloudcafe/objectstorage/objectstorage_api/client.py +++ b/cloudcafe/objectstorage/objectstorage_api/client.py @@ -13,40 +13,68 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - -import cStringIO -import datetime +import hmac import json -import tarfile -import time -import urllib +from time import time +from hashlib import sha1 from cafe.engine.clients.rest import RestClient +from cloudcafe.objectstorage.objectstorage_api.models.responses \ + import AccountContainersList, ContainerObjectsList -class ObjectStorageClient(RestClient): +def _deserialize(response_entity_type): + """ + Auto-deserializes the response from any decorated client method call + that has a 'format' key in it's 'params' dictionary argument, where + 'format' value is either 'json' or 'xml'. + + Deserializes the response into response_entity_type domain object + + response_entity_type must be a Domain Object with a _to_obj() + classmethod defined for every supported format or this won't work. + """ + + def decorator(f): + def wrapper(*args, **kwargs): + response = f(*args, **kwargs) + response.request.__dict__['entity'] = None + response.__dict__['entity'] = None + deserialize_format = None + if isinstance(kwargs, dict): + if isinstance(kwargs.get('params'), dict): + deserialize_format = kwargs['params'].get('format') + + if deserialize_format is not None: + response.__dict__['entity'] = \ + response_entity_type.deserialize( + response.content, deserialize_format) + return response + return wrapper + return decorator + + +class ObjectStorageAPIClient(RestClient): def __init__(self, storage_url, auth_token, base_container_name=None, base_object_name=None): - super(ObjectStorageClient, self).__init__() + super(ObjectStorageAPIClient, self).__init__() self.storage_url = storage_url self.auth_token = auth_token - self.base_container_name = base_container_name - self.base_object_name = base_object_name + self.base_container_name = base_container_name or '' + self.base_object_name = base_object_name or '' self.default_headers['X-Auth-Token'] = self.auth_token def __add_object_metadata_to_headers(self, metadata=None, headers=None): """ Call to __build_metadata specifically for object headers """ - return self.__build_metadata('X-Object-Meta-', metadata, headers) def __add_container_metadata_to_headers(self, metadata=None, headers=None): """ Call to __build_metadata specifically for container headers """ - return self.__build_metadata('X-Container-Meta-', metadata, headers) def __add_account_metadata_to_headers(self, metadata=None, headers=None): @@ -80,11 +108,11 @@ class ObjectStorageClient(RestClient): for key in metadata: try: - meta_key = ''.join([prefix, key]) + meta_key = '{0}{1}'.format(prefix, key) except TypeError as e: self.client_log.error( 'Non-string prefix OR metadata dict value was passed ' - 'to __build_metadata() in object_storage_client.py') + 'to __build_metadata() in object_client.py') self.client_log.exception(e) raise except: @@ -93,82 +121,145 @@ class ObjectStorageClient(RestClient): return dict(metadata_headers, **headers) - def create_container(self, container_name, metadata=None, headers=None, - requestslib_kwargs=None): + #Account------------------------------------------------------------------- - headers = self.__add_container_metadata_to_headers(metadata, headers) - - url = '{0}/{1}'.format(self.storage_url, container_name) - - response = self.request( - 'PUT', - url, - headers=headers, - requestslib_kwargs=requestslib_kwargs) + def retrieve_account_metadata(self, requestslib_kwargs=None): + """4.1.1 View Account Details""" + response = self.head( + self.storage_url, + requestslib_kwargs=requestslib_kwargs) return response - def create_storage_object(self, container_name, object_name, data=None, - metadata=None, headers=None, - requestslib_kwargs=None): + @_deserialize(AccountContainersList) + def list_containers(self, headers=None, params=None, + requestslib_kwargs=None): """ - Creates a storage object in a container via PUT - Optionally adds 'X-Object-Metadata-' prefix to any key in the - metadata dictionary, and then adds that metadata to the headers - dictionary. + Lists all containers for the account. + + If the 'format' variable is passed as part of the 'params' + dictionary, an object representing the deserialized version of + that format (either xml or json) will be appended to the response + as the 'entity' attribute. (ie, response.entity) """ - headers = self.__add_object_metadata_to_headers(metadata, headers) + response = self.get( + self.storage_url, + headers=headers, + params=params, + requestslib_kwargs=requestslib_kwargs) - url = '{0}/{1}/{2}'.format( - self.storage_url, - container_name, - object_name) + return response - response = self.request( - 'PUT', - url, - headers=headers, - data=data, - requestslib_kwargs=requestslib_kwargs) + #Container----------------------------------------------------------------- + + def get_container_metadata(self, container_name, headers=None, + requestslib_kwargs=None): + """4.2.1 View Container Details""" + url = '{0}/{1}'.format(self.storage_url, container_name) + + response = self.head( + url, + headers=headers, + requestslib_kwargs=requestslib_kwargs) + + return response + + def create_container(self, container_name, metadata=None, headers=None, + requestslib_kwargs=None): + url = '{0}/{1}'.format(self.storage_url, container_name) + headers = self.__add_container_metadata_to_headers(metadata, headers) + + response = self.put( + url, + headers=headers, + requestslib_kwargs=requestslib_kwargs) + + return response + + def update_container(self, container_name, headers=None, + requestslib_kwargs=None): + url = '{0}/{1}'.format(self.storage_url, container_name) + + response = self.put( + url, + headers=headers, + requestslib_kwargs=requestslib_kwargs) return response def delete_container(self, container_name, headers=None, requestslib_kwargs=None): - url = '{0}/{1}'.format(self.storage_url, container_name) - response = self.request( - 'DELETE', - url, - headers=headers, - requestslib_kwargs=requestslib_kwargs) + response = self.delete( + url, + headers=headers, + requestslib_kwargs=requestslib_kwargs) return response - def delete_storage_object(self, container_name, object_name, headers=None, + def set_container_metadata(self, container_name, metadata, headers=None, + requestslib_kwargs=None): + url = '{0}/{1}'.format(self.storage_url, container_name) + headers = self.__add_container_metadata_to_headers(metadata, headers) + + response = self.post( + url, + headers=headers, + requestslib_kwargs=requestslib_kwargs) + + return response + + def get_container_options(self, container_name, headers=None, requestslib_kwargs=None): + """4.2.5 CORS Container Headers""" + url = '{0}/{1}'.format(self.storage_url, container_name) - url = '{0}/{1}/{2}'.format( - self.storage_url, - container_name, - object_name) - - response = self.request( - 'DELETE', - url, - headers=headers, - requestslib_kwargs=requestslib_kwargs) + response = self.options( + url, + headers=headers, + requestslib_kwargs=requestslib_kwargs) return response + @_deserialize(ContainerObjectsList) + def list_objects(self, container_name, headers=None, params=None, + requestslib_kwargs=None): + """ + Lists all objects in the specified container. + + If the 'format' variable is passed as part of the 'params' + dictionary, an object representing the deserialized version of + that format (either xml or json) will be appended to the response + as the 'entity' attribute. (ie, response.entity) + """ + url = '{0}/{1}'.format(self.storage_url, container_name) + + response = self.get( + url, + headers=headers, + params=params, + requestslib_kwargs=requestslib_kwargs) + + return response + + def get_object_count(self, container_name): + """ + Returns the number of objects in a container. + """ + response = self.get_container_metadata(container_name) + + obj_count = int(response.headers['x-container-object-count']) + + return obj_count + def _purge_container(self, container_name): params = {'format': 'json'} - r = self.list_objects(container_name, params=params) + response = self.list_objects(container_name, params=params) try: - json_data = json.loads(r.content) + json_data = json.loads(response.content) for entry in json_data: - self.delete_storage_object(container_name, entry['name']) + self.delete_object(container_name, entry['name']) except Exception: pass @@ -177,3 +268,152 @@ class ObjectStorageClient(RestClient): def force_delete_containers(self, container_list): for container_name in container_list: return self._purge_container(container_name) + + #Storage Object------------------------------------------------------------ + + def get_object(self, container_name, object_name, headers=None, + prefetch=True, requestslib_kwargs=None): + """ + optional headers + + If-Match + If-None-Match + If-Modified-Since + If-Unmodified-Since + Range + + If-Match and If-None-Match check the ETag header + 200 on 'If' header success + If none of the entity tags match, or if "*" is given and no current + entity exists, the server MUST NOT perform the requested method, and + MUST return a 412 (Precondition Failed) response. + + 206 (Partial content) for successful range request + If the entity tag does not match, then the server SHOULD + return the entire entity using a 200 (OK) response + see RFC2616 + + If prefetch=False, body download is delayed until response.content is + accessed either directly, via response.iter_content() or .iter_lines() + """ + url = '{0}/{1}/{2}'.format( + self.storage_url, + container_name, + object_name) + + if requestslib_kwargs is None: + requestslib_kwargs = {} + + if requestslib_kwargs.get('prefetch') is None: + requestslib_kwargs['prefetch'] = prefetch + + response = self.get( + url, + headers=headers, + requestslib_kwargs=requestslib_kwargs) + + return response + + def create_object(self, container_name, object_name, data=None, + metadata=None, headers=None, requestslib_kwargs=None): + """ + Creates a storage object in a container via PUT + Optionally adds 'X-Object-Metadata-' prefix to any key in the + metadata dictionary, and then adds that metadata to the headers + dictionary. + """ + url = '{0}/{1}/{2}'.format( + self.storage_url, + container_name, + object_name) + hdrs = self.__add_object_metadata_to_headers(metadata, headers) + + response = self.put( + url, + headers=hdrs, + data=data, + requestslib_kwargs=requestslib_kwargs) + + return response + + def copy_object(self, container_name, object_name, headers=None): + url = '{0}/{1}/{2}'.format( + self.storage_url, + container_name, + object_name) + hdrs = {} + hdrs['X-Auth-Token'] = self.auth_token + + if headers is not None: + if 'X-Copy-From' in headers and 'Content-Length' in headers: + method = 'PUT' + hdrs['X-Copy-From'] = headers['X-Copy-From'] + hdrs['Content-Length'] = headers['Content-Length'] + elif 'Destination' in headers: + method = 'COPY' + hdrs['Destination'] = headers['Destination'] + else: + return None + + response = self.request(method=method, url=url, headers=hdrs) + + return response + + def delete_object(self, container_name, object_name, headers=None, + requestslib_kwargs=None): + url = '{0}/{1}/{2}'.format( + self.storage_url, + container_name, + object_name) + + response = self.delete( + url, + headers=headers, + requestslib_kwargs=requestslib_kwargs) + + return response + + def get_object_metadata(self, container_name, object_name, + headers=None, requestslib_kwargs=None): + url = '{0}/{1}/{2}'.format( + self.storage_url, + container_name, + object_name) + + response = self.head( + url, + headers=headers, + requestslib_kwargs=requestslib_kwargs) + + return response + + def set_object_metadata(self, container_name, object_name, metadata, + headers=None, requestslib_kwargs=None): + url = '{0}/{1}/{2}'.format( + self.storage_url, + container_name, + object_name) + headers = self.__add_object_metadata_to_headers(metadata, headers) + + response = self.post( + url, + headers=headers, + requestslib_kwargs=requestslib_kwargs) + + return response + + def set_temp_url_key(self, headers=None, requestslib_kwargs=None): + return self.post(self.storage_url, headers=headers) + + def create_temp_url(self, method, container, obj, seconds, key, + headers=None, requestslib_kwargs=None): + method = method.upper() + base_url = '{0}/{1}/{2}'.format(self.storage_url, container, obj) + account_hash = self.storage_url.split('/v1/')[1] + object_path = '/v1/{0}/{1}/{2}'.format(account_hash, container, obj) + seconds = int(seconds) + expires = int(time() + seconds) + hmac_body = '{0}\n{1}\n{2}'.format(method, expires, object_path) + sig = hmac.new(key, hmac_body, sha1).hexdigest() + + return {'target_url': base_url, 'signature': sig, 'expires': expires} diff --git a/cloudcafe/objectstorage/objectstorage_api/models/__init__.py b/cloudcafe/objectstorage/objectstorage_api/models/__init__.py new file mode 100644 index 00000000..59ab77fa --- /dev/null +++ b/cloudcafe/objectstorage/objectstorage_api/models/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2013 Rackspace + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/cloudcafe/objectstorage/objectstorage_api/models/responses.py b/cloudcafe/objectstorage/objectstorage_api/models/responses.py new file mode 100644 index 00000000..654d5639 --- /dev/null +++ b/cloudcafe/objectstorage/objectstorage_api/models/responses.py @@ -0,0 +1,98 @@ +""" +Copyright 2013 Rackspace + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import json +from xml.etree import ElementTree +from cafe.engine.models.base import AutoMarshallingModel + + +class AccountContainersList(AutoMarshallingModel): + pass + + class _Container(object): + def __init__(self, name=None, count=None, bytes=None): + self.name = None + self.count = None + self.bytes = None + + def __init__(self): + '''This is a deserializing object only''' + pass + + @classmethod + def _xml_to_obj(cls, serialized_str): + ret = [] + root = ElementTree.fromstring(serialized_str) + setattr(cls, 'name', root.attrib['name']) + for child in root: + container_dict = {} + for sub_child in child: + container_dict[sub_child.tag] = sub_child.text + ret.append(cls._StorageObject(**container_dict)) + return ret + + @classmethod + def _json_to_obj(cls, serialized_str): + ret = [] + data = json.loads(serialized_str) + for container in data: + ret.append( + cls._Container( + name=container.get('name'), + bytes=container.get('bytes'), + count=container.get('count'))) + return ret + + +class ContainerObjectsList(AutoMarshallingModel): + #TODO: make this not use *args and **kwargs + class _StorageObject(dict): + def __init__(self, *args, **kwargs): + self.name = kwargs.get('name', None) + self.bytes = kwargs.get('bytes', None) + self.hash = kwargs.get('hash', None) + self.last_modified = kwargs.get('last_modified', None) + self.content_type = kwargs.get('content_type', None) + + def __init__(self): + '''This is a deserializing object only''' + pass + + @classmethod + def _xml_to_obj(cls, serialized_str): + ret = [] + root = ElementTree.fromstring(serialized_str) + setattr(cls, 'name', root.attrib['name']) + for child in root: + storage_object_dict = {} + for sub_child in child: + storage_object_dict[sub_child.tag] = sub_child.text + ret.append(cls._StorageObject(**storage_object_dict)) + return ret + + @classmethod + def _json_to_obj(cls, serialized_str): + ret = [] + data = json.loads(serialized_str) + for storage_object in data: + storage_obj = cls._StorageObject( + name=storage_object.get('name'), + bytes=storage_object.get('bytes'), + hash=storage_object.get('hash')) + ret.append( + storage_obj, + last_modified=storage_object.get('last_modified'), + content_type=storage_object.get('content_type')) + return ret