diff --git libcloud/common/base.py libcloud/common/base.py index 896c87e..6688138 100644 --- libcloud/common/base.py +++ libcloud/common/base.py @@ -382,7 +382,7 @@ class ConnectionKey(object): self.connection.putrequest(method, url) for key, value in headers.iteritems(): - self.connection.putheader(key, value) + self.connection.putheader(key, str(value)) self.connection.endheaders() else: diff --git libcloud/storage/drivers/atmos.py libcloud/storage/drivers/atmos.py new file mode 100644 index 0000000..2d9a3c7 --- /dev/null +++ libcloud/storage/drivers/atmos.py @@ -0,0 +1,417 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import hashlib +import hmac +import httplib +import time +import urllib +import urlparse + +from xml.etree import ElementTree + +from libcloud import utils +from libcloud.common.base import ConnectionUserAndKey, Response +from libcloud.common.types import LazyList + +from libcloud.storage.base import Object, Container, StorageDriver, CHUNK_SIZE +from libcloud.storage.types import ContainerAlreadyExistsError, \ + ContainerDoesNotExistError, \ + ContainerIsNotEmptyError, \ + ObjectDoesNotExistError + +def collapse(s): + return ' '.join([x for x in s.split(' ') if x]) + +class AtmosError(Exception): + def __init__(self, code, message): + self.code = code + self.message = message + +class AtmosResponse(Response): + def success(self): + return self.status in (httplib.OK, httplib.CREATED, httplib.NO_CONTENT, + httplib.PARTIAL_CONTENT) + + def parse_body(self): + if not self.body: + return None + tree = ElementTree.fromstring(self.body) + return tree + + def parse_error(self): + if not self.body: + return None + tree = ElementTree.fromstring(self.body) + code = int(tree.find('Code').text) + message = tree.find('Message').text + raise AtmosError(code, message) + +class AtmosConnection(ConnectionUserAndKey): + responseCls = AtmosResponse + + def add_default_headers(self, headers): + headers['x-emc-uid'] = self.user_id + headers['Date'] = time.strftime('%a, %d %b %Y %H:%M:%S GMT', + time.gmtime()) + headers['x-emc-date'] = headers['Date'] + + if 'Content-Type' not in headers: + headers['Content-Type'] = 'application/octet-stream' + if 'Accept' not in headers: + headers['Accept'] = '*/*' + + return headers + + def pre_connect_hook(self, params, headers): + headers['x-emc-signature'] = self._calculate_signature(params, headers) + + return params, headers + + def _calculate_signature(self, params, headers): + pathstring = self.action + if pathstring.startswith(self.driver.path): + pathstring = pathstring[len(self.driver.path):] + if params: + if type(params) is dict: + params = params.items() + pathstring += '?' + urllib.urlencode(params) + pathstring = pathstring.lower() + + xhdrs = [(k, v) for k, v in headers.items() if k.startswith('x-emc-')] + xhdrs.sort(key=lambda x: x[0]) + + signature = [ + self.method, + headers.get('Content-Type', ''), + headers.get('Range', ''), + headers.get('Date', ''), + pathstring, + ] + signature.extend([k + ':' + collapse(v) for k, v in xhdrs]) + signature = '\n'.join(signature) + key = base64.b64decode(self.key) + signature = hmac.new(key, signature, hashlib.sha1).digest() + return base64.b64encode(signature) + +class AtmosDriver(StorageDriver): + connectionCls = AtmosConnection + + host = None + path = None + api_name = 'atmos' + + DEFAULT_CDN_TTL = 60 * 60 * 24 * 7 # 1 week + + def __init__(self, key, secret=None, secure=True, host=None, port=None): + host = host or self.host + super(AtmosDriver, self).__init__(key, secret, secure, host, port) + + def list_containers(self): + result = self.connection.request(self._namespace_path('')) + entries = self._list_objects(result.object, object_type='directory') + containers = [] + for entry in entries: + extra = { + 'object_id': entry['id'] + } + containers.append(Container(entry['name'], extra, self)) + return containers + + def get_container(self, container_name): + path = self._namespace_path(container_name + '/?metadata/system') + try: + result = self.connection.request(path) + except AtmosError, e: + if e.code != 1003: + raise + raise ContainerDoesNotExistError(e, self, container_name) + meta = self._emc_meta(result) + extra = { + 'object_id': meta['objectid'] + } + return Container(container_name, extra, self) + + def create_container(self, container_name): + path = self._namespace_path(container_name + '/') + try: + result = self.connection.request(path, method='POST') + except AtmosError, e: + if e.code != 1016: + raise + raise ContainerAlreadyExistsError(e, self, container_name) + return self.get_container(container_name) + + def delete_container(self, container): + try: + self.connection.request(self._namespace_path(container.name + '/'), + method='DELETE') + except AtmosError, e: + if e.code == 1003: + raise ContainerDoesNotExistError(e, self, container.name) + elif e.code == 1023: + raise ContainerIsNotEmptyError(e, self, container.name) + return True + + def get_object(self, container_name, object_name): + container = self.get_container(container_name) + path = container_name + '/' + object_name + path = self._namespace_path(path) + + try: + result = self.connection.request(path + '?metadata/system') + system_meta = self._emc_meta(result) + + result = self.connection.request(path + '?metadata/user') + user_meta = self._emc_meta(result) + except AtmosError, e: + if e.code != 1003: + raise + raise ObjectDoesNotExistError(e, self, object_name) + + last_modified = time.strptime(system_meta['mtime'], + '%Y-%m-%dT%H:%M:%SZ') + last_modified = time.strftime('%a, %d %b %Y %H:%M:%S GMT', + last_modified) + extra = { + 'object_id': system_meta['objectid'], + 'last_modified': last_modified + } + data_hash = user_meta.pop('md5', '') + return Object(object_name, int(system_meta['size']), data_hash, extra, + user_meta, container, self) + + def upload_object(self, file_path, container, object_name, extra=None, + verify_hash=True): + upload_func = self._upload_file + upload_func_kwargs = { 'file_path': file_path } + method = 'PUT' + + extra = extra or {} + request_path = container.name + '/' + object_name + request_path = self._namespace_path(request_path) + content_type = extra.get('content_type', None) + + try: + self.connection.request(request_path + '?metadata/system') + except AtmosError, e: + if e.code != 1003: + raise + method = 'POST' + + result_dict = self._upload_object(object_name=object_name, + content_type=content_type, + upload_func=upload_func, + upload_func_kwargs=upload_func_kwargs, + request_path=request_path, + request_method=method, + headers={}, file_path=file_path) + + response = result_dict['response'].response + bytes_transferred = result_dict['bytes_transferred'] + + if extra is None: + meta_data = {} + else: + meta_data = extra.get('meta_data', {}) + meta_data['md5'] = result_dict['data_hash'] + user_meta = ', '.join([k + '=' + str(v) for k, v in meta_data.items()]) + self.connection.request(request_path + '?metadata/user', method='POST', + headers={'x-emc-meta': user_meta}) + result = self.connection.request(request_path + '?metadata/system') + meta = self._emc_meta(result) + del meta_data['md5'] + extra = { + 'object_id': meta['objectid'], + 'meta_data': meta_data, + } + + return Object(object_name, bytes_transferred, result_dict['data_hash'], + extra, meta_data, container, self) + + def upload_object_via_stream(self, iterator, container, object_name, + extra=None): + if isinstance(iterator, file): + iterator = iter(iterator) + + data_hash = hashlib.md5() + generator = utils.read_in_chunks(iterator, CHUNK_SIZE, True) + bytes_transferred = 0 + try: + chunk = generator.next() + except StopIteration: + chunk = '' + + path = self._namespace_path(container.name + '/' + object_name) + + while True: + end = bytes_transferred + len(chunk) - 1 + data_hash.update(chunk) + headers = { + 'x-emc-meta': 'md5=' + data_hash.hexdigest(), + } + if len(chunk) > 0: + headers['Range'] = 'Bytes=%d-%d' % (bytes_transferred, end) + result = self.connection.request(path, method='PUT', data=chunk, + headers=headers) + bytes_transferred += len(chunk) + + try: + chunk = generator.next() + except StopIteration: + break + if len(chunk) == 0: + break + + data_hash = data_hash.hexdigest() + + if extra is None: + meta_data = {} + else: + meta_data = extra.get('meta_data', {}) + meta_data['md5'] = data_hash + user_meta = ', '.join([k + '=' + str(v) for k, v in meta_data.items()]) + self.connection.request(path + '?metadata/user', method='POST', + headers={'x-emc-meta': user_meta}) + + result = self.connection.request(path + '?metadata/system') + + meta = self._emc_meta(result) + extra = { + 'object_id': meta['objectid'], + 'meta_data': meta_data, + } + + return Object(object_name, bytes_transferred, data_hash, extra, + meta_data, container, self) + + def download_object(self, obj, destination_path, overwrite_existing=False, + delete_on_failure=True): + path = self._namespace_path(obj.container.name + '/' + obj.name) + response = self.connection.request(path, method='GET', raw=True) + + return self._get_object(obj=obj, callback=self._save_object, + response=response, + callback_kwargs={ + 'obj': obj, + 'response': response.response, + 'destination_path': destination_path, + 'overwrite_existing': overwrite_existing, + 'delete_on_failure': delete_on_failure + }, + success_status_code=httplib.OK) + + def download_object_as_stream(self, obj, chunk_size=None): + path = self._namespace_path(obj.container.name + '/' + obj.name) + response = self.connection.request(path, method='GET', raw=True) + + return self._get_object(obj=obj, callback=utils.read_in_chunks, + response=response, + callback_kwargs={ + 'iterator': response.response, + 'chunk_size': chunk_size + }, + success_status_code=httplib.OK) + + def delete_object(self, obj): + path = self._namespace_path(obj.container.name + '/' + obj.name) + try: + self.connection.request(path, method='DELETE') + except AtmosError, e: + if e.code != 1003: + raise + raise ObjectDoesNotExistError(e, self, obj.name) + return True + + def list_container_objects(self, container): + value_dict = {'container': container} + return LazyList(get_more=self._get_more, value_dict=value_dict) + + def enable_object_cdn(self, obj): + return True + + def get_object_cdn_url(self, obj, expiry=None, use_object=False): + if use_object: + path = '/rest/objects' + obj.meta_data['object_id'] + else: + path = '/rest/namespace/' + obj.container.name + '/' + obj.name + + if self.secure: + protocol = 'https' + else: + protocol = 'http' + + expiry = str(expiry or int(time.time()) + self.DEFAULT_CDN_TTL) + params = [ + ('uid', self.key), + ('expires', expiry), + ] + params.append(('signature', self._cdn_signature(path, params))) + + params = urllib.urlencode(params) + path = self.path + path + return urlparse.urlunparse((protocol, self.host, path, '', params, '')) + + def _cdn_signature(self, path, params): + key = base64.b64decode(self.secret) + signature = '\n'.join(['GET', path.lower(), self.key, expiry]) + signature = hmac.new(key, signature, hashlib.sha1).digest() + + return base64.b64encode(signature) + + def _list_objects(self, tree, object_type=None): + listing = tree.find(self._emc_tag('DirectoryList')) + entries = [] + for entry in listing.findall(self._emc_tag('DirectoryEntry')): + file_type = entry.find(self._emc_tag('FileType')).text + if object_type is not None and object_type != file_type: + continue + entries.append({ + 'id': entry.find(self._emc_tag('ObjectID')).text, + 'type': file_type, + 'name': entry.find(self._emc_tag('Filename')).text + }) + return entries + + def _namespace_path(self, path): + return self.path + '/rest/namespace/' + path + + def _object_path(self, object_id): + return self.path + '/rest/objects/' + object_id + + @staticmethod + def _emc_tag(tag): + return '{http://www.emc.com/cos/}' + tag + + def _emc_meta(self, response): + meta = response.headers.get('x-emc-meta', '') + if len(meta) == 0: + return {} + meta = meta.split(', ') + return dict([x.split('=', 1) for x in meta]) + + def _get_more(self, last_key, value_dict): + container = value_dict['container'] + headers = {'x-emc-include-meta': '1'} + path = self._namespace_path(container.name + '/') + result = self.connection.request(path, headers=headers) + entries = self._list_objects(result.object, object_type='regular') + objects = [] + for entry in entries: + metadata = {'object_id': entry['id']} + objects.append(Object(entry['name'], 0, '', {}, metadata, container, + self)) + return objects, None, True diff --git libcloud/storage/drivers/ninefold.py libcloud/storage/drivers/ninefold.py new file mode 100644 index 0000000..f7ee3f2 --- /dev/null +++ libcloud/storage/drivers/ninefold.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libcloud.storage.providers import Provider +from libcloud.storage.drivers.atmos import AtmosDriver + +class NinefoldStorageDriver(AtmosDriver): + host = 'api.ninefold.com' + path = '/storage/v1.0' + + type = Provider.NINEFOLD + name = 'Ninefold' diff --git libcloud/storage/providers.py libcloud/storage/providers.py index 43d9b32..d52e855 100644 --- libcloud/storage/providers.py +++ libcloud/storage/providers.py @@ -33,6 +33,8 @@ DRIVERS = { ('libcloud.storage.drivers.s3', 'S3APSEStorageDriver'), Provider.S3_AP_NORTHEAST: ('libcloud.storage.drivers.s3', 'S3APNEStorageDriver'), + Provider.NINEFOLD: + ('libcloud.storage.drivers.ninefold', 'NinefoldStorageDriver'), } def get_driver(provider): diff --git libcloud/storage/types.py libcloud/storage/types.py index 62f2ec1..1ad9b2f 100644 --- libcloud/storage/types.py +++ libcloud/storage/types.py @@ -37,6 +37,7 @@ class Provider(object): @cvar S3_EU_WEST: Amazon S3 EU West (Ireland) @cvar S3_AP_SOUTHEAST_HOST: Amazon S3 Asia South East (Singapore) @cvar S3_AP_NORTHEAST_HOST: Amazon S3 Asia South East (Tokyo) + @cvar NINEFOLD: Ninefold """ DUMMY = 0 CLOUDFILES_US = 1 @@ -46,6 +47,7 @@ class Provider(object): S3_EU_WEST = 5 S3_AP_SOUTHEAST = 6 S3_AP_NORTHEAST = 7 + NINEFOLD = 8 class ContainerError(LibcloudError): error_type = 'ContainerError' diff --git libcloud/utils.py libcloud/utils.py index a9ab20b..cc36ff3 100644 --- libcloud/utils.py +++ libcloud/utils.py @@ -22,7 +22,7 @@ SHOW_DEPRECATION_WARNING = True SHOW_IN_DEVELOPMENT_WARNING = True OLD_API_REMOVE_VERSION = '0.6.0' -def read_in_chunks(iterator, chunk_size=None): +def read_in_chunks(iterator, chunk_size=None, fill_size=False): """ Return a generator which yields data in chunks. @@ -32,6 +32,10 @@ def read_in_chunks(iterator, chunk_size=None): @type chunk_size: C{int} @param chunk_size: Optional chunk size (defaults to CHUNK_SIZE) + + @type fill_size: C{bool} + @param fill_size: If True, make sure chunks are chunk_size in length + (except for last chunk). """ if isinstance(iterator, (file, HTTPResponse)): @@ -41,13 +45,30 @@ def read_in_chunks(iterator, chunk_size=None): get_data = iterator.next args = () - while True: - chunk = str(get_data(*args)) - - if len(chunk) == 0: + data = '' + empty = False + + while not empty or len(data) > 0: + if not empty: + try: + chunk = str(get_data(*args)) + if len(chunk) > 0: + data += chunk + else: + empty = True + except StopIteration: + empty = True + + if len(data) == 0: raise StopIteration - yield chunk + if fill_size: + if empty or len(data) >= chunk_size: + yield data[:chunk_size] + data = data[chunk_size:] + else: + yield data + data = '' def guess_file_mime_type(file_path): filename = os.path.basename(file_path) diff --git test/storage/fixtures/atmos/already_exists.xml test/storage/fixtures/atmos/already_exists.xml new file mode 100644 index 0000000..2b9d94c --- /dev/null +++ test/storage/fixtures/atmos/already_exists.xml @@ -0,0 +1,5 @@ + + + 1016 + The resource you are trying to create already exists. + diff --git test/storage/fixtures/atmos/empty_directory_listing.xml test/storage/fixtures/atmos/empty_directory_listing.xml new file mode 100644 index 0000000..8040444 --- /dev/null +++ test/storage/fixtures/atmos/empty_directory_listing.xml @@ -0,0 +1,4 @@ + + + + diff --git test/storage/fixtures/atmos/list_containers.xml test/storage/fixtures/atmos/list_containers.xml new file mode 100644 index 0000000..71befd8 --- /dev/null +++ test/storage/fixtures/atmos/list_containers.xml @@ -0,0 +1,45 @@ + + + + + b21cb59a2ba339d1afdd4810010b0a5aba2ab6b9 + directory + container1 + + + 860855a4a445b5e45eeef4024371fd5c73ee3ada + directory + container2 + + + 651eae32634bf84529c74eabd555fda48c7cead6 + regular + not-a-container1 + + + 089293be672782a255498b0b05c4877acf23ef9e + directory + container3 + + + bd804e9f356b51844f93273ec8c94b2e274711d0 + directory + container4 + + + b40b0f3a17fad1d8c8b2085f668f8107bb400fa5 + regular + not-a-container-2 + + + 10bd74388b55a3c8c329ff5dd6d21bd55bfb7370 + directory + container5 + + + c04fa4aa3d0adcdf104baa0cef7b6279680a23c3 + directory + container6 + + + diff --git test/storage/fixtures/atmos/not_empty.xml test/storage/fixtures/atmos/not_empty.xml new file mode 100644 index 0000000..6c46d59 --- /dev/null +++ test/storage/fixtures/atmos/not_empty.xml @@ -0,0 +1,5 @@ + + + 1023 + The directory you are trying to delete is not empty. + diff --git test/storage/fixtures/atmos/not_found.xml test/storage/fixtures/atmos/not_found.xml new file mode 100644 index 0000000..3f157a2 --- /dev/null +++ test/storage/fixtures/atmos/not_found.xml @@ -0,0 +1,5 @@ + + + 1003 + The requested object was not found. + diff --git test/storage/test_atmos.py test/storage/test_atmos.py new file mode 100644 index 0000000..b6682e0 --- /dev/null +++ test/storage/test_atmos.py @@ -0,0 +1,570 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import httplib +import os.path +import sys +import unittest +import urlparse + +from xml.etree import ElementTree + +import libcloud.utils + +from libcloud.common.types import LibcloudError +from libcloud.storage.base import Container, Object +from libcloud.storage.types import ContainerAlreadyExistsError, \ + ContainerDoesNotExistError, \ + ContainerIsNotEmptyError, \ + ObjectDoesNotExistError +from libcloud.storage.drivers.atmos import AtmosConnection, AtmosDriver +from libcloud.storage.drivers.dummy import DummyIterator + +from test import StorageMockHttp, MockRawResponse +from test.file_fixtures import StorageFileFixtures + +class AtmosTests(unittest.TestCase): + def setUp(self): + AtmosDriver.connectionCls.conn_classes = (None, AtmosMockHttp) + AtmosDriver.connectionCls.rawResponseCls = AtmosMockRawResponse + AtmosDriver.path = '' + AtmosMockHttp.type = None + AtmosMockHttp.upload_created = False + AtmosMockRawResponse.type = None + self.driver = AtmosDriver('dummy', base64.b64encode('dummy')) + self._remove_test_file() + + def tearDown(self): + self._remove_test_file() + + def _remove_test_file(self): + file_path = os.path.abspath(__file__) + '.temp' + + try: + os.unlink(file_path) + except OSError: + pass + + def test_list_containers(self): + AtmosMockHttp.type = 'EMPTY' + containers = self.driver.list_containers() + self.assertEqual(len(containers), 0) + + AtmosMockHttp.type = None + containers = self.driver.list_containers() + self.assertEqual(len(containers), 6) + + def test_list_container_objects(self): + container = Container(name='test_container', extra={}, + driver=self.driver) + + AtmosMockHttp.type = 'EMPTY' + objects = self.driver.list_container_objects(container=container) + self.assertEqual(len(objects), 0) + + AtmosMockHttp.type = None + objects = self.driver.list_container_objects(container=container) + self.assertEqual(len(objects), 2) + + obj = [o for o in objects if o.name == 'not-a-container1'][0] + self.assertEqual(obj.meta_data['object_id'], + '651eae32634bf84529c74eabd555fda48c7cead6') + self.assertEqual(obj.container.name, 'test_container') + + def test_get_container(self): + container = self.driver.get_container(container_name='test_container') + self.assertEqual(container.name, 'test_container') + self.assertEqual(container.extra['object_id'], + 'b21cb59a2ba339d1afdd4810010b0a5aba2ab6b9') + + def test_get_container_not_found(self): + try: + self.driver.get_container(container_name='not_found') + except ContainerDoesNotExistError: + pass + else: + self.fail('Exception was not thrown') + + def test_create_container_success(self): + container = self.driver.create_container( + container_name='test_create_container') + self.assertTrue(isinstance(container, Container)) + self.assertEqual(container.name, 'test_create_container') + self.assertEqual(container.extra['object_id'], + '31a27b593629a3fe59f887fd973fd953e80062ce') + + def test_create_container_already_exists(self): + AtmosMockHttp.type = 'ALREADY_EXISTS' + + try: + self.driver.create_container( + container_name='test_create_container') + except ContainerAlreadyExistsError: + pass + else: + self.fail( + 'Container already exists but an exception was not thrown') + + def test_delete_container_success(self): + container = Container(name='foo_bar_container', extra={}, driver=self) + result = self.driver.delete_container(container=container) + self.assertTrue(result) + + def test_delete_container_not_found(self): + AtmosMockHttp.type = 'NOT_FOUND' + container = Container(name='foo_bar_container', extra={}, driver=self) + try: + self.driver.delete_container(container=container) + except ContainerDoesNotExistError: + pass + else: + self.fail( + 'Container does not exist but an exception was not thrown') + + def test_delete_container_not_empty(self): + AtmosMockHttp.type = 'NOT_EMPTY' + container = Container(name='foo_bar_container', extra={}, driver=self) + try: + self.driver.delete_container(container=container) + except ContainerIsNotEmptyError: + pass + else: + self.fail('Container is not empty but an exception was not thrown') + + def test_get_object_success(self): + obj = self.driver.get_object(container_name='test_container', + object_name='test_object') + self.assertEqual(obj.container.name, 'test_container') + self.assertEqual(obj.size, 555) + self.assertEqual(obj.hash, '6b21c4a111ac178feacf9ec9d0c71f17') + self.assertEqual(obj.extra['object_id'], + '322dce3763aadc41acc55ef47867b8d74e45c31d6643') + self.assertEqual( + obj.extra['last_modified'], 'Tue, 25 Jan 2011 22:01:49 GMT') + self.assertEqual(obj.meta_data['foo-bar'], 'test 1') + self.assertEqual(obj.meta_data['bar-foo'], 'test 2') + + + def test_get_object_not_found(self): + try: + self.driver.get_object(container_name='test_container', + object_name='not_found') + except ObjectDoesNotExistError: + pass + else: + self.fail('Exception was not thrown') + + def test_delete_object_success(self): + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=self.driver) + status = self.driver.delete_object(obj=obj) + self.assertTrue(status) + + def test_delete_object_not_found(self): + AtmosMockHttp.type = 'NOT_FOUND' + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=self.driver) + try: + self.driver.delete_object(obj=obj) + except ObjectDoesNotExistError: + pass + else: + self.fail('Object does not exist but an exception was not thrown') + + def test_download_object_success(self): + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=self.driver) + destination_path = os.path.abspath(__file__) + '.temp' + result = self.driver.download_object(obj=obj, + destination_path=destination_path, + overwrite_existing=False, + delete_on_failure=True) + self.assertTrue(result) + + def test_download_object_success_not_found(self): + AtmosMockRawResponse.type = 'NOT_FOUND' + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, + meta_data=None, + driver=self.driver) + destination_path = os.path.abspath(__file__) + '.temp' + try: + self.driver.download_object( + obj=obj, + destination_path=destination_path, + overwrite_existing=False, + delete_on_failure=True) + except ObjectDoesNotExistError: + pass + else: + self.fail('Object does not exist but an exception was not thrown') + + def test_download_object_as_stream(self): + container = Container(name='foo_bar_container', extra={}, + driver=self.driver) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=self.driver) + + stream = self.driver.download_object_as_stream(obj=obj, chunk_size=None) + self.assertTrue(hasattr(stream, '__iter__')) + + def test_upload_object_success(self): + def upload_file(self, response, file_path, chunked=False, + calculate_hash=True): + return True, 'hash343hhash89h932439jsaa89', 1000 + + old_func = AtmosDriver._upload_file + AtmosDriver._upload_file = upload_file + path = os.path.abspath(__file__) + container = Container(name='fbc', extra={}, driver=self) + object_name = 'ftu' + extra = {'meta_data': { 'some-value': 'foobar'}} + obj = self.driver.upload_object(file_path=path, container=container, + extra=extra, object_name=object_name) + self.assertEqual(obj.name, 'ftu') + self.assertEqual(obj.size, 1000) + self.assertTrue('some-value' in obj.meta_data) + AtmosDriver._upload_file = old_func + + def test_upload_object_no_content_type(self): + def no_content_type(name): + return None, None + + old_func = libcloud.utils.guess_file_mime_type + libcloud.utils.guess_file_mime_type = no_content_type + file_path = os.path.abspath(__file__) + container = Container(name='fbc', extra={}, driver=self) + object_name = 'ftu' + try: + self.driver.upload_object(file_path=file_path, container=container, + object_name=object_name) + except AttributeError: + pass + else: + self.fail( + 'File content type not provided' + ' but an exception was not thrown') + finally: + libcloud.utils.guess_file_mime_type = old_func + + def test_upload_object_error(self): + def dummy_content_type(name): + return 'application/zip', None + + def send(instance): + raise Exception('') + + old_func1 = libcloud.utils.guess_file_mime_type + libcloud.utils.guess_file_mime_type = dummy_content_type + old_func2 = AtmosMockHttp.send + AtmosMockHttp.send = send + + file_path = os.path.abspath(__file__) + container = Container(name='fbc', extra={}, driver=self) + object_name = 'ftu' + try: + self.driver.upload_object( + file_path=file_path, + container=container, + object_name=object_name) + except LibcloudError: + pass + else: + self.fail('Timeout while uploading but an exception was not thrown') + finally: + libcloud.utils.guess_file_mime_type = old_func1 + AtmosMockHttp.send = old_func2 + + def test_upload_object_nonexistent_file(self): + def dummy_content_type(name): + return 'application/zip', None + + old_func = libcloud.utils.guess_file_mime_type + libcloud.utils.guess_file_mime_type = dummy_content_type + + file_path = os.path.abspath(__file__ + '.inexistent') + container = Container(name='fbc', extra={}, driver=self) + object_name = 'ftu' + try: + self.driver.upload_object( + file_path=file_path, + container=container, + object_name=object_name) + except OSError: + pass + else: + self.fail('Inesitent but an exception was not thrown') + finally: + libcloud.utils.guess_file_mime_type = old_func + + def test_upload_object_via_stream(self): + def dummy_content_type(name): + return 'application/zip', None + + old_func = libcloud.utils.guess_file_mime_type + libcloud.utils.guess_file_mime_type = dummy_content_type + + container = Container(name='fbc', extra={}, driver=self) + object_name = 'ftsd' + iterator = DummyIterator(data=['2', '3', '5']) + try: + self.driver.upload_object_via_stream(container=container, + object_name=object_name, + iterator=iterator) + finally: + libcloud.utils.guess_file_mime_type = old_func + + def test_signature_algorithm(self): + test_uid = 'fredsmagicuid' + test_key = base64.b64encode('ssssshhhhhmysecretkey') + test_date = 'Mon, 04 Jul 2011 07:39:19 GMT' + test_values = [ + ('GET', '/rest/namespace/foo', '', {}, + 'WfSASIA25TuqO2n0aO9k/dtg6S0='), + ('POST', '/rest/namespace/foo', '', {}, + 'oYKdsF+1DOuUT7iX5CJCDym2EQk='), + ('PUT', '/rest/namespace/foo', '', {}, + 'JleF9dpSWhaT3B2swZI3s41qqs4='), + ('DELETE', '/rest/namespace/foo', '', {}, + '2IX+Bd5XZF5YY+g4P59qXV1uLpo='), + ('GET', '/rest/namespace/foo?metata/system', '', {}, + 'zuHDEAgKM1winGnWn3WBsqnz4ks='), + ('POST', '/rest/namespace/foo?metadata/user', '', { + 'x-emc-meta': 'fakemeta=fake, othermeta=faketoo' + }, '7sLx1nxPIRAtocfv02jz9h1BjbU='), + ] + + class FakeDriver(object): + path = '' + + for method, action, api_path, headers, expected in test_values: + c = AtmosConnection(test_uid, test_key) + c.method = method + c.action = action + d = FakeDriver() + d.path = api_path + c.driver = d + headers = c.add_default_headers(headers) + headers['Date'] = headers['x-emc-date'] = test_date + self.assertEqual(c._calculate_signature({}, headers), expected) + +class AtmosMockHttp(StorageMockHttp, unittest.TestCase): + fixtures = StorageFileFixtures('atmos') + upload_created = False + + def __init__(self, *args, **kwargs): + unittest.TestCase.__init__(self) + + if kwargs.get('host', None) and kwargs.get('port', None): + StorageMockHttp.__init__(self, *args, **kwargs) + + def runTest(self): + pass + + def request(self, method, url, body=None, headers=None, raw=False): + headers = headers or {} + parsed = urlparse.urlparse(url) + if parsed.query.startswith('metadata/'): + parsed = list(parsed) + parsed[2] = parsed[2] + '/' + parsed[4] + parsed[4] = '' + url = urlparse.urlunparse(parsed) + return super(AtmosMockHttp, self).request(method, url, body, headers, + raw) + + def _rest_namespace_EMPTY(self, method, url, body, headers): + body = self.fixtures.load('empty_directory_listing.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _rest_namespace(self, method, url, body, headers): + body = self.fixtures.load('list_containers.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _rest_namespace_test_container_EMPTY(self, method, url, body, headers): + body = self.fixtures.load('empty_directory_listing.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _rest_namespace_test_container(self, method, url, body, headers): + body = self.fixtures.load('list_containers.xml') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _rest_namespace_test_container__metadata_system(self, method, url, body, + headers): + headers = { + 'x-emc-meta': 'objectid=b21cb59a2ba339d1afdd4810010b0a5aba2ab6b9' + } + return (httplib.OK, '', headers, httplib.responses[httplib.OK]) + + def _rest_namespace_not_found__metadata_system(self, method, url, body, + headers): + body = self.fixtures.load('not_found.xml') + return (httplib.NOT_FOUND, body, {}, + httplib.responses[httplib.NOT_FOUND]) + + def _rest_namespace_test_create_container(self, method, url, body, headers): + return (httplib.OK, '', {}, httplib.responses[httplib.OK]) + + def _rest_namespace_test_create_container__metadata_system(self, method, + url, body, + headers): + headers = { + 'x-emc-meta': 'objectid=31a27b593629a3fe59f887fd973fd953e80062ce' + } + return (httplib.OK, '', headers, httplib.responses[httplib.OK]) + + def _rest_namespace_test_create_container_ALREADY_EXISTS(self, method, url, + body, headers): + body = self.fixtures.load('already_exists.xml') + return (httplib.BAD_REQUEST, body, {}, + httplib.responses[httplib.BAD_REQUEST]) + + def _rest_namespace_foo_bar_container(self, method, url, body, headers): + return (httplib.OK, '', {}, httplib.responses[httplib.OK]) + + def _rest_namespace_foo_bar_container_NOT_FOUND(self, method, url, body, + headers): + body = self.fixtures.load('not_found.xml') + return (httplib.NOT_FOUND, body, {}, + httplib.responses[httplib.NOT_FOUND]) + + def _rest_namespace_foo_bar_container_NOT_EMPTY(self, method, url, body, + headers): + body = self.fixtures.load('not_empty.xml') + return (httplib.BAD_REQUEST, body, {}, + httplib.responses[httplib.BAD_REQUEST]) + + def _rest_namespace_test_container_test_object_metadata_system(self, method, + url, body, + headers): + meta = { + 'objectid': '322dce3763aadc41acc55ef47867b8d74e45c31d6643', + 'size': '555', + 'mtime': '2011-01-25T22:01:49Z' + } + headers = { + 'x-emc-meta': ', '.join([k + '=' + v for k, v in meta.items()]) + } + return (httplib.OK, '', headers, httplib.responses[httplib.OK]) + + def _rest_namespace_test_container_test_object_metadata_user(self, method, + url, body, + headers): + meta = { + 'md5': '6b21c4a111ac178feacf9ec9d0c71f17', + 'foo-bar': 'test 1', + 'bar-foo': 'test 2', + } + headers = { + 'x-emc-meta': ', '.join([k + '=' + v for k, v in meta.items()]) + } + return (httplib.OK, '', headers, httplib.responses[httplib.OK]) + + def _rest_namespace_test_container_not_found_metadata_system(self, method, + url, body, + headers): + body = self.fixtures.load('not_found.xml') + return (httplib.NOT_FOUND, body, {}, + httplib.responses[httplib.NOT_FOUND]) + + def _rest_namespace_foo_bar_container_foo_bar_object(self, method, url, + body, headers): + return (httplib.OK, '', {}, httplib.responses[httplib.OK]) + + def _rest_namespace_foo_bar_container_foo_bar_object_NOT_FOUND(self, method, + url, body, + headers): + body = self.fixtures.load('not_found.xml') + return (httplib.NOT_FOUND, body, {}, + httplib.responses[httplib.NOT_FOUND]) + + def _rest_namespace_fbc_ftu_metadata_system(self, method, url, body, + headers): + if not self.upload_created: + self.__class__.upload_created = True + body = self.fixtures.load('not_found.xml') + return (httplib.NOT_FOUND, body, {}, + httplib.responses[httplib.NOT_FOUND]) + + self.__class__.upload_created = False + meta = { + 'objectid': '322dce3763aadc41acc55ef47867b8d74e45c31d6643', + 'size': '555', + 'mtime': '2011-01-25T22:01:49Z' + } + headers = { + 'x-emc-meta': ', '.join([k + '=' + v for k, v in meta.items()]) + } + return (httplib.OK, '', headers, httplib.responses[httplib.OK]) + + def _rest_namespace_fbc_ftu_metadata_user(self, method, url, body, headers): + self.assertTrue('x-emc-meta' in headers) + return (httplib.OK, '', {}, httplib.responses[httplib.OK]) + + def _rest_namespace_fbc_ftsd(self, method, url, body, headers): + self.assertTrue('Range' in headers) + return (httplib.OK, '', {}, httplib.responses[httplib.OK]) + + def _rest_namespace_fbc_ftsd_metadata_user(self, method, url, body, + headers): + self.assertTrue('x-emc-meta' in headers) + return (httplib.OK, '', {}, httplib.responses[httplib.OK]) + + def _rest_namespace_fbc_ftsd_metadata_system(self, method, url, body, + headers): + meta = { + 'objectid': '322dce3763aadc41acc55ef47867b8d74e45c31d6643', + 'size': '555', + 'mtime': '2011-01-25T22:01:49Z' + } + headers = { + 'x-emc-meta': ', '.join([k + '=' + v for k, v in meta.items()]) + } + return (httplib.OK, '', headers, httplib.responses[httplib.OK]) + +class AtmosMockRawResponse(MockRawResponse): + fixtures = StorageFileFixtures('atmos') + + def _rest_namespace_foo_bar_container_foo_bar_object(self, method, url, + body, headers): + body = 'test' + self._data = self._generate_random_data(1000) + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _rest_namespace_foo_bar_container_foo_bar_object_NOT_FOUND(self, method, + url, body, + headers): + body = self.fixtures.load('not_found.xml') + return (httplib.NOT_FOUND, body, {}, + httplib.responses[httplib.NOT_FOUND]) + + def _rest_namespace_fbc_ftu(self, method, url, body, headers): + return (httplib.CREATED, '', {}, httplib.responses[httplib.CREATED]) + +if __name__ == '__main__': + sys.exit(unittest.main()) diff --git test/test_utils.py test/test_utils.py index 3cc074f..aa504c4 100644 --- test/test_utils.py +++ test/test_utils.py @@ -88,5 +88,46 @@ class TestUtils(unittest.TestCase): libcloud.utils.in_development_warning('test_module') self.assertEqual(len(WARNINGS_BUFFER), 1) + def test_read_in_chunks_iterator(self): + def iterator(): + for x in range(0, 1000): + yield 'aa' + + for result in libcloud.utils.read_in_chunks(iterator(), chunk_size=10, + fill_size=False): + self.assertEqual(result, 'aa') + + for result in libcloud.utils.read_in_chunks(iterator(), chunk_size=10, + fill_size=True): + self.assertEqual(result, 'aaaaaaaaaa') + + def test_read_in_chunks_filelike(self): + class FakeFile(file): + def __init__(self): + self.remaining = 500 + + def read(self, size): + self.remaining -= 1 + if self.remaining == 0: + return '' + return 'b' * (size + 1) + + for index, result in enumerate(libcloud.utils.read_in_chunks( + FakeFile(), chunk_size=10, + fill_size=False)): + self.assertEqual(result, 'b' * 11) + + self.assertEqual(index, 498) + + for index, result in enumerate(libcloud.utils.read_in_chunks( + FakeFile(), chunk_size=10, + fill_size=True)): + if index != 548: + self.assertEqual(result, 'b' * 10) + else: + self.assertEqual(result, 'b' * 9) + + self.assertEqual(index, 548) + if __name__ == '__main__': sys.exit(unittest.main())