diff --git libcloud/storage/drivers/atmos.py libcloud/storage/drivers/atmos.py index ca96c62..ee203c4 100644 --- libcloud/storage/drivers/atmos.py +++ libcloud/storage/drivers/atmos.py @@ -20,11 +20,13 @@ import hmac import time from libcloud.utils.py3 import PY3 +from libcloud.utils.py3 import b from libcloud.utils.py3 import httplib +from libcloud.utils.py3 import next from libcloud.utils.py3 import urlparse from libcloud.utils.py3 import urlencode -from libcloud.utils.py3 import next -from libcloud.utils.py3 import b +from libcloud.utils.py3 import urlquote +from libcloud.utils.py3 import urlunquote if PY3: from io import FileIO as file @@ -85,7 +87,7 @@ class AtmosConnection(ConnectionUserAndKey): return params, headers def _calculate_signature(self, params, headers): - pathstring = self.action + pathstring = urlunquote(self.action) if pathstring.startswith(self.driver.path): pathstring = pathstring[len(self.driver.path):] if params: @@ -137,7 +139,7 @@ class AtmosDriver(StorageDriver): return containers def get_container(self, container_name): - path = self._namespace_path(container_name + '/?metadata/system') + path = self._namespace_path(container_name) + '/?metadata/system' try: result = self.connection.request(path) except AtmosError: @@ -152,7 +154,7 @@ class AtmosDriver(StorageDriver): return Container(container_name, extra, self) def create_container(self, container_name): - path = self._namespace_path(container_name + '/') + path = self._namespace_path(container_name) + '/' try: self.connection.request(path, method='POST') except AtmosError: @@ -164,7 +166,7 @@ class AtmosDriver(StorageDriver): def delete_container(self, container): try: - self.connection.request(self._namespace_path(container.name + '/'), + self.connection.request(self._namespace_path(container.name) + '/', method='DELETE') except AtmosError: e = sys.exc_info()[1] @@ -176,8 +178,8 @@ class AtmosDriver(StorageDriver): def get_object(self, container_name, object_name): container = self.get_container(container_name) - path = container_name + '/' + object_name - path = self._namespace_path(path) + object_name_cleaned = self._clean_object_name(object_name) + path = self._namespace_path(container_name) + '/' + object_name_cleaned try: result = self.connection.request(path + '?metadata/system') @@ -206,12 +208,12 @@ class AtmosDriver(StorageDriver): def upload_object(self, file_path, container, object_name, extra=None, verify_hash=True): upload_func = self._upload_file - upload_func_kwargs = { 'file_path': file_path } + upload_func_kwargs = {'file_path': file_path} method = 'PUT' extra = extra or {} - request_path = container.name + '/' + object_name - request_path = self._namespace_path(request_path) + object_name_cleaned = self._clean_object_name(object_name) + request_path = self._namespace_path(container.name) + '/' + object_name_cleaned content_type = extra.get('content_type', None) try: @@ -338,7 +340,7 @@ class AtmosDriver(StorageDriver): success_status_code=httplib.OK) def delete_object(self, obj): - path = self._namespace_path(obj.container.name + '/' + obj.name) + path = self._namespace_path(obj.container.name) + '/' + self._clean_object_name(obj.name) try: self.connection.request(path, method='DELETE') except AtmosError: @@ -398,8 +400,12 @@ class AtmosDriver(StorageDriver): }) return entries + def _clean_object_name(self, name): + name = urlquote(name) + return name + def _namespace_path(self, path): - return self.path + '/rest/namespace/' + path + return self.path + '/rest/namespace/' + urlquote(path) def _object_path(self, object_id): return self.path + '/rest/objects/' + object_id @@ -418,7 +424,7 @@ class AtmosDriver(StorageDriver): def _get_more(self, last_key, value_dict): container = value_dict['container'] headers = {'x-emc-include-meta': '1'} - path = self._namespace_path(container.name + '/') + path = self._namespace_path(container.name) + '/' result = self.connection.request(path, headers=headers) entries = self._list_objects(result.object, object_type='regular') objects = [] diff --git libcloud/test/__init__.py libcloud/test/__init__.py index c9baa81..d63a0cf 100644 --- libcloud/test/__init__.py +++ libcloud/test/__init__.py @@ -158,7 +158,7 @@ class MockHttp(BaseMockHttpObject): meth_name = self._get_method_name(type=self.type, use_param=self.use_param, qs=qs, path=path) - meth = getattr(self, meth_name) + meth = getattr(self, meth_name.replace('%', '_')) if self.test and isinstance(self.test, LibcloudTestCase): self.test._add_visited_url(url=url) @@ -281,7 +281,7 @@ class MockRawResponse(BaseMockHttpObject): meth_name = self._get_method_name(type=self.type, use_param=False, qs=None, path=self.connection.action) - meth = getattr(self, meth_name) + meth = getattr(self, meth_name.replace('%', '_')) result = meth(self.connection.method, None, None, None) self._status, self._body, self._headers, self._reason = result self._response = self.responseCls(self._status, self._body, diff --git libcloud/test/storage/test_atmos.py libcloud/test/storage/test_atmos.py index 84699cc..99b64b4 100644 --- libcloud/test/storage/test_atmos.py +++ libcloud/test/storage/test_atmos.py @@ -90,6 +90,12 @@ class AtmosTests(unittest.TestCase): self.assertEqual(container.extra['object_id'], 'b21cb59a2ba339d1afdd4810010b0a5aba2ab6b9') + def test_get_container_escaped(self): + container = self.driver.get_container(container_name='test & container') + self.assertEqual(container.name, 'test & container') + self.assertEqual(container.extra['object_id'], + 'b21cb59a2ba339d1afdd4810010b0a5aba2ab6b9') + def test_get_container_not_found(self): try: self.driver.get_container(container_name='not_found') @@ -157,6 +163,18 @@ class AtmosTests(unittest.TestCase): self.assertEqual(obj.meta_data['foo-bar'], 'test 1') self.assertEqual(obj.meta_data['bar-foo'], 'test 2') + def test_get_object_escaped(self): + obj = self.driver.get_object(container_name='test & container', + object_name='test & object') + self.assertEqual(obj.container.name, 'test & container') + self.assertEqual(obj.size, 555) + self.assertEqual(obj.hash, '6b21c4a111ac178feacf9ec9d0c71f17') + self.assertEqual(obj.extra['object_id'], + '322dce3763aadc41acc55ef47867b8d74e45c31d6643') + self.assertEqual( + obj.extra['last_modified'], 'Tue, 25 Jan 2011 22:01:49 GMT') + self.assertEqual(obj.meta_data['foo-bar'], 'test 1') + self.assertEqual(obj.meta_data['bar-foo'], 'test 2') def test_get_object_not_found(self): try: @@ -176,6 +194,15 @@ class AtmosTests(unittest.TestCase): status = self.driver.delete_object(obj=obj) self.assertTrue(status) + def test_delete_object_escaped_success(self): + container = Container(name='foo & bar_container', extra={}, + driver=self.driver) + obj = Object(name='foo & bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=self.driver) + status = self.driver.delete_object(obj=obj) + self.assertTrue(status) + def test_delete_object_not_found(self): AtmosMockHttp.type = 'NOT_FOUND' container = Container(name='foo_bar_container', extra={}, @@ -203,6 +230,19 @@ class AtmosTests(unittest.TestCase): delete_on_failure=True) self.assertTrue(result) + def test_download_object_escaped_success(self): + container = Container(name='foo & bar_container', extra={}, + driver=self.driver) + obj = Object(name='foo & bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=self.driver) + destination_path = os.path.abspath(__file__) + '.temp' + result = self.driver.download_object(obj=obj, + destination_path=destination_path, + overwrite_existing=False, + delete_on_failure=True) + self.assertTrue(result) + def test_download_object_success_not_found(self): AtmosMockRawResponse.type = 'NOT_FOUND' container = Container(name='foo_bar_container', extra={}, @@ -234,6 +274,16 @@ class AtmosTests(unittest.TestCase): stream = self.driver.download_object_as_stream(obj=obj, chunk_size=None) self.assertTrue(hasattr(stream, '__iter__')) + def test_download_object_as_stream_escaped(self): + container = Container(name='foo & bar_container', extra={}, + driver=self.driver) + obj = Object(name='foo & bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=self.driver) + + stream = self.driver.download_object_as_stream(obj=obj, chunk_size=None) + self.assertTrue(hasattr(stream, '__iter__')) + def test_upload_object_success(self): def upload_file(self, response, file_path, chunked=False, calculate_hash=True): @@ -244,7 +294,7 @@ class AtmosTests(unittest.TestCase): path = os.path.abspath(__file__) container = Container(name='fbc', extra={}, driver=self) object_name = 'ftu' - extra = {'meta_data': { 'some-value': 'foobar'}} + extra = {'meta_data': {'some-value': 'foobar'}} obj = self.driver.upload_object(file_path=path, container=container, extra=extra, object_name=object_name) self.assertEqual(obj.name, 'ftu') @@ -347,6 +397,8 @@ class AtmosTests(unittest.TestCase): test_values = [ ('GET', '/rest/namespace/foo', '', {}, 'WfSASIA25TuqO2n0aO9k/dtg6S0='), + ('GET', '/rest/namespace/foo%20%26%20bar', '', {}, + 'vmlqXqcInxxoP4YX5mR09BonjX4='), ('POST', '/rest/namespace/foo', '', {}, 'oYKdsF+1DOuUT7iX5CJCDym2EQk='), ('PUT', '/rest/namespace/foo', '', {}, @@ -422,6 +474,13 @@ class AtmosMockHttp(StorageMockHttp, unittest.TestCase): } return (httplib.OK, '', headers, httplib.responses[httplib.OK]) + def _rest_namespace_test_20_26_20container__metadata_system(self, method, url, body, + headers): + headers = { + 'x-emc-meta': 'objectid=b21cb59a2ba339d1afdd4810010b0a5aba2ab6b9' + } + return (httplib.OK, '', headers, httplib.responses[httplib.OK]) + def _rest_namespace_not_found__metadata_system(self, method, url, body, headers): body = self.fixtures.load('not_found.xml') @@ -473,6 +532,19 @@ class AtmosMockHttp(StorageMockHttp, unittest.TestCase): } return (httplib.OK, '', headers, httplib.responses[httplib.OK]) + def _rest_namespace_test_20_26_20container_test_20_26_20object_metadata_system(self, method, + url, body, + headers): + meta = { + 'objectid': '322dce3763aadc41acc55ef47867b8d74e45c31d6643', + 'size': '555', + 'mtime': '2011-01-25T22:01:49Z' + } + headers = { + 'x-emc-meta': ', '.join([k + '=' + v for k, v in list(meta.items())]) + } + return (httplib.OK, '', headers, httplib.responses[httplib.OK]) + def _rest_namespace_test_container_test_object_metadata_user(self, method, url, body, headers): @@ -486,6 +558,19 @@ class AtmosMockHttp(StorageMockHttp, unittest.TestCase): } return (httplib.OK, '', headers, httplib.responses[httplib.OK]) + def _rest_namespace_test_20_26_20container_test_20_26_20object_metadata_user(self, method, + url, body, + headers): + meta = { + 'md5': '6b21c4a111ac178feacf9ec9d0c71f17', + 'foo-bar': 'test 1', + 'bar-foo': 'test 2', + } + headers = { + 'x-emc-meta': ', '.join([k + '=' + v for k, v in list(meta.items())]) + } + return (httplib.OK, '', headers, httplib.responses[httplib.OK]) + def _rest_namespace_test_container_not_found_metadata_system(self, method, url, body, headers): @@ -497,6 +582,10 @@ class AtmosMockHttp(StorageMockHttp, unittest.TestCase): body, headers): return (httplib.OK, '', {}, httplib.responses[httplib.OK]) + def _rest_namespace_foo_20_26_20bar_container_foo_20_26_20bar_object(self, method, url, + body, headers): + return (httplib.OK, '', {}, httplib.responses[httplib.OK]) + def _rest_namespace_foo_bar_container_foo_bar_object_NOT_FOUND(self, method, url, body, headers): @@ -557,6 +646,12 @@ class AtmosMockRawResponse(MockRawResponse): self._data = self._generate_random_data(1000) return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + def _rest_namespace_foo_20_26_20bar_container_foo_20_26_20bar_object(self, method, url, + body, headers): + body = 'test' + self._data = self._generate_random_data(1000) + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + def _rest_namespace_foo_bar_container_foo_bar_object_NOT_FOUND(self, method, url, body, headers): diff --git libcloud/utils/py3.py libcloud/utils/py3.py index 17c1792..d84392a 100644 --- libcloud/utils/py3.py +++ libcloud/utils/py3.py @@ -36,6 +36,7 @@ if sys.version_info >= (3, 0): import urllib.parse as urlparse import xmlrpc.client as xmlrpclib from urllib.parse import quote as urlquote + from urllib.parse import unquote as urlunquote from urllib.parse import urlencode as urlencode basestring = str @@ -67,6 +68,7 @@ else: import urlparse import xmlrpclib from urllib import quote as urlquote + from urllib import unquote as urlunquote from urllib import urlencode as urlencode basestring = unicode = str