From d67049f914f0167bc8b2c8d0d36b23de220260f1 Mon Sep 17 00:00:00 2001 From: Wiktor Kolodziej Date: Sat, 25 Jun 2011 17:22:23 +0200 Subject: [PATCH 01/12] Ignoring pycharm files --- .gitignore | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore index 1c8d957..e5b1ecc 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ MANIFEST /.ropeproject/config.py /.coverage coverage_html_report/ +.idea \ No newline at end of file -- 1.7.3.5 From 85b12eb74000bc42c4e3941e5d32fc6593065425 Mon Sep 17 00:00:00 2001 From: Wiktor Kolodziej Date: Sat, 25 Jun 2011 17:25:24 +0200 Subject: [PATCH 02/12] Issue #LIBCLOUD-78 - first part --- libcloud/common/types.py | 32 +++++++++++++++++++++ libcloud/storage/drivers/s3.py | 9 ++++-- test/test_types.py | 61 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 test/test_types.py diff --git a/libcloud/common/types.py b/libcloud/common/types.py index f2f1b5d..3d8a106 100644 --- a/libcloud/common/types.py +++ b/libcloud/common/types.py @@ -63,3 +63,35 @@ class InvalidCredsError(LibcloudError): # Deprecated alias of L{InvalidCredsError} InvalidCredsException = InvalidCredsError + +class LazyList(object): + + def __init__(self, get_more, container=None): + self._data = [] + self._last_key = None + self._exhausted = False + self._all_loaded = False + self._get_more = get_more + self.container = container + + def __iter__(self): + if self._all_loaded: + for i in self._data: + yield i + else: + while not self._exhausted: + newdata, self._last_key, self._exhausted = self._get_more(self._last_key, self.container) + self._data.append(newdata) + for i in newdata: + yield i + self._all_loaded = True + + def _load_all(self): + while not self._exhausted: + newdata, self._last_key, self._exhausted = self._get_more(self._last_key, self.container) + self._data.extend(newdata) + self._all_loaded = True + + def __len__(self): + self._load_all() + return len(self._data) \ No newline at end of file diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 12cd9a8..79ae537 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -35,6 +35,7 @@ from libcloud.storage.types import InvalidContainerNameError from libcloud.storage.types import ContainerDoesNotExistError from libcloud.storage.types import ObjectDoesNotExistError from libcloud.storage.types import ObjectHashMismatchError +from libcloud.common.types import LazyList in_development_warning('libcloud.storage.drivers.s3') @@ -174,16 +175,18 @@ class S3StorageDriver(StorageDriver): raise LibcloudError('Unexpected status code: %s' % (response.status), driver=self) - def list_container_objects(self, container): + def get_more(self, last_key, container): response = self.connection.request('/%s' % (container.name)) if response.status == httplib.OK: objects = self._to_objs(obj=response.object, xpath='Contents', container=container) - return objects - + return objects, None, True raise LibcloudError('Unexpected status code: %s' % (response.status), driver=self) + def list_container_objects(self, container): + return LazyList(self.get_more, container) + def get_container(self, container_name): # This is very inefficient, but afaik it's the only way to do it containers = self.list_containers() diff --git a/test/test_types.py b/test/test_types.py new file mode 100644 index 0000000..6050431 --- /dev/null +++ b/test/test_types.py @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from libcloud.common.types import LazyList + +class MyLazyList(LazyList): + @staticmethod + def _get_more(last_key, container): + data = [1, 2, 3, 4, 5] + return data, 5, True + +class MyLazyList2(LazyList): + @staticmethod + def _get_more(last_key, container): + if not last_key: + data, last_key, exhausted = [1, 2, 3, 4, 5], 5, False + else: + data, last_key, exhausted = [6, 7, 8, 9, 10], 10, True + + return data, last_key, exhausted + +class TestLazyList(unittest.TestCase): +# def setUp(self): +# super(TestLazyList, self).setUp +# +# def tearDown(self): +# super(TestLazyList, self).tearDown + + def test_init(self): + data = [1, 2, 3, 4, 5] + ll = LazyList(MyLazyList._get_more) + ll_list = list(ll) + self.assertEqual(ll_list, data) + + def test_iterator(self): + data = [1, 2, 3, 4, 5] + ll = LazyList(MyLazyList._get_more) + for i, d in enumerate(ll): + self.assertEqual(d, data[i]) + + def test_iterator_not_exhausted(self): + data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + ll = LazyList(MyLazyList2._get_more) + number_of_iterations = 0 + for i, d in enumerate(ll): + self.assertEqual(d, data[i]) + number_of_iterations += 1 + self.assertEqual(number_of_iterations, 10) -- 1.7.3.5 From ed62d1c2ff0dec62a7963373853157a9381e5379 Mon Sep 17 00:00:00 2001 From: Dan Clark Date: Sat, 25 Jun 2011 18:27:32 +0200 Subject: [PATCH 03/12] Issue #LIBCLOUD-78 - incrementally fetch results for s3 --- libcloud/storage/drivers/s3.py | 14 ++++++++++++-- 1 files changed, 12 insertions(+), 2 deletions(-) diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 79ae537..e784d3e 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -176,11 +176,21 @@ class S3StorageDriver(StorageDriver): driver=self) def get_more(self, last_key, container): - response = self.connection.request('/%s' % (container.name)) + params = {} + if last_key: + params['marker'] = last_key + response = self.connection.request('/%s' % (container.name), params = params) + if response.status == httplib.OK: objects = self._to_objs(obj=response.object, xpath='Contents', container=container) - return objects, None, True + exhausted = "false" == response.object.findtext(fixxpath(xpath='IsTruncated', namespace=NAMESPACE)) + print "called with last_key: %s, returned %d objects" % (last_key, len(objects)) + if (len(objects) > 0): + last_key = objects[-1].name + else: + last_key = None + return objects, last_key, exhausted raise LibcloudError('Unexpected status code: %s' % (response.status), driver=self) -- 1.7.3.5 From c0938987e1e39b2e394fbab32ed9ba89ce1915a3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 25 Jun 2011 21:04:37 +0200 Subject: [PATCH 04/12] Refactor some stuff, implement more iterator methods and add some more tests. --- libcloud/common/types.py | 36 +++++++++++-------- test/test_types.py | 88 +++++++++++++++++++++++++++++++++------------ 2 files changed, 85 insertions(+), 39 deletions(-) diff --git a/libcloud/common/types.py b/libcloud/common/types.py index 3d8a106..879364f 100644 --- a/libcloud/common/types.py +++ b/libcloud/common/types.py @@ -17,7 +17,8 @@ __all__ = [ "LibcloudError", "MalformedResponseError", "InvalidCredsError", - "InvalidCredsException" + "InvalidCredsException", + "LazyList" ] class LibcloudError(Exception): @@ -66,32 +67,37 @@ InvalidCredsException = InvalidCredsError class LazyList(object): - def __init__(self, get_more, container=None): + def __init__(self, get_more, value_dict=None): self._data = [] self._last_key = None self._exhausted = False self._all_loaded = False self._get_more = get_more - self.container = container + self._value_dict = value_dict or {} def __iter__(self): - if self._all_loaded: - for i in self._data: - yield i - else: - while not self._exhausted: - newdata, self._last_key, self._exhausted = self._get_more(self._last_key, self.container) - self._data.append(newdata) - for i in newdata: - yield i - self._all_loaded = True + if not self._all_loaded: + self._load_all() + + data = self._data + for i in data: + yield i def _load_all(self): while not self._exhausted: - newdata, self._last_key, self._exhausted = self._get_more(self._last_key, self.container) + newdata, self._last_key, self._exhausted = \ + self._get_more(last_key=self._last_key, + value_dict=self._value_dict) self._data.extend(newdata) self._all_loaded = True + def __getitem__(self, index): + # TODO: If not exhausted, exhaust it + if index >= len(self._data): + self._load_all() + + return self._data[index] + def __len__(self): self._load_all() - return len(self._data) \ No newline at end of file + return len(self._data) diff --git a/test/test_types.py b/test/test_types.py index 6050431..79c1d1f 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -13,49 +13,89 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys import unittest -from libcloud.common.types import LazyList - -class MyLazyList(LazyList): - @staticmethod - def _get_more(last_key, container): - data = [1, 2, 3, 4, 5] - return data, 5, True - -class MyLazyList2(LazyList): - @staticmethod - def _get_more(last_key, container): - if not last_key: - data, last_key, exhausted = [1, 2, 3, 4, 5], 5, False - else: - data, last_key, exhausted = [6, 7, 8, 9, 10], 10, True - return data, last_key, exhausted +from libcloud.common.types import LazyList class TestLazyList(unittest.TestCase): -# def setUp(self): -# super(TestLazyList, self).setUp -# -# def tearDown(self): -# super(TestLazyList, self).tearDown + def setUp(self): + super(TestLazyList, self).setUp + self._get_more_counter = 0 + + def tearDown(self): + super(TestLazyList, self).tearDown def test_init(self): data = [1, 2, 3, 4, 5] - ll = LazyList(MyLazyList._get_more) + ll = LazyList(get_more=self._get_more_exhausted) ll_list = list(ll) self.assertEqual(ll_list, data) def test_iterator(self): data = [1, 2, 3, 4, 5] - ll = LazyList(MyLazyList._get_more) + ll = LazyList(get_more=self._get_more_exhausted) for i, d in enumerate(ll): self.assertEqual(d, data[i]) + def test_empty_list(self): + ll = LazyList(get_more=self._get_more_empty) + + self.assertEqual(list(ll), []) + self.assertEqual(len(ll), 0) + self.assertTrue(10 not in ll) + def test_iterator_not_exhausted(self): data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - ll = LazyList(MyLazyList2._get_more) + ll = LazyList(get_more=self._get_more_not_exhausted) number_of_iterations = 0 for i, d in enumerate(ll): self.assertEqual(d, data[i]) number_of_iterations += 1 self.assertEqual(number_of_iterations, 10) + + def test_len(self): + ll = LazyList(get_more=self._get_more_not_exhausted) + ll = LazyList(get_more=self._get_more_not_exhausted) + + self.assertEqual(len(ll), 10) + + def test_contains(self): + ll = LazyList(get_more=self._get_more_not_exhausted) + + self.assertTrue(40 not in ll) + self.assertTrue(1 in ll) + self.assertTrue(5 in ll) + self.assertTrue(10 in ll) + + def test_indexing(self): + ll = LazyList(get_more=self._get_more_not_exhausted) + + self.assertEqual(ll[0], 1) + self.assertEqual(ll[9], 10) + + try: + ll[11] + except IndexError: + pass + else: + self.fail('Exception was not thrown') + + def _get_more_empty(self, last_key, value_dict): + return [], None, True + + def _get_more_exhausted(self, last_key, value_dict): + data = [1, 2, 3, 4, 5] + return data, 5, True + + def _get_more_not_exhausted(self, last_key, value_dict): + self._get_more_counter += 1 + if not last_key: + data, last_key, exhausted = [1, 2, 3, 4, 5], 5, False + else: + data, last_key, exhausted = [6, 7, 8, 9, 10], 10, True + + return data, last_key, exhausted + +if __name__ == '__main__': + sys.exit(unittest.main()) -- 1.7.3.5 From 197cc59d658867fe4392d238a9f0fee5cf427ee3 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 25 Jun 2011 21:07:50 +0200 Subject: [PATCH 05/12] pep8. --- libcloud/common/types.py | 8 +++++++- test/test_types.py | 1 + 2 files changed, 8 insertions(+), 1 deletions(-) diff --git a/libcloud/common/types.py b/libcloud/common/types.py index 879364f..16e7e4d 100644 --- a/libcloud/common/types.py +++ b/libcloud/common/types.py @@ -21,6 +21,7 @@ __all__ = [ "LazyList" ] + class LibcloudError(Exception): """The base class for other libcloud exceptions""" @@ -31,9 +32,10 @@ class LibcloudError(Exception): def __str__(self): return ("") + class MalformedResponseError(LibcloudError): """Exception for the cases when a provider returns a malformed response, e.g. you request JSON and provider returns @@ -52,6 +54,7 @@ class MalformedResponseError(LibcloudError): + ">: " + repr(self.body)) + class InvalidCredsError(LibcloudError): """Exception used when invalid credentials are used on a provider.""" @@ -59,12 +62,15 @@ class InvalidCredsError(LibcloudError): driver=None): self.value = value self.driver = driver + def __str__(self): return repr(self.value) + # Deprecated alias of L{InvalidCredsError} InvalidCredsException = InvalidCredsError + class LazyList(object): def __init__(self, get_more, value_dict=None): diff --git a/test/test_types.py b/test/test_types.py index 79c1d1f..fe3d9de 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -18,6 +18,7 @@ import unittest from libcloud.common.types import LazyList + class TestLazyList(unittest.TestCase): def setUp(self): super(TestLazyList, self).setUp -- 1.7.3.5 From f4b993d82009be511f8df85c646dba8ea980211f Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 25 Jun 2011 21:13:52 +0200 Subject: [PATCH 06/12] Rename get_move to _get_move, move it to the end of the file and update it so it uses value_dict... --- libcloud/storage/drivers/s3.py | 50 ++++++++++++++++++++++----------------- 1 files changed, 28 insertions(+), 22 deletions(-) diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index e784d3e..ce6ebb9 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -37,8 +37,6 @@ from libcloud.storage.types import ObjectDoesNotExistError from libcloud.storage.types import ObjectHashMismatchError from libcloud.common.types import LazyList -in_development_warning('libcloud.storage.drivers.s3') - # How long before the token expires EXPIRATION_SECONDS = 15 * 60 @@ -175,27 +173,9 @@ class S3StorageDriver(StorageDriver): raise LibcloudError('Unexpected status code: %s' % (response.status), driver=self) - def get_more(self, last_key, container): - params = {} - if last_key: - params['marker'] = last_key - response = self.connection.request('/%s' % (container.name), params = params) - - if response.status == httplib.OK: - objects = self._to_objs(obj=response.object, - xpath='Contents', container=container) - exhausted = "false" == response.object.findtext(fixxpath(xpath='IsTruncated', namespace=NAMESPACE)) - print "called with last_key: %s, returned %d objects" % (last_key, len(objects)) - if (len(objects) > 0): - last_key = objects[-1].name - else: - last_key = None - return objects, last_key, exhausted - raise LibcloudError('Unexpected status code: %s' % (response.status), - driver=self) - def list_container_objects(self, container): - return LazyList(self.get_more, container) + value_dict = { 'container': container } + return LazyList(get_more=self._get_more, value_dict=value_dict) def get_container(self, container_name): # This is very inefficient, but afaik it's the only way to do it @@ -343,6 +323,32 @@ class S3StorageDriver(StorageDriver): name = urllib.quote(name) return name + def _get_more(self, last_key, value_dict): + container = value_dict['container'] + params = {} + + if last_key: + params['marker'] = last_key + + response = self.connection.request('/%s' % (container.name), + params=params) + + if response.status == httplib.OK: + objects = self._to_objs(obj=response.object, + xpath='Contents', container=container) + is_truncated = response.object.findtext(fixxpath(xpath='IsTruncated', + namespace=NAMESPACE)).lower() + exhausted = (is_truncated == 'false') + + if (len(objects) > 0): + last_key = objects[-1].name + else: + last_key = None + return objects, last_key, exhausted + + raise LibcloudError('Unexpected status code: %s' % (response.status), + driver=self) + def _put_object(self, container, object_name, upload_func, upload_func_kwargs, extra=None, file_path=None, iterator=None, verify_hash=True, storage_class=None): -- 1.7.3.5 From d70601904763d8919dd329656b31a8f3fd19003d Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 25 Jun 2011 21:26:51 +0200 Subject: [PATCH 07/12] Add tests for get_more iterator for the S3 driver. --- .../s3/list_container_objects_not_exhausted1.xml | 38 ++++++++++++++++++++ .../s3/list_container_objects_not_exhausted2.xml | 28 ++++++++++++++ test/storage/test_s3.py | 27 ++++++++++++++ 3 files changed, 93 insertions(+), 0 deletions(-) create mode 100644 test/storage/fixtures/s3/list_container_objects_not_exhausted1.xml create mode 100644 test/storage/fixtures/s3/list_container_objects_not_exhausted2.xml diff --git a/test/storage/fixtures/s3/list_container_objects_not_exhausted1.xml b/test/storage/fixtures/s3/list_container_objects_not_exhausted1.xml new file mode 100644 index 0000000..5a02ed0 --- /dev/null +++ b/test/storage/fixtures/s3/list_container_objects_not_exhausted1.xml @@ -0,0 +1,38 @@ + + + test_container + + + 1000 + true + + 1.zip + 2011-04-09T19:05:18.000Z + "4397da7a7649e8085de9916c240e8166" + 1234567 + + 65a011niqo39cdf8ec533ec3d1ccaafsa932 + + STANDARD + + + 2.zip + 2011-04-09T19:05:18.000Z + "4397da7a7649e8085de9916c240e8166" + 1234567 + + 65a011niqo39cdf8ec533ec3d1ccaafsa932 + + STANDARD + + + 3.zip + 2011-04-09T19:05:18.000Z + "4397da7a7649e8085de9916c240e8166" + 1234567 + + 65a011niqo39cdf8ec533ec3d1ccaafsa932 + + STANDARD + + diff --git a/test/storage/fixtures/s3/list_container_objects_not_exhausted2.xml b/test/storage/fixtures/s3/list_container_objects_not_exhausted2.xml new file mode 100644 index 0000000..0bf5af4 --- /dev/null +++ b/test/storage/fixtures/s3/list_container_objects_not_exhausted2.xml @@ -0,0 +1,28 @@ + + + test_container + + + 3 + false + + 4.zip + 2011-04-09T19:05:18.000Z + "4397da7a7649e8085de9916c240e8166" + 1234567 + + 65a011niqo39cdf8ec533ec3d1ccaafsa932 + + STANDARD + + + 5.zip + 2011-04-09T19:05:18.000Z + "4397da7a7649e8085de9916c240e8166" + 1234567 + + 65a011niqo39cdf8ec533ec3d1ccaafsa932 + + STANDARD + + diff --git a/test/storage/test_s3.py b/test/storage/test_s3.py index 8e31c00..78d1604 100644 --- a/test/storage/test_s3.py +++ b/test/storage/test_s3.py @@ -105,6 +105,20 @@ class S3Tests(unittest.TestCase): self.assertEqual(obj.container.name, 'test_container') self.assertTrue('owner' in obj.meta_data) + def test_list_container_objects_iterator_has_more(self): + S3MockHttp.type = 'ITERATOR' + container = Container(name='test_container', extra={}, + driver=self.driver) + objects = self.driver.list_container_objects(container=container) + + obj = [o for o in objects if o.name == '1.zip'][0] + self.assertEqual(obj.hash, '4397da7a7649e8085de9916c240e8166') + self.assertEqual(obj.size, 1234567) + self.assertEqual(obj.container.name, 'test_container') + + self.assertTrue(obj in objects) + self.assertEqual(len(objects), 5) + def test_get_container_doesnt_exist(self): S3MockHttp.type = 'list_containers' try: @@ -459,6 +473,19 @@ class S3MockHttp(StorageMockHttp): self.base_headers, httplib.responses[httplib.OK]) + def _test_container_ITERATOR(self, method, url, body, headers): + if url.find('3.zip') == -1: + # First part of the response (first 3 objects) + body = self.fixtures.load('list_container_objects_not_exhausted1.xml') + else: + body = self.fixtures.load('list_container_objects_not_exhausted2.xml') + + return (httplib.OK, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _test2_test_list_containers(self, method, url, body, headers): # test_get_object body = self.fixtures.load('list_containers.xml') -- 1.7.3.5 From e817fff71b5a48ab75e75e180bceaea7118a2718 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 25 Jun 2011 22:24:17 +0200 Subject: [PATCH 08/12] Implement the LazyList interface in CloudFiles storage driver. --- libcloud/storage/drivers/cloudfiles.py | 25 ++++++++++++-- .../list_container_objects_not_exhausted1.json | 11 ++++++ .../list_container_objects_not_exhausted2.json | 8 ++++ test/storage/test_cloudfiles.py | 37 ++++++++++++++++++- 4 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted1.json create mode 100644 test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted2.json diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py index 0f882f1..883dee2 100644 --- a/libcloud/storage/drivers/cloudfiles.py +++ b/libcloud/storage/drivers/cloudfiles.py @@ -33,6 +33,7 @@ from libcloud.storage.types import ContainerIsNotEmptyError from libcloud.storage.types import ObjectDoesNotExistError from libcloud.storage.types import ObjectHashMismatchError from libcloud.storage.types import InvalidContainerNameError +from libcloud.common.types import LazyList from libcloud.common.rackspace import ( AUTH_HOST_US, AUTH_HOST_UK, RackspaceBaseConnection) @@ -161,16 +162,34 @@ class CloudFilesStorageDriver(StorageDriver): raise LibcloudError('Unexpected status code: %s' % (response.status)) def list_container_objects(self, container): - response = self.connection.request('/%s' % (container.name)) + value_dict = { 'container': container } + return LazyList(get_more=self._get_more, value_dict=value_dict) + + def _get_more(self, last_key, value_dict): + container = value_dict['container'] + params = {} + + if last_key: + params['marker'] = last_key + + response = self.connection.request('/%s' % (container.name), + params=params) if response.status == httplib.NO_CONTENT: # Empty or inexistent container - return [] + return [], None, True elif response.status == httplib.OK: - return self._to_object_list(json.loads(response.body), container) + objects = self._to_object_list(json.loads(response.body), container) + + # TODO: Is this really needed? + if len(objects) == 0: + return [], None, True + + return objects, objects[-1].name, False raise LibcloudError('Unexpected status code: %s' % (response.status)) + def get_container(self, container_name): response = self.connection.request('/%s' % (container_name), method='HEAD') diff --git a/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted1.json b/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted1.json new file mode 100644 index 0000000..f6382d3 --- /dev/null +++ b/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted1.json @@ -0,0 +1,11 @@ +[ + {"name":"foo-test-1","hash":"16265549b5bda64ecdaa5156de4c97cc", + "bytes":1160520,"content_type":"application/zip", + "last_modified":"2011-01-25T22:01:50.351810"}, + {"name":"foo-test-2","hash":"16265549b5bda64ecdaa5156de4c97bb", + "bytes":1160520,"content_type":"application/zip", + "last_modified":"2011-01-25T22:01:50.351810"}, + {"name":"foo-test-3","hash":"16265549b5bda64ecdaa5156de4c97ee", + "bytes":1160520,"content_type":"application/zip", + "last_modified":"2011-01-25T22:01:46.549890"} +] diff --git a/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted2.json b/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted2.json new file mode 100644 index 0000000..6ad8210 --- /dev/null +++ b/test/storage/fixtures/cloudfiles/list_container_objects_not_exhausted2.json @@ -0,0 +1,8 @@ +[ + {"name":"foo-test-4","hash":"16265549b5bda64ecdaa5156de4c97cc", + "bytes":1160520,"content_type":"application/zip", + "last_modified":"2011-01-25T22:01:50.351810"}, + {"name":"foo-test-5","hash":"16265549b5bda64ecdaa5156de4c97bb", + "bytes":1160520,"content_type":"application/zip", + "last_modified":"2011-01-25T22:01:50.351810"} +] diff --git a/test/storage/test_cloudfiles.py b/test/storage/test_cloudfiles.py index a79a86b..fd87da5 100644 --- a/test/storage/test_cloudfiles.py +++ b/test/storage/test_cloudfiles.py @@ -90,6 +90,18 @@ class CloudFilesTests(unittest.TestCase): self.assertEqual(obj.size, 1160520) self.assertEqual(obj.container.name, 'test_container') + def test_list_container_objects_iterator(self): + CloudFilesMockHttp.type = 'ITERATOR' + container = Container( + name='test_container', extra={}, driver=self.driver) + objects = self.driver.list_container_objects(container=container) + self.assertEqual(len(objects), 5) + + obj = [o for o in objects if o.name == 'foo-test-1'][0] + self.assertEqual(obj.hash, '16265549b5bda64ecdaa5156de4c97cc') + self.assertEqual(obj.size, 1160520) + self.assertEqual(obj.container.name, 'test_container') + def test_get_container(self): container = self.driver.get_container(container_name='test_container') self.assertEqual(container.name, 'test_container') @@ -475,6 +487,7 @@ class CloudFilesMockHttp(StorageMockHttp): httplib.responses[httplib.OK]) def _v1_MossoCloudFS_test_container_EMPTY(self, method, url, body, headers): + ### body = self.fixtures.load('list_container_objects_empty.json') return (httplib.OK, body, @@ -485,8 +498,12 @@ class CloudFilesMockHttp(StorageMockHttp): headers = copy.deepcopy(self.base_headers) if method == 'GET': # list_container_objects - body = self.fixtures.load('list_container_objects.json') - status_code = httplib.OK + if url.find('marker') == -1: + body = self.fixtures.load('list_container_objects.json') + status_code = httplib.OK + else: + body = '' + status_code = httplib.NO_CONTENT elif method == 'HEAD': # get_container body = self.fixtures.load('list_container_objects_empty.json') @@ -496,6 +513,22 @@ class CloudFilesMockHttp(StorageMockHttp): }) return (status_code, body, headers, httplib.responses[httplib.OK]) + def _v1_MossoCloudFS_test_container_ITERATOR(self, method, url, body, headers): + headers = copy.deepcopy(self.base_headers) + # list_container_objects + if url.find('foo-test-3') != -1: + body = self.fixtures.load('list_container_objects_not_exhausted2.json') + status_code = httplib.OK + elif url.find('foo-test-5') != -1: + body = '' + status_code = httplib.NO_CONTENT + else: + # First request + body = self.fixtures.load('list_container_objects_not_exhausted1.json') + status_code = httplib.OK + + return (status_code, body, headers, httplib.responses[httplib.OK]) + def _v1_MossoCloudFS_test_container_not_found( self, method, url, body, headers): # test_get_container_not_found -- 1.7.3.5 From 5dac09cfdd447e422e0309c73353e7caf27d4308 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 26 Jun 2011 11:26:29 +0200 Subject: [PATCH 09/12] Only call load_all if not all of the items have already been loaded, also add another test. --- libcloud/common/types.py | 3 +-- test/test_types.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libcloud/common/types.py b/libcloud/common/types.py index 16e7e4d..8e242fa 100644 --- a/libcloud/common/types.py +++ b/libcloud/common/types.py @@ -98,8 +98,7 @@ class LazyList(object): self._all_loaded = True def __getitem__(self, index): - # TODO: If not exhausted, exhaust it - if index >= len(self._data): + if index >= len(self._data) and not self._all_loaded: self._load_all() return self._data[index] diff --git a/test/test_types.py b/test/test_types.py index fe3d9de..9be2009 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -74,6 +74,7 @@ class TestLazyList(unittest.TestCase): self.assertEqual(ll[0], 1) self.assertEqual(ll[9], 10) + self.assertEqual(ll[-1], 10) try: ll[11] -- 1.7.3.5 From 9ae3d17684c16521e562fbb5ab6f5338c1de49a8 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 26 Jun 2011 11:27:10 +0200 Subject: [PATCH 10/12] Move load all to the end. --- libcloud/common/types.py | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libcloud/common/types.py b/libcloud/common/types.py index 8e242fa..75b289f 100644 --- a/libcloud/common/types.py +++ b/libcloud/common/types.py @@ -89,14 +89,6 @@ class LazyList(object): for i in data: yield i - def _load_all(self): - while not self._exhausted: - newdata, self._last_key, self._exhausted = \ - self._get_more(last_key=self._last_key, - value_dict=self._value_dict) - self._data.extend(newdata) - self._all_loaded = True - def __getitem__(self, index): if index >= len(self._data) and not self._all_loaded: self._load_all() @@ -106,3 +98,11 @@ class LazyList(object): def __len__(self): self._load_all() return len(self._data) + + def _load_all(self): + while not self._exhausted: + newdata, self._last_key, self._exhausted = \ + self._get_more(last_key=self._last_key, + value_dict=self._value_dict) + self._data.extend(newdata) + self._all_loaded = True -- 1.7.3.5 From 1cc29239e8af84def4543f3ff3d50cf505f8e0a8 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 26 Jun 2011 11:45:05 +0200 Subject: [PATCH 11/12] Add __repr__ method to the LazyList class and add tests for it. --- libcloud/common/types.py | 6 ++++++ test/test_types.py | 9 +++++++++ 2 files changed, 15 insertions(+), 0 deletions(-) diff --git a/libcloud/common/types.py b/libcloud/common/types.py index 75b289f..17e7b84 100644 --- a/libcloud/common/types.py +++ b/libcloud/common/types.py @@ -99,6 +99,12 @@ class LazyList(object): self._load_all() return len(self._data) + def __repr__(self): + self._load_all() + repr_string = ', ' .join([repr(item) for item in self._data]) + repr_string = '[%s]' % (repr_string) + return repr_string + def _load_all(self): while not self._exhausted: newdata, self._last_key, self._exhausted = \ diff --git a/test/test_types.py b/test/test_types.py index 9be2009..453ffcc 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -83,6 +83,15 @@ class TestLazyList(unittest.TestCase): else: self.fail('Exception was not thrown') + def test_repr(self): + ll1 = LazyList(get_more=self._get_more_empty) + ll2 = LazyList(get_more=self._get_more_exhausted) + ll3 = LazyList(get_more=self._get_more_not_exhausted) + + self.assertEqual(repr(ll1), '[]') + self.assertEqual(repr(ll2), '[1, 2, 3, 4, 5]') + self.assertEqual(repr(ll3), '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]') + def _get_more_empty(self, last_key, value_dict): return [], None, True -- 1.7.3.5 From a7a6eb3b57180ddb14d9bd91d33cb81f3cb3d5dc Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 26 Jun 2011 12:07:15 +0200 Subject: [PATCH 12/12] Move _get_more to the end of the file. --- libcloud/storage/drivers/cloudfiles.py | 49 +++++++++++++++---------------- 1 files changed, 24 insertions(+), 25 deletions(-) diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py index 883dee2..bbb3993 100644 --- a/libcloud/storage/drivers/cloudfiles.py +++ b/libcloud/storage/drivers/cloudfiles.py @@ -165,31 +165,6 @@ class CloudFilesStorageDriver(StorageDriver): value_dict = { 'container': container } return LazyList(get_more=self._get_more, value_dict=value_dict) - def _get_more(self, last_key, value_dict): - container = value_dict['container'] - params = {} - - if last_key: - params['marker'] = last_key - - response = self.connection.request('/%s' % (container.name), - params=params) - - if response.status == httplib.NO_CONTENT: - # Empty or inexistent container - return [], None, True - elif response.status == httplib.OK: - objects = self._to_object_list(json.loads(response.body), container) - - # TODO: Is this really needed? - if len(objects) == 0: - return [], None, True - - return objects, objects[-1].name, False - - raise LibcloudError('Unexpected status code: %s' % (response.status)) - - def get_container(self, container_name): response = self.connection.request('/%s' % (container_name), method='HEAD') @@ -373,6 +348,30 @@ class CloudFilesStorageDriver(StorageDriver): raise LibcloudError('Unexpected status code: %s' % (response.status)) + def _get_more(self, last_key, value_dict): + container = value_dict['container'] + params = {} + + if last_key: + params['marker'] = last_key + + response = self.connection.request('/%s' % (container.name), + params=params) + + if response.status == httplib.NO_CONTENT: + # Empty or inexistent container + return [], None, True + elif response.status == httplib.OK: + objects = self._to_object_list(json.loads(response.body), container) + + # TODO: Is this really needed? + if len(objects) == 0: + return [], None, True + + return objects, objects[-1].name, False + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + def _put_object(self, container, object_name, upload_func, upload_func_kwargs, extra=None, file_path=None, iterator=None, verify_hash=True): -- 1.7.3.5