diff options
| -rw-r--r-- | src/mailman/docs/NEWS.rst | 2 | ||||
| -rw-r--r-- | src/mailman/rest/docs/lists.rst | 45 | ||||
| -rw-r--r-- | src/mailman/rest/docs/membership.rst | 40 | ||||
| -rw-r--r-- | src/mailman/rest/docs/users.rst | 31 | ||||
| -rw-r--r-- | src/mailman/rest/helpers.py | 34 | ||||
| -rw-r--r-- | src/mailman/rest/lists.py | 5 | ||||
| -rw-r--r-- | src/mailman/rest/members.py | 3 | ||||
| -rw-r--r-- | src/mailman/rest/tests/test_paginate.py | 145 | ||||
| -rw-r--r-- | src/mailman/rest/users.py | 3 |
9 files changed, 305 insertions, 3 deletions
diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index d5e634e7d..65b8cb05e 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -23,6 +23,8 @@ REST ---- * Add ``reply_to_address`` and ``first_strip_reply_to`` as writable attributes of a mailing list's configuration. (LP: #1157881) + * Support pagination of some large collections (lists, users, members). + Given by Florian Fuchs. (LP: #1156529) Configuration ------------- diff --git a/src/mailman/rest/docs/lists.rst b/src/mailman/rest/docs/lists.rst index 5c65a7951..295e8c0b7 100644 --- a/src/mailman/rest/docs/lists.rst +++ b/src/mailman/rest/docs/lists.rst @@ -50,6 +50,51 @@ You can also query for lists from a particular domain. total_size: 1 +Paginating over list records +---------------------------- + +Instead of returning all the list records at once, it's possible to return +them in pages by adding the GET parameters ``count`` and ``page`` to the +request URI. Page 1 is the first page and ``count`` defines the size of the +page. +:: + + >>> mlist = create_list('bird@example.com') + >>> transaction.commit() + + >>> dump_json('http://localhost:9001/3.0/domains/example.com/lists' + ... '?count=1&page=1') + entry 0: + display_name: Ant + fqdn_listname: ant@example.com + http_etag: "..." + list_id: ant.example.com + list_name: ant + mail_host: example.com + member_count: 0 + self_link: http://localhost:9001/3.0/lists/ant.example.com + volume: 1 + http_etag: "..." + start: 0 + total_size: 1 + + >>> dump_json('http://localhost:9001/3.0/domains/example.com/lists' + ... '?count=1&page=2') + entry 0: + display_name: Bird + fqdn_listname: bird@example.com + http_etag: "..." + list_id: bird.example.com + list_name: bird + mail_host: example.com + member_count: 0 + self_link: http://localhost:9001/3.0/lists/bird.example.com + volume: 1 + http_etag: "..." + start: 0 + total_size: 1 + + Creating lists via the API ========================== diff --git a/src/mailman/rest/docs/membership.rst b/src/mailman/rest/docs/membership.rst index 6189990cb..6deed7c48 100644 --- a/src/mailman/rest/docs/membership.rst +++ b/src/mailman/rest/docs/membership.rst @@ -203,6 +203,46 @@ We can also get just the members of a single mailing list. total_size: 2 +Paginating over member records +------------------------------ + +Instead of returning all the member records at once, it's possible to return +them in pages by adding the GET parameters ``count`` and ``page`` to the +request URI. Page 1 is the first page and ``count`` defines the size of the +page. + + >>> dump_json( + ... 'http://localhost:9001/3.0/lists/ant@example.com/roster/member' + ... '?count=1&page=1') + entry 0: + address: aperson@example.com + delivery_mode: regular + http_etag: ... + list_id: ant.example.com + role: member + self_link: http://localhost:9001/3.0/members/4 + user: http://localhost:9001/3.0/users/3 + http_etag: ... + start: 0 + total_size: 1 + +This works with members of a single list as well as with all members. + + >>> dump_json( + ... 'http://localhost:9001/3.0/members?count=1&page=1') + entry 0: + address: aperson@example.com + delivery_mode: regular + http_etag: ... + list_id: ant.example.com + role: member + self_link: http://localhost:9001/3.0/members/4 + user: http://localhost:9001/3.0/users/3 + http_etag: ... + start: 0 + total_size: 1 + + Owners and moderators ===================== diff --git a/src/mailman/rest/docs/users.rst b/src/mailman/rest/docs/users.rst index 36ec28efc..04533f578 100644 --- a/src/mailman/rest/docs/users.rst +++ b/src/mailman/rest/docs/users.rst @@ -62,6 +62,37 @@ returned in the REST API. total_size: 2 +Paginating over user records +---------------------------- + +Instead of returning all the user records at once, it's possible to return +them in pages by adding the GET parameters ``count`` and ``page`` to the +request URI. Page 1 is the first page and ``count`` defines the size of the +page. +:: + + >>> dump_json('http://localhost:9001/3.0/users?count=1&page=1') + entry 0: + created_on: 2005-08-01T07:49:23 + display_name: Anne Person + http_etag: "..." + self_link: http://localhost:9001/3.0/users/1 + user_id: 1 + http_etag: "..." + start: 0 + total_size: 1 + + >>> dump_json('http://localhost:9001/3.0/users?count=1&page=2') + entry 0: + created_on: 2005-08-01T07:49:23 + http_etag: "..." + self_link: http://localhost:9001/3.0/users/2 + user_id: 2 + http_etag: "..." + start: 0 + total_size: 1 + + Creating users ============== diff --git a/src/mailman/rest/helpers.py b/src/mailman/rest/helpers.py index 197e74362..3a548cb10 100644 --- a/src/mailman/rest/helpers.py +++ b/src/mailman/rest/helpers.py @@ -38,6 +38,7 @@ from cStringIO import StringIO from datetime import datetime, timedelta from flufl.enum import Enum from lazr.config import as_boolean +from restish import http from restish.http import Response from restish.resource import MethodDecorator from webob.multidict import MultiDict @@ -101,10 +102,43 @@ def etag(resource): """ assert 'http_etag' not in resource, 'Resource already etagged' etag = hashlib.sha1(repr(resource)).hexdigest() + resource['http_etag'] = '"{0}"'.format(etag) return json.dumps(resource, cls=ExtendedEncoder) +def paginate(method): + """Method decorator to paginate through collection result lists. + + Use this to return only a slice of a collection, specified in the request + itself. The request should use query parameters `count` and `page` to + specify the slice they want. The slice will start at index + ``(page - 1) * count`` and end (exclusive) at ``(page * count)``. + + Decorated methods must take ``self`` and ``request`` as the first two + arguments. + """ + def wrapper(self, request, *args, **kwargs): + try: + count = int(request.GET['count']) + page = int(request.GET['page']) + if count < 0 or page < 0: + return http.bad_request([], b'Invalid parameters') + # Wrong parameter types or no GET attribute in request object. + except (AttributeError, ValueError, TypeError): + return http.bad_request([], b'Invalid parameters') + # No count/page params. + except KeyError: + count = page = None + result = method(self, request, *args, **kwargs) + if count is None and page is None: + return result + list_start = int((page - 1) * count) + list_end = int(page * count) + return result[list_start:list_end] + return wrapper + + class CollectionMixin: """Mixin class for common collection-ish things.""" diff --git a/src/mailman/rest/lists.py b/src/mailman/rest/lists.py index cefd91b7c..2a3c72bfd 100644 --- a/src/mailman/rest/lists.py +++ b/src/mailman/rest/lists.py @@ -40,7 +40,7 @@ from mailman.interfaces.member import MemberRole from mailman.interfaces.subscriptions import ISubscriptionService from mailman.rest.configuration import ListConfiguration from mailman.rest.helpers import ( - CollectionMixin, etag, no_content, path_to, restish_matcher) + CollectionMixin, etag, no_content, paginate, path_to, restish_matcher) from mailman.rest.members import AMember, MemberCollection from mailman.rest.moderation import HeldMessages, SubscriptionRequests from mailman.rest.validator import Validator @@ -115,6 +115,7 @@ class _ListBase(resource.Resource, CollectionMixin): self_link=path_to('lists/{0}'.format(mlist.list_id)), ) + @paginate def _get_collection(self, request): """See `CollectionMixin`.""" return list(getUtility(IListManager)) @@ -229,6 +230,7 @@ class MembersOfList(MemberCollection): self._mlist = mailing_list self._role = role + @paginate def _get_collection(self, request): """See `CollectionMixin`.""" # Overrides _MemberBase._get_collection() because we only want to @@ -250,6 +252,7 @@ class ListsForDomain(_ListBase): resource = self._make_collection(request) return http.ok([], etag(resource)) + @paginate def _get_collection(self, request): """See `CollectionMixin`.""" return list(self._domain.mailing_lists) diff --git a/src/mailman/rest/members.py b/src/mailman/rest/members.py index 5cf357579..76b2ed1a7 100644 --- a/src/mailman/rest/members.py +++ b/src/mailman/rest/members.py @@ -43,7 +43,7 @@ from mailman.interfaces.subscriptions import ISubscriptionService from mailman.interfaces.user import UnverifiedAddressError from mailman.interfaces.usermanager import IUserManager from mailman.rest.helpers import ( - CollectionMixin, PATCH, etag, no_content, path_to) + CollectionMixin, PATCH, etag, no_content, paginate, path_to) from mailman.rest.preferences import Preferences, ReadOnlyPreferences from mailman.rest.validator import ( Validator, enum_validator, subscriber_validator) @@ -69,6 +69,7 @@ class _MemberBase(resource.Resource, CollectionMixin): delivery_mode=member.delivery_mode, ) + @paginate def _get_collection(self, request): """See `CollectionMixin`.""" return list(getUtility(ISubscriptionService)) diff --git a/src/mailman/rest/tests/test_paginate.py b/src/mailman/rest/tests/test_paginate.py new file mode 100644 index 000000000..ab39d0996 --- /dev/null +++ b/src/mailman/rest/tests/test_paginate.py @@ -0,0 +1,145 @@ +# Copyright (C) 2013 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""paginate helper tests.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestPaginateHelper', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.database.transaction import transaction +from mailman.rest.helpers import paginate +from mailman.testing.layers import RESTLayer + + + +class _FakeRequest: + """Fake restish.http.Request object.""" + + def __init__(self, count=None, page=None): + self.GET = {} + if count is not None: + self.GET['count'] = count + if page is not None: + self.GET['page'] = page + + + +class TestPaginateHelper(unittest.TestCase): + """Test the @paginate decorator.""" + + layer = RESTLayer + + def setUp(self): + with transaction(): + self._mlist = create_list('test@example.com') + + def test_no_pagination(self): + # When there is no pagination params in the request, all 5 items in + # the collection are returned. + @paginate + def get_collection(self, request): + return ['one', 'two', 'three', 'four', 'five'] + # Expect 5 items + page = get_collection(None, _FakeRequest()) + self.assertEqual(page, ['one', 'two', 'three', 'four', 'five']) + + def test_valid_pagination_request_page_one(self): + # ?count=2&page=1 returns the first page, with two items in it. + @paginate + def get_collection(self, request): + return ['one', 'two', 'three', 'four', 'five'] + page = get_collection(None, _FakeRequest(2, 1)) + self.assertEqual(page, ['one', 'two']) + + def test_valid_pagination_request_page_two(self): + # ?count=2&page=2 returns the second page, where a page has two items + # in it. + @paginate + def get_collection(self, request): + return ['one', 'two', 'three', 'four', 'five'] + page = get_collection(None, _FakeRequest(2, 2)) + self.assertEqual(page, ['three', 'four']) + + def test_2nd_index_larger_than_total(self): + # ?count=2&page=3 returns the third page with page size 2, but the + # last page only has one item in it. + @paginate + def get_collection(self, request): + return ['one', 'two', 'three', 'four', 'five'] + page = get_collection(None, _FakeRequest(2, 3)) + self.assertEqual(page, ['five']) + + def test_out_of_range_returns_empty_list(self): + # ?count=2&page=4 returns the fourth page, which doesn't exist, so an + # empty collection is returned. + @paginate + def get_collection(self, request): + return ['one', 'two', 'three', 'four', 'five'] + page = get_collection(None, _FakeRequest(2, 4)) + self.assertEqual(page, []) + + def test_count_as_string_returns_bad_request(self): + # ?count=two&page=2 are not valid values, so a bad request occurs. + @paginate + def get_collection(self, request): + return [] + response = get_collection(None, _FakeRequest('two', 1)) + self.assertEqual(response.status, '400 Bad Request') + + def test_no_get_attr_returns_bad_request(self): + # ?count=two&page=2 are not valid values so a bad request is returned. + @paginate + def get_collection(self, request): + return [] + request = _FakeRequest() + del request.GET + # The request object has no GET attribute. + self.assertIsNone(getattr(request, 'GET', None)) + response = get_collection(None, request) + self.assertEqual(response.status, '400 Bad Request') + + def test_negative_count(self): + # ?count=-1&page=1 + @paginate + def get_collection(self, request): + return ['one', 'two', 'three', 'four', 'five'] + response = get_collection(None, _FakeRequest(-1, 1)) + self.assertEqual(response.status, '400 Bad Request') + + def test_negative_page(self): + # ?count=1&page=-1 + @paginate + def get_collection(self, request): + return ['one', 'two', 'three', 'four', 'five'] + response = get_collection(None, _FakeRequest(1, -1)) + self.assertEqual(response.status, '400 Bad Request') + + def test_negative_page_and_count(self): + # ?count=1&page=-1 + @paginate + def get_collection(self, request): + return ['one', 'two', 'three', 'four', 'five'] + response = get_collection(None, _FakeRequest(-1, -1)) + self.assertEqual(response.status, '400 Bad Request') diff --git a/src/mailman/rest/users.py b/src/mailman/rest/users.py index ffe2009b6..6826c92b6 100644 --- a/src/mailman/rest/users.py +++ b/src/mailman/rest/users.py @@ -38,7 +38,7 @@ from mailman.interfaces.address import ExistingAddressError from mailman.interfaces.usermanager import IUserManager from mailman.rest.addresses import UserAddresses from mailman.rest.helpers import ( - CollectionMixin, GetterSetter, PATCH, etag, no_content, path_to) + CollectionMixin, GetterSetter, PATCH, etag, no_content, paginate, path_to) from mailman.rest.preferences import Preferences from mailman.rest.validator import PatchValidator, Validator @@ -86,6 +86,7 @@ class _UserBase(resource.Resource, CollectionMixin): resource['display_name'] = user.display_name return resource + @paginate def _get_collection(self, request): """See `CollectionMixin`.""" return list(getUtility(IUserManager).users) |