Commit 6ca91614 authored by Barry Warsaw's avatar Barry Warsaw

* Support pagination of some large collections (lists, users, members).

  Given by Florian Fuchs.  (LP: #1156529)
parents bc4776ba 4dfa297d
......@@ -23,6 +23,8 @@ REST
----
* Add ``reply_to_address`` and ``first_strip_reply_to`` as writable
attributes of a mailing list's configuration. (LP: #1157881)
* Support pagination of some large collections (lists, users, members).
Given by Florian Fuchs. (LP: #1156529)
Configuration
-------------
......
......@@ -50,6 +50,51 @@ You can also query for lists from a particular domain.
total_size: 1
Paginating over list records
----------------------------
Instead of returning all the list records at once, it's possible to return
them in pages by adding the GET parameters ``count`` and ``page`` to the
request URI. Page 1 is the first page and ``count`` defines the size of the
page.
::
>>> mlist = create_list('[email protected]')
>>> transaction.commit()
>>> dump_json('http://localhost:9001/3.0/domains/example.com/lists'
... '?count=1&page=1')
entry 0:
display_name: Ant
fqdn_listname: [email protected]
http_etag: "..."
list_id: ant.example.com
list_name: ant
mail_host: example.com
member_count: 0
self_link: http://localhost:9001/3.0/lists/ant.example.com
volume: 1
http_etag: "..."
start: 0
total_size: 1
>>> dump_json('http://localhost:9001/3.0/domains/example.com/lists'
... '?count=1&page=2')
entry 0:
display_name: Bird
fqdn_listname: [email protected]
http_etag: "..."
list_id: bird.example.com
list_name: bird
mail_host: example.com
member_count: 0
self_link: http://localhost:9001/3.0/lists/bird.example.com
volume: 1
http_etag: "..."
start: 0
total_size: 1
Creating lists via the API
==========================
......
......@@ -203,6 +203,46 @@ We can also get just the members of a single mailing list.
total_size: 2
Paginating over member records
------------------------------
Instead of returning all the member records at once, it's possible to return
them in pages by adding the GET parameters ``count`` and ``page`` to the
request URI. Page 1 is the first page and ``count`` defines the size of the
page.
>>> dump_json(
... 'http://localhost:9001/3.0/lists/[email protected]/roster/member'
... '?count=1&page=1')
entry 0:
address: [email protected]
delivery_mode: regular
http_etag: ...
list_id: ant.example.com
role: member
self_link: http://localhost:9001/3.0/members/4
user: http://localhost:9001/3.0/users/3
http_etag: ...
start: 0
total_size: 1
This works with members of a single list as well as with all members.
>>> dump_json(
... 'http://localhost:9001/3.0/members?count=1&page=1')
entry 0:
address: [email protected]
delivery_mode: regular
http_etag: ...
list_id: ant.example.com
role: member
self_link: http://localhost:9001/3.0/members/4
user: http://localhost:9001/3.0/users/3
http_etag: ...
start: 0
total_size: 1
Owners and moderators
=====================
......
......@@ -62,6 +62,37 @@ returned in the REST API.
total_size: 2
Paginating over user records
----------------------------
Instead of returning all the user records at once, it's possible to return
them in pages by adding the GET parameters ``count`` and ``page`` to the
request URI. Page 1 is the first page and ``count`` defines the size of the
page.
::
>>> dump_json('http://localhost:9001/3.0/users?count=1&page=1')
entry 0:
created_on: 2005-08-01T07:49:23
display_name: Anne Person
http_etag: "..."
self_link: http://localhost:9001/3.0/users/1
user_id: 1
http_etag: "..."
start: 0
total_size: 1
>>> dump_json('http://localhost:9001/3.0/users?count=1&page=2')
entry 0:
created_on: 2005-08-01T07:49:23
http_etag: "..."
self_link: http://localhost:9001/3.0/users/2
user_id: 2
http_etag: "..."
start: 0
total_size: 1
Creating users
==============
......
......@@ -38,6 +38,7 @@ from cStringIO import StringIO
from datetime import datetime, timedelta
from flufl.enum import Enum
from lazr.config import as_boolean
from restish import http
from restish.http import Response
from restish.resource import MethodDecorator
from webob.multidict import MultiDict
......@@ -101,10 +102,43 @@ def etag(resource):
"""
assert 'http_etag' not in resource, 'Resource already etagged'
etag = hashlib.sha1(repr(resource)).hexdigest()
resource['http_etag'] = '"{0}"'.format(etag)
return json.dumps(resource, cls=ExtendedEncoder)
def paginate(method):
"""Method decorator to paginate through collection result lists.
Use this to return only a slice of a collection, specified in the request
itself. The request should use query parameters `count` and `page` to
specify the slice they want. The slice will start at index
``(page - 1) * count`` and end (exclusive) at ``(page * count)``.
Decorated methods must take ``self`` and ``request`` as the first two
arguments.
"""
def wrapper(self, request, *args, **kwargs):
try:
count = int(request.GET['count'])
page = int(request.GET['page'])
if count < 0 or page < 0:
return http.bad_request([], b'Invalid parameters')
# Wrong parameter types or no GET attribute in request object.
except (AttributeError, ValueError, TypeError):
return http.bad_request([], b'Invalid parameters')
# No count/page params.
except KeyError:
count = page = None
result = method(self, request, *args, **kwargs)
if count is None and page is None:
return result
list_start = int((page - 1) * count)
list_end = int(page * count)
return result[list_start:list_end]
return wrapper
class CollectionMixin:
"""Mixin class for common collection-ish things."""
......
......@@ -40,7 +40,7 @@ from mailman.interfaces.member import MemberRole
from mailman.interfaces.subscriptions import ISubscriptionService
from mailman.rest.configuration import ListConfiguration
from mailman.rest.helpers import (
CollectionMixin, etag, no_content, path_to, restish_matcher)
CollectionMixin, etag, no_content, paginate, path_to, restish_matcher)
from mailman.rest.members import AMember, MemberCollection
from mailman.rest.moderation import HeldMessages, SubscriptionRequests
from mailman.rest.validator import Validator
......@@ -115,6 +115,7 @@ class _ListBase(resource.Resource, CollectionMixin):
self_link=path_to('lists/{0}'.format(mlist.list_id)),
)
@paginate
def _get_collection(self, request):
"""See `CollectionMixin`."""
return list(getUtility(IListManager))
......@@ -229,6 +230,7 @@ class MembersOfList(MemberCollection):
self._mlist = mailing_list
self._role = role
@paginate
def _get_collection(self, request):
"""See `CollectionMixin`."""
# Overrides _MemberBase._get_collection() because we only want to
......@@ -250,6 +252,7 @@ class ListsForDomain(_ListBase):
resource = self._make_collection(request)
return http.ok([], etag(resource))
@paginate
def _get_collection(self, request):
"""See `CollectionMixin`."""
return list(self._domain.mailing_lists)
......@@ -43,7 +43,7 @@ from mailman.interfaces.subscriptions import ISubscriptionService
from mailman.interfaces.user import UnverifiedAddressError
from mailman.interfaces.usermanager import IUserManager
from mailman.rest.helpers import (
CollectionMixin, PATCH, etag, no_content, path_to)
CollectionMixin, PATCH, etag, no_content, paginate, path_to)
from mailman.rest.preferences import Preferences, ReadOnlyPreferences
from mailman.rest.validator import (
Validator, enum_validator, subscriber_validator)
......@@ -69,6 +69,7 @@ class _MemberBase(resource.Resource, CollectionMixin):
delivery_mode=member.delivery_mode,
)
@paginate
def _get_collection(self, request):
"""See `CollectionMixin`."""
return list(getUtility(ISubscriptionService))
......
# Copyright (C) 2013 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
# GNU Mailman is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
"""paginate helper tests."""
from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'TestPaginateHelper',
]
import unittest
from mailman.app.lifecycle import create_list
from mailman.database.transaction import transaction
from mailman.rest.helpers import paginate
from mailman.testing.layers import RESTLayer
class _FakeRequest:
"""Fake restish.http.Request object."""
def __init__(self, count=None, page=None):
self.GET = {}
if count is not None:
self.GET['count'] = count
if page is not None:
self.GET['page'] = page
class TestPaginateHelper(unittest.TestCase):
"""Test the @paginate decorator."""
layer = RESTLayer
def setUp(self):
with transaction():
self._mlist = create_list('[email protected]')
def test_no_pagination(self):
# When there is no pagination params in the request, all 5 items in
# the collection are returned.
@paginate
def get_collection(self, request):
return ['one', 'two', 'three', 'four', 'five']
# Expect 5 items
page = get_collection(None, _FakeRequest())
self.assertEqual(page, ['one', 'two', 'three', 'four', 'five'])
def test_valid_pagination_request_page_one(self):
# ?count=2&page=1 returns the first page, with two items in it.
@paginate
def get_collection(self, request):
return ['one', 'two', 'three', 'four', 'five']
page = get_collection(None, _FakeRequest(2, 1))
self.assertEqual(page, ['one', 'two'])
def test_valid_pagination_request_page_two(self):
# ?count=2&page=2 returns the second page, where a page has two items
# in it.
@paginate
def get_collection(self, request):
return ['one', 'two', 'three', 'four', 'five']
page = get_collection(None, _FakeRequest(2, 2))
self.assertEqual(page, ['three', 'four'])
def test_2nd_index_larger_than_total(self):
# ?count=2&page=3 returns the third page with page size 2, but the
# last page only has one item in it.
@paginate
def get_collection(self, request):
return ['one', 'two', 'three', 'four', 'five']
page = get_collection(None, _FakeRequest(2, 3))
self.assertEqual(page, ['five'])
def test_out_of_range_returns_empty_list(self):
# ?count=2&page=4 returns the fourth page, which doesn't exist, so an
# empty collection is returned.
@paginate
def get_collection(self, request):
return ['one', 'two', 'three', 'four', 'five']
page = get_collection(None, _FakeRequest(2, 4))
self.assertEqual(page, [])
def test_count_as_string_returns_bad_request(self):
# ?count=two&page=2 are not valid values, so a bad request occurs.
@paginate
def get_collection(self, request):
return []
response = get_collection(None, _FakeRequest('two', 1))
self.assertEqual(response.status, '400 Bad Request')
def test_no_get_attr_returns_bad_request(self):
# ?count=two&page=2 are not valid values so a bad request is returned.
@paginate
def get_collection(self, request):
return []
request = _FakeRequest()
del request.GET
# The request object has no GET attribute.
self.assertIsNone(getattr(request, 'GET', None))
response = get_collection(None, request)
self.assertEqual(response.status, '400 Bad Request')
def test_negative_count(self):
# ?count=-1&page=1
@paginate
def get_collection(self, request):
return ['one', 'two', 'three', 'four', 'five']
response = get_collection(None, _FakeRequest(-1, 1))
self.assertEqual(response.status, '400 Bad Request')
def test_negative_page(self):
# ?count=1&page=-1
@paginate
def get_collection(self, request):
return ['one', 'two', 'three', 'four', 'five']
response = get_collection(None, _FakeRequest(1, -1))
self.assertEqual(response.status, '400 Bad Request')
def test_negative_page_and_count(self):
# ?count=1&page=-1
@paginate
def get_collection(self, request):
return ['one', 'two', 'three', 'four', 'five']
response = get_collection(None, _FakeRequest(-1, -1))
self.assertEqual(response.status, '400 Bad Request')
......@@ -38,7 +38,7 @@ from mailman.interfaces.address import ExistingAddressError
from mailman.interfaces.usermanager import IUserManager
from mailman.rest.addresses import UserAddresses
from mailman.rest.helpers import (
CollectionMixin, GetterSetter, PATCH, etag, no_content, path_to)
CollectionMixin, GetterSetter, PATCH, etag, no_content, paginate, path_to)
from mailman.rest.preferences import Preferences
from mailman.rest.validator import PatchValidator, Validator
......@@ -86,6 +86,7 @@ class _UserBase(resource.Resource, CollectionMixin):
resource['display_name'] = user.display_name
return resource
@paginate
def _get_collection(self, request):
"""See `CollectionMixin`."""
return list(getUtility(IUserManager).users)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment