Skip to content

Commit fb31add

Browse files
authored
Merge pull request #2 from python-scim/cursor-pagination
Support for RFC 9865 cursors
2 parents 20192c4 + 298b64a commit fb31add

16 files changed

Lines changed: 647 additions & 142 deletions

‎doc/changelog.rst‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,9 @@ Added
1414
- :class:`~scim2_sqlalchemy.SqlAlchemyStorage` and
1515
:class:`~scim2_sqlalchemy.AsyncSqlAlchemyStorage`, which serve the mappings to
1616
:doc:`scim2-server <scim2_server:index>`. They filter, sort and page the resources in the
17-
database, on one resource type or at the root of the server. The asynchronous storage needs
18-
the ``asyncio`` extra.
17+
database, on one resource type or at the root of the server. They page with ``startIndex``, or
18+
with the cursors of :rfc:`9865`, which give stable pages. The asynchronous storage needs the
19+
``asyncio`` extra.
1920
- :meth:`ResourceMapping.schemas <scim2_sqlalchemy.ResourceMapping.schemas>`, which publishes the
2021
attributes a mapping stores.
2122
- Filters, sorts and uniqueness checks compare strings with the comparison key of the policy of

‎doc/explanation.rst‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,31 @@ the identifier of each resource of the page. It then loads the records of each t
103103
type without the sort attribute sorts its resources as having no value. Between two resources of
104104
different types that share a sort value, the order of the resource types decides.
105105

106+
Cursors give stable pages
107+
-------------------------
108+
109+
The storage pages with ``startIndex``, and with the cursors of :rfc:`9865`. With a cursor, the
110+
position of a page holds the sort value and the identifier of a resource: the last one of the
111+
page for the next page, and the first one for the previous page. The next page selects the
112+
resources after them in the sort order. A resource created or deleted between two pages does not
113+
move the other ones. :doc:`scim2_server:explanation/pagination` explains what these stable pages
114+
guarantee, and :doc:`scim2_server:how-to/page-with-cursors` explains how to announce the
115+
cursors.
116+
117+
The sort value of a position is the value the database compares, after the comparison key of the
118+
policy. The storage writes the condition on the sort value by hand, since a row comparison would
119+
not place ``NULL`` where the sort does. At the root, the position also holds the resource type,
120+
which orders the resources of different types that share a sort value.
121+
122+
The storage reads one resource more than ``count``, to know whether a page follows. It only knows
123+
it in the direction it reads: forward for a next page, backward for a previous page. In the other
124+
direction, it assumes that the page the client comes from still exists. When all the resources of
125+
that page were deleted meanwhile, the client gets an empty page without cursors, and starts again
126+
from the first page.
127+
128+
Each page shows the resources as they are when it is read. The storage does not give a snapshot
129+
of the collection as it was on the first page.
130+
106131
Writes that cannot be stored are refused
107132
----------------------------------------
108133

@@ -173,9 +198,6 @@ send the same queries.
173198
Limits
174199
------
175200

176-
The storage pages with ``startIndex`` and ``count``. It does not support the cursors of
177-
:rfc:`9865`.
178-
179201
A search or a read only loads the collections its response keeps, following the ``attributes``
180202
and ``excludedAttributes`` parameters. With ``excludedAttributes=members``, the members of the
181203
groups are not loaded. The storage still loads every column of the records, which costs little

‎pyproject.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ authors = [
1515
]
1616
requires-python = ">=3.11"
1717
dependencies = [
18-
"scim2-models>=0.12.1",
19-
"scim2-server>=0.7.1",
18+
"scim2-models>=0.12.2",
19+
"scim2-server>=0.8.0",
2020
"sqlalchemy>=2.0.41",
2121
]
2222

@@ -41,7 +41,7 @@ dev = [
4141
"pytest-cov>=7.1.0",
4242
"pytest-postgresql>=9.1.0",
4343
"scim2-client>=0.12.0",
44-
"scim2-server[testing]>=0.7.1",
44+
"scim2-server[testing]>=0.8.0",
4545
"scim2-tester>=0.5.2",
4646
"tox-uv>=1.36.0",
4747
"werkzeug>=3.1.8",

‎src/scim2_sqlalchemy/keyset.py‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
import datetime
2+
import decimal
3+
import uuid
4+
from collections.abc import Callable
5+
from collections.abc import Sequence
6+
from dataclasses import dataclass
7+
from typing import Any
8+
9+
from scim2_models import InvalidCursorException
10+
from sqlalchemy import ColumnElement
11+
from sqlalchemy import and_
12+
from sqlalchemy import false
13+
from sqlalchemy import nulls_first
14+
from sqlalchemy import nulls_last
15+
from sqlalchemy import or_
16+
from sqlalchemy import tuple_
17+
18+
_TAGGED: dict[str, tuple[type, Callable[[Any], str], Callable[[str], Any]]] = {
19+
"datetime": (
20+
datetime.datetime,
21+
datetime.datetime.isoformat,
22+
datetime.datetime.fromisoformat,
23+
),
24+
"date": (datetime.date, datetime.date.isoformat, datetime.date.fromisoformat),
25+
"decimal": (decimal.Decimal, str, decimal.Decimal),
26+
"uuid": (uuid.UUID, str, uuid.UUID),
27+
}
28+
29+
30+
def _encode(value: Any) -> Any:
31+
"""Return a sort value as a value JSON can hold, tagged with its type when JSON has none."""
32+
if value is None or isinstance(value, bool | int | float | str):
33+
return value
34+
for tag, (python_type, dump, _) in _TAGGED.items():
35+
if isinstance(value, python_type):
36+
return {tag: dump(value)}
37+
raise TypeError(f"Cannot page on a sort value of type {type(value).__name__}")
38+
39+
40+
def _decode(value: Any) -> Any:
41+
"""Return the sort value a position holds."""
42+
if value is None or isinstance(value, bool | int | float | str):
43+
return value
44+
if not isinstance(value, dict) or len(value) != 1:
45+
raise InvalidCursorException
46+
((tag, text),) = value.items()
47+
if tag not in _TAGGED or not isinstance(text, str):
48+
raise InvalidCursorException
49+
try:
50+
return _TAGGED[tag][2](text)
51+
except (ValueError, decimal.InvalidOperation):
52+
raise InvalidCursorException from None
53+
54+
55+
@dataclass(frozen=True)
56+
class _Anchor:
57+
"""The row a cursor page starts after, or ends before.
58+
59+
ties are the values that order the rows sharing a sort value: the
60+
identifier as text, preceded at the root by the position of the resource
61+
type.
62+
"""
63+
64+
forward: bool
65+
key: Any
66+
ties: tuple[Any, ...]
67+
68+
69+
def _anchor(position: Any, root: bool) -> _Anchor | None:
70+
"""Read the position a storage gave, or return None for the first page."""
71+
if position is None:
72+
return None
73+
names = {"d", "k", "t", "i"} if root else {"d", "k", "i"}
74+
if not isinstance(position, dict) or set(position) != names:
75+
raise InvalidCursorException
76+
if position["d"] not in ("next", "previous") or not isinstance(position["i"], str):
77+
raise InvalidCursorException
78+
ties: tuple[Any, ...] = (position["i"],)
79+
if root:
80+
if type(position["t"]) is not int:
81+
raise InvalidCursorException
82+
ties = (position["t"], position["i"])
83+
return _Anchor(position["d"] == "next", _decode(position["k"]), ties)
84+
85+
86+
def _position(direction: str, key: Any, ties: Sequence[Any]) -> Any:
87+
"""Return the position of the page before or after a row, as a value JSON can hold.
88+
89+
The sort value is null when the search has no sort.
90+
"""
91+
position: dict[str, Any] = {"d": direction, "k": _encode(key)}
92+
if len(ties) == 2:
93+
position["t"] = ties[0]
94+
position["i"] = ties[-1]
95+
return position
96+
97+
98+
def _key_order(key: Any, ascending: bool) -> Any:
99+
"""Per RFC 7644 §3.4.2.3, resources without a value come last when ascending and first when descending."""
100+
if ascending:
101+
return nulls_last(key.asc())
102+
return nulls_first(key.desc())
103+
104+
105+
def _keyset_order(
106+
key: Any, ties: Sequence[Any], descending: bool, forward: bool
107+
) -> list[Any]:
108+
"""Return the ORDER BY terms of a search: the sort value, then the ties.
109+
110+
A previous page is read backward, in the reverse order of the sort.
111+
"""
112+
terms = [] if key is None else [_key_order(key, descending != forward)]
113+
return [*terms, *(tie.asc() if forward else tie.desc() for tie in ties)]
114+
115+
116+
def _beyond(
117+
key: Any,
118+
ties: Sequence[Any],
119+
anchor: _Anchor,
120+
values: Sequence[Any],
121+
descending: bool,
122+
) -> ColumnElement[bool]:
123+
"""Return the condition keeping the rows past the anchor, in the reading direction.
124+
125+
values are the ties of the anchor, in the types of the ties. The
126+
condition on the sort value is written out, as a row comparison would not
127+
place NULL where the sort does.
128+
"""
129+
tie_after: ColumnElement[bool]
130+
if len(ties) == 1:
131+
tie_after = ties[0] > values[0] if anchor.forward else ties[0] < values[0]
132+
else:
133+
row, bound = tuple_(*ties), tuple_(*values)
134+
tie_after = row > bound if anchor.forward else row < bound
135+
if key is None:
136+
return tie_after
137+
138+
ascending = descending != anchor.forward
139+
if anchor.key is None:
140+
key_after = key.is_not(None) if not ascending else false()
141+
return or_(key_after, and_(key.is_(None), tie_after))
142+
key_after = or_(key > anchor.key, key.is_(None)) if ascending else key < anchor.key
143+
return or_(key_after, and_(key == anchor.key, tie_after))

‎src/scim2_sqlalchemy/query.py‎

Lines changed: 66 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
from typing import Any
44

55
from scim2_models import AttributeBinding
6+
from scim2_models import InvalidCursorException
67
from scim2_models import InvalidFilterException
78
from scim2_models import InvalidPathException
89
from scim2_models import ResponseParameters
@@ -29,7 +30,6 @@
2930
from sqlalchemy import literal
3031
from sqlalchemy import not_
3132
from sqlalchemy import null
32-
from sqlalchemy import nulls_first
3333
from sqlalchemy import nulls_last
3434
from sqlalchemy import or_
3535
from sqlalchemy import select
@@ -40,6 +40,9 @@
4040
from .comparison import _SUBSTRINGS
4141
from .comparison import _Comparator
4242
from .conversion import _as_stored
43+
from .keyset import _Anchor
44+
from .keyset import _beyond
45+
from .keyset import _keyset_order
4346
from .mapping import ResourceMapping
4447
from .mapping import _Collection
4548
from .mapping import _Column
@@ -270,28 +273,6 @@ def _sort_key(
270273
return expression
271274

272275

273-
def _sorted(expression: Any, search_request: SearchRequest[Any]) -> Any:
274-
"""Per RFC 7644 §3.4.2.3, resources without a value come last when ascending and first when descending."""
275-
if search_request.sort_order == SearchRequest.SortOrder.descending:
276-
return nulls_first(expression.desc())
277-
return nulls_last(expression.asc())
278-
279-
280-
def _order_by(
281-
mapping: ResourceMapping,
282-
search_request: SearchRequest[Any],
283-
comparator: _Comparator,
284-
) -> list[Any]:
285-
"""Return the ORDER BY terms of a search, the record identifier last.
286-
287-
Without the identifier, records sharing a sort value could change pages
288-
from a request to the next.
289-
"""
290-
key = _sort_key(mapping, search_request, comparator)
291-
terms = [] if key is None else [_sorted(key, search_request)]
292-
return [*terms, mapping._id.expression]
293-
294-
295276
def _first_entry(collection: _Collection, expression: Any) -> Any:
296277
"""Return the value of the primary entry of a collection, or else of its first one."""
297278
relationship = collection.relationship_property
@@ -321,39 +302,74 @@ def _where(
321302
return _FilterTranslator(mapping, scim_filter, comparator).visit(scim_filter.ast)
322303

323304

305+
def _cursor_page(page: Select[Any], search_request: SearchRequest[Any]) -> Select[Any]:
306+
"""Limit a cursor page to count + 1 rows, the last one telling whether a page follows."""
307+
if search_request.count is None:
308+
return page
309+
return page.limit(search_request.count + 1)
310+
311+
312+
def _index_page(page: Select[Any], search_request: SearchRequest[Any]) -> Select[Any]:
313+
page = page.offset(search_request.start_index_0 or 0)
314+
if search_request.count is None:
315+
return page
316+
return page.limit(search_request.count)
317+
318+
324319
def _search_statements(
325320
mapping: ResourceMapping,
326321
search_request: SearchRequest[Any],
327322
comparator: _Comparator,
328-
) -> tuple[Select[tuple[int]], Select[Any]]:
329-
"""Return the statement counting the matching records, and the one selecting a page of them."""
323+
anchor: _Anchor | None = None,
324+
) -> tuple[Select[tuple[int]], Select[tuple[Any, Any, str]]]:
325+
"""Return the statement counting the matching records, and the one selecting a page of them.
326+
327+
The page lists each record with its sort value and its identifier as
328+
text. The records are ordered by sort value, then by identifier: without
329+
the identifier, records sharing a sort value could change pages from a
330+
request to the next. A cursor page is read from the anchor, in the reverse
331+
order for a previous page.
332+
"""
330333
condition = _where(mapping, search_request, comparator)
331334
count = select(func.count()).select_from(mapping.record).where(condition)
335+
key = _sort_key(mapping, search_request, comparator)
336+
record_id = mapping._id.expression
337+
descending = search_request.sort_order == SearchRequest.SortOrder.descending
338+
forward = anchor is None or anchor.forward
332339
page = (
333-
select(mapping.record)
340+
select(
341+
mapping.record,
342+
(null() if key is None else key).label("key"),
343+
cast(record_id, String).label("id"),
344+
)
334345
.where(condition)
335-
.order_by(*_order_by(mapping, search_request, comparator))
346+
.order_by(*_keyset_order(key, [record_id], descending, forward))
336347
.options(
337348
*mapping._loader_options(mapping._returned_collections(search_request))
338349
)
339-
.offset(search_request.start_index_0 or 0)
340350
)
341-
if search_request.count is not None:
342-
page = page.limit(search_request.count)
343-
return count, page
351+
if search_request.cursor is None:
352+
return count, _index_page(page, search_request)
353+
if anchor is not None:
354+
value = _as_key(mapping._id, anchor.ties[0])
355+
if value is None:
356+
raise InvalidCursorException
357+
page = page.where(_beyond(key, [record_id], anchor, [value], descending))
358+
return count, _cursor_page(page, search_request)
344359

345360

346361
def _root_statements(
347362
mappings: list[ResourceMapping],
348363
search_request: SearchRequest[Any],
349364
comparator: _Comparator,
350-
) -> tuple[Select[tuple[int]], Select[tuple[int, str]]]:
365+
anchor: _Anchor | None = None,
366+
) -> tuple[Select[tuple[int]], Select[tuple[int, str, Any]]]:
351367
"""Return the statement counting the matching records of several mappings, and the one selecting a page of them.
352368
353-
The page lists the position of the mapping and the identifier of each
354-
record, as text, so that the identifiers of every table fit in one column.
355-
A mapping whose model does not declare the sort attribute sorts its
356-
records as having no value.
369+
The page lists the position of the mapping, the identifier of each
370+
record as text, so that the identifiers of every table fit in one column,
371+
and its sort value. A mapping whose model does not declare the sort
372+
attribute sorts its records as having no value.
357373
"""
358374
conditions = [_where(mapping, search_request, comparator) for mapping in mappings]
359375
keys = [_sort_key(mapping, search_request, comparator) for mapping in mappings]
@@ -382,15 +398,20 @@ def _root_statements(
382398
columns.append(key.label("key"))
383399
branches.append(select(*columns).where(condition))
384400
rows = union_all(*branches).subquery()
385-
terms = [] if key_type is None else [_sorted(rows.c.key, search_request)]
386-
page = (
387-
select(rows.c.position, rows.c.id)
388-
.order_by(*terms, rows.c.position, rows.c.id)
389-
.offset(search_request.start_index_0 or 0)
390-
)
391-
if search_request.count is not None:
392-
page = page.limit(search_request.count)
393-
return count, page
401+
row_key = None if key_type is None else rows.c.key
402+
ties = [rows.c.position, rows.c.id]
403+
descending = search_request.sort_order == SearchRequest.SortOrder.descending
404+
forward = anchor is None or anchor.forward
405+
page = select(
406+
rows.c.position, rows.c.id, (null() if row_key is None else row_key)
407+
).order_by(*_keyset_order(row_key, ties, descending, forward))
408+
if search_request.cursor is None:
409+
return count, _index_page(page, search_request)
410+
if anchor is not None:
411+
if anchor.ties[0] not in range(len(mappings)):
412+
raise InvalidCursorException
413+
page = page.where(_beyond(row_key, ties, anchor, anchor.ties, descending))
414+
return count, _cursor_page(page, search_request)
394415

395416

396417
def _records_statement(

0 commit comments

Comments
 (0)