Link: HTTP header pagination, closes #1014

datasette/renderer.py

@@ -5,6 +5,7 @@ from datasette.utils import (
     CustomJSONEncoder,
     path_from_row_pks,
 )
+from datasette.utils.asgi import Response
 
 
 def convert_specific_columns_to_json(rows, columns, json_cols):
@@ -44,6 +45,9 @@ def json_renderer(args, data, view_name):
 
     # Deal with the _shape option
     shape = args.get("_shape", "arrays")
+
+    next_url = data.get("next_url")
+
     if shape == "arrayfirst":
         data = [row[0] for row in data["rows"]]
     elif shape in ("objects", "object", "array"):
@@ -71,6 +75,7 @@ def json_renderer(args, data, view_name):
                 data = {"ok": False, "error": error}
         elif shape == "array":
             data = data["rows"]
+
     elif shape == "arrays":
         pass
     else:
@@ -89,4 +94,9 @@ def json_renderer(args, data, view_name):
     else:
         body = json.dumps(data, cls=CustomJSONEncoder)
         content_type = "application/json; charset=utf-8"
-    return {"body": body, "status_code": status_code, "content_type": content_type}
+    headers = {}
+    if next_url:
+        headers["link"] = '<{}>; rel="next"'.format(next_url)
+    return Response(
+        body, status=status_code, headers=headers, content_type=content_type
+    )

docs/json_api.rst

@@ -1,3 +1,5 @@
+.. _json_api:
+
 JSON API
 ========
 
@@ -18,6 +20,8 @@ requests to fetch the data.
 If you start Datasette without the ``--cors`` option only JavaScript running on
 the same domain as Datasette will be able to access the API.
 
+.. _json_api_shapes:
+
 Different shapes
 ----------------
 
@@ -138,6 +142,34 @@ this format.
 The ``object`` keys are always strings. If your table has a compound primary
 key, the ``object`` keys will be a comma-separated string.
 
+.. _json_api_pagination:
+
+Pagination
+----------
+
+The default JSON representation includes a ``"next_url"`` key which can be used to access the next page of results. If that key is null or missing then it means you have reached the final page of results.
+
+Other representations include pagination information in the ``link`` HTTP header. That header will look something like this::
+
+    link: <https://latest.datasette.io/fixtures/sortable.json?_next=d%2Cv>; rel="next"
+
+Here is an example Python function built using `requests <https://requests.readthedocs.io/>`__ that returns a list of all of the paginated items from one of these API endpoints:
+
+.. code-block:: python
+
+    def paginate(url):
+        items = []
+        while url:
+            response = requests.get(url)
+            try:
+                url = response.links.get("next").get("url")
+            except AttributeError:
+                url = None
+            items.extend(response.json())
+        return items
+
+.. _json_api_special:
+
 Special JSON arguments
 ----------------------
 

tests/test_api.py

@@ -1828,3 +1828,31 @@ def test_binary_data_in_json(app_client, path, expected_json, expected_text):
         assert response.json == expected_json
     else:
         assert response.text == expected_text
+
+
+@pytest.mark.parametrize(
+    "qs",
+    [
+        "",
+        "?_shape=arrays",
+        "?_shape=arrayfirst",
+        "?_shape=object",
+        "?_shape=objects",
+        "?_shape=array",
+        "?_shape=array&_nl=on",
+    ],
+)
+def test_paginate_using_link_header(app_client, qs):
+    path = "/fixtures/compound_three_primary_keys.json{}".format(qs)
+    num_pages = 0
+    while path:
+        response = app_client.get(path)
+        num_pages += 1
+        link = response.headers.get("link")
+        if link:
+            assert link.startswith("<")
+            assert link.endswith('>; rel="next"')
+            path = link[1:].split(">")[0]
+        else:
+            path = None
+    assert num_pages == 21