diff --git a/datasette/views/table.py b/datasette/views/table.py index ef9831b2..44919281 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -62,6 +62,7 @@ from .table_create_alter import ( from .table_extras import ( TABLE_EXTRA_BUNDLES, TableExtraContext, + count_is_truncated, precompute_database_action_permissions, precompute_table_action_permissions, resolve_table_extras, @@ -2257,6 +2258,8 @@ async def table_view_data( extras.add("facet_results") if request.args.get("_shape") == "object": extras.add("primary_keys") + if "count" in extras: + extras.add("count_truncated") if extra_extras: extras.update(extra_extras) @@ -2335,7 +2338,7 @@ async def table_view_data( data["rows"] = transformed_rows if context_for_html_hack: - data["count_truncated"] = _count_truncated_for_table_page( + data["count_truncated"] = count_is_truncated( datasette, db, database_name, table_name, count_sql, data.get("count") ) data.update(extra_context_from_filters) @@ -2403,24 +2406,6 @@ async def table_view_data( return data, rows[:page_size], columns, expanded_columns, sql, next_url -def _count_truncated_for_table_page( - datasette, db, database_name, table_name, count_sql, count -): - if count != db.count_limit + 1: - return False - if ( - not db.is_mutable - and datasette.inspect_data - and count_sql == f"select count(*) from {table_name} " - ): - try: - datasette.inspect_data[database_name]["tables"][table_name]["count"] - return False - except KeyError: - pass - return True - - async def _next_value_and_url( datasette, db, diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index db659c80..f8ece55b 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -140,6 +140,39 @@ class CountExtra(Extra): return count +def count_is_truncated(datasette, db, database_name, table_name, count_sql, count): + if count != db.count_limit + 1: + return False + if ( + not db.is_mutable + and datasette.inspect_data + and count_sql == f"select count(*) from {table_name} " + ): + try: + datasette.inspect_data[database_name]["tables"][table_name]["count"] + return False + except KeyError: + pass + return True + + +class CountTruncatedExtra(Extra): + description = "True if the count hit Datasette's counting limit, meaning the real number of matching rows is at least the reported count." + example = ExtraExample("/fixtures/facetable.json?_extra=count,count_truncated") + scopes = {ExtraScope.TABLE} + expensive = True + + async def resolve(self, context, count): + return count_is_truncated( + context.datasette, + context.db, + context.database_name, + context.table_name, + context.count_sql, + count, + ) + + class FacetInstancesProvider(Provider): scopes = {ExtraScope.TABLE} @@ -1196,6 +1229,7 @@ TABLE_EXTRA_BUNDLES = { TABLE_EXTRA_CLASSES = [ CountExtra, + CountTruncatedExtra, CountSqlExtra, FacetResultsExtra, FacetsTimedOutExtra, diff --git a/docs/json_api.rst b/docs/json_api.rst index b8287a64..2a859ba2 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -302,6 +302,15 @@ The available table extras are listed below. 15 +``count_truncated`` + True if the count hit Datasette's counting limit, meaning the real number of matching rows is at least the reported count. (May execute additional queries.) + + ``GET /fixtures/facetable.json?_extra=count,count_truncated`` + + .. code-block:: json + + false + ``count_sql`` SQL query string used to calculate the total count for the current table view, including active filters. diff --git a/existing-api.md b/existing-api.md index eb7b5c20..a8a2237f 100644 --- a/existing-api.md +++ b/existing-api.md @@ -658,7 +658,8 @@ views/table_extras.py:1197-1235; unknown names silently ignored): | `_extra=` | Returns | |---|---| -| `count` | total matching-row count, computed with a `limit 10001` subquery so it caps at 10001; `null` with `_nocount` or on count timeout | +| `count` | total matching-row count, computed with a `limit 10001` subquery so it caps at 10001; `null` with `_nocount` or on count timeout. Requesting `count` implicitly includes `count_truncated` | +| `count_truncated` | `true` when `count` hit the counting limit (the real count is at least the reported value) | | `count_sql` | the SQL used for the count | | `facet_results` | `{"results": {name: facet}, "timed_out": [...]}`; each facet: `{name, type, hideable, toggle_url, results: [{value, label, count, toggle_url, selected}], truncated}` | | `facets_timed_out` | facet names that exceeded `facet_time_limit_ms` | diff --git a/stable-api-recommendations.md b/stable-api-recommendations.md index a4dbb344..64b2acbc 100644 --- a/stable-api-recommendations.md +++ b/stable-api-recommendations.md @@ -178,7 +178,10 @@ is a table/row/query feature. Also decide the contract for **unknown silent ignoring means typos return the default payload with no signal; recommend a 400 or a `warnings` key. -### 2c. Count truncation is invisible in JSON (P2) +### 2c. Count truncation is invisible in JSON (P2) — ✅ IMPLEMENTED + +> **Status:** implemented — a public `count_truncated` extra now exists and +> is implicitly included whenever `count` is requested. The `count` extra is computed with a `limit 10001` subquery, so `count: 10001` actually means "at least 10001" — the `count_truncated` flag exists diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 41c89f39..e3737ee6 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1504,6 +1504,7 @@ async def test_col_nocol_errors(ds_client, path, expected_error): "rows": [{"id": "1", "content": "hey", "content2": "world"}], "truncated": False, "count": 1, + "count_truncated": False, }, ), ), @@ -1590,3 +1591,39 @@ async def test_extra_render_cell(): finally: ds.pm.unregister(name="TestRenderCellPlugin") + + +@pytest.mark.asyncio +async def test_count_truncated_included_with_count_extra(tmp_path_factory): + from datasette.app import Datasette + from datasette.utils import sqlite3 + + db_directory = tmp_path_factory.mktemp("dbs") + db_path = str(db_directory / "counts.db") + conn = sqlite3.connect(db_path) + conn.execute("vacuum") + conn.execute("create table big (id integer primary key)") + conn.execute("create table small (id integer primary key)") + conn.executemany("insert into big (id) values (?)", [(i,) for i in range(10)]) + conn.executemany("insert into small (id) values (?)", [(i,) for i in range(3)]) + conn.commit() + conn.close() + ds = Datasette([db_path]) + ds.get_database("counts").count_limit = 5 + try: + response = await ds.client.get("/counts/big.json?_extra=count") + data = response.json() + # Count is capped at count_limit + 1 and flagged as truncated + assert data["count"] == 6 + assert data["count_truncated"] is True + + response = await ds.client.get("/counts/small.json?_extra=count") + data = response.json() + assert data["count"] == 3 + assert data["count_truncated"] is False + + # count_truncated can also be requested on its own + response = await ds.client.get("/counts/big.json?_extra=count_truncated") + assert response.json()["count_truncated"] is True + finally: + ds.close()