From 15388328307d20843820c56d96e3055f3868ccfa Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 15 Sep 2026 10:00:53 -0700 Subject: [PATCH] /db/table/-/count? endpoint, fixed 'count all' button Closes #2914 --- datasette/app.py | 5 ++ datasette/static/app.css | 31 +++++++++ datasette/static/table.js | 35 ++++++++++ datasette/templates/table.html | 7 +- datasette/views/table.py | 114 ++++++++++++++++++++++++--------- docs/changelog.rst | 1 + docs/json_api.rst | 17 +++++ tests/test_playwright.py | 47 ++++++++++++++ 8 files changed, 224 insertions(+), 33 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 3251ed47..93cd6933 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -145,6 +145,7 @@ from .views.stored_queries import ( ) from .views.table import ( TableAutocompleteView, + TableCountView, TableDropView, TableFragmentView, TableInsertView, @@ -2923,6 +2924,10 @@ ORDER BY allowed.parent, allowed.child TableSetColumnTypeView.as_view(self), r"/(?P[^\/\.]+)/(?P[^\/\.]+)/-/set-column-type$", ) + add_route( + TableCountView.as_view(self), + r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/count$", + ) add_route( TableFragmentView.as_view(self), r"/(?P[^\/\.]+)/(?P
[^\/\.]+)/-/fragment$", diff --git a/datasette/static/app.css b/datasette/static/app.css index d101e4b7..f7e6c24f 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -216,6 +216,37 @@ a:active { text-decoration: underline; } +button.count-all { + background: none; + border: none; + padding: 3px 0; + margin-left: 0.25rem; + color: #276890; + font-family: inherit; + font-size: 0.8125rem; + font-weight: 400; + line-height: 1.5; + cursor: pointer; +} + +button.count-all:hover, +button.count-all:focus-visible { + text-decoration: underline; +} + +button.count-all:disabled { + color: #596478; + cursor: wait; +} + +@media (pointer: coarse) { + button.count-all { + min-height: 44px; + padding-left: 7px; + padding-right: 7px; + } +} + button.button-as-link { background: none; border: none; diff --git a/datasette/static/table.js b/datasette/static/table.js index 74a96d8e..143e976f 100644 --- a/datasette/static/table.js +++ b/datasette/static/table.js @@ -860,10 +860,45 @@ function openColumnChooser() { }); } +function initCountAll() { + var button = document.querySelector(".count-all"); + if (!button) { + return; + } + button.addEventListener("click", async function () { + var count = document.querySelector(".table-count"); + var error = document.querySelector(".count-error"); + button.disabled = true; + button.textContent = "Counting…"; + error.textContent = ""; + try { + var response = await fetch(button.dataset.countUrl + location.search, { + method: "POST", + headers: { + Accept: "application/json", + }, + }); + var data = await response.json(); + if (!response.ok || !data.ok) { + throw new Error((data.errors || ["Count failed"]).join(" ")); + } + count.textContent = + data.count.toLocaleString("en-US") + + (data.count === 1 ? " row" : " rows"); + button.remove(); + } catch (ex) { + error.textContent = ex.message || "Count failed"; + button.disabled = false; + button.textContent = "count all"; + } + }); +} + // Ensures Table UI is initialized only after the Manager is ready. document.addEventListener("datasette_init", function (evt) { const { detail: manager } = evt; + initCountAll(); initializeColumnActions(manager); // Main table diff --git a/datasette/templates/table.html b/datasette/templates/table.html index c2131360..e8d8ae65 100644 --- a/datasette/templates/table.html +++ b/datasette/templates/table.html @@ -1,6 +1,6 @@ {% extends "base.html" %} -{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} +{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}{{ "{:,}".format(count - 1) }}+ rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %} {% block extra_head %} {{- super() -}} @@ -48,8 +48,9 @@ {% if count or human_description_en %}

- {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows - {% if allow_execute_sql and query.sql %} count all{% endif %} + {% if count_truncated %}{{ "{:,}".format(count - 1) }}+ rows + + {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %} {% if human_description_en %}{{ human_description_en }}{% endif %}

diff --git a/datasette/views/table.py b/datasette/views/table.py index bae82cc7..a53d7701 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1428,6 +1428,42 @@ class TableDropView(BaseView): return Response.json({"ok": True}, status=200) +class TableCountView(BaseView): + name = "table-count" + + async def post(self, request): + try: + return await self.count(request) + except (NotFound, Forbidden, BadRequest, DatasetteError) as ex: + return Response.error(str(ex), status=ex.status) + + async def count(self, request): + resolved = await self.ds.resolve_table(request) + visible, _private = await self.ds.check_visibility( + request.actor, + action="view-table", + resource=TableResource(database=resolved.db.name, table=resolved.table), + ) + if not visible: + raise Forbidden("You do not have permission to view this table") + _, where_clauses, params, _, _ = await _table_filters( + self.ds, request, resolved.db.name, resolved.table + ) + sql = f"select count(*) from {escape_sqlite(resolved.table)}" + if where_clauses: + sql += " where " + " and ".join(where_clauses) + try: + results = await resolved.db.execute(sql, params) + except QueryInterrupted: + return Response.error("Count query timed out", status=400) + except (sqlite3.OperationalError, InvalidSql) as ex: + return Response.error(str(ex), status=400) + return Response.json( + {"ok": True, "count": results.single_value()}, + headers={"Cache-Control": "no-store"}, + ) + + class TableFragmentView(BaseView): name = "table-fragment" @@ -1953,6 +1989,47 @@ async def table_view_traced(datasette, request): return r +async def _table_filters(datasette, request, database_name, table_name): + # Arguments that start with _ and don't contain a __ are + # special - things like ?_search= - and should not be + # treated as filters. + filter_args = [] + for key in request.args: + if not (key.startswith("_") and "__" not in key): + for v in request.args.getlist(key): + filter_args.append((key, v)) + + # Build where clauses from query string arguments + filters = Filters(sorted(filter_args)) + where_clauses, params = filters.build_where_clauses(table_name) + + # Execute filters_from_request plugin hooks - including the default + # ones that live in datasette/filters.py + extra_context_from_filters = {} + extra_human_descriptions = [] + + for hook in pm.hook.filters_from_request( + request=request, + table=table_name, + database=database_name, + datasette=datasette, + ): + filter_arguments = await await_me_maybe(hook) + if filter_arguments: + where_clauses.extend(filter_arguments.where_clauses) + params.update(filter_arguments.params) + extra_human_descriptions.extend(filter_arguments.human_descriptions) + extra_context_from_filters.update(filter_arguments.extra_context) + + return ( + filters, + where_clauses, + params, + extra_human_descriptions, + extra_context_from_filters, + ) + + async def table_view_data( datasette, request, @@ -2031,36 +2108,13 @@ async def table_view_data( table_metadata = await datasette.table_config(database_name, table_name) - # Arguments that start with _ and don't contain a __ are - # special - things like ?_search= - and should not be - # treated as filters. - filter_args = [] - for key in request.args: - if not (key.startswith("_") and "__" not in key): - for v in request.args.getlist(key): - filter_args.append((key, v)) - - # Build where clauses from query string arguments - filters = Filters(sorted(filter_args)) - where_clauses, params = filters.build_where_clauses(table_name) - - # Execute filters_from_request plugin hooks - including the default - # ones that live in datasette/filters.py - extra_context_from_filters = {} - extra_human_descriptions = [] - - for hook in pm.hook.filters_from_request( - request=request, - table=table_name, - database=database_name, - datasette=datasette, - ): - filter_arguments = await await_me_maybe(hook) - if filter_arguments: - where_clauses.extend(filter_arguments.where_clauses) - params.update(filter_arguments.params) - extra_human_descriptions.extend(filter_arguments.human_descriptions) - extra_context_from_filters.update(filter_arguments.extra_context) + ( + filters, + where_clauses, + params, + extra_human_descriptions, + extra_context_from_filters, + ) = await _table_filters(datasette, request, database_name, table_name) # Deal with custom sort orders sortable_columns = await _sortable_columns_for_table( diff --git a/docs/changelog.rst b/docs/changelog.rst index dcde1900..db5e3553 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,7 @@ Changelog Unreleased ---------- +- Fixed incorrect counts when clicking **count all** on filtered table pages. The button now uses a new :ref:`POST count endpoint `. (:issue:`2914`) - Datasette now uses `httpx2 `__, the Pydantic-maintained continuation of `httpx `__, in place of ``httpx``. The public API is the same, but responses returned by :ref:`internals_datasette_client` are now ``httpx2.Response`` objects rather than ``httpx.Response``. Plugins that use ``isinstance()`` checks against ``httpx.Response`` should be updated to use ``httpx2``. **Plugins that use httpx without explicitly depending on it** will need to add an explicit dependency or switch to `httpx2`. .. _v1_0_a39: diff --git a/docs/json_api.rst b/docs/json_api.rst index e57cac2d..0a12fa74 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -1326,6 +1326,23 @@ The following extras are available for arbitrary SQL query responses and stored, .. [[[end]]] +.. _TableCountView: + +Counting all matching rows +-------------------------- + +``POST //
/-/count`` returns an exact count of the rows matching the table's query string filters:: + + POST /fixtures/facetable/-/count?state=CA + + {"ok": true, "count": 10} + +The endpoint supports the same column, search and plugin filters as the table page. Pagination and display options such as ``_next``, ``_size`` and ``_sort`` do not affect the count. + +This requires ``view-table`` permission. ``execute-sql`` permission is only needed if using ``_where`` filters. + +Unlike the ``count`` extra, this count is not capped by the row count limit. The usual SQL time limit still applies; a timed-out count returns a 400 JSON error. + .. _TableAutocompleteView: Table autocomplete diff --git a/tests/test_playwright.py b/tests/test_playwright.py index 75429835..2b3dedc2 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -108,6 +108,11 @@ def write_playwright_database(db_path): conn = sqlite3.connect(db_path) try: conn.executescript(""" + create table count_numbers (id integer primary key); + with recursive sequence(id) as ( + select 1 union all select id + 1 from sequence where id < 10002 + ) + insert into count_numbers select id from sequence; create table projects ( id integer primary key, title text not null, @@ -1604,3 +1609,45 @@ def test_delete_row_flow_removes_row(page, datasette_server): page.locator(".row-mutation-status", has_text="Deleted row 1").wait_for() page.locator('tr[data-row="1"]').wait_for(state="detached") assert project_rows(datasette_server, id=1) == [] + + +@pytest.mark.playwright +def test_count_all(page, datasette_server): + page.goto(datasette_server + "data/count_numbers?id__gt=1&_sort=id") + assert page.locator(".table-count").inner_text() == "10,000+ rows" + with page.expect_response("**/count_numbers/-/count?*") as response: + page.get_by_role("button", name="count all", exact=True).click() + assert response.value.request.method == "POST" + assert response.value.request.post_data is None + assert "content-type" not in response.value.request.headers + assert response.value.json() == {"ok": True, "count": 10001} + page.wait_for_function( + 'document.querySelector(".table-count").textContent === "10,001 rows"' + ) + assert page.locator(".count-all").count() == 0 + assert "id" in page.locator("h3").first.inner_text() + + +@pytest.mark.playwright +def test_count_all_error_retry(page, datasette_server): + page.goto(datasette_server + "data/count_numbers?id__gt=1") + page.route( + "**/count_numbers/-/count?*", + lambda route: route.fulfill( + status=400, + content_type="application/json", + body=json.dumps({"ok": False, "errors": ["Count query timed out"]}), + ), + ) + button = page.get_by_role("button", name="count all", exact=True) + button.click() + page.wait_for_function( + 'document.querySelector(".count-error").textContent === "Count query timed out"' + ) + assert button.is_enabled() + page.unroute("**/count_numbers/-/count?*") + button.click() + page.wait_for_function( + 'document.querySelector(".table-count").textContent === "10,001 rows"' + ) + assert page.locator(".count-error").inner_text() == ""