/db/table/-/count? endpoint, fixed 'count all' button

Closes #2914

datasette/app.py

@@ -145,6 +145,7 @@ from .views.stored_queries import (
 )
 from .views.table import (
     TableAutocompleteView,
+    TableCountView,
     TableDropView,
     TableFragmentView,
     TableInsertView,
@@ -2923,6 +2924,10 @@ ORDER BY allowed.parent, allowed.child
             TableSetColumnTypeView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/set-column-type$",
         )
+        add_route(
+            TableCountView.as_view(self),
+            r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/count$",
+        )
         add_route(
             TableFragmentView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/fragment$",

datasette/static/app.css

@@ -216,6 +216,37 @@ a:active  {
 	text-decoration: underline;
 }
 
+button.count-all {
+    background: none;
+    border: none;
+    padding: 3px 0;
+    margin-left: 0.25rem;
+    color: #276890;
+    font-family: inherit;
+    font-size: 0.8125rem;
+    font-weight: 400;
+    line-height: 1.5;
+    cursor: pointer;
+}
+
+button.count-all:hover,
+button.count-all:focus-visible {
+    text-decoration: underline;
+}
+
+button.count-all:disabled {
+    color: #596478;
+    cursor: wait;
+}
+
+@media (pointer: coarse) {
+    button.count-all {
+        min-height: 44px;
+        padding-left: 7px;
+        padding-right: 7px;
+    }
+}
+
 button.button-as-link {
     background: none;
     border: none;

datasette/static/table.js

@@ -860,10 +860,45 @@ function openColumnChooser() {
   });
 }
 
+function initCountAll() {
+  var button = document.querySelector(".count-all");
+  if (!button) {
+    return;
+  }
+  button.addEventListener("click", async function () {
+    var count = document.querySelector(".table-count");
+    var error = document.querySelector(".count-error");
+    button.disabled = true;
+    button.textContent = "Counting…";
+    error.textContent = "";
+    try {
+      var response = await fetch(button.dataset.countUrl + location.search, {
+        method: "POST",
+        headers: {
+          Accept: "application/json",
+        },
+      });
+      var data = await response.json();
+      if (!response.ok || !data.ok) {
+        throw new Error((data.errors || ["Count failed"]).join(" "));
+      }
+      count.textContent =
+        data.count.toLocaleString("en-US") +
+        (data.count === 1 ? " row" : " rows");
+      button.remove();
+    } catch (ex) {
+      error.textContent = ex.message || "Count failed";
+      button.disabled = false;
+      button.textContent = "count all";
+    }
+  });
+}
+
 // Ensures Table UI is initialized only after the Manager is ready.
 document.addEventListener("datasette_init", function (evt) {
   const { detail: manager } = evt;
 
+  initCountAll();
   initializeColumnActions(manager);
 
   // Main table

datasette/templates/table.html

@@ -1,6 +1,6 @@
 {% extends "base.html" %}
 
-{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
+{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}{{ "{:,}".format(count - 1) }}+ rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
 
 {% block extra_head %}
 {{- super() -}}
@@ -48,8 +48,9 @@
 
 {% if count or human_description_en %}
     <h3>
-        {% if count_truncated %}&gt;{{ "{:,}".format(count - 1) }} rows
-        {% if allow_execute_sql and query.sql %} <a class="count-sql" style="font-size: 0.8em;" href="{{ urls.database_query(database, count_sql) }}">count all</a>{% endif %}
+        {% if count_truncated %}<span class="table-count" aria-live="polite">{{ "{:,}".format(count - 1) }}+ rows</span>
+        <button type="button" class="count-all" data-count-url="{{ urls.table(database, table) }}/-/count">count all</button>
+        <span class="count-error" role="alert"></span>
         {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}
         {% if human_description_en %}{{ human_description_en }}{% endif %}
     </h3>

datasette/views/table.py

@@ -1428,6 +1428,42 @@ class TableDropView(BaseView):
         return Response.json({"ok": True}, status=200)
 
 
+class TableCountView(BaseView):
+    name = "table-count"
+
+    async def post(self, request):
+        try:
+            return await self.count(request)
+        except (NotFound, Forbidden, BadRequest, DatasetteError) as ex:
+            return Response.error(str(ex), status=ex.status)
+
+    async def count(self, request):
+        resolved = await self.ds.resolve_table(request)
+        visible, _private = await self.ds.check_visibility(
+            request.actor,
+            action="view-table",
+            resource=TableResource(database=resolved.db.name, table=resolved.table),
+        )
+        if not visible:
+            raise Forbidden("You do not have permission to view this table")
+        _, where_clauses, params, _, _ = await _table_filters(
+            self.ds, request, resolved.db.name, resolved.table
+        )
+        sql = f"select count(*) from {escape_sqlite(resolved.table)}"
+        if where_clauses:
+            sql += " where " + " and ".join(where_clauses)
+        try:
+            results = await resolved.db.execute(sql, params)
+        except QueryInterrupted:
+            return Response.error("Count query timed out", status=400)
+        except (sqlite3.OperationalError, InvalidSql) as ex:
+            return Response.error(str(ex), status=400)
+        return Response.json(
+            {"ok": True, "count": results.single_value()},
+            headers={"Cache-Control": "no-store"},
+        )
+
+
 class TableFragmentView(BaseView):
     name = "table-fragment"
 
@@ -1953,6 +1989,47 @@ async def table_view_traced(datasette, request):
     return r
 
 
+async def _table_filters(datasette, request, database_name, table_name):
+    # Arguments that start with _ and don't contain a __ are
+    # special - things like ?_search= - and should not be
+    # treated as filters.
+    filter_args = []
+    for key in request.args:
+        if not (key.startswith("_") and "__" not in key):
+            for v in request.args.getlist(key):
+                filter_args.append((key, v))
+
+    # Build where clauses from query string arguments
+    filters = Filters(sorted(filter_args))
+    where_clauses, params = filters.build_where_clauses(table_name)
+
+    # Execute filters_from_request plugin hooks - including the default
+    # ones that live in datasette/filters.py
+    extra_context_from_filters = {}
+    extra_human_descriptions = []
+
+    for hook in pm.hook.filters_from_request(
+        request=request,
+        table=table_name,
+        database=database_name,
+        datasette=datasette,
+    ):
+        filter_arguments = await await_me_maybe(hook)
+        if filter_arguments:
+            where_clauses.extend(filter_arguments.where_clauses)
+            params.update(filter_arguments.params)
+            extra_human_descriptions.extend(filter_arguments.human_descriptions)
+            extra_context_from_filters.update(filter_arguments.extra_context)
+
+    return (
+        filters,
+        where_clauses,
+        params,
+        extra_human_descriptions,
+        extra_context_from_filters,
+    )
+
+
 async def table_view_data(
     datasette,
     request,
@@ -2031,36 +2108,13 @@ async def table_view_data(
 
     table_metadata = await datasette.table_config(database_name, table_name)
 
-    # Arguments that start with _ and don't contain a __ are
-    # special - things like ?_search= - and should not be
-    # treated as filters.
-    filter_args = []
-    for key in request.args:
-        if not (key.startswith("_") and "__" not in key):
-            for v in request.args.getlist(key):
-                filter_args.append((key, v))
-
-    # Build where clauses from query string arguments
-    filters = Filters(sorted(filter_args))
-    where_clauses, params = filters.build_where_clauses(table_name)
-
-    # Execute filters_from_request plugin hooks - including the default
-    # ones that live in datasette/filters.py
-    extra_context_from_filters = {}
-    extra_human_descriptions = []
-
-    for hook in pm.hook.filters_from_request(
-        request=request,
-        table=table_name,
-        database=database_name,
-        datasette=datasette,
-    ):
-        filter_arguments = await await_me_maybe(hook)
-        if filter_arguments:
-            where_clauses.extend(filter_arguments.where_clauses)
-            params.update(filter_arguments.params)
-            extra_human_descriptions.extend(filter_arguments.human_descriptions)
-            extra_context_from_filters.update(filter_arguments.extra_context)
+    (
+        filters,
+        where_clauses,
+        params,
+        extra_human_descriptions,
+        extra_context_from_filters,
+    ) = await _table_filters(datasette, request, database_name, table_name)
 
     # Deal with custom sort orders
     sortable_columns = await _sortable_columns_for_table(

docs/changelog.rst

@@ -9,6 +9,7 @@ Changelog
 Unreleased
 ----------
 
+- Fixed incorrect counts when clicking **count all** on filtered table pages. The button now uses a new :ref:`POST count endpoint <TableCountView>`. (:issue:`2914`)
 - Datasette now uses `httpx2 <https://httpx2.pydantic.dev/>`__, the Pydantic-maintained continuation of `httpx <https://www.python-httpx.org/>`__, in place of ``httpx``. The public API is the same, but responses returned by :ref:`internals_datasette_client` are now ``httpx2.Response`` objects rather than ``httpx.Response``. Plugins that use ``isinstance()`` checks against ``httpx.Response`` should be updated to use ``httpx2``. **Plugins that use httpx without explicitly depending on it** will need to add an explicit dependency or switch to `httpx2`.
 
 .. _v1_0_a39:

docs/json_api.rst

@@ -1326,6 +1326,23 @@ The following extras are available for arbitrary SQL query responses and stored,
 
 .. [[[end]]]
 
+.. _TableCountView:
+
+Counting all matching rows
+--------------------------
+
+``POST /<database>/<table>/-/count`` returns an exact count of the rows matching the table's query string filters::
+
+    POST /fixtures/facetable/-/count?state=CA
+
+    {"ok": true, "count": 10}
+
+The endpoint supports the same column, search and plugin filters as the table page. Pagination and display options such as ``_next``, ``_size`` and ``_sort`` do not affect the count.
+
+This requires ``view-table`` permission. ``execute-sql`` permission is only needed if using ``_where`` filters.
+
+Unlike the ``count`` extra, this count is not capped by the row count limit. The usual SQL time limit still applies; a timed-out count returns a 400 JSON error.
+
 .. _TableAutocompleteView:
 
 Table autocomplete

tests/test_playwright.py

@@ -108,6 +108,11 @@ def write_playwright_database(db_path):
     conn = sqlite3.connect(db_path)
     try:
         conn.executescript("""
+        create table count_numbers (id integer primary key);
+        with recursive sequence(id) as (
+            select 1 union all select id + 1 from sequence where id < 10002
+        )
+        insert into count_numbers select id from sequence;
         create table projects (
             id integer primary key,
             title text not null,
@@ -1604,3 +1609,45 @@ def test_delete_row_flow_removes_row(page, datasette_server):
     page.locator(".row-mutation-status", has_text="Deleted row 1").wait_for()
     page.locator('tr[data-row="1"]').wait_for(state="detached")
     assert project_rows(datasette_server, id=1) == []
+
+
+@pytest.mark.playwright
+def test_count_all(page, datasette_server):
+    page.goto(datasette_server + "data/count_numbers?id__gt=1&_sort=id")
+    assert page.locator(".table-count").inner_text() == "10,000+ rows"
+    with page.expect_response("**/count_numbers/-/count?*") as response:
+        page.get_by_role("button", name="count all", exact=True).click()
+    assert response.value.request.method == "POST"
+    assert response.value.request.post_data is None
+    assert "content-type" not in response.value.request.headers
+    assert response.value.json() == {"ok": True, "count": 10001}
+    page.wait_for_function(
+        'document.querySelector(".table-count").textContent === "10,001 rows"'
+    )
+    assert page.locator(".count-all").count() == 0
+    assert "id" in page.locator("h3").first.inner_text()
+
+
+@pytest.mark.playwright
+def test_count_all_error_retry(page, datasette_server):
+    page.goto(datasette_server + "data/count_numbers?id__gt=1")
+    page.route(
+        "**/count_numbers/-/count?*",
+        lambda route: route.fulfill(
+            status=400,
+            content_type="application/json",
+            body=json.dumps({"ok": False, "errors": ["Count query timed out"]}),
+        ),
+    )
+    button = page.get_by_role("button", name="count all", exact=True)
+    button.click()
+    page.wait_for_function(
+        'document.querySelector(".count-error").textContent === "Count query timed out"'
+    )
+    assert button.is_enabled()
+    page.unroute("**/count_numbers/-/count?*")
+    button.click()
+    page.wait_for_function(
+        'document.querySelector(".table-count").textContent === "10,001 rows"'
+    )
+    assert page.locator(".count-error").inner_text() == ""