Database page from the catalog; allowed_resources().all() in one query

- The database page's columns, primary keys, FTS tables, hidden flags and
  foreign keys come from the _internal catalog (catalog_all_foreign_keys()
  added); only row counts query the database. Removes three round trips
  per table and the O(tables^2) hidden-table check from that page.
- PaginatedResources.all() fetches everything after the first page in a
  single query: each page re-ran the whole permission query, so iterating
  87,000 tables cost O(N^2 / 1000) (21 s of the index page at N=5000).
- catalog_generation is bumped after the per-database state it covers.
- Docs: index page row-count behaviour (pages.rst), immutable counts are
  computed on first use (performance.rst), catalog_tables.type and "pages
  that cover many databases read the catalog" (internals.rst).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EwkLo2rJWidpkDVT3HkvNv

datasette/app.py

@@ -2057,7 +2057,29 @@ ORDER BY allowed.parent, allowed.child
 
         # Validate and cap limit
         limit = min(max(1, limit), 1000)
+        return await self._allowed_resources_page(
+            action,
+            actor,
+            parent=parent,
+            include_is_private=include_is_private,
+            include_reasons=include_reasons,
+            limit=limit,
+            next=next,
+        )
 
+    async def _allowed_resources_page(
+        self,
+        action,
+        actor,
+        *,
+        parent,
+        include_is_private,
+        include_reasons,
+        limit,
+        next,
+    ):
+        """allowed_resources() without the limit cap: ``limit=None`` returns
+        every remaining resource in one query (PaginatedResources.all())."""
         # Get base SQL query
         query, params = await self.allowed_resources_sql(
             action=action,
@@ -2087,15 +2109,16 @@ ORDER BY allowed.parent, allowed.child
 
         # Add LIMIT (fetch limit+1 to detect if there are more results)
         # Note: query from allowed_resources_sql() already includes ORDER BY parent, child
-        query = f"{query} LIMIT :limit"
-        params["limit"] = limit + 1
+        if limit is not None:
+            query = f"{query} LIMIT :limit"
+            params["limit"] = limit + 1
 
         # Execute query
         result = await self.get_internal_database().execute(query, params)
         rows = list(result.rows)
 
         # Check if truncated (got more than limit rows)
-        truncated = len(rows) > limit
+        truncated = limit is not None and len(rows) > limit
         if truncated:
             rows = rows[:limit]  # Remove the extra row
 

datasette/schema_watcher.py

@@ -870,7 +870,6 @@ class SchemaWatcher:
                 write_catalog_entries(conn, live)
 
             await self.ds.get_internal_database().execute_write_fn(_write)
-            self.catalog_generation += 1
             for r in stored:
                 state = r["state"]
                 state.catalog_version = r["version"]
@@ -885,6 +884,9 @@ class SchemaWatcher:
                 remember_derived_table_dependencies(
                     state, [(t[1], t[3]) for t in r["schema"]["tables"]]
                 )
+            # After the states: a reader that sees the new generation must
+            # also see their new catalog versions
+            self.catalog_generation += 1
         if missing:
             for state in missing:
                 if not state.missing:
@@ -906,9 +908,9 @@ class SchemaWatcher:
                 )
 
         await self.ds.get_internal_database().execute_write_fn(_clear)
-        self.catalog_generation += 1
         for state in states:
             state.catalog_version = None
+        self.catalog_generation += 1
 
     async def _delete_catalog(self, names):
         watcher = self

datasette/utils/__init__.py

@@ -34,10 +34,6 @@ if typing.TYPE_CHECKING:
     from datasette.permissions import Resource
 
 
-# Page size PaginatedResources.all() uses after the first page
-ALL_RESOURCES_PAGE_SIZE = 1000
-
-
 @dataclasses.dataclass
 class PaginatedResources:
     """Paginated results from allowed_resources query."""
@@ -71,24 +67,22 @@ class PaginatedResources:
         for resource in self.resources:
             yield resource
 
-        # Continue fetching subsequent pages if there are more. Every page
-        # re-runs the whole permission query, so fetch the rest in the
-        # largest pages allowed_resources() supports - the caller sees the
-        # same sequence of resources whatever the page size.
-        next_token = self.next
-        while next_token:
-            page = await self._datasette.allowed_resources(
+        # Fetch everything after the first page in one query. Each page
+        # re-runs the whole permission query, so paging through N resources
+        # 1000 at a time cost O(N^2 / 1000) - 21 seconds for the index page
+        # with 87,000 tables. The caller sees the same sequence either way.
+        if self.next:
+            rest = await self._datasette._allowed_resources_page(
                 self._action,
                 self._actor,
                 parent=self._parent,
                 include_is_private=self._include_is_private,
                 include_reasons=self._include_reasons,
-                limit=max(self._limit, ALL_RESOURCES_PAGE_SIZE),
-                next=next_token,
+                limit=None,
+                next=self.next,
             )
-            for resource in page.resources:
+            for resource in rest.resources:
                 yield resource
-            next_token = page.next
 
 
 # From https://www.sqlite.org/lang_keywords.html

datasette/utils/catalog.py

@@ -334,3 +334,55 @@ async def catalog_relationship_counts(datasette, database_names):
     for database_name, table_name, n in rows.rows:
         counts.setdefault(database_name, {})[table_name] = n
     return counts
+
+
+async def catalog_all_foreign_keys(datasette, database_name):
+    """``get_all_foreign_keys()`` for one database, from the catalog:
+    ``{table: {"incoming": [...], "outgoing": [...]}}`` for every table."""
+    internal = datasette.get_internal_database()
+    tables = await internal.execute(
+        "select table_name from catalog_tables where database_name = ? order by table_name",
+        [database_name],
+    )
+    table_to_foreign_keys = {
+        row[0]: {"incoming": [], "outgoing": []} for row in tables.rows
+    }
+    rows = await internal.execute(
+        """
+        select table_name, id, "table", "from", "to" from catalog_foreign_keys
+        where database_name = ?
+        order by table_name, id, seq
+        """,
+        [database_name],
+    )
+    by_table = {}
+    for table_name, fk_id, other_table, from_, to_ in rows.rows:
+        by_table.setdefault(table_name, {}).setdefault(fk_id, []).append(
+            (other_table, from_, to_)
+        )
+    for table, fks in by_table.items():
+        if table not in table_to_foreign_keys:
+            continue
+        for parts in fks.values():
+            if len(parts) != 1:
+                # Compound foreign keys are left out, as get_outbound_foreign_keys() does
+                continue
+            other_table, from_, to_ = parts[0]
+            if other_table not in table_to_foreign_keys:
+                # Weird edge case where something refers to a table that does
+                # not actually exist
+                continue
+            table_to_foreign_keys[other_table]["incoming"].append(
+                {"other_table": table, "column": to_, "other_column": from_}
+            )
+            table_to_foreign_keys[table]["outgoing"].append(
+                {"other_table": other_table, "column": from_, "other_column": to_}
+            )
+    for foreign_keys in table_to_foreign_keys.values():
+        foreign_keys["incoming"].sort(
+            key=lambda fk: (fk["other_table"], fk["column"], fk["other_column"])
+        )
+        foreign_keys["outgoing"].sort(
+            key=lambda fk: (fk["other_table"], fk["column"], fk["other_column"])
+        )
+    return table_to_foreign_keys

datasette/views/database.py

@@ -36,6 +36,11 @@ from datasette.utils import (
     named_parameters as derive_named_parameters,
 )
 from datasette.utils.asgi import AsgiFileDownload, Forbidden, NotFound, Response
+from datasette.utils.catalog import (
+    catalog_all_foreign_keys,
+    catalog_summaries,
+    catalog_table_details,
+)
 from datasette.write_sql import QueryWriteRejected
 
 from . import Context
@@ -497,25 +502,39 @@ async def get_tables(datasette, request, db, allowed_dict) -> list[DatabaseTable
     """
     tables = []
     table_counts = await db.table_counts(100)
-    hidden_table_names = set(await db.hidden_table_names())
-    all_foreign_keys = await db.get_all_foreign_keys()
+    # Everything except the row counts comes from the _internal catalog - the
+    # same catalog allowed_dict was built from - in a few queries, instead of
+    # three introspection round trips per table (and an O(tables^2)
+    # hidden-table check) against the database itself
+    summary = (await catalog_summaries(datasette, [db.name]))[db.name]
+    db_config = datasette.config.get("databases", {}).get(db.name, {})
+    config_hidden = {
+        t
+        for t, table_config in db_config.get("tables", {}).items()
+        if table_config.get("hidden")
+    }
+    all_foreign_keys = await catalog_all_foreign_keys(datasette, db.name)
+    shown = [table for table in table_counts if table in allowed_dict]
+    details = await catalog_table_details(
+        datasette, [(db.name, table) for table in shown]
+    )
 
-    for table in table_counts:
-        if table not in allowed_dict:
+    for table in shown:
+        if table not in all_foreign_keys:
+            # Not in the catalog (yet), so not in allowed_dict either
             continue
-
-        table_columns = await db.table_columns(table)
+        table_details = details[(db.name, table)]
         tables.append(
             DatabaseTable(
                 name=table,
-                columns=table_columns,
-                primary_keys=await db.primary_keys(table),
+                columns=table_details["columns"],
+                primary_keys=table_details["primary_keys"],
                 count=table_counts[table],
                 count_truncated=_table_count_truncated(
                     datasette, db, table, table_counts[table]
                 ),
-                hidden=table in hidden_table_names,
-                fts_table=await db.fts_table(table),
+                hidden=summary.is_hidden(table, config_hidden),
+                fts_table=table_details["fts_table"],
                 foreign_keys=all_foreign_keys[table],
                 private=allowed_dict[table].private,
             )

docs/internals.rst

@@ -2069,6 +2069,8 @@ Datasette is designed to serve anything from one database to thousands of databa
 
 **Schema changes** made by Datasette's own writes are detected as soon as the write finishes, and databases that other processes may change are polled - see :ref:`setting_schema_watch_interval_ms`. If a database file is replaced (for example by renaming a new file over it) or deleted, connections to the old file are discarded. Once Datasette has seen a database file exist it will not create it again: writes to a deleted file fail with an error rather than silently creating a new empty database. A database added with :ref:`datasette_add_database` whose file does not exist yet is created by its first write.
 
+**Pages that cover many databases** - the index page, ``allowed_resources()`` listings and their derived-table permission rules, ``/-/api`` and the like - read what they need about each database's tables from the :ref:`catalog tables in the internal database <internals_internal>` instead of opening every database. Plugins that summarise many databases should do the same: a query against ``catalog_tables``, ``catalog_columns`` or ``catalog_foreign_keys`` costs the same however many databases are attached, while calling methods such as ``db.table_names()`` on every database opens a connection to each of them. Use the live methods on a single :ref:`Database <internals_database>` when you need the database's own current answer.
+
 With :ref:`setting_num_sql_threads` set to ``0`` (for example in Pyodide) there are no threads: reads and writes run on the event loop, read connections are still pooled and closed when idle, and the write connection stays open until the database is closed.
 
 Every new connection is set up by the :ref:`plugin_hook_prepare_connection` plugin hook, except isolated connections, the brief connections used to check a file's schema version, and connections to the internal database. Because connections are opened and closed as needed, anything set up on a connection - functions, temporary tables, ``ATTACH``, ``PRAGMA`` settings - only lasts as long as that connection, and is not seen by other connections to the same database. Use ``prepare_connection`` for anything every connection needs.
@@ -2829,7 +2831,7 @@ You can also set the ``DATASETTE_INTERNAL`` environment variable to specify this
     export DATASETTE_INTERNAL=/path/to/internal.db
     datasette mydatabase.db
 
-Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_views``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases.
+Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_views``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. ``catalog_tables.type`` is the type ``PRAGMA table_list`` reports for each table (``table``, ``virtual`` or ``shadow``), or ``NULL`` if unknown. These tables should not be considered a stable API - they may change between Datasette releases.
 
 Metadata is stored in tables ``metadata_instance``, ``metadata_databases``, ``metadata_resources`` and ``metadata_columns``. Plugins can interact with these tables via the :ref:`get_*_metadata() and set_*_metadata() methods <datasette_get_set_metadata>`.
 

docs/pages.rst

@@ -23,6 +23,10 @@ Add ``/.json`` to the end of the URL for the JSON version of the underlying data
 
 The index page can also be accessed at ``/-/``, useful for if the default index page has been replaced using an :ref:`index.html custom template <customization_custom_templates>`. The ``/-/`` page will always render the default Datasette ``index.html`` template.
 
+.. _IndexView_row_counts:
+
+The index page reads table names, columns, primary keys, hidden tables and foreign key relationships from the :ref:`internal database catalog <internals_internal>`, so it does not need to open each attached database. Row counts are the exception: they need a ``count(*)`` per table. The index page shows row counts when it lists at most 20 databases that would need counting, each smaller than 100MB, and the counts for a database are dropped if any of them takes longer than 10ms. With more databases than that it shows the number of tables in each database, sorts the most interesting tables first by their number of foreign key relationships, and only includes row counts that are already known without opening a database: counts for :ref:`immutable databases <performance_immutable_mode>` from ``--inspect-file`` or calculated by an earlier page. Visit a database's page for its row counts.
+
 .. _DatabaseView:
 
 Database

docs/performance.rst

@@ -22,14 +22,14 @@ To open a file in immutable mode pass it to the datasette command using the ``-i
 
     datasette -i data.db
 
-When you open a file in immutable mode like this Datasette will also calculate and cache the row counts for each table in that database when it first starts up, further improving performance.
+When you open a file in immutable mode like this Datasette will also calculate and cache the row counts for each table in that database the first time a page needs them, further improving performance. (Earlier versions counted every immutable database at startup.)
 
 .. _performance_inspect:
 
 Using "datasette inspect"
 -------------------------
 
-Counting the rows in a table can be a very expensive operation on larger databases. In immutable mode Datasette performs this count only once and caches the results, but this can still cause server startup time to increase by several seconds or more.
+Counting the rows in a table can be a very expensive operation on larger databases. In immutable mode Datasette performs this count only once and caches the results, but this can still make the first page that shows those counts take several seconds or more.
 
 If you know that a database is never going to change you can precalculate the table row counts once and store then in a JSON file, then use that file when you later start the server.
 
@@ -37,7 +37,7 @@ To create a JSON file containing the calculated row counts for a database, use t
 
     datasette inspect data.db --inspect-file=counts.json
 
-Then later you can start Datasette against the ``counts.json`` file and use it to skip the row counting step and speed up server startup::
+Then later you can start Datasette against the ``counts.json`` file and use it to skip the row counting step entirely::
 
     datasette -i data.db --inspect-file=counts.json
 

tests/test_connection_budget.py

@@ -417,7 +417,14 @@ async def test_startup_with_crossdb_attaches_nothing(db_files):
 
 
 @pytest.mark.asyncio
-async def test_restart_with_persisted_catalog_opens_nothing(db_files, tmp_path):
+async def test_restart_with_persisted_catalog_opens_nothing(
+    db_files, tmp_path, monkeypatch
+):
+    from datasette import schema_watcher
+
+    # The files' ctimes are recent (os.utime() in the fixture): without this
+    # their stored fingerprints would be racy, and so rescanned
+    monkeypatch.setattr(schema_watcher, "RACY_WINDOW_NS", 0)
     internal = str(tmp_path / "internal.db")
     ds = Datasette(db_files, internal=internal)
     await ds.invoke_startup()
@@ -506,62 +513,74 @@ async def test_check_databases_reuses_catalog_scan(db_files, tmp_path):
 # Parity: the catalog helpers answer exactly what the live helpers do
 
 
+MISC_SQL = """
+    create table docs (id integer primary key, title text, body text);
+    create virtual table docs_fts using fts4(title, body, content="docs");
+    create virtual table docs_vocab using fts4aux(docs_fts);
+    create virtual table plain using fts5(x);
+    create virtual table geo using rtree(id, minx, maxx);
+    create table _private (x);
+    create table geometry_columns (f_table_name text);
+    create table idx_foo_bar (x);
+    create table IDXA (x);
+    create table parent (id integer primary key);
+    create table child (
+        id integer primary key,
+        p integer references parent(id),
+        q integer references missing(id)
+    );
+    create table comp (a, b, foreign key (a, b) references parent(id, id));
+    create table self_ref (id integer primary key, up integer references self_ref(id));
+    create table wide (b text, a integer, c, primary key (a, b));
+    create view v_docs as select * from docs;
+"""
+
+
 @pytest.mark.asyncio
-async def test_catalog_helpers_match_live_introspection(tmp_path):
-    path = str(tmp_path / "misc.db")
-    conn = sqlite3.connect(path)
-    conn.executescript("""
-        create table docs (id integer primary key, title text, body text);
-        create virtual table docs_fts using fts4(title, body, content="docs");
-        create virtual table docs_vocab using fts4aux(docs_fts);
-        create virtual table plain using fts5(x);
-        create virtual table geo using rtree(id, minx, maxx);
-        create table _private (x);
-        create table geometry_columns (f_table_name text);
-        create table idx_foo_bar (x);
-        create table IDXA (x);
-        create table parent (id integer primary key);
-        create table child (
-            id integer primary key,
-            p integer references parent(id),
-            q integer references missing(id)
-        );
-        create table comp (a, b, foreign key (a, b) references parent(id, id));
-        create table self_ref (id integer primary key, up integer references self_ref(id));
-        create table wide (b text, a integer, c, primary key (a, b));
-        create view v_docs as select * from docs;
-        """)
-    conn.close()
-    ds = Datasette(
-        [path],
-        config={"databases": {"misc": {"tables": {"docs": {"hidden": True}}}}},
-        settings={"schema_watch_interval_ms": 0},
+@pytest.mark.parametrize("name", ["misc", "fixtures"])
+async def test_catalog_helpers_match_live_introspection(tmp_path, name):
+    from datasette.fixtures import write_fixture_database
+    from datasette.utils.catalog import (
+        all_derived_table_dependencies,
+        catalog_all_foreign_keys,
     )
+
+    path = str(tmp_path / f"{name}.db")
+    if name == "fixtures":
+        write_fixture_database(path)
+        config_hidden = set()
+        config = {}
+    else:
+        conn = sqlite3.connect(path)
+        conn.executescript(MISC_SQL)
+        conn.close()
+        config_hidden = {"docs"}
+        config = {"databases": {"misc": {"tables": {"docs": {"hidden": True}}}}}
+    ds = Datasette([path], config=config, settings={"schema_watch_interval_ms": 0})
     await ds.invoke_startup()
-    db = ds.get_database("misc")
+    db = ds.get_database(name)
     tables = await db.table_names()
-    summary = (await catalog_summaries(ds, ["misc"]))["misc"]
+    summary = (await catalog_summaries(ds, [name]))[name]
     # hidden_table_names() also lists SpatiaLite names that do not exist
     hidden = set(await db.hidden_table_names()) & set(tables)
-    config_hidden = {"docs"}
     assert {t for t in tables if summary.is_hidden(t, config_hidden)} == hidden
     assert summary.views == set(await db.view_names())
 
-    details = await catalog_table_details(ds, [("misc", t) for t in tables])
+    details = await catalog_table_details(ds, [(name, t) for t in tables])
     for table in tables:
-        assert details[("misc", table)]["columns"] == await db.table_columns(table)
-        assert details[("misc", table)]["primary_keys"] == await db.primary_keys(table)
-        assert details[("misc", table)]["fts_table"] == await db.fts_table(table)
+        assert details[(name, table)]["columns"] == await db.table_columns(table)
+        assert details[(name, table)]["primary_keys"] == await db.primary_keys(table)
+        assert details[(name, table)]["fts_table"] == await db.fts_table(table)
 
-    relationships = (await catalog_relationship_counts(ds, ["misc"])).get("misc", {})
-    for table, fks in (await db.get_all_foreign_keys()).items():
+    all_foreign_keys = await db.get_all_foreign_keys()
+    assert await catalog_all_foreign_keys(ds, name) == all_foreign_keys
+    relationships = (await catalog_relationship_counts(ds, [name])).get(name, {})
+    for table, fks in all_foreign_keys.items():
         assert relationships.get(table, 0) == len(fks["incoming"] + fks["outgoing"])
 
-    from datasette.utils.catalog import all_derived_table_dependencies
-
     derived = await all_derived_table_dependencies(ds)
     live = await db.execute_fn(sqlite_derived_table_dependencies)
-    assert derived["misc"] == live
+    assert derived.get(name, {}) == live
     # Restored from the catalog rather than remembered from the scan
     db._cached_derived_table_dependencies = None
     assert await db.derived_table_dependencies() == live