diff --git a/datasette/app.py b/datasette/app.py index c3ef4d90..212c97bd 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -2057,7 +2057,29 @@ ORDER BY allowed.parent, allowed.child # Validate and cap limit limit = min(max(1, limit), 1000) + return await self._allowed_resources_page( + action, + actor, + parent=parent, + include_is_private=include_is_private, + include_reasons=include_reasons, + limit=limit, + next=next, + ) + async def _allowed_resources_page( + self, + action, + actor, + *, + parent, + include_is_private, + include_reasons, + limit, + next, + ): + """allowed_resources() without the limit cap: ``limit=None`` returns + every remaining resource in one query (PaginatedResources.all()).""" # Get base SQL query query, params = await self.allowed_resources_sql( action=action, @@ -2087,15 +2109,16 @@ ORDER BY allowed.parent, allowed.child # Add LIMIT (fetch limit+1 to detect if there are more results) # Note: query from allowed_resources_sql() already includes ORDER BY parent, child - query = f"{query} LIMIT :limit" - params["limit"] = limit + 1 + if limit is not None: + query = f"{query} LIMIT :limit" + params["limit"] = limit + 1 # Execute query result = await self.get_internal_database().execute(query, params) rows = list(result.rows) # Check if truncated (got more than limit rows) - truncated = len(rows) > limit + truncated = limit is not None and len(rows) > limit if truncated: rows = rows[:limit] # Remove the extra row diff --git a/datasette/schema_watcher.py b/datasette/schema_watcher.py index 3c8333f6..8a11f48b 100644 --- a/datasette/schema_watcher.py +++ b/datasette/schema_watcher.py @@ -870,7 +870,6 @@ class SchemaWatcher: write_catalog_entries(conn, live) await self.ds.get_internal_database().execute_write_fn(_write) - self.catalog_generation += 1 for r in stored: state = r["state"] state.catalog_version = r["version"] @@ -885,6 +884,9 @@ class SchemaWatcher: remember_derived_table_dependencies( state, [(t[1], t[3]) for t in r["schema"]["tables"]] ) + # After the states: a reader that sees the new generation must + # also see their new catalog versions + self.catalog_generation += 1 if missing: for state in missing: if not state.missing: @@ -906,9 +908,9 @@ class SchemaWatcher: ) await self.ds.get_internal_database().execute_write_fn(_clear) - self.catalog_generation += 1 for state in states: state.catalog_version = None + self.catalog_generation += 1 async def _delete_catalog(self, names): watcher = self diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index c36f8a58..c98f92d2 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -34,10 +34,6 @@ if typing.TYPE_CHECKING: from datasette.permissions import Resource -# Page size PaginatedResources.all() uses after the first page -ALL_RESOURCES_PAGE_SIZE = 1000 - - @dataclasses.dataclass class PaginatedResources: """Paginated results from allowed_resources query.""" @@ -71,24 +67,22 @@ class PaginatedResources: for resource in self.resources: yield resource - # Continue fetching subsequent pages if there are more. Every page - # re-runs the whole permission query, so fetch the rest in the - # largest pages allowed_resources() supports - the caller sees the - # same sequence of resources whatever the page size. - next_token = self.next - while next_token: - page = await self._datasette.allowed_resources( + # Fetch everything after the first page in one query. Each page + # re-runs the whole permission query, so paging through N resources + # 1000 at a time cost O(N^2 / 1000) - 21 seconds for the index page + # with 87,000 tables. The caller sees the same sequence either way. + if self.next: + rest = await self._datasette._allowed_resources_page( self._action, self._actor, parent=self._parent, include_is_private=self._include_is_private, include_reasons=self._include_reasons, - limit=max(self._limit, ALL_RESOURCES_PAGE_SIZE), - next=next_token, + limit=None, + next=self.next, ) - for resource in page.resources: + for resource in rest.resources: yield resource - next_token = page.next # From https://www.sqlite.org/lang_keywords.html diff --git a/datasette/utils/catalog.py b/datasette/utils/catalog.py index d5d1e4ec..344b16b6 100644 --- a/datasette/utils/catalog.py +++ b/datasette/utils/catalog.py @@ -334,3 +334,55 @@ async def catalog_relationship_counts(datasette, database_names): for database_name, table_name, n in rows.rows: counts.setdefault(database_name, {})[table_name] = n return counts + + +async def catalog_all_foreign_keys(datasette, database_name): + """``get_all_foreign_keys()`` for one database, from the catalog: + ``{table: {"incoming": [...], "outgoing": [...]}}`` for every table.""" + internal = datasette.get_internal_database() + tables = await internal.execute( + "select table_name from catalog_tables where database_name = ? order by table_name", + [database_name], + ) + table_to_foreign_keys = { + row[0]: {"incoming": [], "outgoing": []} for row in tables.rows + } + rows = await internal.execute( + """ + select table_name, id, "table", "from", "to" from catalog_foreign_keys + where database_name = ? + order by table_name, id, seq + """, + [database_name], + ) + by_table = {} + for table_name, fk_id, other_table, from_, to_ in rows.rows: + by_table.setdefault(table_name, {}).setdefault(fk_id, []).append( + (other_table, from_, to_) + ) + for table, fks in by_table.items(): + if table not in table_to_foreign_keys: + continue + for parts in fks.values(): + if len(parts) != 1: + # Compound foreign keys are left out, as get_outbound_foreign_keys() does + continue + other_table, from_, to_ = parts[0] + if other_table not in table_to_foreign_keys: + # Weird edge case where something refers to a table that does + # not actually exist + continue + table_to_foreign_keys[other_table]["incoming"].append( + {"other_table": table, "column": to_, "other_column": from_} + ) + table_to_foreign_keys[table]["outgoing"].append( + {"other_table": other_table, "column": from_, "other_column": to_} + ) + for foreign_keys in table_to_foreign_keys.values(): + foreign_keys["incoming"].sort( + key=lambda fk: (fk["other_table"], fk["column"], fk["other_column"]) + ) + foreign_keys["outgoing"].sort( + key=lambda fk: (fk["other_table"], fk["column"], fk["other_column"]) + ) + return table_to_foreign_keys diff --git a/datasette/views/database.py b/datasette/views/database.py index 14674c93..3034a891 100644 --- a/datasette/views/database.py +++ b/datasette/views/database.py @@ -36,6 +36,11 @@ from datasette.utils import ( named_parameters as derive_named_parameters, ) from datasette.utils.asgi import AsgiFileDownload, Forbidden, NotFound, Response +from datasette.utils.catalog import ( + catalog_all_foreign_keys, + catalog_summaries, + catalog_table_details, +) from datasette.write_sql import QueryWriteRejected from . import Context @@ -497,25 +502,39 @@ async def get_tables(datasette, request, db, allowed_dict) -> list[DatabaseTable """ tables = [] table_counts = await db.table_counts(100) - hidden_table_names = set(await db.hidden_table_names()) - all_foreign_keys = await db.get_all_foreign_keys() + # Everything except the row counts comes from the _internal catalog - the + # same catalog allowed_dict was built from - in a few queries, instead of + # three introspection round trips per table (and an O(tables^2) + # hidden-table check) against the database itself + summary = (await catalog_summaries(datasette, [db.name]))[db.name] + db_config = datasette.config.get("databases", {}).get(db.name, {}) + config_hidden = { + t + for t, table_config in db_config.get("tables", {}).items() + if table_config.get("hidden") + } + all_foreign_keys = await catalog_all_foreign_keys(datasette, db.name) + shown = [table for table in table_counts if table in allowed_dict] + details = await catalog_table_details( + datasette, [(db.name, table) for table in shown] + ) - for table in table_counts: - if table not in allowed_dict: + for table in shown: + if table not in all_foreign_keys: + # Not in the catalog (yet), so not in allowed_dict either continue - - table_columns = await db.table_columns(table) + table_details = details[(db.name, table)] tables.append( DatabaseTable( name=table, - columns=table_columns, - primary_keys=await db.primary_keys(table), + columns=table_details["columns"], + primary_keys=table_details["primary_keys"], count=table_counts[table], count_truncated=_table_count_truncated( datasette, db, table, table_counts[table] ), - hidden=table in hidden_table_names, - fts_table=await db.fts_table(table), + hidden=summary.is_hidden(table, config_hidden), + fts_table=table_details["fts_table"], foreign_keys=all_foreign_keys[table], private=allowed_dict[table].private, ) diff --git a/docs/internals.rst b/docs/internals.rst index 3e5ccad2..5b6d92b6 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2069,6 +2069,8 @@ Datasette is designed to serve anything from one database to thousands of databa **Schema changes** made by Datasette's own writes are detected as soon as the write finishes, and databases that other processes may change are polled - see :ref:`setting_schema_watch_interval_ms`. If a database file is replaced (for example by renaming a new file over it) or deleted, connections to the old file are discarded. Once Datasette has seen a database file exist it will not create it again: writes to a deleted file fail with an error rather than silently creating a new empty database. A database added with :ref:`datasette_add_database` whose file does not exist yet is created by its first write. +**Pages that cover many databases** - the index page, ``allowed_resources()`` listings and their derived-table permission rules, ``/-/api`` and the like - read what they need about each database's tables from the :ref:`catalog tables in the internal database ` instead of opening every database. Plugins that summarise many databases should do the same: a query against ``catalog_tables``, ``catalog_columns`` or ``catalog_foreign_keys`` costs the same however many databases are attached, while calling methods such as ``db.table_names()`` on every database opens a connection to each of them. Use the live methods on a single :ref:`Database ` when you need the database's own current answer. + With :ref:`setting_num_sql_threads` set to ``0`` (for example in Pyodide) there are no threads: reads and writes run on the event loop, read connections are still pooled and closed when idle, and the write connection stays open until the database is closed. Every new connection is set up by the :ref:`plugin_hook_prepare_connection` plugin hook, except isolated connections, the brief connections used to check a file's schema version, and connections to the internal database. Because connections are opened and closed as needed, anything set up on a connection - functions, temporary tables, ``ATTACH``, ``PRAGMA`` settings - only lasts as long as that connection, and is not seen by other connections to the same database. Use ``prepare_connection`` for anything every connection needs. @@ -2829,7 +2831,7 @@ You can also set the ``DATASETTE_INTERNAL`` environment variable to specify this export DATASETTE_INTERNAL=/path/to/internal.db datasette mydatabase.db -Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_views``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases. +Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_views``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. ``catalog_tables.type`` is the type ``PRAGMA table_list`` reports for each table (``table``, ``virtual`` or ``shadow``), or ``NULL`` if unknown. These tables should not be considered a stable API - they may change between Datasette releases. Metadata is stored in tables ``metadata_instance``, ``metadata_databases``, ``metadata_resources`` and ``metadata_columns``. Plugins can interact with these tables via the :ref:`get_*_metadata() and set_*_metadata() methods `. diff --git a/docs/pages.rst b/docs/pages.rst index 65c03a49..de2e57bd 100644 --- a/docs/pages.rst +++ b/docs/pages.rst @@ -23,6 +23,10 @@ Add ``/.json`` to the end of the URL for the JSON version of the underlying data The index page can also be accessed at ``/-/``, useful for if the default index page has been replaced using an :ref:`index.html custom template `. The ``/-/`` page will always render the default Datasette ``index.html`` template. +.. _IndexView_row_counts: + +The index page reads table names, columns, primary keys, hidden tables and foreign key relationships from the :ref:`internal database catalog `, so it does not need to open each attached database. Row counts are the exception: they need a ``count(*)`` per table. The index page shows row counts when it lists at most 20 databases that would need counting, each smaller than 100MB, and the counts for a database are dropped if any of them takes longer than 10ms. With more databases than that it shows the number of tables in each database, sorts the most interesting tables first by their number of foreign key relationships, and only includes row counts that are already known without opening a database: counts for :ref:`immutable databases ` from ``--inspect-file`` or calculated by an earlier page. Visit a database's page for its row counts. + .. _DatabaseView: Database diff --git a/docs/performance.rst b/docs/performance.rst index 4427757c..3cf2e123 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -22,14 +22,14 @@ To open a file in immutable mode pass it to the datasette command using the ``-i datasette -i data.db -When you open a file in immutable mode like this Datasette will also calculate and cache the row counts for each table in that database when it first starts up, further improving performance. +When you open a file in immutable mode like this Datasette will also calculate and cache the row counts for each table in that database the first time a page needs them, further improving performance. (Earlier versions counted every immutable database at startup.) .. _performance_inspect: Using "datasette inspect" ------------------------- -Counting the rows in a table can be a very expensive operation on larger databases. In immutable mode Datasette performs this count only once and caches the results, but this can still cause server startup time to increase by several seconds or more. +Counting the rows in a table can be a very expensive operation on larger databases. In immutable mode Datasette performs this count only once and caches the results, but this can still make the first page that shows those counts take several seconds or more. If you know that a database is never going to change you can precalculate the table row counts once and store then in a JSON file, then use that file when you later start the server. @@ -37,7 +37,7 @@ To create a JSON file containing the calculated row counts for a database, use t datasette inspect data.db --inspect-file=counts.json -Then later you can start Datasette against the ``counts.json`` file and use it to skip the row counting step and speed up server startup:: +Then later you can start Datasette against the ``counts.json`` file and use it to skip the row counting step entirely:: datasette -i data.db --inspect-file=counts.json diff --git a/tests/test_connection_budget.py b/tests/test_connection_budget.py index 8ecb00f0..63853816 100644 --- a/tests/test_connection_budget.py +++ b/tests/test_connection_budget.py @@ -417,7 +417,14 @@ async def test_startup_with_crossdb_attaches_nothing(db_files): @pytest.mark.asyncio -async def test_restart_with_persisted_catalog_opens_nothing(db_files, tmp_path): +async def test_restart_with_persisted_catalog_opens_nothing( + db_files, tmp_path, monkeypatch +): + from datasette import schema_watcher + + # The files' ctimes are recent (os.utime() in the fixture): without this + # their stored fingerprints would be racy, and so rescanned + monkeypatch.setattr(schema_watcher, "RACY_WINDOW_NS", 0) internal = str(tmp_path / "internal.db") ds = Datasette(db_files, internal=internal) await ds.invoke_startup() @@ -506,62 +513,74 @@ async def test_check_databases_reuses_catalog_scan(db_files, tmp_path): # Parity: the catalog helpers answer exactly what the live helpers do +MISC_SQL = """ + create table docs (id integer primary key, title text, body text); + create virtual table docs_fts using fts4(title, body, content="docs"); + create virtual table docs_vocab using fts4aux(docs_fts); + create virtual table plain using fts5(x); + create virtual table geo using rtree(id, minx, maxx); + create table _private (x); + create table geometry_columns (f_table_name text); + create table idx_foo_bar (x); + create table IDXA (x); + create table parent (id integer primary key); + create table child ( + id integer primary key, + p integer references parent(id), + q integer references missing(id) + ); + create table comp (a, b, foreign key (a, b) references parent(id, id)); + create table self_ref (id integer primary key, up integer references self_ref(id)); + create table wide (b text, a integer, c, primary key (a, b)); + create view v_docs as select * from docs; +""" + + @pytest.mark.asyncio -async def test_catalog_helpers_match_live_introspection(tmp_path): - path = str(tmp_path / "misc.db") - conn = sqlite3.connect(path) - conn.executescript(""" - create table docs (id integer primary key, title text, body text); - create virtual table docs_fts using fts4(title, body, content="docs"); - create virtual table docs_vocab using fts4aux(docs_fts); - create virtual table plain using fts5(x); - create virtual table geo using rtree(id, minx, maxx); - create table _private (x); - create table geometry_columns (f_table_name text); - create table idx_foo_bar (x); - create table IDXA (x); - create table parent (id integer primary key); - create table child ( - id integer primary key, - p integer references parent(id), - q integer references missing(id) - ); - create table comp (a, b, foreign key (a, b) references parent(id, id)); - create table self_ref (id integer primary key, up integer references self_ref(id)); - create table wide (b text, a integer, c, primary key (a, b)); - create view v_docs as select * from docs; - """) - conn.close() - ds = Datasette( - [path], - config={"databases": {"misc": {"tables": {"docs": {"hidden": True}}}}}, - settings={"schema_watch_interval_ms": 0}, +@pytest.mark.parametrize("name", ["misc", "fixtures"]) +async def test_catalog_helpers_match_live_introspection(tmp_path, name): + from datasette.fixtures import write_fixture_database + from datasette.utils.catalog import ( + all_derived_table_dependencies, + catalog_all_foreign_keys, ) + + path = str(tmp_path / f"{name}.db") + if name == "fixtures": + write_fixture_database(path) + config_hidden = set() + config = {} + else: + conn = sqlite3.connect(path) + conn.executescript(MISC_SQL) + conn.close() + config_hidden = {"docs"} + config = {"databases": {"misc": {"tables": {"docs": {"hidden": True}}}}} + ds = Datasette([path], config=config, settings={"schema_watch_interval_ms": 0}) await ds.invoke_startup() - db = ds.get_database("misc") + db = ds.get_database(name) tables = await db.table_names() - summary = (await catalog_summaries(ds, ["misc"]))["misc"] + summary = (await catalog_summaries(ds, [name]))[name] # hidden_table_names() also lists SpatiaLite names that do not exist hidden = set(await db.hidden_table_names()) & set(tables) - config_hidden = {"docs"} assert {t for t in tables if summary.is_hidden(t, config_hidden)} == hidden assert summary.views == set(await db.view_names()) - details = await catalog_table_details(ds, [("misc", t) for t in tables]) + details = await catalog_table_details(ds, [(name, t) for t in tables]) for table in tables: - assert details[("misc", table)]["columns"] == await db.table_columns(table) - assert details[("misc", table)]["primary_keys"] == await db.primary_keys(table) - assert details[("misc", table)]["fts_table"] == await db.fts_table(table) + assert details[(name, table)]["columns"] == await db.table_columns(table) + assert details[(name, table)]["primary_keys"] == await db.primary_keys(table) + assert details[(name, table)]["fts_table"] == await db.fts_table(table) - relationships = (await catalog_relationship_counts(ds, ["misc"])).get("misc", {}) - for table, fks in (await db.get_all_foreign_keys()).items(): + all_foreign_keys = await db.get_all_foreign_keys() + assert await catalog_all_foreign_keys(ds, name) == all_foreign_keys + relationships = (await catalog_relationship_counts(ds, [name])).get(name, {}) + for table, fks in all_foreign_keys.items(): assert relationships.get(table, 0) == len(fks["incoming"] + fks["outgoing"]) - from datasette.utils.catalog import all_derived_table_dependencies - derived = await all_derived_table_dependencies(ds) live = await db.execute_fn(sqlite_derived_table_dependencies) - assert derived["misc"] == live + assert derived.get(name, {}) == live # Restored from the catalog rather than remembered from the scan db._cached_derived_table_dependencies = None assert await db.derived_table_dependencies() == live