From db7ba1d30c0fb9e9e6eb2e39403a062fd93a973d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Tue, 7 Jul 2026 13:58:48 -0700 Subject: [PATCH] Switch to sqlite-utils migrations for internal.db, closes #2827 --- datasette/utils/internal_db.py | 165 +++++++++++++++++++++------------ tests/test_internal_db.py | 49 ++++++++++ 2 files changed, 153 insertions(+), 61 deletions(-) diff --git a/datasette/utils/internal_db.py b/datasette/utils/internal_db.py index bf172667..10ca32a5 100644 --- a/datasette/utils/internal_db.py +++ b/datasette/utils/internal_db.py @@ -1,9 +1,30 @@ import textwrap + +from sqlite_utils import Database as SQLiteUtilsDatabase +from sqlite_utils import Migrations + from datasette.utils import table_column_details +INTERNAL_DB_SCHEMA_TABLES = { + "catalog_databases", + "catalog_tables", + "catalog_views", + "catalog_columns", + "catalog_indexes", + "catalog_foreign_keys", + "metadata_instance", + "metadata_databases", + "metadata_resources", + "metadata_columns", + "column_types", + "queries", +} -async def init_internal_db(db): - create_tables_sql = textwrap.dedent(""" +INTERNAL_DB_SCHEMA_INDEXES = { + "queries_owner_idx", +} + +INTERNAL_DB_SCHEMA_SQL = textwrap.dedent(""" CREATE TABLE IF NOT EXISTS catalog_databases ( database_name TEXT PRIMARY KEY, path TEXT, @@ -67,74 +88,96 @@ async def init_internal_db(db): FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name), FOREIGN KEY (database_name, table_name) REFERENCES catalog_tables(database_name, table_name) ); + + CREATE TABLE IF NOT EXISTS metadata_instance ( + key text, + value text, + unique(key) + ); + + CREATE TABLE IF NOT EXISTS metadata_databases ( + database_name text, + key text, + value text, + unique(database_name, key) + ); + + CREATE TABLE IF NOT EXISTS metadata_resources ( + database_name text, + resource_name text, + key text, + value text, + unique(database_name, resource_name, key) + ); + + CREATE TABLE IF NOT EXISTS metadata_columns ( + database_name text, + resource_name text, + column_name text, + key text, + value text, + unique(database_name, resource_name, column_name, key) + ); + + CREATE TABLE IF NOT EXISTS column_types ( + database_name TEXT NOT NULL, + resource_name TEXT NOT NULL, + column_name TEXT NOT NULL, + column_type TEXT NOT NULL, + config TEXT, + PRIMARY KEY (database_name, resource_name, column_name) + ); + + CREATE TABLE IF NOT EXISTS queries ( + database_name TEXT NOT NULL, + name TEXT NOT NULL, + sql TEXT NOT NULL, + title TEXT, + description TEXT, + description_html TEXT, + options TEXT NOT NULL DEFAULT '{}', + parameters TEXT NOT NULL DEFAULT '[]', + is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), + is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), + is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), + source TEXT NOT NULL DEFAULT 'user', + owner_id TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (database_name, name) + ); + + CREATE INDEX IF NOT EXISTS queries_owner_idx + ON queries(owner_id); """).strip() - await db.execute_write_script(create_tables_sql) - await initialize_metadata_tables(db) -async def initialize_metadata_tables(db): - await db.execute_write_script(textwrap.dedent(""" - CREATE TABLE IF NOT EXISTS metadata_instance ( - key text, - value text, - unique(key) - ); +internal_migrations = Migrations("datasette_internal") - CREATE TABLE IF NOT EXISTS metadata_databases ( - database_name text, - key text, - value text, - unique(database_name, key) - ); - CREATE TABLE IF NOT EXISTS metadata_resources ( - database_name text, - resource_name text, - key text, - value text, - unique(database_name, resource_name, key) - ); +def _internal_schema_exists(db): + table_names = set(db.table_names()) + if not INTERNAL_DB_SCHEMA_TABLES.issubset(table_names): + return False + index_names = { + row[0] + for row in db.execute("select name from sqlite_master where type = 'index'") + } + return INTERNAL_DB_SCHEMA_INDEXES.issubset(index_names) - CREATE TABLE IF NOT EXISTS metadata_columns ( - database_name text, - resource_name text, - column_name text, - key text, - value text, - unique(database_name, resource_name, column_name, key) - ); - CREATE TABLE IF NOT EXISTS column_types ( - database_name TEXT NOT NULL, - resource_name TEXT NOT NULL, - column_name TEXT NOT NULL, - column_type TEXT NOT NULL, - config TEXT, - PRIMARY KEY (database_name, resource_name, column_name) - ); +@internal_migrations(name="0001_initial") +def initial_internal_schema(db): + if _internal_schema_exists(db): + return + db.executescript(INTERNAL_DB_SCHEMA_SQL) - CREATE TABLE IF NOT EXISTS queries ( - database_name TEXT NOT NULL, - name TEXT NOT NULL, - sql TEXT NOT NULL, - title TEXT, - description TEXT, - description_html TEXT, - options TEXT NOT NULL DEFAULT '{}', - parameters TEXT NOT NULL DEFAULT '[]', - is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)), - is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)), - is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)), - source TEXT NOT NULL DEFAULT 'user', - owner_id TEXT, - created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (database_name, name) - ); - CREATE INDEX IF NOT EXISTS queries_owner_idx - ON queries(owner_id); - """)) +async def init_internal_db(db): + def apply_migrations(conn): + internal_migrations.apply(SQLiteUtilsDatabase(conn, execute_plugins=False)) + + await db.execute_write_fn(apply_migrations, transaction=False) async def populate_schema_tables(internal_db, db): diff --git a/tests/test_internal_db.py b/tests/test_internal_db.py index 340c4813..e1ab51bb 100644 --- a/tests/test_internal_db.py +++ b/tests/test_internal_db.py @@ -1,6 +1,8 @@ import pytest +import sqlite3 from datasette.utils import escape_sqlite +from datasette.utils.internal_db import INTERNAL_DB_SCHEMA_SQL # ensure refresh_schemas() gets called before interacting with internal_db @@ -17,6 +19,53 @@ async def test_internal_databases(ds_client): assert databases.rows[0]["database_name"] == "fixtures" +@pytest.mark.asyncio +async def test_internal_migrations_recorded(ds_client): + internal_db = await ensure_internal(ds_client) + migrations = await internal_db.execute(""" + select migration_set, name + from _sqlite_migrations + order by id + """) + assert [tuple(row) for row in migrations.rows] == [ + ("datasette_internal", "0001_initial") + ] + + +@pytest.mark.asyncio +async def test_internal_migrations_adopt_existing_internal_db(tmp_path): + from datasette.app import Datasette + + internal_db_path = str(tmp_path / "internal.db") + conn = sqlite3.connect(internal_db_path) + conn.executescript(INTERNAL_DB_SCHEMA_SQL) + conn.execute( + "insert into metadata_instance (key, value) values (?, ?)", + ("legacy", "preserved"), + ) + conn.commit() + conn.close() + + ds = Datasette(internal=internal_db_path) + await ds.invoke_startup() + internal_db = ds.get_internal_database() + + metadata = await internal_db.execute( + "select key, value from metadata_instance where key = 'legacy'" + ) + assert [tuple(row) for row in metadata.rows] == [("legacy", "preserved")] + migrations = await internal_db.execute(""" + select migration_set, name + from _sqlite_migrations + order by id + """) + assert [tuple(row) for row in migrations.rows] == [ + ("datasette_internal", "0001_initial") + ] + + ds.close() + + @pytest.mark.asyncio async def test_internal_tables(ds_client): internal_db = await ensure_internal(ds_client)