mirror of
https://github.com/simonw/datasette.git
synced 2026-07-08 08:34:42 +02:00
Switch to sqlite-utils migrations for internal.db, closes #2827
This commit is contained in:
Simon Willison
parent
96e8b85523
commit
db7ba1d30c
2 changed files with 153 additions and 61 deletions
|
|
@ -1,9 +1,30 @@
|
|||
import textwrap
|
||||
|
||||
from sqlite_utils import Database as SQLiteUtilsDatabase
|
||||
from sqlite_utils import Migrations
|
||||
|
||||
from datasette.utils import table_column_details
|
||||
|
||||
INTERNAL_DB_SCHEMA_TABLES = {
|
||||
"catalog_databases",
|
||||
"catalog_tables",
|
||||
"catalog_views",
|
||||
"catalog_columns",
|
||||
"catalog_indexes",
|
||||
"catalog_foreign_keys",
|
||||
"metadata_instance",
|
||||
"metadata_databases",
|
||||
"metadata_resources",
|
||||
"metadata_columns",
|
||||
"column_types",
|
||||
"queries",
|
||||
}
|
||||
|
||||
async def init_internal_db(db):
|
||||
create_tables_sql = textwrap.dedent("""
|
||||
INTERNAL_DB_SCHEMA_INDEXES = {
|
||||
"queries_owner_idx",
|
||||
}
|
||||
|
||||
INTERNAL_DB_SCHEMA_SQL = textwrap.dedent("""
|
||||
CREATE TABLE IF NOT EXISTS catalog_databases (
|
||||
database_name TEXT PRIMARY KEY,
|
||||
path TEXT,
|
||||
|
|
@ -67,74 +88,96 @@ async def init_internal_db(db):
|
|||
FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name),
|
||||
FOREIGN KEY (database_name, table_name) REFERENCES catalog_tables(database_name, table_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metadata_instance (
|
||||
key text,
|
||||
value text,
|
||||
unique(key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metadata_databases (
|
||||
database_name text,
|
||||
key text,
|
||||
value text,
|
||||
unique(database_name, key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metadata_resources (
|
||||
database_name text,
|
||||
resource_name text,
|
||||
key text,
|
||||
value text,
|
||||
unique(database_name, resource_name, key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metadata_columns (
|
||||
database_name text,
|
||||
resource_name text,
|
||||
column_name text,
|
||||
key text,
|
||||
value text,
|
||||
unique(database_name, resource_name, column_name, key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS column_types (
|
||||
database_name TEXT NOT NULL,
|
||||
resource_name TEXT NOT NULL,
|
||||
column_name TEXT NOT NULL,
|
||||
column_type TEXT NOT NULL,
|
||||
config TEXT,
|
||||
PRIMARY KEY (database_name, resource_name, column_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS queries (
|
||||
database_name TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
sql TEXT NOT NULL,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
description_html TEXT,
|
||||
options TEXT NOT NULL DEFAULT '{}',
|
||||
parameters TEXT NOT NULL DEFAULT '[]',
|
||||
is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)),
|
||||
is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)),
|
||||
is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)),
|
||||
source TEXT NOT NULL DEFAULT 'user',
|
||||
owner_id TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (database_name, name)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS queries_owner_idx
|
||||
ON queries(owner_id);
|
||||
""").strip()
|
||||
await db.execute_write_script(create_tables_sql)
|
||||
await initialize_metadata_tables(db)
|
||||
|
||||
|
||||
async def initialize_metadata_tables(db):
|
||||
await db.execute_write_script(textwrap.dedent("""
|
||||
CREATE TABLE IF NOT EXISTS metadata_instance (
|
||||
key text,
|
||||
value text,
|
||||
unique(key)
|
||||
);
|
||||
internal_migrations = Migrations("datasette_internal")
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metadata_databases (
|
||||
database_name text,
|
||||
key text,
|
||||
value text,
|
||||
unique(database_name, key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metadata_resources (
|
||||
database_name text,
|
||||
resource_name text,
|
||||
key text,
|
||||
value text,
|
||||
unique(database_name, resource_name, key)
|
||||
);
|
||||
def _internal_schema_exists(db):
|
||||
table_names = set(db.table_names())
|
||||
if not INTERNAL_DB_SCHEMA_TABLES.issubset(table_names):
|
||||
return False
|
||||
index_names = {
|
||||
row[0]
|
||||
for row in db.execute("select name from sqlite_master where type = 'index'")
|
||||
}
|
||||
return INTERNAL_DB_SCHEMA_INDEXES.issubset(index_names)
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metadata_columns (
|
||||
database_name text,
|
||||
resource_name text,
|
||||
column_name text,
|
||||
key text,
|
||||
value text,
|
||||
unique(database_name, resource_name, column_name, key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS column_types (
|
||||
database_name TEXT NOT NULL,
|
||||
resource_name TEXT NOT NULL,
|
||||
column_name TEXT NOT NULL,
|
||||
column_type TEXT NOT NULL,
|
||||
config TEXT,
|
||||
PRIMARY KEY (database_name, resource_name, column_name)
|
||||
);
|
||||
@internal_migrations(name="0001_initial")
|
||||
def initial_internal_schema(db):
|
||||
if _internal_schema_exists(db):
|
||||
return
|
||||
db.executescript(INTERNAL_DB_SCHEMA_SQL)
|
||||
|
||||
CREATE TABLE IF NOT EXISTS queries (
|
||||
database_name TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
sql TEXT NOT NULL,
|
||||
title TEXT,
|
||||
description TEXT,
|
||||
description_html TEXT,
|
||||
options TEXT NOT NULL DEFAULT '{}',
|
||||
parameters TEXT NOT NULL DEFAULT '[]',
|
||||
is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)),
|
||||
is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)),
|
||||
is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)),
|
||||
source TEXT NOT NULL DEFAULT 'user',
|
||||
owner_id TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (database_name, name)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS queries_owner_idx
|
||||
ON queries(owner_id);
|
||||
"""))
|
||||
async def init_internal_db(db):
|
||||
def apply_migrations(conn):
|
||||
internal_migrations.apply(SQLiteUtilsDatabase(conn, execute_plugins=False))
|
||||
|
||||
await db.execute_write_fn(apply_migrations, transaction=False)
|
||||
|
||||
|
||||
async def populate_schema_tables(internal_db, db):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import pytest
|
||||
import sqlite3
|
||||
|
||||
from datasette.utils import escape_sqlite
|
||||
from datasette.utils.internal_db import INTERNAL_DB_SCHEMA_SQL
|
||||
|
||||
|
||||
# ensure refresh_schemas() gets called before interacting with internal_db
|
||||
|
|
@ -17,6 +19,53 @@ async def test_internal_databases(ds_client):
|
|||
assert databases.rows[0]["database_name"] == "fixtures"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_internal_migrations_recorded(ds_client):
|
||||
internal_db = await ensure_internal(ds_client)
|
||||
migrations = await internal_db.execute("""
|
||||
select migration_set, name
|
||||
from _sqlite_migrations
|
||||
order by id
|
||||
""")
|
||||
assert [tuple(row) for row in migrations.rows] == [
|
||||
("datasette_internal", "0001_initial")
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_internal_migrations_adopt_existing_internal_db(tmp_path):
|
||||
from datasette.app import Datasette
|
||||
|
||||
internal_db_path = str(tmp_path / "internal.db")
|
||||
conn = sqlite3.connect(internal_db_path)
|
||||
conn.executescript(INTERNAL_DB_SCHEMA_SQL)
|
||||
conn.execute(
|
||||
"insert into metadata_instance (key, value) values (?, ?)",
|
||||
("legacy", "preserved"),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
ds = Datasette(internal=internal_db_path)
|
||||
await ds.invoke_startup()
|
||||
internal_db = ds.get_internal_database()
|
||||
|
||||
metadata = await internal_db.execute(
|
||||
"select key, value from metadata_instance where key = 'legacy'"
|
||||
)
|
||||
assert [tuple(row) for row in metadata.rows] == [("legacy", "preserved")]
|
||||
migrations = await internal_db.execute("""
|
||||
select migration_set, name
|
||||
from _sqlite_migrations
|
||||
order by id
|
||||
""")
|
||||
assert [tuple(row) for row in migrations.rows] == [
|
||||
("datasette_internal", "0001_initial")
|
||||
]
|
||||
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_internal_tables(ds_client):
|
||||
internal_db = await ensure_internal(ds_client)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue