remote/datasette
1
0
Fork 0
mirror of https://github.com/simonw/datasette.git synced 2026-07-16 20:44:35 +02:00
Code Issues Projects Releases Packages Wiki Activity

Make internal catalog refresh atomic

Browse source 
Refs #2831
This commit is contained in:
Simon Willison 2026-07-14 08:41:27 -07:00 • committed by GitHub
commit 9cfc252394
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 72 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

14
datasette/app.py
View file

@ -753,19 +753,7 @@ class Datasette:
# Compare schema versions to see if we should skip it
if schema_version == current_schema_versions.get(database_name):
continue
placeholders = "(?, ?, ?, ?)"
values = [database_name, str(db.path), db.is_memory, schema_version]
if db.path is None:
placeholders = "(?, null, ?, ?)"
values = [database_name, db.is_memory, schema_version]
await internal_db.execute_write(
"""
INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version)
VALUES {}
""".format(placeholders),
values,
)
await populate_schema_tables(internal_db, db)
await populate_schema_tables(internal_db, db, schema_version)
@property
def urls(self):

133
datasette/utils/internal_db.py
View file

@ -180,29 +180,9 @@ async def init_internal_db(db):
await db.execute_write_fn(apply_migrations, transaction=False)
async def populate_schema_tables(internal_db, db):
async def populate_schema_tables(internal_db, db, schema_version):
database_name = db.name
def delete_everything(conn):
conn.execute(
"DELETE FROM catalog_tables WHERE database_name = ?", [database_name]
)
conn.execute(
"DELETE FROM catalog_views WHERE database_name = ?", [database_name]
)
conn.execute(
"DELETE FROM catalog_columns WHERE database_name = ?", [database_name]
)
conn.execute(
"DELETE FROM catalog_foreign_keys WHERE database_name = ?",
[database_name],
)
conn.execute(
"DELETE FROM catalog_indexes WHERE database_name = ?", [database_name]
)
await internal_db.execute_write_fn(delete_everything)
tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows
views = (await db.execute("select * from sqlite_master WHERE type = 'view'")).rows
@ -266,47 +246,76 @@ async def populate_schema_tables(internal_db, db):
indexes_to_insert,
) = await db.execute_fn(collect_info)
await internal_db.execute_write_many(
"""
INSERT INTO catalog_tables (database_name, table_name, rootpage, sql)
values (?, ?, ?, ?)
""",
tables_to_insert,
)
await internal_db.execute_write_many(
"""
INSERT INTO catalog_views (database_name, view_name, rootpage, sql)
values (?, ?, ?, ?)
""",
views_to_insert,
)
await internal_db.execute_write_many(
"""
INSERT INTO catalog_columns (
database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
) VALUES (
:database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
def replace_catalog(conn):
# Delete child rows before their catalog_tables parents so this also
# works if a prepare_connection plugin enables foreign key enforcement.
for table in (
"catalog_columns",
"catalog_foreign_keys",
"catalog_indexes",
"catalog_views",
"catalog_tables",
):
conn.execute(
"DELETE FROM {} WHERE database_name = ?".format(table),
[database_name],
)
conn.execute(
"""
INSERT OR REPLACE INTO catalog_databases (
database_name, path, is_memory, schema_version
) VALUES (?, ?, ?, ?)
""",
[
database_name,
str(db.path) if db.path is not None else None,
db.is_memory,
schema_version,
],
)
""",
columns_to_insert,
)
await internal_db.execute_write_many(
"""
INSERT INTO catalog_foreign_keys (
database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
) VALUES (
:database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
conn.executemany(
"""
INSERT INTO catalog_tables (database_name, table_name, rootpage, sql)
values (?, ?, ?, ?)
""",
tables_to_insert,
)
""",
foreign_keys_to_insert,
)
await internal_db.execute_write_many(
"""
INSERT INTO catalog_indexes (
database_name, table_name, seq, name, "unique", origin, partial
) VALUES (
:database_name, :table_name, :seq, :name, :unique, :origin, :partial
conn.executemany(
"""
INSERT INTO catalog_views (database_name, view_name, rootpage, sql)
values (?, ?, ?, ?)
""",
views_to_insert,
)
""",
indexes_to_insert,
)
conn.executemany(
"""
INSERT INTO catalog_columns (
database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
) VALUES (
:database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
)
""",
columns_to_insert,
)
conn.executemany(
"""
INSERT INTO catalog_foreign_keys (
database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
) VALUES (
:database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
)
""",
foreign_keys_to_insert,
)
conn.executemany(
"""
INSERT INTO catalog_indexes (
database_name, table_name, seq, name, "unique", origin, partial
) VALUES (
:database_name, :table_name, :seq, :name, :unique, :origin, :partial
)
""",
indexes_to_insert,
)
await internal_db.execute_write_fn(replace_catalog)