github/datasette
1
0
Fork 0
mirror of https://github.com/simonw/datasette.git synced 2025-12-10 16:51:24 +01:00
Code Issues Projects Releases Packages Wiki Activity

Use execute_write_many to optimize internal DB, refs #1555, #1570

Browse source
This commit is contained in:
Simon Willison 2021-12-18 11:11:08 -08:00
commit d637ed4676
2 changed files with 81 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

142
datasette/utils/internal_db.py
View file

@ -70,84 +70,102 @@ async def populate_schema_tables(internal_db, db):
"DELETE FROM tables WHERE database_name = ?", [database_name], block=True
)
tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows
tables_to_insert = []
columns_to_delete = []
columns_to_insert = []
foreign_keys_to_delete = []
foreign_keys_to_insert = []
indexes_to_delete = []
indexes_to_insert = []
for table in tables:
table_name = table["name"]
await internal_db.execute_write(
"""
INSERT INTO tables (database_name, table_name, rootpage, sql)
values (?, ?, ?, ?)
""",
[database_name, table_name, table["rootpage"], table["sql"]],
block=True,
)
# And the columns
await internal_db.execute_write(
"DELETE FROM columns WHERE database_name = ? and table_name = ?",
[database_name, table_name],
block=True,
tables_to_insert.append(
(database_name, table_name, table["rootpage"], table["sql"])
)
columns_to_delete.append((database_name, table_name))
columns = await db.table_column_details(table_name)
for column in columns:
params = {
columns_to_insert.extend(
{
**{"database_name": database_name, "table_name": table_name},
**column._asdict(),
}
await internal_db.execute_write(
"""
INSERT INTO columns (
database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
) VALUES (
:database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
)
""",
params,
block=True,
)
# And the foreign_keys
await internal_db.execute_write(
"DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?",
[database_name, table_name],
block=True,
for column in columns
)
foreign_keys_to_delete.append((database_name, table_name))
foreign_keys = (
await db.execute(f"PRAGMA foreign_key_list([{table_name}])")
).rows
for foreign_key in foreign_keys:
params = {
foreign_keys_to_insert.extend(
{
**{"database_name": database_name, "table_name": table_name},
**dict(foreign_key),
}
await internal_db.execute_write(
"""
INSERT INTO foreign_keys (
database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
) VALUES (
:database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
)
""",
params,
block=True,
)
# And the indexes
await internal_db.execute_write(
"DELETE FROM indexes WHERE database_name = ? and table_name = ?",
[database_name, table_name],
block=True,
for foreign_key in foreign_keys
)
indexes_to_delete.append((database_name, table_name))
indexes = (await db.execute(f"PRAGMA index_list([{table_name}])")).rows
for index in indexes:
params = {
indexes_to_insert.extend(
{
**{"database_name": database_name, "table_name": table_name},
**dict(index),
}
await internal_db.execute_write(
"""
INSERT INTO indexes (
database_name, table_name, seq, name, "unique", origin, partial
) VALUES (
:database_name, :table_name, :seq, :name, :unique, :origin, :partial
)
""",
params,
block=True,
)
for index in indexes
)
await internal_db.execute_write_many(
"""
INSERT INTO tables (database_name, table_name, rootpage, sql)
values (?, ?, ?, ?)
""",
tables_to_insert,
block=True,
)
await internal_db.execute_write_many(
"DELETE FROM columns WHERE database_name = ? and table_name = ?",
columns_to_delete,
block=True,
)
await internal_db.execute_write_many(
"""
INSERT INTO columns (
database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
) VALUES (
:database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
)
""",
columns_to_insert,
block=True,
)
await internal_db.execute_write_many(
"DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?",
foreign_keys_to_delete,
block=True,
)
await internal_db.execute_write_many(
"""
INSERT INTO foreign_keys (
database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
) VALUES (
:database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
)
""",
foreign_keys_to_insert,
block=True,
)
await internal_db.execute_write_many(
"DELETE FROM indexes WHERE database_name = ? and table_name = ?",
indexes_to_delete,
block=True,
)
await internal_db.execute_write_many(
"""
INSERT INTO indexes (
database_name, table_name, seq, name, "unique", origin, partial
) VALUES (
:database_name, :table_name, :seq, :name, :unique, :origin, :partial
)
""",
indexes_to_insert,
block=True,
)

2
tests/test_api.py
View file

@ -937,7 +937,7 @@ def test_trace(trace_debug):
assert isinstance(trace["traceback"], list)
assert isinstance(trace["database"], str)
assert isinstance(trace["sql"], str)
assert isinstance(trace["params"], (list, dict, None.__class__))
assert isinstance(trace.get("params"), (list, dict, None.__class__))
sqls = [trace["sql"] for trace in trace_info["traces"] if "sql" in trace]
# There should be a mix of different types of SQL statement