Use execute_write_many to optimize internal DB, refs #1555, #1570

datasette/utils/internal_db.py

@@ -70,29 +70,63 @@ async def populate_schema_tables(internal_db, db):
         "DELETE FROM tables WHERE database_name = ?", [database_name], block=True
     )
     tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows
+    tables_to_insert = []
+    columns_to_delete = []
+    columns_to_insert = []
+    foreign_keys_to_delete = []
+    foreign_keys_to_insert = []
+    indexes_to_delete = []
+    indexes_to_insert = []
+
     for table in tables:
         table_name = table["name"]
-        await internal_db.execute_write(
+        tables_to_insert.append(
+            (database_name, table_name, table["rootpage"], table["sql"])
+        )
+        columns_to_delete.append((database_name, table_name))
+        columns = await db.table_column_details(table_name)
+        columns_to_insert.extend(
+            {
+                **{"database_name": database_name, "table_name": table_name},
+                **column._asdict(),
+            }
+            for column in columns
+        )
+        foreign_keys_to_delete.append((database_name, table_name))
+        foreign_keys = (
+            await db.execute(f"PRAGMA foreign_key_list([{table_name}])")
+        ).rows
+        foreign_keys_to_insert.extend(
+            {
+                **{"database_name": database_name, "table_name": table_name},
+                **dict(foreign_key),
+            }
+            for foreign_key in foreign_keys
+        )
+        indexes_to_delete.append((database_name, table_name))
+        indexes = (await db.execute(f"PRAGMA index_list([{table_name}])")).rows
+        indexes_to_insert.extend(
+            {
+                **{"database_name": database_name, "table_name": table_name},
+                **dict(index),
+            }
+            for index in indexes
+        )
+
+    await internal_db.execute_write_many(
         """
         INSERT INTO tables (database_name, table_name, rootpage, sql)
         values (?, ?, ?, ?)
     """,
-            [database_name, table_name, table["rootpage"], table["sql"]],
+        tables_to_insert,
         block=True,
     )
-        # And the columns
+    await internal_db.execute_write_many(
-        await internal_db.execute_write(
         "DELETE FROM columns WHERE database_name = ? and table_name = ?",
-            [database_name, table_name],
+        columns_to_delete,
         block=True,
     )
-        columns = await db.table_column_details(table_name)
+    await internal_db.execute_write_many(
-        for column in columns:
-            params = {
-                **{"database_name": database_name, "table_name": table_name},
-                **column._asdict(),
-            }
-            await internal_db.execute_write(
         """
         INSERT INTO columns (
             database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
@@ -100,24 +134,15 @@ async def populate_schema_tables(internal_db, db):
             :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
         )
     """,
-                params,
+        columns_to_insert,
         block=True,
     )
-        # And the foreign_keys
+    await internal_db.execute_write_many(
-        await internal_db.execute_write(
         "DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?",
-            [database_name, table_name],
+        foreign_keys_to_delete,
         block=True,
     )
-        foreign_keys = (
+    await internal_db.execute_write_many(
-            await db.execute(f"PRAGMA foreign_key_list([{table_name}])")
-        ).rows
-        for foreign_key in foreign_keys:
-            params = {
-                **{"database_name": database_name, "table_name": table_name},
-                **dict(foreign_key),
-            }
-            await internal_db.execute_write(
         """
         INSERT INTO foreign_keys (
             database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
@@ -125,22 +150,15 @@ async def populate_schema_tables(internal_db, db):
             :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
         )
     """,
-                params,
+        foreign_keys_to_insert,
         block=True,
     )
-        # And the indexes
+    await internal_db.execute_write_many(
-        await internal_db.execute_write(
         "DELETE FROM indexes WHERE database_name = ? and table_name = ?",
-            [database_name, table_name],
+        indexes_to_delete,
         block=True,
     )
-        indexes = (await db.execute(f"PRAGMA index_list([{table_name}])")).rows
+    await internal_db.execute_write_many(
-        for index in indexes:
-            params = {
-                **{"database_name": database_name, "table_name": table_name},
-                **dict(index),
-            }
-            await internal_db.execute_write(
         """
         INSERT INTO indexes (
             database_name, table_name, seq, name, "unique", origin, partial
@@ -148,6 +166,6 @@ async def populate_schema_tables(internal_db, db):
             :database_name, :table_name, :seq, :name, :unique, :origin, :partial
         )
     """,
-                params,
+        indexes_to_insert,
         block=True,
     )

tests/test_api.py

@@ -937,7 +937,7 @@ def test_trace(trace_debug):
         assert isinstance(trace["traceback"], list)
         assert isinstance(trace["database"], str)
         assert isinstance(trace["sql"], str)
-        assert isinstance(trace["params"], (list, dict, None.__class__))
+        assert isinstance(trace.get("params"), (list, dict, None.__class__))
 
     sqls = [trace["sql"] for trace in trace_info["traces"] if "sql" in trace]
     # There should be a mix of different types of SQL statement