Use execute_write_many to optimize internal DB, refs #1555, #1570

datasette/utils/internal_db.py

@@ -70,84 +70,102 @@ async def populate_schema_tables(internal_db, db):
         "DELETE FROM tables WHERE database_name = ?", [database_name], block=True
     )
     tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows
+    tables_to_insert = []
+    columns_to_delete = []
+    columns_to_insert = []
+    foreign_keys_to_delete = []
+    foreign_keys_to_insert = []
+    indexes_to_delete = []
+    indexes_to_insert = []
+
     for table in tables:
         table_name = table["name"]
-        await internal_db.execute_write(
+        tables_to_insert.append(
-            """
+            (database_name, table_name, table["rootpage"], table["sql"])
-            INSERT INTO tables (database_name, table_name, rootpage, sql)
-            values (?, ?, ?, ?)
-        """,
-            [database_name, table_name, table["rootpage"], table["sql"]],
-            block=True,
-        )
-        # And the columns
-        await internal_db.execute_write(
-            "DELETE FROM columns WHERE database_name = ? and table_name = ?",
-            [database_name, table_name],
-            block=True,
         )
+        columns_to_delete.append((database_name, table_name))
         columns = await db.table_column_details(table_name)
-        for column in columns:
+        columns_to_insert.extend(
-            params = {
+            {
                 **{"database_name": database_name, "table_name": table_name},
                 **column._asdict(),
             }
-            await internal_db.execute_write(
+            for column in columns
-                """
-                INSERT INTO columns (
-                    database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
-                ) VALUES (
-                    :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
-                )
-            """,
-                params,
-                block=True,
-            )
-        # And the foreign_keys
-        await internal_db.execute_write(
-            "DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?",
-            [database_name, table_name],
-            block=True,
         )
+        foreign_keys_to_delete.append((database_name, table_name))
         foreign_keys = (
             await db.execute(f"PRAGMA foreign_key_list([{table_name}])")
         ).rows
-        for foreign_key in foreign_keys:
+        foreign_keys_to_insert.extend(
-            params = {
+            {
                 **{"database_name": database_name, "table_name": table_name},
                 **dict(foreign_key),
             }
-            await internal_db.execute_write(
+            for foreign_key in foreign_keys
-                """
-                INSERT INTO foreign_keys (
-                    database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
-                ) VALUES (
-                    :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
-                )
-            """,
-                params,
-                block=True,
-            )
-        # And the indexes
-        await internal_db.execute_write(
-            "DELETE FROM indexes WHERE database_name = ? and table_name = ?",
-            [database_name, table_name],
-            block=True,
         )
+        indexes_to_delete.append((database_name, table_name))
         indexes = (await db.execute(f"PRAGMA index_list([{table_name}])")).rows
-        for index in indexes:
+        indexes_to_insert.extend(
-            params = {
+            {
                 **{"database_name": database_name, "table_name": table_name},
                 **dict(index),
             }
-            await internal_db.execute_write(
+            for index in indexes
-                """
+        )
-                INSERT INTO indexes (
+
-                    database_name, table_name, seq, name, "unique", origin, partial
+    await internal_db.execute_write_many(
-                ) VALUES (
+        """
-                    :database_name, :table_name, :seq, :name, :unique, :origin, :partial
+        INSERT INTO tables (database_name, table_name, rootpage, sql)
-                )
+        values (?, ?, ?, ?)
-            """,
+    """,
-                params,
+        tables_to_insert,
-                block=True,
+        block=True,
-            )
+    )
+    await internal_db.execute_write_many(
+        "DELETE FROM columns WHERE database_name = ? and table_name = ?",
+        columns_to_delete,
+        block=True,
+    )
+    await internal_db.execute_write_many(
+        """
+        INSERT INTO columns (
+            database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
+        ) VALUES (
+            :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
+        )
+    """,
+        columns_to_insert,
+        block=True,
+    )
+    await internal_db.execute_write_many(
+        "DELETE FROM foreign_keys WHERE database_name = ? and table_name = ?",
+        foreign_keys_to_delete,
+        block=True,
+    )
+    await internal_db.execute_write_many(
+        """
+        INSERT INTO foreign_keys (
+            database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
+        ) VALUES (
+            :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
+        )
+    """,
+        foreign_keys_to_insert,
+        block=True,
+    )
+    await internal_db.execute_write_many(
+        "DELETE FROM indexes WHERE database_name = ? and table_name = ?",
+        indexes_to_delete,
+        block=True,
+    )
+    await internal_db.execute_write_many(
+        """
+        INSERT INTO indexes (
+            database_name, table_name, seq, name, "unique", origin, partial
+        ) VALUES (
+            :database_name, :table_name, :seq, :name, :unique, :origin, :partial
+        )
+    """,
+        indexes_to_insert,
+        block=True,
+    )

tests/test_api.py

@@ -937,7 +937,7 @@ def test_trace(trace_debug):
         assert isinstance(trace["traceback"], list)
         assert isinstance(trace["database"], str)
         assert isinstance(trace["sql"], str)
-        assert isinstance(trace["params"], (list, dict, None.__class__))
+        assert isinstance(trace.get("params"), (list, dict, None.__class__))
 
     sqls = [trace["sql"] for trace in trace_info["traces"] if "sql" in trace]
     # There should be a mix of different types of SQL statement