diff --git a/datasette/schema_watcher.py b/datasette/schema_watcher.py index 8a11f48b..ba98f20f 100644 --- a/datasette/schema_watcher.py +++ b/datasette/schema_watcher.py @@ -63,6 +63,7 @@ import asyncio import json import logging import os +import threading import time from .utils import sqlite3 @@ -243,8 +244,11 @@ class SchemaWatcher: self._spawned = set() self._last_sweep = 0.0 # Bumped after every write to the catalog tables, so readers can - # cache things derived from the catalog (see datasette.utils.catalog) + # cache things derived from the catalog (see datasette.utils.catalog). + # Locked: several event loops (threads) can store scans at once, and + # a lost increment would leave a stale cache in place self.catalog_generation = 0 + self._generation_lock = threading.Lock() self.counters = { "sweeps": 0, "sweep_seconds": 0.0, @@ -259,6 +263,10 @@ class SchemaWatcher: "restored_from_persisted": 0, } + def _bump_catalog_generation(self): + with self._generation_lock: + self.catalog_generation += 1 + # ------------------------------------------------------------------ # configuration # ------------------------------------------------------------------ @@ -886,7 +894,7 @@ class SchemaWatcher: ) # After the states: a reader that sees the new generation must # also see their new catalog versions - self.catalog_generation += 1 + self._bump_catalog_generation() if missing: for state in missing: if not state.missing: @@ -910,7 +918,7 @@ class SchemaWatcher: await self.ds.get_internal_database().execute_write_fn(_clear) for state in states: state.catalog_version = None - self.catalog_generation += 1 + self._bump_catalog_generation() async def _delete_catalog(self, names): watcher = self @@ -924,7 +932,7 @@ class SchemaWatcher: conn.execute(f"DELETE FROM {table} WHERE database_name = ?", [name]) await self.ds.get_internal_database().execute_write_fn(_delete) - self.catalog_generation += 1 + self._bump_catalog_generation() for name in names: self._pending_removals.discard(name) @@ -952,7 +960,7 @@ class SchemaWatcher: ) await internal.execute_write_fn(_delete_stale) - self.catalog_generation += 1 + self._bump_catalog_generation() self._pending_removals.clear() states = [s for s in self.states.values() if not s.removed] persisted = {} diff --git a/datasette/utils/sqlite.py b/datasette/utils/sqlite.py index b5d9cfae..f4b88be9 100644 --- a/datasette/utils/sqlite.py +++ b/datasette/utils/sqlite.py @@ -354,6 +354,13 @@ def _quote_identifier(value: str) -> str: def _virtual_table_module(sql: str | None) -> str | None: if not sql: return None + if "virtual" not in sql.lower(): + # Fast path: removing comments only ever replaces text with spaces, + # so it cannot produce the VIRTUAL keyword. Skips the + # character-by-character comment scan for ordinary tables, which + # cost ~1.4s of event loop time per 17,500 tables when the + # SchemaWatcher computes derived-table dependencies at startup. + return None match = _VIRTUAL_TABLE_MODULE_RE.search(_strip_sql_comments(sql)) if match is None: return None