remote/datasette
1
0
Fork 0
mirror of https://github.com/simonw/datasette.git synced 2026-10-19 07:47:00 +02:00
Code Issues Projects Releases Packages Wiki Activity

Torture harness, and the fixes it found

Browse source 
tests/connection_torture.py drives one Datasette from several event loops
(threads) with the smallest pool, 1ms idle timeout and 10ms schema polling
through every connection operation, plus add/remove_database, scratch
databases, external schema changes, replaced/deleted files, cancellation,
Database.close()/Datasette.close() mid-flight and non-threaded mode. One
scenario per process, so crashes and hangs are results. It checks for hangs,
crashes, undocumented exceptions, per-database write order, lost writes,
read-your-writes, catalog freshness, and fds/threads/the instance released
after close(). tests/test_connection_torture.py runs a short fixed-seed
subset of every scenario.

Fixes for what it found:
- A cursor that left the thread owning its connection (the cursor returned
  by execute_write_many()/execute_write_script(), one returned by a write
  callback, one rejected by execute_fn(), or one held by an exception's
  frames) could be freed on the event loop thread while the connection ran
  the same SQL again; on Python < 3.12 freeing it resets the shared cached
  statement, and the other query failed with "bad parameter or other API
  misuse". Such cursors are now closed on the owning thread. Pre-existing on
  main (51 failures per 1,800 multi-loop execute_write_many calls).
- invoke_startup() is single-flight across event loops (a concurrent.futures
  future, datasette.utils.inflight) instead of _startup_sequence() holding
  an asyncio.Lock: a second loop got "bound to a different event loop" or
  hung, and two loops could run the startup hooks twice. A request made
  from a startup hook no longer re-runs or deadlocks startup.
- The index page takes one snapshot of ds.databases: a database removed,
  renamed or closed mid-request, or whose file was deleted, no longer turns
  the page into a 500. Database.size is 0 for a deleted file.

The scan-coalescing helpers move to datasette/utils/inflight.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EwkLo2rJWidpkDVT3HkvNv
This commit is contained in:
Claude 2026-10-04 09:07:48 +00:00
commit da523c30e5
No known key found for this signature in database
11 changed files with 1763 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

69
datasette/app.py
View file

@ -116,6 +116,7 @@ from .utils.asgi import (
asgi_static,
)
from .utils.catalog import all_derived_table_dependencies
from .utils.inflight import InFlight, wait_for_concurrent
from .utils.internal_db import init_internal_db
from .utils.sqlite import (
sqlite3,
@ -499,17 +500,11 @@ class Datasette:
# Datasette can be driven by several event loops, and an
# asyncio.Lock binds to whichever loop first waits on it
self._refresh_schemas_lock = threading.Lock()
try:
self._startup_lock = asyncio.Lock()
except RuntimeError as rex:
# Workaround for intermittent test failure, see:
# https://github.com/simonw/datasette/issues/1802
if "There is no current event loop in thread" in str(rex):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
self._startup_lock = asyncio.Lock()
else:
raise
# invoke_startup() in progress (datasette.utils.inflight.InFlight),
# claimed under this lock: concurrent callers on any event loop wait
# for it. Not an asyncio.Lock, which binds to one event loop
self._startup_running = None
self._startup_state_lock = threading.Lock()
self._background_tasks = BackgroundTaskSupervisor(self)
self.crossdb = crossdb
self.nolock = nolock
@ -834,9 +829,40 @@ class Datasette:
return pm
async def invoke_startup(self):
# This must be called for Datasette to be in a usable state
if self._startup_invoked:
# This must be called for Datasette to be in a usable state.
# Several event loops (threads) may call it at once: one runs
# startup, the others wait for it to finish - and try again
# themselves if it failed or its event loop went away.
while not self._startup_invoked:
loop = asyncio.get_running_loop()
task = asyncio.current_task()
with self._startup_state_lock:
running = self._startup_running
if running is not None and (running.done() or running.abandoned()):
running.finish()
running = self._startup_running = None
if running is None:
running = self._startup_running = InFlight(loop, task)
mine = True
elif running.task is task:
# Called from inside startup itself (a startup hook
# making a request through datasette.client)
return
else:
mine = False
if not mine:
await wait_for_concurrent(running.future)
continue
try:
await self._invoke_startup()
finally:
with self._startup_state_lock:
if self._startup_running is running:
self._startup_running = None
running.finish()
return
async def _invoke_startup(self):
# Group spans created during startup under a single parent span
with tracer.start_as_current_span(STARTUP):
# Register event classes
@ -3293,16 +3319,13 @@ ORDER BY allowed.parent, allowed.child
"""
if self._startup_invoked and self._setup_db_done:
return
async with self._startup_lock:
if self._startup_invoked and self._setup_db_done:
return
if not self._setup_db_done:
# Immutable databases used to have their table counts
# computed here, opening every one of them at startup. They
# are now computed the first time a page needs them (see
# Database.table_counts()) and cached from then on.
self._setup_db_done = True
await self.invoke_startup()
# Immutable databases used to have their table counts computed
# here, opening every one of them at startup. They are now computed
# the first time a page needs them (see Database.table_counts()) and
# cached from then on. invoke_startup() makes concurrent callers -
# on any event loop - wait for the one running it.
self._setup_db_done = True
await self.invoke_startup()
def add_background_task(self, func, name=None) -> BackgroundTask:
"""Register a piece of supervised background work, typically from

61
datasette/connection_pool.py
View file

@ -154,6 +154,58 @@ class LeasedConnection:
return f"<LeasedConnection {self._kind} database={self._db_name!r} {state}>"
def close_cursors(value):
"""Close the cursors in a callback's result (a cursor, or a tuple or
list holding one), in a generator's locals, or - for an exception - in
the locals of the frames it was raised through.
Call it on the thread that owns the connection, before the connection
can be used by anyone else. A cursor keeps a reference to the
connection's cached statement for its SQL; on Python before 3.12,
deallocating the cursor resets that statement. If the cursor dies on
another thread (the caller's event loop) while the connection is running
the same SQL again - the write thread's next execute_write_many(), or
another worker that has leased the connection - the statement is reset
mid-step and that query fails with "bad parameter or other API misuse".
A closed cursor keeps its rowcount, lastrowid and description.
"""
if isinstance(value, BaseException):
tb = value.__traceback__
while tb is not None:
try:
values = list(tb.tb_frame.f_locals.values())
except Exception: # noqa: BLE001
values = []
for item in values:
if isinstance(item, sqlite3.Cursor):
_close_cursor(item)
tb = tb.tb_next
return
if isinstance(value, sqlite3.Cursor):
_close_cursor(value)
elif isinstance(value, (tuple, list)):
for item in value:
if isinstance(item, sqlite3.Cursor):
_close_cursor(item)
elif inspect.isgenerator(value):
try:
values = list(inspect.getgeneratorlocals(value).values())
except Exception: # noqa: BLE001
values = []
for item in values:
if isinstance(item, sqlite3.Cursor):
_close_cursor(item)
value.close()
def _close_cursor(cursor):
try:
cursor.close()
except Exception: # noqa: BLE001, S110
# Its connection is closed already
pass
def _generator_uses_connection(result):
"""True for a generator that would step the leased connection after
the callback returned: one holding the connection (a closure over
@ -219,12 +271,19 @@ class ReadConnectionPool:
"""Call fn(conn) with a pooled connection leased for the duration."""
entry = self.acquire(db)
lease = LeasedConnection(entry.conn, db.name, "read")
rejected = False
try:
result = fn(lease)
if isinstance(result, sqlite3.Cursor) or _generator_uses_connection(result):
rejected = True
close_cursors(result)
except BaseException as e:
close_cursors(e)
raise
finally:
lease._expire()
self.release(entry)
if isinstance(result, sqlite3.Cursor) or _generator_uses_connection(result):
if rejected:
raise ConnectionLeaseError(
f"An execute_fn() callback for database {db.name!r} returned a {type(result).__name__}, which "
"would keep using the pooled connection after the callback "

29
datasette/database.py
View file

@ -16,7 +16,11 @@ import sqlite_utils
from opentelemetry import context as otel_context_api
from opentelemetry.trace import Status, StatusCode
from .connection_pool import ConnectionLeaseError, LeasedConnection # noqa: F401
from .connection_pool import ( # noqa: F401
ConnectionLeaseError,
LeasedConnection,
close_cursors,
)
from .inspect import inspect_hash
from .telemetry import (
callback_name,
@ -1119,7 +1123,12 @@ class Database:
elif self.is_memory:
return 0
elif self.is_mutable:
return Path(self.path).stat().st_size
try:
return Path(self.path).stat().st_size
except FileNotFoundError:
# Deleted from under Datasette: report it as empty rather
# than failing every page that lists databases
return 0
elif self.ds.inspect_data and self.ds.inspect_data.get(self.name):
self.cached_size = self.ds.inspect_data[self.name]["size"]
return self.cached_size
@ -1440,12 +1449,22 @@ def _call_with_lease(fn, conn, db_name, kind):
Unlike read callbacks, write callbacks may return a cursor (for example
``lambda conn: conn.execute(...)``, and execute_write_many() and
execute_write_script() return theirs); reading attributes such as
``lastrowid`` from it afterwards is fine, fetching rows is not.
execute_write_script() return theirs); it is closed before it leaves
this thread, so reading attributes such as ``lastrowid`` from it
afterwards is fine, fetching rows is not.
"""
lease = LeasedConnection(conn, db_name, kind)
try:
return fn(lease)
result = fn(lease)
except BaseException as e:
close_cursors(e)
raise
else:
# A returned cursor is closed here, on the thread that owns the
# connection (see close_cursors()); its rowcount and lastrowid can
# still be read
close_cursors(result)
return result
finally:
lease._expire()

68
datasette/schema_watcher.py
View file

@ -74,7 +74,6 @@ open.
"""
import asyncio
import concurrent.futures
import json
import logging
import os
@ -83,6 +82,7 @@ import time
from .utils import sqlite3
from .utils.catalog import remember_derived_table_dependencies
from .utils.inflight import InFlight, wait_for_concurrent
from .utils.internal_db import CATALOG_TABLES, collect_schema, write_catalog_entries
logger = logging.getLogger("datasette.schema_watcher")
@ -144,7 +144,8 @@ class WatchState:
self.catalog_version = None
self.notified_version = None
self.needs_scan = True
# An _InflightScan while a catalog scan of this database runs
# An InFlight (datasette.utils.inflight) while a catalog scan of
# this database runs
self.scan_future = None
# time.monotonic() before which a failed scan is not retried, and
# the delay used for the next failure (doubles, see RETRY_DELAY_S)
@ -248,61 +249,6 @@ def header_schema_version(path):
return int.from_bytes(header[40:44], "big")
class _InflightScan:
"""One catalog scan in progress for one database.
Not bound to an event loop: one Datasette is driven by several loops
(``--get``, TestClient, pytest-asyncio), each in its own thread, and a
write on any of them may need to wait for a scan started on another.
Waiters use :func:`_wait_for` on ``future``, a ``concurrent.futures``
future that the scan's owner resolves in a ``finally`` block.
"""
__slots__ = ("future", "loop")
def __init__(self, loop):
self.future = concurrent.futures.Future()
self.loop = loop
def done(self):
return self.future.done()
def abandoned(self):
"""True if nothing will ever finish this scan: the loop running it
has closed, or has stopped (``run_until_complete()`` returned while
the scan task was still pending)."""
loop = self.loop
return loop.is_closed() or not loop.is_running()
def finish(self):
try:
self.future.set_result(None)
except concurrent.futures.InvalidStateError:
pass
def _wait_for(cf_future):
"""An asyncio future on the running loop that completes when the
concurrent future does. Cancelling it does not cancel cf_future, which
other waiters (maybe on other loops) share."""
loop = asyncio.get_running_loop()
waiter = loop.create_future()
def _set():
if not waiter.done():
waiter.set_result(None)
def _done(_):
try:
loop.call_soon_threadsafe(_set)
except RuntimeError:
# That loop has closed; nobody is waiting any more
pass
cf_future.add_done_callback(_done)
return waiter
def _is_transient(error):
if isinstance(error, sqlite3.OperationalError):
message = str(error)
@ -987,7 +933,7 @@ class SchemaWatcher:
inflight.finish()
if state.needs_scan and not state.removed and state not in failed:
state.needs_scan = False
state.scan_future = _InflightScan(loop)
state.scan_future = InFlight(loop)
mine.append(state)
if mine:
records = {state: state.scan_future for state in mine}
@ -1018,7 +964,7 @@ class SchemaWatcher:
for record in records.values():
record.finish()
if waits:
await asyncio.wait([_wait_for(f) for f in waits])
await asyncio.wait([wait_for_concurrent(f) for f in waits])
pending = [
s
for s in pending
@ -1042,7 +988,7 @@ class SchemaWatcher:
"""Wait for a catalog scan of state that is in flight, if any."""
inflight = state.scan_future
if inflight is not None and not inflight.done() and not inflight.abandoned():
await _wait_for(inflight.future)
await wait_for_concurrent(inflight.future)
async def _scan_and_store(self, states, records, handled):
memory = [s for s in states if not s.is_file and s.db.memory_name]
@ -1069,7 +1015,7 @@ class SchemaWatcher:
async def _store(self, results, records=None, handled=None):
"""Write scan results to the catalog. ``records`` maps each state to
the _InflightScan this scan was started as: results for a state whose
the InFlight this scan was started as: results for a state whose
scan has since been taken over by another loop are dropped. States
whose outcome was recorded are added to ``handled``; the others
(transient failures) are retried later by refresh()."""

63
datasette/utils/inflight.py Normal file
View file

@ -0,0 +1,63 @@
"""Waiting for work started on another event loop.
One Datasette can be driven by several event loops at once, each in its own
thread (``datasette --get``, TestClient, pytest-asyncio, an embedding app).
asyncio primitives (``asyncio.Lock``, ``asyncio.Future``) belong to one loop,
so "one call does the work, concurrent callers wait for it" is built from a
``concurrent.futures.Future`` that any loop can wait for, and a
``threading.Lock`` held only for the moment it takes to claim the work.
"""
import asyncio
import concurrent.futures
class InFlight:
"""One piece of work in progress, started by ``task`` on ``loop``."""
__slots__ = ("future", "loop", "task")
def __init__(self, loop, task=None):
self.future = concurrent.futures.Future()
self.loop = loop
self.task = task
def done(self):
return self.future.done()
def abandoned(self):
"""True if nothing will ever finish this work: the loop running it
has closed, or has stopped (``run_until_complete()`` returned while
the task doing it was still pending)."""
loop = self.loop
return loop.is_closed() or not loop.is_running()
def finish(self):
"""Wake every waiter (they check for themselves whether the work
succeeded). Safe to call more than once, from any thread."""
try:
self.future.set_result(None)
except concurrent.futures.InvalidStateError:
pass
def wait_for_concurrent(cf_future):
"""An asyncio future on the running loop that completes when
``cf_future`` does. Cancelling it does not cancel ``cf_future``, which
other waiters - maybe on other loops - share."""
loop = asyncio.get_running_loop()
waiter = loop.create_future()
def _set():
if not waiter.done():
waiter.set_result(None)
def _done(_):
try:
loop.call_soon_threadsafe(_set)
except RuntimeError:
# That loop has closed; nobody is waiting any more
pass
cf_future.add_done_callback(_done)
return waiter

32
datasette/views/index.py
View file

@ -1,5 +1,6 @@
import json
from datasette.database import DatasetteClosedError
from datasette.plugins import pm
from datasette.utils import (
UNSTABLE_API_MESSAGE,
@ -7,6 +8,8 @@ from datasette.utils import (
add_cors_headers,
await_me_maybe,
make_slot_function,
sqlite3,
tilde_encode,
)
from datasette.utils.asgi import Response
from datasette.utils.catalog import (
@ -46,8 +49,13 @@ class IndexView(BaseView):
"view-database", request.actor, include_is_private=True
)
allowed_databases = [r async for r in db_page.all()]
# One snapshot for the whole request: add_database() and
# remove_database() replace ds.databases rather than changing it, so
# a database removed while this page is being built is still listed
# instead of failing the page with a KeyError
all_databases = self.ds.databases
allowed_db_dict = {
r.parent: r for r in allowed_databases if r.parent in self.ds.databases
r.parent: r for r in allowed_databases if r.parent in all_databases
}
# Group tables by database
@ -62,7 +70,7 @@ class IndexView(BaseView):
names = list(allowed_db_dict)
summaries = await catalog_summaries(self.ds, names)
all_counts = await self._table_counts(names)
all_counts = await self._table_counts(names, all_databases)
sort_by_relationships = request.args.get("_sort") == "relationships"
need_relationships = {
name for name in names if sort_by_relationships or not all_counts[name]
@ -149,7 +157,7 @@ class IndexView(BaseView):
table_counts,
tables_and_views_truncated,
) in prepared:
db = self.ds.databases[name]
db = all_databases[name]
tables_and_views_truncated = [
{
"name": t["name"],
@ -173,7 +181,8 @@ class IndexView(BaseView):
"name": name,
"hash": db.hash,
"color": db.color,
"path": self.ds.urls.database(name),
# From the snapshot: name may have been removed since
"path": self.ds.urls.path(tilde_encode(db.route)),
"tables_and_views_truncated": tables_and_views_truncated,
"tables_and_views_more": (len(visible_tables) + len(views))
> TRUNCATE_AT,
@ -189,6 +198,10 @@ class IndexView(BaseView):
}
)
# Leave out databases removed while this page was being built: the
# HTML template links to them by name
databases = [d for d in databases if d["name"] in self.ds.databases]
if as_format:
headers = {}
if self.ds.cors:
@ -235,13 +248,13 @@ class IndexView(BaseView):
},
)
async def _table_counts(self, names):
async def _table_counts(self, names, all_databases):
"""``{database: {table: count}}`` - empty for a database whose
counts are skipped or timed out (see COUNT_MAX_DATABASES)."""
counts = {}
to_count = []
for name in names:
db = self.ds.databases[name]
db = all_databases[name]
if not db.is_mutable and db.cached_table_counts is not None:
# Known without opening the database
counts[name] = db.cached_table_counts
@ -258,7 +271,12 @@ class IndexView(BaseView):
except OSError:
# The file has gone: there is nothing to count
continue
table_counts = await db.table_counts(10)
try:
table_counts = await db.table_counts(10)
except (DatasetteClosedError, sqlite3.Error):
# Closed or removed while this page was being built, or its
# file deleted or replaced: show it without counts
continue
# If any of these are None it means at least one timed out - ignore them all
if any(v is None for v in table_counts.values()):
table_counts = {}

4
docs/internals.rst
View file

@ -1951,7 +1951,7 @@ Application lifecycle
Datasette guarantees a fixed sequence of events between the moment a ``Datasette`` instance is constructed and the moment its resources are released:
1. ``Datasette(...)`` — the constructor runs synchronously and does not run plugin hooks.
2. **Startup** — ``await datasette.invoke_startup()`` runs once: it populates the internal database's catalog of table schemas (:ref:`internals_internal`), loads canned queries and column type configuration, then calls every registered :ref:`plugin_hook_startup` hook, in plugin registration order. When Datasette is being served, table-count precomputation for immutable databases runs immediately before this, as part of the same startup sequence.
2. **Startup** — ``await datasette.invoke_startup()`` runs once, even if several event loops call it at the same time (the others wait for it to finish): it populates the internal database's catalog of table schemas (:ref:`internals_internal`), loads canned queries and column type configuration, then calls every registered :ref:`plugin_hook_startup` hook, in plugin registration order. When Datasette is being served, table-count precomputation for immutable databases runs immediately before this, as part of the same startup sequence.
3. **Background-task launch** — once *every* ``startup`` hook has finished (not before), every task registered with :ref:`datasette_add_background_task` — by any plugin — is launched. A task registered by one plugin's ``startup`` hook can safely depend on state set up by another plugin's ``startup`` hook, because launch only happens after the whole round of hooks completes.
4. **Serving** — the instance handles requests (or, for headless or CLI use, does whatever the embedding program does with it).
5. **Shutdown** — triggered by the ASGI ``lifespan.shutdown`` event (Ctrl-C, ``SIGTERM``) or the end of a ``datasette serve`` process: every :ref:`plugin_hook_shutdown` hook runs first, while background tasks are still alive, so a plugin can tell its own task to wind down gracefully; every still-running background task is then cancelled and given a five-second grace period to actually stop; finally every database connection is released via :ref:`datasette_close`.
@ -2239,7 +2239,7 @@ The write connection belongs to Datasette: calling ``conn.close()`` inside an ``
Datasette cannot detect every misuse. These are unsupported and may fail unpredictably, or read from a connection that is now being used by someone else:
- Iterating over or fetching from a **cursor** created inside the function after the function has returned. Write functions may return a cursor (``lambda conn: conn.execute(...)`` is a common pattern) and reading attributes such as ``.lastrowid`` and ``.rowcount`` from it afterwards is fine; fetching rows from it is not.
- Iterating over or fetching from a **cursor** created inside the function after the function has returned. Write functions may return a cursor (``lambda conn: conn.execute(...)`` is a common pattern) and reading attributes such as ``.lastrowid`` and ``.rowcount`` from it afterwards is fine; fetching rows from it is not. A cursor returned by a write or isolated function - including the ones returned by ``execute_write_many()`` and ``execute_write_script()`` - is closed before it is handed back, on the thread that owns the connection, and so are cursors left in the local variables of a function that raised an exception. (On Python versions before 3.12, a cursor that was freed on another thread could reset the same statement while the connection was running it again, making that query fail with "bad parameter or other API misuse".)
- Reaching the underlying connection through ``cursor.connection``.
- Relying on state set up on the connection by an earlier call (see above).

1297
tests/connection_torture.py Normal file
View file

File diff suppressed because it is too large Load diff

173
tests/test_connection_review_fixes.py
View file

@ -668,22 +668,22 @@ async def test_stale_owned_catalog_is_not_persisted_at_close(tmp_path):
make_db(path)
config = {"databases": {"data": {"schema_watch": "owned"}}}
async def tables_after_start(change=None):
async def tables_after_start(change_before_close=None):
ds = Datasette([path], internal=internal, config=config)
await ds.invoke_startup()
if change:
change()
response = await ds.client.get("/data.json")
if change_before_close:
# Behind Datasette's back, after the last request: an owned
# database is not polled, so the catalog is now stale
change_before_close()
ds.close()
return [t["name"] for t in response.json()["tables"]]
assert await tables_after_start() == ["t"]
# Owned: a change made behind Datasette's back is not noticed while
# it runs (documented) ...
assert await tables_after_start(
lambda: make_db(path, "create table t2 (id integer)")
) == ["t"]
# ... but it is not persisted as current either
# The stale catalog was not persisted as current: the restart rescans
assert await tables_after_start() == ["t", "t2"]
@ -744,3 +744,164 @@ async def test_execute_fn_may_return_generator_over_fetched_rows():
with pytest.raises(ConnectionLeaseError):
await db.execute_fn(closure)
ds.close()
# ----------------------------------------------------------------------
# Found by the torture harness (tests/connection_torture.py)
# ----------------------------------------------------------------------
@pytest.mark.asyncio
async def test_index_survives_database_removed_mid_request(tmp_path, monkeypatch):
# The index page looked databases up by name again after listing them:
# one removed meanwhile (remove_database(), a scratch delete or rename)
# failed the whole page with a KeyError
from datasette.views import index as index_module
paths = []
for name in ("keep", "gone"):
path = str(tmp_path / f"{name}.db")
make_db(path)
paths.append(path)
ds = Datasette(paths)
await ds.invoke_startup()
original = index_module.catalog_summaries
async def remove_during(datasette, names):
result = await original(datasette, names)
if "gone" in datasette.databases:
datasette.remove_database("gone")
return result
monkeypatch.setattr(index_module, "catalog_summaries", remove_during)
response = await ds.client.get("/.json")
assert response.status_code == 200
assert [d["name"] for d in response.json()["databases"]] == ["keep"]
ds.close()
@pytest.mark.asyncio
async def test_index_and_databases_with_closed_or_deleted_database(tmp_path):
paths = []
for name in ("keep", "closed", "deleted"):
path = str(tmp_path / f"{name}.db")
make_db(path)
paths.append(path)
ds = Datasette(paths, settings={"schema_watch_interval_ms": 0})
await ds.invoke_startup()
# Closed by a plugin but still attached
ds.get_database("closed").close()
# Deleted from under Datasette before any sweep noticed
os.unlink(paths[2])
for path in ("/.json", "/", "/-/databases.json"):
response = await ds.client.get(path)
assert response.status_code == 200, (path, response.text[:300])
databases = (await ds.client.get("/-/databases.json")).json()
if isinstance(databases, dict):
databases = databases["databases"]
sizes = {d["name"]: d["size"] for d in databases}
assert sizes["deleted"] == 0
ds.close()
@pytest.mark.asyncio
async def test_cursors_are_closed_before_leaving_their_thread(owned_db):
# A cursor still references its connection's cached statement, and on
# Python < 3.12 freeing it resets that statement. execute_write_many()
# handed its cursor to the caller's event loop thread; freed there while
# the write thread ran the same SQL again, the write failed with
# "bad parameter or other API misuse"
_ds, db, _path = owned_db
cursor = await db.execute_write_many(
"insert into t (v) values (?)", [("a",), ("b",)]
)
assert cursor.rowcount == 2
with pytest.raises(sqlite3.ProgrammingError):
cursor.fetchall()
cursor = await db.execute_write_fn(
lambda conn: conn.execute("insert into t (v) values ('c')")
)
assert cursor.lastrowid == 3
with pytest.raises(sqlite3.ProgrammingError):
cursor.fetchone()
# Cursors held by the frames of an exception raised by a callback are
# closed before the connection goes back to the pool (or write thread)
captured = []
def read_then_fail(conn):
cursor = conn.execute("select * from t")
captured.append(cursor)
raise ValueError("boom")
with pytest.raises(ValueError):
await db.execute_fn(read_then_fail)
with pytest.raises(ValueError):
await db.execute_write_fn(read_then_fail)
for cursor in captured:
with pytest.raises(sqlite3.ProgrammingError):
cursor.fetchone()
def test_startup_from_several_event_loops_runs_once(monkeypatch):
# _startup_sequence() serialized startup with an asyncio.Lock: callers
# on a second event loop got "is bound to a different event loop" (or,
# free-threaded, waited forever), and invoke_startup() itself let two
# loops run every startup hook twice
ds = Datasette(memory=True)
calls = []
original = Datasette._apply_column_types_config
async def slow(self):
calls.append(threading.current_thread().name)
await asyncio.sleep(0.3)
return await original(self)
monkeypatch.setattr(Datasette, "_apply_column_types_config", slow)
def starter(use_sequence):
async def run():
if use_sequence:
await ds._startup_sequence()
else:
await ds.invoke_startup()
assert ds._startup_invoked
return (await ds.client.get("/_memory.json")).status_code
return run
statuses = run_in_threads(starter(True), starter(True), starter(False))
assert statuses == [200, 200, 200]
assert len(calls) == 1
ds.close()
@pytest.mark.asyncio
async def test_startup_hook_can_make_requests():
# A request made from inside startup (same task) must not wait for
# startup to finish - that would wait forever
from datasette import hookimpl
from datasette.plugins import pm
statuses = []
class Plugin:
__name__ = "StartupRequestPlugin"
@hookimpl
def startup(self, datasette):
async def inner():
statuses.append(
(await datasette.client.get("/-/versions.json")).status_code
)
return inner
pm.register(Plugin(), name="startup_request_plugin")
try:
ds = Datasette(memory=True)
await asyncio.wait_for(ds.invoke_startup(), 10)
assert statuses == [200]
ds.close()
finally:
pm.unregister(name="startup_request_plugin")

69
tests/test_connection_torture.py Normal file
View file

@ -0,0 +1,69 @@
"""
A fast, fixed-seed subset of the connection torture harness
(tests/connection_torture.py). Each scenario runs in its own process, so a
segfault or a hang fails one test instead of killing the test run.
Several event loops (threads) drive one Datasette with extreme settings
(smallest pool, 1ms idle timeout, 10ms schema polling) through reads,
writes of every kind, DDL, isolated functions, analyze_sql, cancellation,
add/remove_database, scratch databases, external schema changes, files
replaced and deleted, Database.close() and Datasette.close() mid-flight,
and non-threaded mode. See the harness docstring for what is checked.
"""
import json
import os
import signal
import subprocess
import sys
import pytest
HARNESS = os.path.join(os.path.dirname(__file__), "connection_torture.py")
@pytest.mark.parametrize(
"scenario,seed",
[
("mixed", 101),
("lifecycle", 102),
("close", 103),
("cancel", 104),
("pool", 105),
("nothreads", 106),
("writes", 107),
],
)
def test_connection_torture(scenario, seed):
try:
proc = subprocess.run(
[
sys.executable,
HARNESS,
scenario,
"--seed",
str(seed),
"--duration",
"1.5",
],
capture_output=True,
text=True,
timeout=150,
check=False,
)
except subprocess.TimeoutExpired as e: # pragma: no cover
pytest.fail(f"{scenario} hung:\n{(e.stderr or '')[-4000:]}")
if proc.returncode < 0: # pragma: no cover
pytest.fail(
f"{scenario} crashed with {signal.Signals(-proc.returncode).name}:\n"
f"{proc.stderr[-4000:]}"
)
summary = json.loads(proc.stdout.strip().splitlines()[-1])
problems = "\n".join(
f"[{p['kind']}] {p['message'][:600]}" for p in summary["problems"]
)
assert proc.returncode == 0, f"{scenario} seed {seed}:\n{problems}"
assert summary["ok"]
# It did something (the slow-query scenarios do a few dozen operations
# in 1.5s on a busy machine, the others hundreds)
assert sum(summary["counts"].values()) > 10

4
tests/test_schema_watcher.py
View file

@ -625,7 +625,9 @@ async def test_default_schema_watch_setting(tmp_path, db_path, setting, expected
Database(ds, path=other), name="polled", schema_watch="external"
)
assert polled._watch_state.mode == "external"
owned = ds.add_database(Database(ds, path=other), name="owned", schema_watch="owned")
owned = ds.add_database(
Database(ds, path=other), name="owned", schema_watch="owned"
)
assert owned._watch_state.mode == "owned"
ds.close()