This commit is contained in:
Simon Willison 2026-10-12 18:18:59 +00:00 • committed by GitHub
commit 5d797878b9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
50 changed files with 11348 additions and 590 deletions

204
tests/connection_budget.py Normal file
View file

@ -0,0 +1,204 @@
"""Record which databases a block of code touches, and assert a budget.
With a capped, idle-reaped connection pool, anything that loops over every
attached database opens (and evicts) a connection per database at a 0% hit
rate. These helpers make "this request only touches the databases it is
about" a testable property::
async with connection_budget() as budget:
await ds.client.get("/db1/t1.json")
budget.assert_only("db1") # plus _internal, always allowed
What counts as touching a database:
* ``opened``: ``Database.connect()`` calls - pooled read connections, write
connections, isolated connections and the SchemaWatcher's short-lived
scan/pragma connections (``track=False``)
* ``reads``: ``Database._execute_fn()`` calls (every ``execute()``,
``execute_fn()`` and introspection helper), whether or not the pool had an
idle connection - so a warm pool does not hide a loop over every database
* ``writes``: ``execute_write*()``, ``execute_isolated_fn()`` and work run on
the write connection
* ``attached``: databases ATTACHed to a connection while it was prepared
(``--crossdb``)
* ``probes``: ``sqlite3.connect()`` calls that bypass ``Database.connect()``
(``:memory:`` feature probes and the like), keyed by filename
The patches are installed process-wide for the duration of the block, so
background work (the SchemaWatcher's polling task, the read pool's reaper)
that runs during the block is counted too. Tests that want to measure one
request should turn polling off (``schema_watch_interval_ms=0``) or measure
it separately.
"""
import collections
import contextlib
import threading
from datasette import app as app_module
from datasette.database import Database
from datasette.utils import sqlite3
INTERNAL = "__INTERNAL__"
_local = threading.local()
class ConnectionBudget:
def __init__(self):
self._lock = threading.Lock()
self.opened = collections.Counter()
self.reads = collections.Counter()
self.writes = collections.Counter()
self.attached = collections.Counter()
self.probes = collections.Counter()
def _add(self, counter, key):
with self._lock:
counter[key] += 1
def touched(self):
"""Names of every database opened, read, written or ATTACHed."""
names = set()
for counter in (self.opened, self.reads, self.writes, self.attached):
names.update(counter)
return names
def summary(self):
return {
"opened": dict(self.opened),
"reads": dict(self.reads),
"writes": dict(self.writes),
"attached": dict(self.attached),
"probes": dict(self.probes),
}
def assert_only(self, *names, internal=True, probes=0):
"""Assert that nothing outside ``names`` was touched.
``_internal`` is allowed unless ``internal=False``. ``probes`` is the
number of direct ``sqlite3.connect()`` calls allowed.
"""
allowed = set(names)
if internal:
allowed.add(INTERNAL)
extra = self.touched() - allowed
assert not extra, (
f"touched {len(extra)} database(s) outside the budget "
f"{sorted(allowed)}: {sorted(extra)[:10]}"
f"{' ...' if len(extra) > 10 else ''}\n{self.summary()}"
)
total_probes = sum(self.probes.values())
assert (
total_probes <= probes
), f"{total_probes} direct sqlite3.connect() call(s): {dict(self.probes)}"
def assert_at_most(self, n, *, internal=True):
"""Assert that at most ``n`` user databases were touched."""
touched = self.touched()
if internal:
touched.discard(INTERNAL)
assert len(touched) <= n, (
f"touched {len(touched)} databases, budget {n}: "
f"{sorted(touched)[:10]}\n{self.summary()}"
)
def assert_no_connections(self, internal=True):
"""Assert that no connection was opened at all (``_internal`` aside)."""
opened = {
k: v for k, v in self.opened.items() if not (internal and k == INTERNAL)
}
assert not opened, f"opened connections: {opened}"
assert not self.attached, f"attached: {dict(self.attached)}"
assert not self.probes, f"probes: {dict(self.probes)}"
@contextlib.contextmanager
def _patched(budget):
original_connect = Database.connect
original_execute_fn = Database._execute_fn
original_execute_write_fn = Database._execute_write_fn
original_isolated = Database.execute_isolated_fn
original_on_write_conn = Database._execute_on_write_connection
original_prepare = app_module.Datasette._prepare_connection
original_sqlite_connect = sqlite3.connect
def connect(self, *args, **kwargs):
budget._add(budget.opened, self.name)
_local.depth = getattr(_local, "depth", 0) + 1
try:
return original_connect(self, *args, **kwargs)
finally:
_local.depth -= 1
async def _execute_fn(self, fn):
budget._add(budget.reads, self.name)
return await original_execute_fn(self, fn)
async def _execute_write_fn(self, fn, *args, **kwargs):
budget._add(budget.writes, self.name)
return await original_execute_write_fn(self, fn, *args, **kwargs)
async def execute_isolated_fn(self, fn):
budget._add(budget.writes, self.name)
return await original_isolated(self, fn)
async def _execute_on_write_connection(self, fn):
budget._add(budget.writes, self.name)
return await original_on_write_conn(self, fn)
def _prepare_connection(self, conn, database, **kwargs):
result = original_prepare(self, conn, database, **kwargs)
try:
rows = conn.execute("PRAGMA database_list").fetchall()
except Exception: # noqa: BLE001
rows = []
for row in rows:
name = row[1]
if name not in ("main", "temp"):
budget._add(budget.attached, name)
return result
def sqlite_connect(database, *args, **kwargs):
if not getattr(_local, "depth", 0):
budget._add(budget.probes, str(database))
return original_sqlite_connect(database, *args, **kwargs)
Database.connect = connect
Database._execute_fn = _execute_fn
Database._execute_write_fn = _execute_write_fn
Database.execute_isolated_fn = execute_isolated_fn
Database._execute_on_write_connection = _execute_on_write_connection
app_module.Datasette._prepare_connection = _prepare_connection
sqlite3.connect = sqlite_connect
try:
yield budget
finally:
Database.connect = original_connect
Database._execute_fn = original_execute_fn
Database._execute_write_fn = original_execute_write_fn
Database.execute_isolated_fn = original_isolated
Database._execute_on_write_connection = original_on_write_conn
app_module.Datasette._prepare_connection = original_prepare
sqlite3.connect = original_sqlite_connect
class connection_budget:
"""Context manager (sync or async) yielding a ConnectionBudget."""
def __init__(self):
self.budget = ConnectionBudget()
self._cm = None
def __enter__(self):
self._cm = _patched(self.budget)
return self._cm.__enter__()
def __exit__(self, *exc):
return self._cm.__exit__(*exc)
async def __aenter__(self):
return self.__enter__()
async def __aexit__(self, *exc):
return self.__exit__(*exc)

1251
tests/connection_torture.py Normal file

File diff suppressed because it is too large Load diff

View file

@ -759,6 +759,13 @@ async def test_settings_json(ds_client):
"suggest_facets": True,
"default_cache_ttl": 5,
"num_sql_threads": 1,
"max_open_connections": 32,
"max_write_connections": 8,
"max_pending_writes": 256,
"write_queue_timeout_ms": 5000,
"connection_idle_timeout_ms": 30000,
"schema_watch_interval_ms": 1000,
"default_schema_watch": "external",
"cache_size_kb": 0,
"allow_csv_stream": True,
"max_csv_mb": 100,

View file

@ -0,0 +1,652 @@
"""Requests, startup and background work only touch the databases they need.
With a capped, idle-reaped connection pool, a loop over every attached
database opens (and evicts) one connection per database. These tests attach
50 databases and use tests/connection_budget.py to assert which databases
each endpoint opens, reads or writes. Every test uses a fresh instance, so
caches that are only warm after a first request do not hide a sweep.
"""
import json
import os
import shutil
import sqlite3
import time
import pytest
from datasette.app import Datasette
from datasette.utils.catalog import (
catalog_relationship_counts,
catalog_summaries,
catalog_table_details,
)
from datasette.utils.sqlite import (
sqlite_derived_table_dependencies,
supports_table_list,
)
from .connection_budget import INTERNAL, connection_budget
N = 50
@pytest.fixture(autouse=True)
def only_default_plugins():
"""Plugins other tests leave registered (tests/plugins/my_plugin.py and
friends query datasette.get_database() from their hooks) are not part
of what these tests measure: unregister them for the duration."""
from datasette.plugins import DEFAULT_PLUGINS, pm
removed = [
(name, plugin)
for name, plugin in pm.list_name_plugin()
if name not in DEFAULT_PLUGINS
]
for name, _ in removed:
pm.unregister(name=name)
try:
yield
finally:
for name, plugin in removed:
if pm.get_plugin(name) is None:
pm.register(plugin, name=name)
def _make_db(path, i):
conn = sqlite3.connect(path)
conn.executescript("""
create table t2 (id integer primary key, title text);
create table t1 (
id integer primary key, name text, other_id integer references t2(id)
);
""")
conn.executemany(
"insert into t1 (name, other_id) values (?, ?)",
[(f"n{j}", j % 3) for j in range(20)],
)
conn.executemany(
"insert into t2 (title) values (?)", [(f"t{j}",) for j in range(3)]
)
if i % 5 == 0:
conn.executescript("""
create virtual table t1_fts using fts5(
name, content="t1", content_rowid='id'
);
insert into t1_fts(t1_fts) values('rebuild');
""")
if i % 3 == 0:
conn.execute("create view v1 as select * from t1")
if i % 7 == 0:
conn.execute("create table _hidden (x)")
conn.commit()
conn.close()
@pytest.fixture(scope="module")
def db_files(tmp_path_factory):
directory = tmp_path_factory.mktemp("budget")
paths = []
for i in range(N):
path = str(directory / f"db{i:03d}.db")
_make_db(path, i)
paths.append(path)
# Older than the SchemaWatcher's racy window, so a sweep is stat-only
old = time.time() - 60
for path in paths:
os.utime(path, (old, old))
return paths
def _ds(paths, **kwargs):
settings = {"schema_watch_interval_ms": 0}
settings.update(kwargs.pop("settings", {}))
ds = Datasette(paths, settings=settings, **kwargs)
ds.root_enabled = True
return ds
def _root(ds):
return {"ds_actor": ds.client.actor_cookie({"id": "root"})}
async def _get(ds, path, cookies=None):
async with connection_budget() as budget:
response = await ds.client.get(path, cookies=cookies or {})
assert response.status_code == 200, (path, response.status_code, response.text)
return response, budget
@pytest.mark.asyncio
@pytest.mark.parametrize("path", ["/", "/.json", "/-/", "/-/.json"])
@pytest.mark.parametrize("as_root", [False, True])
async def test_index_page_opens_no_database(db_files, path, as_root):
ds = _ds(db_files)
await ds.invoke_startup()
response, budget = await _get(ds, path, _root(ds) if as_root else None)
budget.assert_only()
budget.assert_no_connections()
if path.endswith(".json"):
data = response.json()
assert len(data["databases"]) == N
# More databases than COUNT_MAX_DATABASES: no row counts
assert not any(d["show_table_row_counts"] for d in data["databases"])
db000 = next(d for d in data["databases"] if d["name"] == "db000")
assert db000["tables_count"] == 2 # t1, t2
# _hidden, and t1_fts (an external content FTS table is hidden).
# Its fts5 shadow tables are not listed at all: they derive from
# t1_fts, which is itself derived (the one-hop permission policy)
assert db000["hidden_tables_count"] == 2
assert db000["views_count"] == 1
shown = {t["name"]: t for t in db000["tables_and_views_truncated"]}
assert shown["t1"]["columns"] == ["id", "name", "other_id"]
assert shown["t1"]["primary_keys"] == ["id"]
assert shown["t1"]["fts_table"] == "t1_fts"
# Sorted by foreign key relationships when there are no counts
assert shown["t1"]["num_relationships_for_sorting"] == 1
ds.close()
@pytest.mark.asyncio
async def test_index_page_counts_rows_for_few_databases(db_files):
# At most COUNT_MAX_DATABASES databases: rows are counted, and only the
# databases listed are touched
ds = _ds(db_files[:3])
await ds.invoke_startup()
response, budget = await _get(ds, "/.json")
budget.assert_only("db000", "db001", "db002")
data = response.json()
assert all(d["show_table_row_counts"] for d in data["databases"])
assert data["databases"][0]["table_rows_sum"] == 23
ds.close()
@pytest.mark.asyncio
async def test_index_page_uses_cached_immutable_counts(db_files):
# Immutable databases' counts are computed once, when first needed, and
# reused even when there are too many databases to count
ds = Datasette(
db_files[3:],
immutables=db_files[:3],
settings={"schema_watch_interval_ms": 0},
)
await ds.invoke_startup()
for name in ("db000", "db001", "db002"):
await ds.get_database(name).table_counts()
response, budget = await _get(ds, "/.json")
budget.assert_no_connections()
budget.assert_only()
by_name = {d["name"]: d for d in response.json()["databases"]}
assert by_name["db000"]["show_table_row_counts"]
assert not by_name["db010"]["show_table_row_counts"]
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("path", ["/-/databases", "/-/databases.json"])
async def test_databases_endpoint(db_files, path):
ds = _ds(db_files)
await ds.invoke_startup()
_, budget = await _get(ds, path)
budget.assert_only()
budget.assert_no_connections()
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"path",
[
"/db001",
"/db001.json",
"/db000.json?_search=n",
"/db001/t1",
"/db001/t1.json",
"/db001/t1?_facet=name",
"/db001/t1.json?_facet=name",
"/db000/t1_fts.json?_search=n1",
"/db001/-/query?sql=select+1",
"/db001/-/query.json?sql=select+1",
"/db001/-/query",
"/db001/t1/1",
"/db001/t1/1.json",
"/db001/-/schema.json",
"/db001/t1/-/autocomplete?q=n",
],
)
@pytest.mark.parametrize("as_root", [False, True])
async def test_database_scoped_pages_touch_only_their_database(db_files, path, as_root):
ds = _ds(db_files)
await ds.invoke_startup()
database = path.split("/")[1].split(".")[0]
_, budget = await _get(ds, path, _root(ds) if as_root else None)
budget.assert_only(database)
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"path,as_root",
[
("/-/versions", False),
("/-/versions.json", False),
("/-/plugins.json", False),
("/-/settings.json", False),
("/-/config.json", False),
("/-/actor.json", False),
("/-/threads", True),
("/-/threads.json", True),
("/-/tasks.json", True),
("/-/actions.json", True),
("/-/jump?q=t1", False),
("/-/jump.json?q=t1", False),
("/-/jump.json", False),
("/-/allowed.json?action=view-table", False),
("/-/allowed.json?action=view-table", True),
("/-/allowed.json?action=view-database", False),
("/-/allowed?action=view-table", True),
("/-/rules.json?action=view-table", True),
("/-/check.json?action=view-table&parent=db001&child=t1", True),
("/-/permissions", True),
("/-/allow-debug", False),
("/-/api", False),
("/-/api", True),
("/-/create-token", True),
("/-/queries.json", False),
("/-/patterns", False),
],
)
async def test_instance_pages_open_no_user_database(db_files, path, as_root):
ds = _ds(db_files)
await ds.invoke_startup()
# /-/versions probes the SQLite library once per instance
await ds.client.get("/-/versions.json")
_, budget = await _get(ds, path, _root(ds) if as_root else None)
budget.assert_only()
budget.assert_no_connections()
ds.close()
@pytest.mark.asyncio
async def test_versions_probes_sqlite_once(db_files):
ds = _ds(db_files[:12], crossdb=True)
await ds.invoke_startup()
async with connection_budget() as budget:
for _ in range(3):
assert (await ds.client.get("/-/versions.json")).status_code == 200
# One :memory: connection for the first request, none after; it is not
# given the --crossdb ATTACHes
assert sum(budget.probes.values()) == 1
assert not budget.attached
budget.assert_only(probes=1)
ds.close()
@pytest.mark.asyncio
async def test_detect_json1_is_cached(db_files):
ds = _ds(db_files[:2])
await ds.invoke_startup()
await ds.client.get("/db001/t1")
_, budget = await _get(ds, "/db001/t1?_facet=name")
assert not budget.probes
ds.close()
@pytest.mark.asyncio
async def test_instance_schema_reads_each_visible_database_once(db_files):
# /-/schema dumps every visible database's sqlite_master: one read each,
# nothing for databases the actor cannot see
ds = _ds(
db_files[:10],
config={"databases": {"db003": {"allow": False}}},
)
await ds.invoke_startup()
_, budget = await _get(ds, "/-/schema.json")
expected = {f"db{i:03d}" for i in range(10)} - {"db003"}
assert budget.touched() - {INTERNAL} == expected
assert all(budget.reads[name] == 1 for name in expected)
ds.close()
@pytest.mark.asyncio
async def test_autocomplete_debug_is_bounded(db_files):
ds = _ds(db_files)
await ds.invoke_startup()
_, budget = await _get(ds, "/-/debug/autocomplete")
# Only databases holding one of the (at most 5) suggestions
budget.assert_at_most(5)
ds.close()
@pytest.mark.asyncio
async def test_execute_write_touches_only_its_database(db_files, tmp_path):
paths = []
for path in db_files[:20]:
target = tmp_path / os.path.basename(path)
shutil.copy(path, target)
paths.append(str(target))
ds = _ds(paths)
await ds.invoke_startup()
async with connection_budget() as budget:
response = await ds.client.post(
"/db004/-/execute-write",
actor={"id": "root"},
data={"sql": "create table new_table (id integer primary key)"},
)
assert response.status_code == 200, response.text
assert "Query executed" in response.text
budget.assert_only("db004")
# The catalog already has the new table
rows = await ds.get_internal_database().execute(
"select 1 from catalog_tables where database_name = 'db004' and table_name = 'new_table'"
)
assert rows.rows
ds.close()
@pytest.mark.asyncio
async def test_telemetry_metric_callbacks_open_nothing(db_files):
from datasette import telemetry
ds = _ds(db_files)
await ds.invoke_startup()
telemetry.register_datasette(ds)
try:
with connection_budget() as budget:
observations = 0
for callback in (
telemetry.observe_sql_thread_limit,
telemetry.observe_sql_thread_queue_depth,
telemetry.observe_pending_queries,
telemetry.observe_write_queue_depth,
telemetry.observe_open_connections,
):
observations += len(list(callback()))
assert observations
budget.assert_only(internal=False)
budget.assert_no_connections(internal=False)
finally:
telemetry.unregister_datasette(ds)
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 3])
async def test_startup_opens_each_database_at_most_once(db_files, num_sql_threads):
imm = db_files[:5]
with connection_budget() as budget:
ds = Datasette(
db_files[5:],
immutables=imm,
settings={
"schema_watch_interval_ms": 0,
"num_sql_threads": num_sql_threads,
},
)
await ds._startup_sequence()
# One catalog scan connection per database, no pooled reads or writes
# (immutable table counts are no longer precomputed at startup)
assert set(budget.opened) - {INTERNAL, "_memory"} == {
f"db{i:03d}" for i in range(N)
}
assert all(
count == 1 for name, count in budget.opened.items() if name != INTERNAL
), budget.opened
assert set(budget.reads) <= {INTERNAL}, budget.reads
assert set(budget.writes) <= {INTERNAL}, budget.writes
assert not budget.attached
ds.close()
@pytest.mark.asyncio
async def test_startup_with_crossdb_attaches_nothing(db_files):
with connection_budget() as budget:
ds = Datasette(
db_files[:12],
settings={"schema_watch_interval_ms": 0},
crossdb=True,
)
await ds._startup_sequence()
assert not budget.attached
# A crossdb query still sees the attached databases
response = await ds.client.get(
"/_memory/-/query.json?sql=select+count(*)+from+db001.t1&_shape=array"
)
assert response.status_code == 200
ds.close()
@pytest.mark.asyncio
async def test_restart_with_persisted_catalog_opens_nothing(
db_files, tmp_path, monkeypatch
):
from datasette import schema_watcher
# The files' ctimes are recent (os.utime() in the fixture): without this
# their stored fingerprints would be racy, and so rescanned
monkeypatch.setattr(schema_watcher, "RACY_WINDOW_NS", 0)
internal = str(tmp_path / "internal.db")
ds = Datasette(db_files, internal=internal)
await ds.invoke_startup()
ds.close()
with connection_budget() as budget:
ds = Datasette(
db_files, internal=internal, settings={"schema_watch_interval_ms": 0}
)
await ds._startup_sequence()
response = await ds.client.get("/.json")
assert response.status_code == 200
budget.assert_only()
ds.close()
@pytest.mark.asyncio
async def test_idle_sweeps_open_nothing(db_files):
ds = _ds(db_files)
await ds.invoke_startup()
with connection_budget() as budget:
for _ in range(3):
await ds._refresh_schemas(background=True)
await ds.refresh_schemas(force=True)
budget.assert_only(internal=False)
budget.assert_no_connections(internal=False)
ds.close()
@pytest.mark.asyncio
async def test_failed_scan_is_not_retried_by_idle_sweeps(
db_files, tmp_path, monkeypatch
):
from datasette import schema_watcher
# The file was just written: do not wait out the racy window
monkeypatch.setattr(schema_watcher, "RACY_WINDOW_NS", 0)
bad = tmp_path / "bad.db"
bad.write_bytes(b"this is not a SQLite database" * 10)
old = time.time() - 60
os.utime(bad, (old, old))
ds = _ds(db_files[:3] + [str(bad)])
await ds.invoke_startup()
assert ds.get_database("bad")._watch_state.error is not None
with connection_budget() as budget:
for _ in range(3):
await ds._refresh_schemas(background=True)
assert "bad" not in budget.touched()
# ...until the file changes
good = sqlite3.connect(tmp_path / "good.db")
good.execute("create table fixed (id integer primary key)")
good.commit()
good.close()
os.replace(tmp_path / "good.db", bad)
await ds._refresh_schemas(background=True)
rows = await ds.get_internal_database().execute(
"select table_name from catalog_tables where database_name = 'bad'"
)
assert [r[0] for r in rows.rows] == ["fixed"]
ds.close()
@pytest.mark.asyncio
async def test_check_databases_reuses_catalog_scan(db_files, tmp_path):
from datasette.cli import check_databases
from datasette.utils import ConnectionProblem
ds = _ds(db_files)
with connection_budget() as budget:
await check_databases(ds)
# The catalog scan opened each database once; no second pass
assert all(count == 1 for name, count in budget.opened.items() if name != INTERNAL)
assert set(budget.reads) <= {INTERNAL, "_memory"}, budget.reads
ds.close()
bad = tmp_path / "bad.db"
bad.write_bytes(b"this is not a SQLite database" * 10)
ds = _ds(db_files[:3] + [str(bad)])
with pytest.raises(Exception) as excinfo:
await check_databases(ds)
assert "bad.db" in str(excinfo.value)
assert "file is not a database" in str(excinfo.value)
assert not isinstance(excinfo.value, ConnectionProblem)
ds.close()
# Parity: the catalog helpers answer exactly what the live helpers do
MISC_SQL = """
create table docs (id integer primary key, title text, body text);
create virtual table docs_fts using fts4(title, body, content="docs");
create virtual table docs_vocab using fts4aux(docs_fts);
create virtual table plain using fts5(x);
create virtual table geo using rtree(id, minx, maxx);
create table _private (x);
create table geometry_columns (f_table_name text);
create table idx_foo_bar (x);
create table IDXA (x);
create table parent (id integer primary key);
create table child (
id integer primary key,
p integer references parent(id),
q integer references missing(id)
);
create table comp (a, b, foreign key (a, b) references parent(id, id));
create table self_ref (id integer primary key, up integer references self_ref(id));
create table wide (b text, a integer, c, primary key (a, b));
create view v_docs as select * from docs;
"""
@pytest.mark.asyncio
@pytest.mark.parametrize("name", ["misc", "fixtures"])
async def test_catalog_helpers_match_live_introspection(tmp_path, name):
from datasette.fixtures import write_fixture_database
from datasette.utils.catalog import (
all_derived_table_dependencies,
catalog_all_foreign_keys,
)
path = str(tmp_path / f"{name}.db")
if name == "fixtures":
write_fixture_database(path)
config_hidden = set()
config = {}
else:
conn = sqlite3.connect(path)
conn.executescript(MISC_SQL)
conn.close()
config_hidden = {"docs"}
config = {"databases": {"misc": {"tables": {"docs": {"hidden": True}}}}}
ds = Datasette([path], config=config, settings={"schema_watch_interval_ms": 0})
await ds.invoke_startup()
db = ds.get_database(name)
tables = await db.table_names()
summary = (await catalog_summaries(ds, [name]))[name]
# hidden_table_names() also lists SpatiaLite names that do not exist
hidden = set(await db.hidden_table_names()) & set(tables)
assert {t for t in tables if summary.is_hidden(t, config_hidden)} == hidden
assert summary.views == set(await db.view_names())
details = await catalog_table_details(ds, [(name, t) for t in tables])
for table in tables:
assert details[(name, table)]["columns"] == await db.table_columns(table)
assert details[(name, table)]["primary_keys"] == await db.primary_keys(table)
assert details[(name, table)]["fts_table"] == await db.fts_table(table)
all_foreign_keys = await db.get_all_foreign_keys()
assert await catalog_all_foreign_keys(ds, name) == all_foreign_keys
relationships = (await catalog_relationship_counts(ds, [name])).get(name, {})
for table, fks in all_foreign_keys.items():
assert relationships.get(table, 0) == len(fks["incoming"] + fks["outgoing"])
derived = await all_derived_table_dependencies(ds)
live = await db.execute_fn(sqlite_derived_table_dependencies)
assert derived.get(name, {}) == live
# Restored from the catalog rather than remembered from the scan
db._cached_derived_table_dependencies = None
assert await db.derived_table_dependencies() == live
ds.close()
@pytest.mark.asyncio
@pytest.mark.skipif(not supports_table_list(), reason="needs PRAGMA table_list")
async def test_catalog_records_table_types(tmp_path):
path = str(tmp_path / "types.db")
conn = sqlite3.connect(path)
conn.executescript("""
create table plain (id integer primary key);
create virtual table f using fts5(x);
""")
conn.close()
ds = Datasette([path], settings={"schema_watch_interval_ms": 0})
await ds.invoke_startup()
rows = await ds.get_internal_database().execute(
"select table_name, type from catalog_tables where database_name = 'types'"
)
types = dict(rows.rows)
assert types["plain"] == "table"
assert types["f"] == "virtual"
assert types["f_data"] == "shadow"
ds.close()
@pytest.mark.asyncio
async def test_catalog_derived_dependencies_follow_catalog_changes(tmp_path):
path = str(tmp_path / "live.db")
sqlite3.connect(path).execute("create table docs (body text)").connection.close()
ds = Datasette([path], settings={"schema_watch_interval_ms": 0})
await ds.invoke_startup()
from datasette.utils.catalog import all_derived_table_dependencies
assert "live" not in await all_derived_table_dependencies(ds)
db = ds.get_database("live")
await db.execute_write(
"create virtual table docs_fts using fts5(body, content='docs')"
)
derived = await all_derived_table_dependencies(ds)
assert derived["live"]["docs_fts"] == "docs"
assert derived["live"]["docs_fts_data"] == "docs_fts"
ds.close()
def test_connection_budget_helper_counts(db_files):
# The helper itself: a direct execute_fn is a read, connect() an open
import asyncio
async def run():
ds = _ds(db_files[:2])
await ds.invoke_startup()
db = ds.get_database("db001")
async with connection_budget() as budget:
await db.execute("select 1")
conn = sqlite3.connect(":memory:")
conn.close()
assert budget.reads["db001"] == 1
assert budget.touched() == {"db001"}
with pytest.raises(AssertionError, match="db001"):
budget.assert_only("db000")
ds.close()
return budget
budget = asyncio.run(run())
assert sum(budget.probes.values()) == 1
assert json.dumps(budget.summary())

View file

@ -0,0 +1,283 @@
"""
Connections handed to execute_write_fn(), the SQL write methods and
execute_isolated_fn() callbacks are lease proxies, like read connections:
they stop working once the callback returns.
"""
import asyncio
import sqlite3
import threading
import time
import pytest
import sqlite_utils
from datasette.app import Datasette
from datasette.database import ConnectionLeaseError
def _make_db(tmp_path, name="leases"):
path = str(tmp_path / f"{name}.db")
conn = sqlite3.connect(path)
conn.execute("create table t (id integer primary key, v text)")
conn.execute("insert into t (v) values ('x')")
conn.commit()
conn.close()
return path
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 2])
@pytest.mark.parametrize("transaction", [True, False])
async def test_write_callback_connection_expires(
tmp_path, num_sql_threads, transaction
):
ds = Datasette([_make_db(tmp_path)], settings={"num_sql_threads": num_sql_threads})
db = ds.get_database("leases")
stash = {}
def write(conn):
stash["conn"] = conn
assert isinstance(conn, sqlite3.Connection)
conn.execute("insert into t (v) values ('y')")
if not transaction:
# transaction=False DML is not committed for you (unchanged)
conn.commit()
return conn.execute("select count(*) from t").fetchone()[0]
assert await db.execute_write_fn(write, transaction=transaction) == 2
with pytest.raises(ConnectionLeaseError, match="execute_write_fn"):
stash["conn"].execute("select 1")
with pytest.raises(ConnectionLeaseError):
stash["conn"].row_factory = None
# The write connection itself is fine for the next write
await db.execute_write("insert into t (v) values ('z')")
assert (await db.execute("select count(*) from t")).single_value() == 3
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 2])
async def test_write_callback_may_return_cursor(tmp_path, num_sql_threads):
# A common plugin pattern: lambda conn: conn.execute(...). The returned
# cursor's attributes stay readable; only the connection is leased.
ds = Datasette([_make_db(tmp_path)], settings={"num_sql_threads": num_sql_threads})
db = ds.get_database("leases")
cursor = await db.execute_write_fn(
lambda conn: conn.execute("insert into t (v) values ('y')")
)
assert cursor.lastrowid == 2
assert cursor.rowcount == 1
many = await db.execute_write_many("insert into t (v) values (?)", [["a"], ["b"]])
assert many.rowcount == 2
await db.execute_write_script("insert into t (v) values ('c');")
assert (await db.execute("select count(*) from t")).single_value() == 5
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 2])
async def test_with_conn_inside_write_callback_yields_proxy(tmp_path, num_sql_threads):
ds = Datasette([_make_db(tmp_path)], settings={"num_sql_threads": num_sql_threads})
db = ds.get_database("leases")
stash = {}
def write(conn):
with conn as inner:
stash["inner"] = inner
inner.execute("insert into t (v) values ('y')")
await db.execute_write_fn(write, transaction=False)
with pytest.raises(ConnectionLeaseError):
stash["inner"].execute("select 1")
assert (await db.execute("select count(*) from t")).single_value() == 2
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 2])
async def test_sqlite_utils_on_write_lease(tmp_path, num_sql_threads):
ds = Datasette([_make_db(tmp_path)], settings={"num_sql_threads": num_sql_threads})
db = ds.get_database("leases")
def write(conn):
sdb = sqlite_utils.Database(conn)
sdb["dogs"].insert_all([{"id": 1, "name": "Cleo"}], pk="id")
sdb["dogs"].transform(rename={"name": "dog_name"})
return sdb["dogs"].columns_dict
assert await db.execute_write_fn(write) == {"id": int, "dog_name": str}
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 2])
@pytest.mark.parametrize("immutable", [False, True])
async def test_isolated_callback_connection_expires(
tmp_path, num_sql_threads, immutable
):
path = _make_db(tmp_path)
if immutable:
ds = Datasette(immutables=[path], settings={"num_sql_threads": num_sql_threads})
else:
ds = Datasette([path], settings={"num_sql_threads": num_sql_threads})
db = ds.get_database("leases")
stash = {}
def isolated(conn):
stash["conn"] = conn
return conn.execute("select count(*) from t").fetchone()[0]
assert await db.execute_isolated_fn(isolated) == 1
with pytest.raises(ConnectionLeaseError, match="execute_isolated_fn"):
stash["conn"].execute("select 1")
ds.close()
@pytest.mark.asyncio
async def test_write_wrapper_receives_lease(tmp_path):
from datasette import hookimpl
from datasette.plugins import pm
stash = {}
class WrapperPlugin:
__name__ = "WrapperPlugin"
@hookimpl
def write_wrapper(self, datasette, database, request, transaction):
if database != "leases":
return None
def wrapper(conn):
stash["conn"] = conn
yield
return wrapper
pm.register(WrapperPlugin(), name="test_write_wrapper_receives_lease")
try:
ds = Datasette([_make_db(tmp_path)])
db = ds.get_database("leases")
await db.execute_write("insert into t (v) values ('y')")
with pytest.raises(ConnectionLeaseError):
stash["conn"].execute("select 1")
ds.close()
finally:
pm.unregister(name="test_write_wrapper_receives_lease")
@pytest.mark.asyncio
async def test_close_waits_for_immutable_isolated_fn(tmp_path):
# On main this ran on the executor untracked, so close() could close
# the connection while the callback was still stepping it (a segfault
# with a progress handler installed)
ds = Datasette(immutables=[_make_db(tmp_path)], settings={"num_sql_threads": 2})
db = ds.get_database("leases")
started = threading.Event()
def slow(conn):
started.set()
time.sleep(0.3)
return conn.execute("select count(*) from t").fetchone()[0]
task = asyncio.ensure_future(db.execute_isolated_fn(slow))
assert await asyncio.to_thread(started.wait, 5)
assert len(db._pending_execute_futures) == 1
db.close()
assert await task == 1
ds.close()
NO_THREADS_SCRIPT = r"""
import asyncio, os, sqlite3, sys, threading
def no_threads(self):
raise RuntimeError("can't start new thread")
threading.Thread.start = no_threads
from datasette.app import Datasette
from datasette.database import Database
path, scratch = sys.argv[1], sys.argv[2]
conn = sqlite3.connect(path)
conn.execute("create table t (id integer primary key, v text)")
conn.execute("insert into t (v) values ('x')")
conn.commit()
conn.close()
async def main():
ds = Datasette(
[path],
settings={
"num_sql_threads": 0,
"connection_idle_timeout_ms": 50,
"schema_watch_interval_ms": 50,
},
)
await ds.invoke_startup()
db = ds.get_database("nothreads")
assert (await db.execute("select count(*) from t")).single_value() == 1
await db.execute_write("insert into t (v) values ('y')")
await db.execute_write("create table t2 (id)")
tables = await ds.get_internal_database().execute(
"select table_name from catalog_tables where database_name = 'nothreads' "
"order by table_name"
)
assert [r[0] for r in tables.rows] == ["t", "t2"], tables.rows
assert await db.execute_isolated_fn(
lambda conn: conn.execute("select count(*) from t").fetchone()[0]
) == 2
await db.execute_write("insert into t (v) values ('z')", block=False)
response = await ds.client.get("/nothreads/t.json?_shape=array")
assert response.status_code == 200, response.text
assert len(response.json()) == 3
# External change detected by the polling task, inline on the loop
other = sqlite3.connect(path)
other.execute("create table external_t (id)")
other.commit()
other.close()
for _ in range(100):
await asyncio.sleep(0.02)
tables = await ds.get_internal_database().execute(
"select 1 from catalog_tables where table_name = 'external_t'"
)
if tables.rows:
break
else:
raise AssertionError("external change not seen")
# Scratch database added later, created by its first write
sdb = ds.add_database(Database(ds, path=scratch), name="scratch")
await sdb.execute_write("create table s (id)")
assert os.path.exists(scratch)
ds.close()
print("ok", threading.active_count())
asyncio.run(main())
"""
def test_num_sql_threads_zero_starts_no_threads(tmp_path):
# Simulates Pyodide, where no thread can be started: nothing in the
# read pool, write path or schema watcher may need one
import subprocess
import sys
result = subprocess.run(
[
sys.executable,
"-c",
NO_THREADS_SCRIPT,
str(tmp_path / "nothreads.db"),
str(tmp_path / "scratch.db"),
],
capture_output=True,
text=True,
timeout=60,
check=False,
)
assert result.returncode == 0, result.stdout + result.stderr
assert result.stdout.strip() == "ok 1"

View file

@ -0,0 +1,92 @@
"""Exercise real HTTP sockets and WAL files under a small OS descriptor limit."""
import json
import subprocess
import sys
import pytest
@pytest.mark.skipif(sys.platform == "win32", reason="Requires RLIMIT_NOFILE")
def test_500_wal_databases_with_http_and_writes_under_256_fds():
script = r"""
import asyncio, concurrent.futures, json, resource, socket, sqlite3, tempfile, threading, time, urllib.request
from pathlib import Path
import psutil
import uvicorn
from datasette.app import Datasette
async def main():
with tempfile.TemporaryDirectory() as directory:
paths = []
for i in range(500):
path = str(Path(directory) / f"db{i}.db")
conn = sqlite3.connect(path)
conn.execute("pragma journal_mode=wal")
conn.execute("create table t(id integer primary key)")
conn.close()
paths.append(path)
_, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (256, hard))
ds = Datasette(paths)
await ds.invoke_startup()
process = psutil.Process()
baseline_rss = process.memory_info().rss
peak = {"fds": 0, "rss": 0, "writers": 0}
stopped = threading.Event()
def monitor():
while not stopped.wait(0.002):
peak["fds"] = max(peak["fds"], process.num_fds())
peak["rss"] = max(peak["rss"], process.memory_info().rss)
peak["writers"] = max(peak["writers"], ds._write_budget.snapshot()["writers"])
watcher = threading.Thread(target=monitor)
watcher.start()
sock = socket.socket()
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
server = uvicorn.Server(uvicorn.Config(ds.app(), log_level="error"))
thread = threading.Thread(target=lambda: server.run(sockets=[sock]))
thread.start()
try:
deadline = time.monotonic() + 10
while not server.started:
assert time.monotonic() < deadline
await asyncio.sleep(0.01)
def request(i):
with urllib.request.urlopen(f"http://127.0.0.1:{port}/db{i}/-/query.json?sql=select+count(*)+from+t", timeout=20) as response:
assert response.status == 200
assert json.load(response)["ok"]
def reads():
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as clients:
list(clients.map(request, range(500)))
async def writes():
for start in range(0, 500, 16):
await asyncio.gather(*(ds.get_database(f"db{i}").execute_write("insert into t values (1)") for i in range(start, min(start + 16, 500))))
await asyncio.gather(asyncio.to_thread(reads), writes())
for i in range(500):
assert (await ds.get_database(f"db{i}").execute("select count(*) from t")).single_value() == 1
assert ds._read_pool.stats["peak_open"] <= 32
assert ds._write_budget.snapshot()["stats"]["peak_writers"] <= 8
print(json.dumps({"peak": peak, "baseline_rss": baseline_rss, "writes": 500, "http_reads": 500}), flush=True)
finally:
server.should_exit = True
await asyncio.to_thread(thread.join, 15)
assert not thread.is_alive()
stopped.set()
watcher.join()
sock.close()
ds.close()
asyncio.run(main())
"""
result = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
timeout=90,
check=False,
)
assert result.returncode == 0, result.stdout + result.stderr
measurements = json.loads(result.stdout.strip().splitlines()[-1])
assert measurements["peak"]["fds"] < 256
assert measurements["peak"]["writers"] <= 8
print(measurements)

View file

@ -0,0 +1,112 @@
import errno
import sqlite3
import pytest
from datasette.app import Datasette
from datasette.connection_pool import ConnectionLeaseError
@pytest.fixture
def database_file(tmp_path):
path = tmp_path / "data.db"
conn = sqlite3.connect(path)
conn.executescript("create table t(id); insert into t values (1), (2)")
conn.close()
return str(path)
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["read", "write", "isolated"])
async def test_saved_methods_and_cursors_cannot_escape_callback(database_file, kind):
ds = Datasette([database_file], settings={"schema_watch_interval_ms": 0})
db = ds.get_database("data")
saved = {}
def callback(conn):
saved["commit"] = conn.commit
saved["cursor"] = conn.execute("select * from t")
saved["fetch"] = saved["cursor"].fetchone
saved["connection"] = saved["cursor"].connection
assert saved["fetch"]()[0] == 1
method = {
"read": db.execute_fn,
"write": db.execute_write_fn,
"isolated": db.execute_isolated_fn,
}[kind]
try:
await method(callback)
for call in (saved["commit"], lambda: saved["connection"].execute("select 1")):
with pytest.raises(ConnectionLeaseError):
call()
for call in (saved["fetch"], lambda: next(saved["cursor"])):
with pytest.raises((ConnectionLeaseError, sqlite3.ProgrammingError)):
call()
assert (await db.execute("select count(*) from t")).single_value() == 2
finally:
ds.close()
@pytest.mark.asyncio
async def test_read_result_containing_cursor_is_rejected(database_file):
ds = Datasette([database_file])
db = ds.get_database("data")
try:
with pytest.raises(ConnectionLeaseError):
await db.execute_fn(lambda c: {"nested": [c.execute("select * from t")]})
result = await db.execute_write_fn(
lambda c: {"nested": [c.execute("insert into t values (3)")]}
)
assert result["nested"][0].rowcount == 1
with pytest.raises((ConnectionLeaseError, sqlite3.ProgrammingError)):
result["nested"][0].fetchone()
assert (await db.execute("select count(*) from t")).single_value() == 3
finally:
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failure",
[
sqlite3.OperationalError("unable to open database file"),
OSError(errno.EMFILE, "Too many open files"),
],
)
async def test_schema_recovers_after_resource_error_without_file_change(
database_file, monkeypatch, failure
):
ds = Datasette([database_file], settings={"schema_watch_interval_ms": 0})
try:
await ds.invoke_startup()
db = ds.get_database("data")
conn = sqlite3.connect(database_file)
conn.execute("create table added_externally(id)")
conn.close()
watcher = ds._schema_watcher
original = watcher._connect
def fail(database, prepare):
if database is db and prepare:
raise failure
return original(database, prepare)
with monkeypatch.context() as patch:
patch.setattr(watcher, "_connect", fail)
await ds.refresh_schemas()
state = db._watch_state
assert state.needs_scan
# Move past the retry backoff and fingerprint race window without
# touching the file or relying on timing-sensitive sleeps.
state.retry_at = 0
state.fp_racy = False
await watcher.sweep(background=True)
tables = await ds.get_internal_database().execute(
"select table_name from catalog_tables where database_name=?", ["data"]
)
assert {row[0] for row in tables.rows} == {"t", "added_externally"}
assert state.error is None
assert not state.needs_scan
finally:
ds.close()

View file

@ -0,0 +1,326 @@
"""SQLite resources must not retain locks or run after their callback ends."""
import operator
import sqlite3
import pytest
from datasette.app import Datasette
from datasette.connection_pool import ConnectionLeaseError
@pytest.fixture
def resource_db(tmp_path):
path = tmp_path / "resources.db"
conn = sqlite3.connect(path)
conn.executescript(
"create table t(id integer primary key, data blob);"
"insert into t(data) values (x'616263646566'), (zeroblob(6)), (zeroblob(6));"
"create table another(id);"
)
conn.close()
return str(path)
def callback_method(db, kind):
return {
"read": db.execute_fn,
"write": db.execute_write_fn,
"isolated": db.execute_isolated_fn,
}[kind]
def check_no_read_lock(path):
# A retained cursor/blob would block this commit in rollback journal mode.
conn = sqlite3.connect(path, timeout=0)
try:
conn.execute("insert into another values (1)")
conn.commit()
finally:
conn.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 2])
@pytest.mark.parametrize("kind", ["read", "write", "isolated", "immutable_isolated"])
@pytest.mark.parametrize("started", [False, True])
async def test_stashed_dump_iterator_expires_and_releases_lock(
resource_db, num_sql_threads, kind, started
):
ds = Datasette(
**{("immutables" if kind == "immutable_isolated" else "files"): [resource_db]},
settings={"num_sql_threads": num_sql_threads},
)
db = ds.get_database("resources")
saved = {}
def callback(conn):
saved["dump"] = dump = conn.iterdump()
saved["next"] = dump.__next__
saved["send"] = dump.send
saved["close"] = dump.close
assert iter(dump) is dump
if started:
for statement in dump:
if statement.startswith('INSERT INTO "t"'):
break
else:
pytest.fail("dump did not reach a data row")
return "done"
try:
assert (
await callback_method(db, kind.replace("immutable_", ""))(callback)
== "done"
)
for call in (
saved["next"],
lambda: iter(saved["dump"]),
lambda: saved["send"](None),
lambda: saved["dump"].throw(ValueError),
saved["close"],
):
with pytest.raises(ConnectionLeaseError):
call()
if kind != "immutable_isolated":
check_no_read_lock(resource_db)
assert (await db.execute("select count(*) from t")).single_value() == 3
finally:
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 2])
@pytest.mark.parametrize("kind", ["read", "write", "isolated"])
async def test_dump_can_be_consumed_inside_callback(resource_db, num_sql_threads, kind):
ds = Datasette([resource_db], settings={"num_sql_threads": num_sql_threads})
try:
statements = await callback_method(ds.get_database("resources"), kind)(
lambda conn: list(conn.iterdump())
)
restored = sqlite3.connect(":memory:")
try:
restored.executescript("\n".join(statements))
assert (
restored.execute("select data from t where id=1").fetchone()[0]
== b"abcdef"
)
finally:
restored.close()
finally:
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", [False, True])
async def test_returning_dump_from_read_callback_is_rejected(resource_db, nested):
ds = Datasette([resource_db])
def callback(conn):
dump = conn.iterdump()
return {"dump": [dump]} if nested else dump
try:
with pytest.raises(ConnectionLeaseError, match="database resources"):
await ds.get_database("resources").execute_fn(callback)
check_no_read_lock(resource_db)
finally:
ds.close()
@pytest.mark.asyncio
async def test_generator_over_dump_from_read_callback_is_rejected(resource_db):
ds = Datasette([resource_db])
try:
with pytest.raises(ConnectionLeaseError, match="database resources"):
await ds.get_database("resources").execute_fn(
lambda conn: (line for line in conn.iterdump())
)
check_no_read_lock(resource_db)
finally:
ds.close()
requires_blob = pytest.mark.skipif(
not hasattr(sqlite3.Connection, "blobopen"),
reason="Incremental BLOB access requires Python 3.11",
)
@requires_blob
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 2])
@pytest.mark.parametrize("kind", ["read", "write", "isolated", "immutable_isolated"])
async def test_stashed_blob_expires_and_releases_lock(
resource_db, num_sql_threads, kind
):
ds = Datasette(
**{("immutables" if kind == "immutable_isolated" else "files"): [resource_db]},
settings={"num_sql_threads": num_sql_threads},
)
saved = {}
def callback(conn):
saved["blob"] = blob = conn.blobopen("t", "data", 1, readonly=True)
saved["read"] = blob.read
saved["close"] = blob.close
assert isinstance(blob, sqlite3.Blob)
assert len(blob) == 6
assert blob[0] == ord("a")
assert blob[1:3] == b"bc"
assert blob.read(2) == b"ab"
assert blob.tell() == 2
blob.seek(-1, 2)
assert saved["read"]() == b"f"
try:
await callback_method(
ds.get_database("resources"), kind.replace("immutable_", "")
)(callback)
blob = saved["blob"]
for call in (
saved["read"],
saved["close"],
lambda: blob.write(b"x"),
lambda: blob.tell(),
lambda: blob.seek(0),
lambda: len(blob),
lambda: blob[0],
lambda: blob[:],
lambda: operator.setitem(blob, 0, 120),
lambda: operator.setitem(blob, slice(0, 1), b"x"),
blob.__enter__,
lambda: blob.__exit__(None, None, None),
):
with pytest.raises(ConnectionLeaseError):
call()
if kind != "immutable_isolated":
check_no_read_lock(resource_db)
finally:
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["write", "isolated"])
@pytest.mark.parametrize(
"resource", ["dump", pytest.param("blob", marks=requires_blob)]
)
async def test_returned_write_resource_is_expired(resource_db, kind, resource):
ds = Datasette([resource_db])
def callback(conn):
if resource == "dump":
return conn.iterdump()
return conn.blobopen("t", "data", 1, readonly=True)
try:
result = await callback_method(ds.get_database("resources"), kind)(callback)
with pytest.raises(ConnectionLeaseError):
if resource == "dump":
next(result)
else:
result.read()
check_no_read_lock(resource_db)
finally:
ds.close()
@requires_blob
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 2])
async def test_writable_blob_closes_before_transaction_commits(
resource_db, num_sql_threads
):
ds = Datasette([resource_db], settings={"num_sql_threads": num_sql_threads})
db = ds.get_database("resources")
saved = {}
def write(conn):
saved["blob"] = blob = conn.blobopen("t", "data", 1)
blob.write(b"xy")
blob[2] = ord("z")
blob[3:] = b"123"
# Leaving this BLOB open on SQLite makes the enclosing COMMIT fail
# with "SQL statements in progress"; expiry must close it first.
return "written"
try:
assert await db.execute_write_fn(write) == "written"
assert (
await db.execute("select data from t where id=1")
).single_value() == b"xyz123"
with pytest.raises(ConnectionLeaseError):
saved["blob"].read()
finally:
ds.close()
@requires_blob
@pytest.mark.asyncio
async def test_blob_context_manager_returns_lease(resource_db):
ds = Datasette([resource_db])
saved = {}
def write(conn):
with conn.blobopen("t", "data", 1) as blob:
saved["blob"] = blob
blob.write(b"xyz")
with pytest.raises(sqlite3.ProgrammingError):
blob.read()
try:
await ds.get_database("resources").execute_write_fn(write)
with pytest.raises(ConnectionLeaseError):
saved["blob"].read()
finally:
ds.close()
@requires_blob
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", [False, True])
async def test_returning_blob_from_read_callback_is_rejected(resource_db, nested):
ds = Datasette([resource_db])
def callback(conn):
blob = conn.blobopen("t", "data", 1, readonly=True)
return {"blob": [blob]} if nested else blob
try:
with pytest.raises(ConnectionLeaseError, match="database resources"):
await ds.get_database("resources").execute_fn(callback)
check_no_read_lock(resource_db)
finally:
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["read", "write", "isolated"])
@pytest.mark.parametrize(
"resource", ["dump", pytest.param("blob", marks=requires_blob)]
)
async def test_resources_close_when_callback_raises(resource_db, kind, resource):
ds = Datasette([resource_db])
saved = {}
def fail(conn):
if resource == "dump":
saved["resource"] = value = conn.iterdump()
for statement in value:
if statement.startswith('INSERT INTO "t"'):
break
else:
saved["resource"] = conn.blobopen("t", "data", 1, readonly=True)
raise ValueError("callback failed")
try:
with pytest.raises(ValueError, match="callback failed"):
await callback_method(ds.get_database("resources"), kind)(fail)
with pytest.raises(ConnectionLeaseError):
if resource == "dump":
next(saved["resource"])
else:
saved["resource"].read()
check_no_read_lock(resource_db)
finally:
ds.close()

View file

@ -0,0 +1,862 @@
"""
Regression tests for problems found by the second review of the connection
management redesign (concurrency, lifecycle and public-contract findings).
Each test names the problem it pins down.
"""
import asyncio
import gc
import os
import sqlite3
import threading
import time
import weakref
import pytest
from datasette import schema_watcher as schema_watcher_module
from datasette.app import Datasette
from datasette.connection_pool import ConnectionLeaseError
from datasette.database import Database
def make_db(path, sql="create table t (id integer primary key, v text)"):
conn = sqlite3.connect(path)
conn.executescript(sql)
conn.commit()
conn.close()
async def catalog_tables(ds, name):
rows = await ds.get_internal_database().execute(
"select table_name from catalog_tables where database_name = ? "
"order by table_name",
[name],
)
return [r[0] for r in rows.rows]
def live_tables(path):
conn = sqlite3.connect(path)
try:
return sorted(
r[0]
for r in conn.execute(
"select name from sqlite_master where type = 'table' "
"and name not like 'sqlite_%'"
)
)
finally:
conn.close()
def run_in_threads(*coroutine_functions, timeout=20):
"""Run each coroutine function with asyncio.run() in its own thread:
several event loops driving one Datasette at the same time."""
results = [None] * len(coroutine_functions)
def runner(i, fn):
try:
results[i] = ("ok", asyncio.run(fn()))
except BaseException as e: # noqa: BLE001
results[i] = ("error", e)
threads = [
threading.Thread(target=runner, args=(i, fn), daemon=True)
for i, fn in enumerate(coroutine_functions)
]
for thread in threads:
thread.start()
deadline = time.monotonic() + timeout
for thread in threads:
thread.join(max(0.0, deadline - time.monotonic()))
hung = [i for i, thread in enumerate(threads) if thread.is_alive()]
assert not hung, f"event loop threads {hung} did not finish within {timeout}s"
for status, value in results:
if status == "error":
raise value
return [value for _, value in results]
@pytest.fixture
def owned_db(tmp_path):
path = str(tmp_path / "data.db")
make_db(path)
ds = Datasette(settings={"schema_watch_interval_ms": 0})
db = ds.add_database(Database(ds, path=path), name="data", schema_watch="owned")
asyncio.run(ds.invoke_startup())
yield ds, db, path
ds.close()
# ----------------------------------------------------------------------
# Concurrency and lifecycle
# ----------------------------------------------------------------------
def test_cross_loop_ddl_writes_do_not_hang(owned_db, monkeypatch):
# C1: refresh() treated a scan started on another *live* event loop as
# abandoned, stole it, and each loop then resolved or cleared the
# other's future: blocking writes on one loop never returned
ds, db, path = owned_db
original = schema_watcher_module.SchemaWatcher._scan_chunk_sync
def slow_scan(self, states):
time.sleep(0.2)
return original(self, states)
monkeypatch.setattr(
schema_watcher_module.SchemaWatcher, "_scan_chunk_sync", slow_scan
)
def writer(prefix, count):
async def run():
for i in range(count):
await asyncio.wait_for(
db.execute_write(f"create table {prefix}{i} (id integer)"), 10
)
return run
def inserter():
async def run():
async def worker():
for _ in range(20):
await asyncio.wait_for(
db.execute_write("insert into t (v) values ('x')"), 10
)
await asyncio.gather(*(worker() for _ in range(3)))
return run
run_in_threads(writer("a", 3), writer("b", 3), inserter())
async def check():
return await catalog_tables(ds, "data")
assert asyncio.run(check()) == live_tables(path)
state = db._watch_state
assert state.scan_future is None
assert not state.needs_scan
def test_scan_from_stopped_loop_is_taken_over(owned_db):
# The rule for taking over a scan: its loop has closed or stopped, so
# nothing will finish it. A loop stopped by run_until_complete()
# returning while a scan task is pending must not hang other loops.
ds, db, _path = owned_db
loop = asyncio.new_event_loop()
try:
gate = threading.Event()
original = schema_watcher_module.SchemaWatcher._scan_chunk_sync
def gated_scan(self, states):
gate.wait(5)
return original(self, states)
schema_watcher_module.SchemaWatcher._scan_chunk_sync = gated_scan
try:
async def start_scan():
# block=False: the scan task is spawned on this loop and
# left pending when run_until_complete() returns
await db.execute_write(
"create table stopped1 (id integer)", block=False
)
await asyncio.sleep(0.2)
loop.run_until_complete(start_scan())
finally:
schema_watcher_module.SchemaWatcher._scan_chunk_sync = original
gate.set()
async def other_loop():
await asyncio.wait_for(
db.execute_write("create table other1 (id integer)"), 10
)
return await catalog_tables(ds, "data")
tables = run_in_threads(other_loop)[0]
assert {"stopped1", "other1"} <= set(tables)
finally:
loop.close()
def test_one_poller_across_live_event_loops(tmp_path):
# C2: every switch between two live event loops started another
# poller task, each sweeping every interval
path = str(tmp_path / "ext.db")
make_db(path)
ds = Datasette([path], settings={"schema_watch_interval_ms": 100})
asyncio.run(ds.invoke_startup())
watcher = ds._schema_watcher
turns = [threading.Event(), threading.Event()]
done = threading.Event()
counts = {}
def client(me, other, rounds):
async def run():
for _ in range(rounds):
turns[me].wait(5)
turns[me].clear()
assert (await ds.client.get("/ext.json")).status_code == 200
turns[other].set()
if me == 0:
sweeps = watcher.counters["sweeps"]
await asyncio.sleep(1.0)
counts["sweeps_per_s"] = watcher.counters["sweeps"] - sweeps
counts["pollers"] = len([t for t in watcher._pollers if not t.done()])
done.set()
else:
done.wait(10)
return run
turns[0].set()
run_in_threads(client(0, 1, 10), client(1, 0, 10))
assert counts["pollers"] == 1
# One poller at a 100ms interval: about 10 sweeps a second, not 10 per
# loop switch
assert counts["sweeps_per_s"] <= 15
ds.close()
@pytest.mark.asyncio
async def test_failed_catalog_write_is_retried(owned_db, monkeypatch):
# C3: refresh() cleared needs_scan before scanning and never restored
# it, so an owned database lost a catalog update for good if the
# catalog write failed (internal database locked)
ds, db, _path = owned_db
original = schema_watcher_module.write_catalog_entries
locked = threading.Event()
locked.set()
def flaky(conn, entries):
if locked.is_set():
raise sqlite3.OperationalError("database is locked")
return original(conn, entries)
monkeypatch.setattr(schema_watcher_module, "write_catalog_entries", flaky)
await db.execute_write("create table added_while_locked (id integer)")
# Let the refresh task spawned by the write's notification fail too
for _ in range(5):
await asyncio.sleep(0.01)
state = db._watch_state
assert state.needs_scan
assert state in ds._schema_watcher._pending
assert await catalog_tables(ds, "data") == ["t"]
# While still locked, a request is not failed by the catalog update
state.retry_at = 0.0
assert (await ds.client.get("/data.json")).status_code == 200
assert state.retry_at > time.monotonic()
# Once unlocked, the next request retries (after the back-off)
locked.clear()
state.retry_at = 0.0
await ds.client.get("/data.json")
assert await catalog_tables(ds, "data") == ["added_while_locked", "t"]
response = await ds.client.get("/data.json")
assert {t["name"] for t in response.json()["tables"]} == {
"added_while_locked",
"t",
}
def test_block_false_write_from_closed_loop_reaches_catalog(owned_db):
# C3: the schema-change notification went to the caller's loop, which
# had closed: notified_version was recorded anyway and the change was
# never picked up
ds, db, path = owned_db
async def write():
await db.execute_write(
"create table added_nonblocking (id integer)", block=False
)
asyncio.run(write())
# Let the write thread finish (it delivers to a closed loop)
deadline = time.monotonic() + 5
while "added_nonblocking" not in live_tables(path):
assert time.monotonic() < deadline
time.sleep(0.01)
time.sleep(0.05)
async def later():
await ds.client.get("/data.json")
return await catalog_tables(ds, "data")
assert asyncio.run(later()) == ["added_nonblocking", "t"]
def test_unclosed_datasette_is_collected():
# C4: the temp internal database's atexit.register(bound method) pinned
# every Datasette even after the caller released its last reference.
ds = Datasette()
ref = weakref.ref(ds)
del ds
gc.collect()
assert ref() is None
def test_closed_datasette_is_collected(tmp_path):
# C4 + C5: atexit and the read-pool reaper's loop variable both kept a
# closed Datasette (and every Database, cache and plugin state) alive
path = str(tmp_path / "data.db")
make_db(path)
ds = Datasette([path], settings={"connection_idle_timeout_ms": 20})
async def use(ds):
await ds.invoke_startup()
await ds.get_database("data").execute("select 1")
await ds.get_database("data").execute_write("insert into t (v) values (1)")
asyncio.run(use(ds))
# Let the reaper scan (and close) the idle connection
time.sleep(0.3)
ds.close()
ref = weakref.ref(ds)
del ds
gc.collect()
assert ref() is None
@pytest.mark.asyncio
async def test_write_callback_cannot_close_write_connection(owned_db):
# C6: closing the write lease closed the write thread's connection;
# every later write failed until the thread idled out
_ds, db, _path = owned_db
def close_it(conn):
conn.close()
with pytest.raises(ConnectionLeaseError):
await db.execute_write_fn(close_it)
await db.execute_write("insert into t (v) values ('after')")
# cursor.connection has the same lease protections as the connection.
def close_raw(conn):
conn.execute("select 1").connection.close()
with pytest.raises(ConnectionLeaseError):
await db.execute_write_fn(close_raw)
for _ in range(3):
await db.execute_write("insert into t (v) values ('again')")
rows = await db.execute("select count(*) from t")
assert rows.first()[0] == 4
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", (3, 0))
async def test_isolated_lease_close_is_allowed(tmp_path, num_sql_threads):
path = str(tmp_path / "data.db")
make_db(path)
ds = Datasette([path], settings={"num_sql_threads": num_sql_threads})
db = ds.get_database("data")
await db.execute_isolated_fn(lambda conn: conn.close())
await db.execute_write("insert into t (v) values (1)")
ds.close()
class Abort(BaseException):
pass
@pytest.mark.asyncio
@pytest.mark.parametrize("exception_class", (Abort, SystemExit))
async def test_write_callback_base_exception_is_delivered(owned_db, exception_class):
# C7: a BaseException from a write callback killed the write thread
# without answering the caller, who waited forever
_ds, db, _path = owned_db
def boom(conn):
raise exception_class("stop")
with pytest.raises(RuntimeError) as excinfo:
await asyncio.wait_for(db.execute_write_fn(boom), 5)
assert isinstance(excinfo.value.__cause__, exception_class)
# Same for an isolated function
with pytest.raises(RuntimeError):
await asyncio.wait_for(db.execute_isolated_fn(boom), 5)
await asyncio.wait_for(db.execute_write("insert into t (v) values (1)"), 5)
# ----------------------------------------------------------------------
# Public contract
# ----------------------------------------------------------------------
def _fts_setup(path):
make_db(path, "create table secret (id integer primary key, body text)")
conn = sqlite3.connect(path)
conn.execute("insert into secret values (1, 'TOP SECRET PAYLOAD')")
conn.commit()
conn.close()
def _add_fts(path):
conn = sqlite3.connect(path)
conn.executescript("""
create virtual table secret_fts using fts5(body, content='secret', content_rowid='id');
insert into secret_fts(secret_fts) values ('rebuild');
""")
conn.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ("external", "owned"))
async def test_derived_table_rule_uses_current_schema(tmp_path, mode):
# Contract C1: the "external-content FTS tables require access to their
# content table" rule was decided from a catalog that had not seen an
# FTS table created outside Datasette: its rows were served to
# anonymous users (until the next poll, or for good for owned DBs)
path = str(tmp_path / "data.db")
_fts_setup(path)
ds = Datasette(
config={
"databases": {"data": {"tables": {"secret": {"allow": {"id": "root"}}}}}
},
settings={"default_allow_sql": False, "schema_watch_interval_ms": 0},
)
ds.add_database(Database(ds, path=path), name="data", schema_watch=mode)
await ds.invoke_startup()
assert (await ds.client.get("/data/secret.json")).status_code == 403
_add_fts(path)
response = await ds.client.get("/data/secret_fts.json")
assert response.status_code == 403
assert "TOP SECRET" not in response.text
# Finding the catalog behind schedules a rescan
await ds.client.get("/data.json")
assert "secret_fts" in await catalog_tables(ds, "data")
ds.close()
@pytest.mark.asyncio
async def test_add_database_file_changes_are_picked_up(tmp_path):
# Contract C2: add_database() defaulted to "owned" (never polled), so a
# table created by another process never appeared in listings
path = str(tmp_path / "added.db")
make_db(path)
ds = Datasette(settings={"schema_watch_interval_ms": 50})
await ds.invoke_startup()
ds.add_database(Database(ds, path=path), name="added")
await ds.client.get("/added.json")
make_db(path, "create table t_ext (id integer)")
async def listed():
response = await ds.client.get("/added.json")
return {t["name"] for t in response.json()["tables"]}
deadline = time.monotonic() + 5
while "t_ext" not in await listed():
assert time.monotonic() < deadline, "t_ext never appeared"
await asyncio.sleep(0.05)
ds.close()
@pytest.mark.asyncio
async def test_named_memory_database_out_of_band_changes(tmp_path):
# Contract C2: named in-memory databases could not be watched at all;
# datasette-app-support restores one with backup() into
# db.connect(write=True) and then shows it
ds = Datasette(settings={"schema_watch_interval_ms": 50})
await ds.invoke_startup()
memory = ds.add_memory_database("review_fixes_temporary", name="temporary")
await memory.execute_write("create table m0 (id integer)")
source_path = str(tmp_path / "backup.db")
make_db(source_path, "create table restored_table (id integer)")
source = sqlite3.connect(source_path)
target = memory.connect(write=True)
source.backup(target)
source.close()
target.close()
async def listed():
response = await ds.client.get("/temporary.json")
return {t["name"] for t in response.json()["tables"]}
deadline = time.monotonic() + 5
while "restored_table" not in await listed():
assert time.monotonic() < deadline, "restored_table never appeared"
await asyncio.sleep(0.05)
ds.close()
FTS_SETUP = """
create table docs (id integer primary key, body text);
create virtual table docs_fts using fts5(body, content='docs', content_rowid='id');
create trigger docs_ai after insert on docs begin
insert into docs_fts (rowid, body) values (new.id, new.body);
end;
create trigger docs_ad after delete on docs begin
insert into docs_fts (docs_fts, rowid, body) values ('delete', old.id, old.body);
end;
insert into docs (id, body) values (1, 'oldword');
"""
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", (3, 0))
async def test_write_connection_pins_recursive_triggers(tmp_path, num_sql_threads):
# Contract C4: INSERT OR REPLACE on a sqlite-utils style FTS table only
# cleaned up the old index entry once some code had wrapped the write
# connection in sqlite_utils.Database (recursive_triggers drift) - and
# the idle write thread's fresh connections lost the drift again
path = str(tmp_path / "fts.db")
make_db(path, FTS_SETUP)
ds = Datasette(
[path],
settings={
"num_sql_threads": num_sql_threads,
"connection_idle_timeout_ms": 50,
},
)
db = ds.get_database("fts")
for word in ("newword", "newerword"):
flag = await db.execute_write_fn(
lambda conn: conn.execute("PRAGMA recursive_triggers").fetchone()[0]
)
assert flag == 1
await db.execute_write(
"insert or replace into docs (id, body) values (1, ?)", [word]
)
# Let the idle write thread exit: the next write opens a new
# connection
await asyncio.sleep(0.3)
stale = await db.execute(
"select rowid from docs_fts where docs_fts match 'oldword OR newword'"
)
assert stale.rows == []
ds.close()
@pytest.mark.asyncio
async def test_new_memory_database_is_not_scanned_at_registration():
# Contract C5: add_memory_database() after startup scanned the
# shared-cache database straight away, racing plugins that fill it from
# their own connection (datasette-copy-to-memory's VACUUM INTO failed
# with "database table is locked")
ds = Datasette(settings={"schema_watch_interval_ms": 0})
await ds.invoke_startup()
watcher = ds._schema_watcher
scans = watcher.counters["scans"]
memory = ds.add_memory_database("review_fixes_copy", name="copy")
await memory.execute("select 1 + 1")
await asyncio.sleep(0.05)
assert watcher.counters["scans"] == scans
state = memory._watch_state
assert state.scan_future is None
assert state in watcher._pending
# Fill it from a connection of its own, as the plugin does. (The plugin
# uses VACUUM INTO from a connection opened with uri=True; a plain
# connection is simpler and does not depend on whether the SQLite build
# treats URI file names as URIs without that flag)
conn = sqlite3.connect("file:review_fixes_copy?mode=memory&cache=shared", uri=True)
conn.execute("create table copied (id integer)")
conn.commit()
conn.close()
# The next request scans it
response = await ds.client.get("/copy.json")
assert {t["name"] for t in response.json()["tables"]} == {"copied"}
ds.close()
@pytest.mark.asyncio
async def test_locked_memory_scan_is_retried(monkeypatch):
ds = Datasette(settings={"schema_watch_interval_ms": 0})
await ds.invoke_startup()
memory = ds.add_memory_database("review_fixes_locked", name="locked")
original = schema_watcher_module.collect_schema
locked = threading.Event()
locked.set()
def locked_scan(conn, name):
if locked.is_set():
raise sqlite3.OperationalError("database table is locked: sqlite_master")
return original(conn, name)
monkeypatch.setattr(schema_watcher_module, "collect_schema", locked_scan)
await memory.execute_write("create table m (id integer)")
for _ in range(5):
await asyncio.sleep(0.01)
state = memory._watch_state
# Transient: not recorded as a broken database
assert state.error is None
assert state.needs_scan
locked.clear()
state.retry_at = 0.0
await ds.client.get("/locked.json")
assert await catalog_tables(ds, "locked") == ["m"]
ds.close()
@pytest.mark.asyncio
async def test_stale_owned_catalog_is_not_persisted_at_close(tmp_path):
# Contract C6: close() stored a fresh fingerprint for an owned database
# whose catalog was stale, so a restart restored the stale catalog
path = str(tmp_path / "data.db")
internal = str(tmp_path / "internal.db")
make_db(path)
config = {"databases": {"data": {"schema_watch": "owned"}}}
async def tables_after_start(change_before_close=None):
ds = Datasette([path], internal=internal, config=config)
await ds.invoke_startup()
response = await ds.client.get("/data.json")
if change_before_close:
# Behind Datasette's back, after the last request: an owned
# database is not polled, so the catalog is now stale
change_before_close()
ds.close()
return [t["name"] for t in response.json()["tables"]]
assert await tables_after_start() == ["t"]
assert await tables_after_start(
lambda: make_db(path, "create table t2 (id integer)")
) == ["t"]
# The stale catalog was not persisted as current: the restart rescans
assert await tables_after_start() == ["t", "t2"]
@pytest.mark.asyncio
async def test_crossdb_memory_connections_are_reaped_and_refreshed(tmp_path):
# Contract C7: --crossdb _memory connections ATTACH up to ten files but
# were never counted, reaped or refreshed: the fds stayed open and a
# removed database stayed queryable
paths = []
for i in range(3):
path = str(tmp_path / f"db{i}.db")
make_db(path)
paths.append(path)
ds = Datasette(paths[:2], crossdb=True, settings={"connection_idle_timeout_ms": 50})
await ds.invoke_startup()
memory = ds.get_database("_memory")
assert (await memory.execute("select count(*) from db0.t")).first()[0] == 0
pool = ds._read_pool
assert pool.snapshot()["open"] >= 1
deadline = time.monotonic() + 5
while pool.snapshot()["open"]:
assert time.monotonic() < deadline, "crossdb connections were not reaped"
await asyncio.sleep(0.05)
# Added and removed databases are reflected immediately
await memory.execute("select 1")
ds.add_database(Database(ds, path=paths[2]), name="db2")
assert (await memory.execute("select count(*) from db2.t")).first()[0] == 0
ds.remove_database("db0")
with pytest.raises(sqlite3.OperationalError):
await memory.execute("select count(*) from db0.t")
ds.close()
@pytest.mark.asyncio
async def test_execute_fn_may_return_generator_over_fetched_rows():
# Contract C8: every generator was rejected, including harmless ones
ds = Datasette(memory=True)
db = ds.get_database("_memory")
def fetched(conn):
rows = conn.execute("select 1 union all select 2").fetchall()
return (row[0] for row in rows)
assert list(await db.execute_fn(fetched)) == [1, 2]
def live_cursor(conn):
return (row[0] for row in conn.execute("select 1"))
with pytest.raises(ConnectionLeaseError):
await db.execute_fn(live_cursor)
def closure(conn):
def rows():
yield from conn.execute("select 1")
return rows()
with pytest.raises(ConnectionLeaseError):
await db.execute_fn(closure)
ds.close()
# ----------------------------------------------------------------------
# Found by the torture harness (tests/connection_torture.py)
# ----------------------------------------------------------------------
@pytest.mark.asyncio
async def test_index_survives_database_removed_mid_request(tmp_path, monkeypatch):
# The index page looked databases up by name again after listing them:
# one removed meanwhile (remove_database())
# failed the whole page with a KeyError
from datasette.views import index as index_module
paths = []
for name in ("keep", "gone"):
path = str(tmp_path / f"{name}.db")
make_db(path)
paths.append(path)
ds = Datasette(paths)
await ds.invoke_startup()
original = index_module.catalog_summaries
async def remove_during(datasette, names):
result = await original(datasette, names)
if "gone" in datasette.databases:
datasette.remove_database("gone")
return result
monkeypatch.setattr(index_module, "catalog_summaries", remove_during)
response = await ds.client.get("/.json")
assert response.status_code == 200
assert [d["name"] for d in response.json()["databases"]] == ["keep"]
ds.close()
@pytest.mark.asyncio
async def test_index_and_databases_with_closed_or_deleted_database(tmp_path):
paths = []
for name in ("keep", "closed", "deleted"):
path = str(tmp_path / f"{name}.db")
make_db(path)
paths.append(path)
ds = Datasette(paths, settings={"schema_watch_interval_ms": 0})
await ds.invoke_startup()
# Closed by a plugin but still attached
ds.get_database("closed").close()
# Deleted from under Datasette before any sweep noticed
os.unlink(paths[2])
for path in ("/.json", "/", "/-/databases.json"):
response = await ds.client.get(path)
assert response.status_code == 200, (path, response.text[:300])
databases = (await ds.client.get("/-/databases.json")).json()
if isinstance(databases, dict):
databases = databases["databases"]
sizes = {d["name"]: d["size"] for d in databases}
assert sizes["deleted"] == 0
ds.close()
@pytest.mark.asyncio
async def test_cursors_are_closed_before_leaving_their_thread(owned_db):
# A cursor still references its connection's cached statement, and on
# Python < 3.12 freeing it resets that statement. execute_write_many()
# handed its cursor to the caller's event loop thread; freed there while
# the write thread ran the same SQL again, the write failed with
# "bad parameter or other API misuse"
_ds, db, _path = owned_db
cursor = await db.execute_write_many(
"insert into t (v) values (?)", [("a",), ("b",)]
)
assert cursor.rowcount == 2
with pytest.raises(ConnectionLeaseError):
cursor.fetchall()
cursor = await db.execute_write_fn(
lambda conn: conn.execute("insert into t (v) values ('c')")
)
assert cursor.lastrowid == 3
with pytest.raises(ConnectionLeaseError):
cursor.fetchone()
# Cursors held by the frames of an exception raised by a callback are
# closed before the connection goes back to the pool (or write thread)
captured = []
def read_then_fail(conn):
cursor = conn.execute("select * from t")
captured.append(cursor)
raise ValueError("boom")
with pytest.raises(ValueError):
await db.execute_fn(read_then_fail)
with pytest.raises(ValueError):
await db.execute_write_fn(read_then_fail)
for cursor in captured:
with pytest.raises(ConnectionLeaseError):
cursor.fetchone()
def test_startup_from_several_event_loops_runs_once(monkeypatch):
# _startup_sequence() serialized startup with an asyncio.Lock: callers
# on a second event loop got "is bound to a different event loop" (or,
# free-threaded, waited forever), and invoke_startup() itself let two
# loops run every startup hook twice
ds = Datasette(memory=True)
calls = []
original = Datasette._apply_column_types_config
async def slow(self):
calls.append(threading.current_thread().name)
await asyncio.sleep(0.3)
return await original(self)
monkeypatch.setattr(Datasette, "_apply_column_types_config", slow)
def starter(use_sequence):
async def run():
if use_sequence:
await ds._startup_sequence()
else:
await ds.invoke_startup()
assert ds._startup_invoked
return (await ds.client.get("/_memory.json")).status_code
return run
statuses = run_in_threads(starter(True), starter(True), starter(False))
assert statuses == [200, 200, 200]
assert len(calls) == 1
ds.close()
@pytest.mark.asyncio
async def test_startup_hook_can_make_requests():
# A request made from inside startup (same task) must not wait for
# startup to finish - that would wait forever
from datasette import hookimpl
from datasette.plugins import pm
statuses = []
class Plugin:
__name__ = "StartupRequestPlugin"
@hookimpl
def startup(self, datasette):
async def inner():
statuses.append(
(await datasette.client.get("/-/versions.json")).status_code
)
return inner
pm.register(Plugin(), name="startup_request_plugin")
try:
ds = Datasette(memory=True)
await asyncio.wait_for(ds.invoke_startup(), 10)
assert statuses == [200]
ds.close()
finally:
pm.unregister(name="startup_request_plugin")
def test_execute_write_many_from_several_event_loops(owned_db):
# T1 end to end: on Python 3.11 before the fix, dozens of these calls
# failed with "bad parameter or other API misuse" (3 of 3 runs)
_ds, db, _path = owned_db
errors = []
def loop(i):
async def run():
async def worker(t):
for seq in range(60):
try:
await db.execute_write_many(
"insert into t (v) values (?)", [(f"{i}-{t}-{seq}",)]
)
except Exception as e: # noqa: BLE001
errors.append(repr(e))
await asyncio.gather(*(worker(t) for t in range(5)))
return run
run_in_threads(loop(0), loop(1), loop(2))
assert errors == []

View file

@ -0,0 +1,75 @@
"""
A fast, fixed-seed subset of the connection torture harness
(tests/connection_torture.py). Each scenario runs in its own process, so a
segfault or a hang fails one test instead of killing the test run.
Several event loops (threads) drive one Datasette with extreme settings
(smallest pool, 1ms idle timeout, 10ms schema polling) through reads,
writes of every kind, DDL, isolated functions, analyze_sql, cancellation,
add/remove_database, external schema changes, files
replaced and deleted, Database.close() and Datasette.close() mid-flight,
and non-threaded mode. See the harness docstring for what is checked.
"""
import json
import os
import signal
import subprocess
import sys
import pytest
HARNESS = os.path.join(os.path.dirname(__file__), "connection_torture.py")
@pytest.mark.parametrize(
"scenario,seed",
[
("mixed", 101),
("lifecycle", 102),
("close", 103),
("cancel", 104),
("pool", 105),
("nothreads", 106),
("writes", 107),
],
)
def test_connection_torture(scenario, seed):
try:
proc = subprocess.run(
[
sys.executable,
HARNESS,
scenario,
"--seed",
str(seed),
"--duration",
"1.5",
],
capture_output=True,
text=True,
timeout=150,
check=False,
)
except subprocess.TimeoutExpired as e: # pragma: no cover
pytest.fail(f"{scenario} hung:\n{(e.stderr or '')[-4000:]}")
if proc.returncode < 0: # pragma: no cover
pytest.fail(
f"{scenario} crashed with {signal.Signals(-proc.returncode).name}:\n"
f"{proc.stderr[-4000:]}"
)
lines = proc.stdout.strip().splitlines()
if not lines:
pytest.fail(
f"{scenario} exited {proc.returncode} without a summary:\n"
f"{proc.stderr[-4000:]}"
)
summary = json.loads(lines[-1])
problems = "\n".join(
f"[{p['kind']}] {p['message'][:600]}" for p in summary["problems"]
)
assert proc.returncode == 0, f"{scenario} seed {seed}:\n{problems}"
assert summary["ok"]
# It did something (the slow-query scenarios do a few dozen operations
# in 1.5s on a busy machine, the others hundreds)
assert sum(summary["counts"].values()) > 10

View file

@ -95,6 +95,11 @@ async def test_derived_permission_discovery_error_is_retried(monkeypatch, listin
action="view-table", resource=TableResource("data", "documents")
)
# While the catalog is current, dependencies come from its rows and
# nothing is discovered live. Live discovery - the path that can fail -
# happens while a rescan of the database is pending.
db._watch_state.needs_scan = True
db._cached_derived_table_dependencies = None
token = _permission_check_cache.set({})
try:
with monkeypatch.context() as patch:

View file

@ -29,7 +29,9 @@ async def test_internal_migrations_recorded(ds_client):
order by id
""")
assert [tuple(row) for row in migrations.rows] == [
("datasette_internal", "0001_initial")
("datasette_internal", "0001_initial"),
("datasette_internal", "0002_catalog_fingerprint"),
("datasette_internal", "0003_catalog_table_type"),
]
@ -61,7 +63,9 @@ async def test_internal_migrations_adopt_existing_internal_db(tmp_path):
order by id
""")
assert [tuple(row) for row in migrations.rows] == [
("datasette_internal", "0001_initial")
("datasette_internal", "0001_initial"),
("datasette_internal", "0002_catalog_fingerprint"),
("datasette_internal", "0003_catalog_table_type"),
]
ds.close()
@ -73,7 +77,13 @@ async def test_internal_tables(ds_client):
tables = await internal_db.execute("select * from catalog_tables")
assert len(tables) > 5
table = tables.rows[0]
assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"}
assert set(table.keys()) == {
"rootpage",
"table_name",
"database_name",
"sql",
"type",
}
@pytest.mark.asyncio

View file

@ -13,6 +13,7 @@ from opentelemetry import context as otel_context_api
from datasette.app import Datasette
from datasette.database import (
ConnectionLeaseError,
Database,
DatasetteClosedError,
ExecuteWriteResult,
@ -1263,10 +1264,11 @@ async def test_database_close_shuts_down_write_thread(tmpdir):
await db.execute_write("insert into t (id) values (1)")
assert db._write_thread is not None
assert db._write_thread.is_alive()
thread = db._write_thread
db.close()
# Wait briefly for the thread to exit — the sentinel should cause it to return.
db._write_thread.join(timeout=5)
assert not db._write_thread.is_alive()
thread.join(timeout=5)
assert not thread.is_alive()
assert db._write_thread is None
ds._internal_database.close()
@ -1312,10 +1314,18 @@ async def test_database_close_is_idempotent(tmpdir):
async def test_close_releases_memory_connections(num_sql_threads, named):
ds = Datasette(memory=True, settings={"num_sql_threads": num_sql_threads})
db = ds.add_memory_database(uuid.uuid4().hex) if named else ds.get_database()
read_connection = await db.execute_fn(lambda conn: conn)
write_connection = await db.execute_write_fn(lambda conn: conn)
leased = await db.execute_fn(lambda conn: conn)
leased_write = await db.execute_write_fn(lambda conn: conn)
# Inspect the owned raw connections to verify physical closure as well
# as lease expiry. cursor.connection intentionally no longer leaks them.
connections = list(db._all_connections)
assert len(connections) >= 2
with pytest.raises(ConnectionLeaseError):
leased.execute("select 1")
with pytest.raises(ConnectionLeaseError):
leased_write.execute("select 1")
ds.close()
for conn in (read_connection, write_connection):
for conn in connections:
with pytest.raises(sqlite3.ProgrammingError, match="closed"):
conn.execute("select 1")

View file

@ -0,0 +1,116 @@
import sqlite3
import uuid
import pytest
from datasette.app import Datasette
from datasette.write_budget import DatabaseResourceError
@pytest.mark.asyncio
@pytest.mark.parametrize("threads", [0, 2])
@pytest.mark.parametrize("raise_after_commit", [False, True])
async def test_first_isolated_memory_write_keeps_committed_data(
threads, raise_after_commit
):
ds = Datasette(settings={"num_sql_threads": threads, "schema_watch_interval_ms": 0})
db = ds.add_memory_database(uuid.uuid4().hex)
try:
def create(conn):
conn.execute("create table t(id)")
conn.execute("insert into t values (123)")
conn.commit()
if raise_after_commit:
raise ValueError("after commit")
return conn.execute("select id from t").fetchone()[0]
if raise_after_commit:
with pytest.raises(ValueError, match="after commit"):
await db.execute_isolated_fn(create)
else:
assert await db.execute_isolated_fn(create) == 123
# Only the keeper remains after the isolated callback, on both
# success and failure. Do not open a reader before checking this.
assert len(db._all_connections) == 1
assert (await db.execute("select id from t")).single_value() == 123
assert (
await db.execute_isolated_fn(
lambda conn: conn.execute("select id from t").fetchone()[0]
)
== 123
)
await db.execute_write("insert into t values (456)")
assert (await db.execute("select count(*) from t")).single_value() == 2
connections = list(db._all_connections)
finally:
ds.close()
for conn in connections:
with pytest.raises(sqlite3.ProgrammingError):
conn.execute("select 1")
@pytest.mark.asyncio
@pytest.mark.parametrize("threads", [0, 2])
async def test_isolated_memory_keeper_occupies_writer_slot(threads, tmp_path):
path = tmp_path / "file.db"
conn = sqlite3.connect(path)
conn.execute("create table t(id)")
conn.close()
ds = Datasette(
[str(path)],
settings={
"num_sql_threads": threads,
"max_write_connections": 1,
"write_queue_timeout_ms": 50,
"schema_watch_interval_ms": 0,
},
)
memory = ds.add_memory_database(uuid.uuid4().hex)
file_db = ds.get_database("file")
try:
await memory.execute_isolated_fn(
lambda conn: conn.execute("create table t(id)")
)
with pytest.raises(DatabaseResourceError):
await file_db.execute_write("insert into t values (1)")
assert "t" in await memory.table_names()
memory.close()
await file_db.execute_write("insert into t values (1)")
finally:
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("threads", [0, 2])
async def test_keeper_preparation_failure_does_not_run_isolated_callback(
threads, monkeypatch
):
ds = Datasette(
settings={
"num_sql_threads": threads,
"max_write_connections": 1,
"schema_watch_interval_ms": 0,
}
)
db = ds.add_memory_database(uuid.uuid4().hex)
original = ds._prepare_connection
ran = []
def prepare(conn, name):
if name == db.name:
raise ValueError("keeper failed")
return original(conn, name)
try:
with monkeypatch.context() as patch:
patch.setattr(ds, "_prepare_connection", prepare)
with pytest.raises(ValueError, match="keeper failed"):
await db.execute_isolated_fn(lambda conn: ran.append(True))
assert not ran
assert not db._all_connections
db.close()
another = ds.add_memory_database(uuid.uuid4().hex)
await another.execute_write("create table t(id)")
finally:
ds.close()

View file

@ -215,18 +215,15 @@ async def test_concurrent_first_requests_all_wait_for_slow_startup():
@pytest.mark.asyncio
async def test_setup_db_still_runs_when_invoke_startup_ran_first(tmp_path, monkeypatch):
# Regression test: `datasette serve` (cli.py _serve_async) calls
# ds.invoke_startup() directly, before uvicorn ever sends a
# lifespan.startup event that drives _startup_sequence(). If
# _startup_sequence()'s fast path only checked `_startup_invoked`, it
# would see startup already done and skip the immutable-database
# table-count precompute (setup_db) entirely - a silent regression
# versus main, where AsgiRunOnFirstRequest ran setup_db unconditionally
# on request #1.
async def test_immutable_table_counts_computed_when_first_needed(tmp_path, monkeypatch):
# Immutable databases used to have their table counts precomputed by
# _startup_sequence(), opening every immutable database at startup.
# They are now computed the first time a page needs them, with the same
# generous time limit, and cached.
db_path = tmp_path / "immutable.db"
conn = sqlite3.connect(str(db_path))
conn.execute("create table t (id integer primary key)")
conn.execute("insert into t default values")
conn.commit()
conn.close()
@ -241,22 +238,22 @@ async def test_setup_db_still_runs_when_invoke_startup_ran_first(tmp_path, monke
monkeypatch.setattr(Database, "table_counts", counting_table_counts)
# Simulate the CLI path: invoke_startup() runs directly and completes
# BEFORE _startup_sequence() ever gets a chance to run setup_db.
await ds.invoke_startup()
await ds._startup_sequence()
assert ds._startup_invoked is True
assert call_count["n"] == 0
# The lifespan/first-request path (or the CLI itself, per the fix)
# calling the shared entry point afterwards must still precompute
# table counts for immutable databases.
await ds._startup_sequence()
assert call_count["n"] == 1
assert ds._setup_db_done is True
assert call_count["n"] == 0
assert ds.get_database("immutable").cached_table_counts is None
# Idempotency: a second call must not recompute.
await ds._startup_sequence()
response = await ds.client.get("/.json")
assert response.json()["databases"][0]["table_rows_sum"] == 1
assert call_count["n"] == 1
assert ds.get_database("immutable").cached_table_counts == {"t": 1}
# Cached from then on
await ds.client.get("/immutable.json")
assert ds.get_database("immutable").cached_table_counts == {"t": 1}
ds.close()
@pytest.mark.asyncio

View file

@ -48,6 +48,11 @@ def test_plugin_hooks_have_tests(plugin_hook):
assert ok, f"Plugin hook is missing tests: {plugin_hook}"
def test_hook_write_task_completed():
# Success and expiry delivery are exercised in tests/test_write_budget.py.
assert "write_task_completed" in dir(pm.hook)
def test_hook_jump_items_sql():
# Detailed behavior is covered in tests/test_jump.py.
assert "jump_items_sql" in dir(pm.hook)

378
tests/test_read_pool.py Normal file
View file

@ -0,0 +1,378 @@
import asyncio
import sqlite3
import threading
import time
import pytest
import sqlite_utils
from datasette.app import Datasette
from datasette.connection_pool import ConnectionLeaseError
def _make_dbs(tmp_path, n):
paths = []
for i in range(n):
path = tmp_path / f"db{i}.db"
conn = sqlite3.connect(path)
conn.execute("create table t (id integer primary key, v text)")
conn.execute("insert into t (v) values ('x')")
conn.commit()
conn.close()
paths.append(str(path))
return paths
def _file_conns(db):
return [c for c in db._all_connections]
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 2])
async def test_lease_expires_after_callback(tmp_path, num_sql_threads):
(path,) = _make_dbs(tmp_path, 1)
ds = Datasette([path], settings={"num_sql_threads": num_sql_threads})
db = ds.get_database("db0")
stash = {}
def callback(conn):
stash["conn"] = conn
assert isinstance(conn, sqlite3.Connection)
return conn.execute("select count(*) from t").fetchone()[0]
assert await db.execute_fn(callback) == 1
with pytest.raises(ConnectionLeaseError, match="db0"):
stash["conn"].execute("select 1")
with pytest.raises(ConnectionLeaseError):
stash["conn"].row_factory = None
# A second lease of the same underlying connection gets a new proxy, so
# the stale one still fails while another callback is using it
seen = {}
def callback2(conn):
with pytest.raises(ConnectionLeaseError):
stash["conn"].execute("select 1")
seen["ok"] = True
await db.execute_fn(callback2)
assert seen["ok"]
ds.close()
@pytest.mark.asyncio
async def test_returning_cursor_raises(tmp_path):
(path,) = _make_dbs(tmp_path, 1)
ds = Datasette([path])
db = ds.get_database("db0")
with pytest.raises(ConnectionLeaseError, match="returned a Cursor"):
await db.execute_fn(lambda conn: conn.execute("select * from t"))
# Pool is still usable afterwards
assert (await db.execute("select count(*) from t")).single_value() == 1
ds.close()
@pytest.mark.asyncio
async def test_leased_connection_compatibility(tmp_path):
(path,) = _make_dbs(tmp_path, 1)
ds = Datasette([path])
db = ds.get_database("db0")
def callback(conn):
out = {}
out["tables"] = sqlite_utils.Database(conn).table_names()
out["columns"] = list(sqlite_utils.Database(conn)["t"].columns_dict)
target = sqlite3.connect(":memory:")
conn.backup(target)
out["backup"] = target.execute("select count(*) from t").fetchone()[0]
conn.create_function("double", 1, lambda x: x * 2)
out["fn"] = conn.execute("select double(21)").fetchone()[0]
cursor = conn.cursor()
cursor.execute("select v from t")
out["cursor"] = cursor.fetchall()[0][0]
return out
assert await db.execute_fn(callback) == {
"tables": ["t"],
"columns": ["id", "v"],
"backup": 1,
"fn": 42,
"cursor": "x",
}
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 3])
async def test_global_cap_evicts_least_recently_used(tmp_path, num_sql_threads):
paths = _make_dbs(tmp_path, 6)
ds = Datasette(paths, settings={"num_sql_threads": num_sql_threads})
pool = ds._read_pool
# Below the 4 x num_sql_threads clamp, so set the private attribute
pool.max_open = 2
for i in range(6):
await ds.get_database(f"db{i}").execute("select 1")
assert pool.snapshot()["open"] <= 2
# The two most recently used databases are the ones still open
open_dbs = {
name
for name in (f"db{i}" for i in range(6))
if ds.get_database(name)._read_pool_state.open
}
assert open_dbs == {"db4", "db5"}
assert pool.stats["evicted_lru"] >= 4
for i in range(6):
db = ds.get_database(f"db{i}")
assert len(_file_conns(db)) == db._read_pool_state.open
ds.close()
@pytest.mark.asyncio
async def test_prepare_connection_runs_for_every_pooled_connection(tmp_path):
paths = _make_dbs(tmp_path, 3)
ds = Datasette(paths)
ds._read_pool.max_open = 1
calls = []
original = ds._prepare_connection
def counting(conn, database):
calls.append(database)
return original(conn, database)
ds._prepare_connection = counting
# Pool may have been created already with the original bound method
ds._read_pool._prepare_connection = counting
for _ in range(2):
for i in range(3):
assert (
await ds.get_database(f"db{i}").execute("select v from t")
).single_value() == "x"
# Cap of 1 means every switch of database reopens and re-prepares
assert calls == ["db0", "db1", "db2"] * 2
ds.close()
def test_idle_reaper_closes_connections_without_event_loop(tmp_path):
paths = _make_dbs(tmp_path, 3)
ds = Datasette(paths, settings={"num_sql_threads": 0})
pool = ds._read_pool
pool.idle_timeout = 0.2
async def query_all():
for i in range(3):
await ds.get_database(f"db{i}").execute("select 1")
asyncio.run(query_all())
assert pool.snapshot()["open"] == 3
# No event loop is running now - the reaper thread does the work
deadline = time.monotonic() + 5
while pool.snapshot()["open"] and time.monotonic() < deadline:
time.sleep(0.05)
assert pool.snapshot()["open"] == 0
assert pool.stats["expired"] == 3
for i in range(3):
assert _file_conns(ds.get_database(f"db{i}")) == []
ds.close()
@pytest.mark.parametrize(
"num_sql_threads,configured,expected",
[
(3, 128, 128),
(3, 5, 5),
(8, 20, 20),
(0, 1, 1),
(3, 0, 0),
],
)
def test_max_open_connections_respects_configured_limit(
num_sql_threads, configured, expected
):
ds = Datasette(
settings={
"num_sql_threads": num_sql_threads,
"max_open_connections": configured,
}
)
assert ds._read_pool.max_open == expected
# /-/settings.json reports the configured value
assert ds.setting("max_open_connections") == configured
ds.close()
def test_connection_idle_timeout_ms_setting():
ds = Datasette(settings={"connection_idle_timeout_ms": 1500})
assert ds._read_pool.idle_timeout == 1.5
ds.close()
ds = Datasette(settings={"connection_idle_timeout_ms": 0})
assert ds._read_pool.idle_timeout == 0
ds.close()
@pytest.mark.asyncio
async def test_hard_cap_when_every_connection_is_leased(tmp_path):
paths = _make_dbs(tmp_path, 3)
ds = Datasette(paths, settings={"num_sql_threads": 3, "max_open_connections": 1})
entered = threading.Event()
release = threading.Event()
def slow(conn):
entered.set()
assert release.wait(5)
return conn.execute("select 1").fetchone()[0]
tasks = [
asyncio.create_task(ds.get_database(f"db{i}").execute_fn(slow))
for i in range(3)
]
try:
assert await asyncio.to_thread(entered.wait, 5)
await asyncio.sleep(0.05)
assert ds._read_pool.snapshot()["open"] == 1
assert not any(task.done() for task in tasks)
release.set()
assert await asyncio.gather(*tasks) == [1, 1, 1]
assert ds._read_pool.stats["peak_open"] == 1
finally:
release.set()
await asyncio.gather(*tasks, return_exceptions=True)
ds.close()
@pytest.mark.asyncio
async def test_memory_databases_not_counted(tmp_path):
ds = Datasette(memory=True, settings={"max_open_connections": 1})
db = ds.add_memory_database("pool_mem_test")
await db.execute_write("create table t (id integer)")
await db.execute_write("insert into t values (1)")
await ds.get_database("_memory").execute("select 1")
assert (await db.execute("select count(*) from t")).single_value() == 1
assert ds._read_pool.snapshot()["open"] == 0
ds.close()
@pytest.mark.asyncio
async def test_database_close_with_leased_connection(tmp_path):
(path,) = _make_dbs(tmp_path, 1)
ds = Datasette([path], settings={"num_sql_threads": 0})
db = ds.get_database("db0")
await db.execute("select 1")
def closes_database(conn):
conn.execute("select 1")
db.close()
return "done"
assert await db.execute_fn(closes_database) == "done"
assert _file_conns(db) == []
assert ds._read_pool.snapshot()["open"] == 0
ds.close()
@pytest.mark.asyncio
async def test_callback_that_closes_connection_is_discarded(tmp_path):
(path,) = _make_dbs(tmp_path, 1)
ds = Datasette([path])
db = ds.get_database("db0")
await db.execute_fn(lambda conn: conn.close())
assert ds._read_pool.stats["discarded_broken"] == 1
assert (await db.execute("select count(*) from t")).single_value() == 1
ds.close()
@pytest.mark.asyncio
async def test_open_transaction_rolled_back_on_release(tmp_path):
(path,) = _make_dbs(tmp_path, 1)
ds = Datasette([path], settings={"num_sql_threads": 1})
db = ds.get_database("db0")
def begin(conn):
conn.execute("begin")
conn.execute("select * from t").fetchall()
await db.execute_fn(begin)
assert ds._read_pool.stats["rolled_back"] == 1
assert not await db.execute_fn(lambda conn: conn.in_transaction)
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", [0, 2])
async def test_invalidate_database_discards_stale_connections(
tmp_path, num_sql_threads
):
paths = _make_dbs(tmp_path, 2)
ds = Datasette(paths, settings={"num_sql_threads": num_sql_threads})
db0, db1 = ds.get_database("db0"), ds.get_database("db1")
pool = ds._read_pool
await db0.execute("select 1")
await db1.execute("select 1")
old = db0._read_pool_state.idle[-1].conn
assert db0._read_pool_state.open == 1
# What the SchemaWatcher does when it sees db0's file replaced
db0._invalidate_connections()
# Idle connections for db0 are closed at once; db1 is untouched
assert db0._read_pool_state.open == 0
assert db1._read_pool_state.open == 1
assert old not in db0._all_connections
with pytest.raises(sqlite3.ProgrammingError):
old.execute("select 1")
# The next read opens a new connection
await db0.execute("select 1")
assert db0._read_pool_state.idle[-1].conn is not old
assert pool.stats["discarded_stale"] == 1
ds.close()
@pytest.mark.asyncio
async def test_leased_connection_discarded_on_release_after_invalidation(tmp_path):
(path,) = _make_dbs(tmp_path, 1)
ds = Datasette([path], settings={"num_sql_threads": 2})
db = ds.get_database("db0")
pool = ds._read_pool
seen = {}
def callback(conn):
seen["raw"] = conn.execute("select 1").connection
# The file is replaced while this callback holds the connection:
# it must not be closed under us, only discarded on release
db._invalidate_connections()
return conn.execute("select count(*) from t").fetchone()[0]
assert await db.execute_fn(callback) == 1
assert db._read_pool_state.open == 0
assert pool.stats["discarded_stale"] == 1
with pytest.raises(ConnectionLeaseError):
seen["raw"].execute("select 1")
assert (await db.execute("select count(*) from t")).single_value() == 1
ds.close()
def test_reaper_unavailable_falls_back_to_checkout_reaping(tmp_path, monkeypatch):
# Pyodide cannot start threads: idle connections must then be reaped by
# the next checkout instead, and reads must keep working
from datasette import connection_pool
reaper = connection_pool._Reaper()
monkeypatch.setattr(connection_pool, "_reaper", reaper)
def no_threads(self):
raise RuntimeError("can't start new thread")
monkeypatch.setattr(threading.Thread, "start", no_threads)
paths = _make_dbs(tmp_path, 2)
ds = Datasette(paths, settings={"num_sql_threads": 0})
pool = ds._read_pool
pool.idle_timeout = 0.05
async def read(name):
return (await ds.get_database(name).execute("select 1")).single_value()
assert asyncio.run(read("db0")) == 1
assert reaper.unavailable
assert pool.snapshot()["open"] == 1
time.sleep(0.1)
assert asyncio.run(read("db1")) == 1
# db0's expired connection was closed during db1's checkout
assert pool.stats["expired"] == 1
assert ds.get_database("db0")._read_pool_state.open == 0
ds.close()

View file

@ -0,0 +1,718 @@
"""
SchemaWatcher tests. Changes to "external" databases are made from a
separate OS process (a child Python interpreter using plain sqlite3, or the
sqlite-utils CLI), the way other tools change files Datasette is serving.
"""
import asyncio
import os
import sqlite3
import subprocess
import sys
import textwrap
import time
import pytest
from datasette import schema_watcher as schema_watcher_module
from datasette.app import Datasette
from datasette.database import Database
INTERVAL_MS = 50
def run_external(path, sql, journal_mode=None, keep_wal=False):
"""Run sql against path in a separate Python process; returns the
child's time.monotonic() right after COMMIT."""
script = textwrap.dedent("""
import sqlite3, sys, time
path, sql, journal_mode, keep_wal = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
conn = sqlite3.connect(path, isolation_level=None, timeout=10)
if journal_mode:
conn.execute("PRAGMA journal_mode=" + journal_mode)
if keep_wal == "1":
# Never checkpoint: changes stay in the -wal file
conn.execute("PRAGMA wal_autocheckpoint=0")
conn.executescript(sql)
print(time.monotonic(), flush=True)
if keep_wal == "1":
# Keep the connection open so close() does not checkpoint
sys.stdin.read()
conn.close()
""")
if keep_wal:
proc = subprocess.Popen(
[sys.executable, "-c", script, path, sql, journal_mode or "", "1"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
text=True,
)
line = proc.stdout.readline()
return float(line), proc
out = subprocess.run(
[sys.executable, "-c", script, path, sql, journal_mode or "", "0"],
check=True,
capture_output=True,
text=True,
)
return float(out.stdout.strip())
def run_process(*args):
subprocess.run([sys.executable, *args], check=True)
def make_db(path, sql="create table t (id integer primary key, a text)", wal=False):
conn = sqlite3.connect(path)
if wal:
conn.execute("PRAGMA journal_mode=wal")
conn.executescript(sql)
conn.execute("insert into t (a) values ('one')")
conn.commit()
conn.close()
async def catalog_columns(ds, db_name, table):
rows = await ds.get_internal_database().execute(
"select name from catalog_columns where database_name = ? and table_name = ? order by cid",
[db_name, table],
)
return [r[0] for r in rows.rows]
async def catalog_tables(ds, db_name):
rows = await ds.get_internal_database().execute(
"select table_name from catalog_tables where database_name = ? order by table_name",
[db_name],
)
return [r[0] for r in rows.rows]
async def wait_for(check, timeout=5.0):
t0 = time.monotonic()
while True:
result = await check()
if result:
return time.monotonic() - t0
if time.monotonic() - t0 > timeout:
raise AssertionError(f"timed out after {timeout}s")
await asyncio.sleep(0.01)
@pytest.fixture
def db_path(tmp_path):
path = str(tmp_path / "ext.db")
make_db(path)
return path
async def start_ds(paths, interval_ms=INTERVAL_MS, settings=None, **kwargs):
ds = Datasette(
files=paths,
settings={"schema_watch_interval_ms": interval_ms, **(settings or {})},
**kwargs,
)
await ds.invoke_startup()
await ds.start_background_tasks()
return ds
@pytest.mark.asyncio
async def test_modes_and_defaults(tmp_path, db_path):
immutable_path = str(tmp_path / "imm.db")
make_db(immutable_path)
ds = Datasette(files=[db_path], immutables=[immutable_path])
await ds.invoke_startup()
watcher = ds._schema_watcher
assert watcher.states["ext"].mode == "external"
assert watcher.states["imm"].mode == "immutable"
# add_database(): files and named in-memory databases can be changed
# by code other than Datasette's write methods (a plugin's own
# connection, a backup() into a named in-memory database), so they get
# the default_schema_watch mode too unless the caller says otherwise
added = ds.add_database(Database(ds, memory_name="sw_owned_mem"), name="added")
assert watcher.states["added"].mode == "external"
assert added._watch_state is watcher.states["added"]
added_file = str(tmp_path / "added_file.db")
make_db(added_file)
ds.add_database(Database(ds, path=added_file), name="added_file")
assert watcher.states["added_file"].mode == "external"
ds.add_database(
Database(ds, memory_name="sw_owned_mem2"), name="owned", schema_watch="owned"
)
assert watcher.states["owned"].mode == "owned"
# A private ":memory:" database: nothing else can change it
ds.add_database(Database(ds, is_memory=True), name="private")
assert watcher.states["private"].mode == "owned"
ds.close()
# default_schema_watch=owned applies to add_database() too
ds = Datasette(files=[db_path], settings={"default_schema_watch": "owned"})
ds.add_database(Database(ds, memory_name="sw_owned_mem3"), name="added")
assert ds._schema_watcher.states["added"].mode == "owned"
assert ds._schema_watcher.states["ext"].mode == "owned"
ds.close()
@pytest.mark.asyncio
async def test_mode_from_config(tmp_path, db_path):
ds = Datasette(
files=[db_path], config={"databases": {"ext": {"schema_watch": "owned"}}}
)
assert ds._schema_watcher.states["ext"].mode == "owned"
ds.close()
@pytest.mark.asyncio
async def test_no_request_path_sweep(db_path):
ds = await start_ds([db_path], interval_ms=0)
watcher = ds._schema_watcher
sweeps = watcher.counters["sweeps"]
for _ in range(5):
response = await ds.client.get("/ext/t.json")
assert response.status_code == 200
assert watcher.counters["sweeps"] == sweeps
ds.close()
@pytest.mark.asyncio
async def test_owned_write_path_detects_schema_change(tmp_path):
ds = Datasette(settings={"schema_watch_interval_ms": 0})
await ds.invoke_startup()
db = ds.add_database(Database(ds, memory_name="sw_owned"), name="owned")
await ds.refresh_schemas(force=True)
watcher = ds._schema_watcher
scans = watcher.counters["scans"]
# Data-only writes: no catalog rebuild
await db.execute_write("create table if not exists t (id integer primary key)")
assert await catalog_tables(ds, "owned") == ["t"]
scans = watcher.counters["scans"]
for i in range(5):
await db.execute_write("insert into t default values")
assert watcher.counters["scans"] == scans
# Each write API: catalog is current as soon as the call returns
await db.execute_write("create table t2 (id integer primary key)")
assert "t2" in await catalog_tables(ds, "owned")
await db.execute_write_script("create table t3 (id integer); create table t4 (id);")
assert {"t3", "t4"} <= set(await catalog_tables(ds, "owned"))
scans = watcher.counters["scans"]
await db.execute_write_many("insert into t2 (id) values (?)", [[1], [2]])
assert watcher.counters["scans"] == scans
def plugin_write(conn):
conn.execute("alter table t add column extra text")
await db.execute_write_fn(plugin_write)
assert await catalog_columns(ds, "owned", "t") == ["id", "extra"]
def isolated(conn):
conn.execute("create table t6 (id)")
await db.execute_isolated_fn(isolated)
assert "t6" in await catalog_tables(ds, "owned")
# block=False: refreshed in the background
await db.execute_write("create table t7 (id)", block=False)
await wait_for(lambda: _contains(ds, "owned", "t7"))
ds.close()
async def _contains(ds, db_name, table):
return table in await catalog_tables(ds, db_name)
@pytest.mark.asyncio
async def test_owned_write_path_non_threaded(tmp_path):
path = str(tmp_path / "nt.db")
make_db(path)
ds = Datasette(settings={"num_sql_threads": 0, "schema_watch_interval_ms": 0})
await ds.invoke_startup()
db = ds.add_database(Database(ds, path=path), name="nt")
await ds._schema_watcher.flush_pending()
await db.execute_write("create table t2 (id)")
assert await catalog_tables(ds, "nt") == ["t", "t2"]
ds.close()
@pytest.mark.asyncio
async def test_external_data_only_insert_does_not_rescan(db_path):
ds = await start_ds([db_path])
watcher = ds._schema_watcher
state = watcher.states["ext"]
scans = state.stats["scans"]
checks = state.stats["checks"]
for i in range(3):
run_external(db_path, f"insert into t (a) values ('x{i}')")
await asyncio.sleep(INTERVAL_MS * 3 / 1000)
# The fingerprint changed, so schema_version was checked...
assert state.stats["checks"] > checks
# ... but the catalog was not rebuilt
assert state.stats["scans"] == scans
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("use_header", (True, False))
async def test_external_add_column(db_path, monkeypatch, use_header):
monkeypatch.setattr(schema_watcher_module, "USE_HEADER_CHECK", use_header)
ds = await start_ds([db_path])
run_external(db_path, "alter table t add column b text")
async def has_b():
return await catalog_columns(ds, "ext", "t") == ["id", "a", "b"]
await wait_for(has_b)
ds.close()
@pytest.mark.asyncio
async def test_external_sqlite_utils_transform(db_path):
ds = await start_ds([db_path])
run_process(
"-m", "sqlite_utils", "transform", db_path, "t", "--rename", "a", "renamed"
)
async def renamed():
return await catalog_columns(ds, "ext", "t") == ["id", "renamed"]
await wait_for(renamed)
# And the table page uses the new column too
response = await ds.client.get("/ext/t.json?_shape=array")
assert response.json() == [{"id": 1, "renamed": "one"}]
ds.close()
@pytest.mark.asyncio
async def test_external_create_table_in_wal_before_checkpoint(tmp_path):
path = str(tmp_path / "walled.db")
make_db(path, wal=True)
ds = await start_ds([path])
_, proc = run_external(
path, "create table in_wal (id integer)", journal_mode="wal", keep_wal=True
)
try:
assert os.path.getsize(path + "-wal") > 0
async def seen():
return "in_wal" in await catalog_tables(ds, "walled")
await wait_for(seen)
# Main file still does not contain it: it was only in the WAL
assert os.path.exists(path + "-wal")
finally:
proc.stdin.close()
proc.wait()
ds.close()
@pytest.mark.asyncio
async def test_external_atomic_replace_by_rename(tmp_path, db_path):
ds = await start_ds([db_path])
# Warm a cached read connection on the old inode
response = await ds.client.get("/ext/t.json?_shape=array")
assert response.json() == [{"id": 1, "a": "one"}]
replacement = str(tmp_path / "replacement.db")
conn = sqlite3.connect(replacement)
# Same schema_version as the original file (1 CREATE TABLE) but a
# different table: only the inode change gives it away
conn.execute("create table t (id integer primary key, different text)")
conn.execute("insert into t (different) values ('new file')")
conn.commit()
assert conn.execute("pragma schema_version").fetchone()[0] == 1
conn.close()
old_inode = os.stat(db_path).st_ino
run_process(
"-c",
"import os, sys; os.replace(sys.argv[1], sys.argv[2])",
replacement,
db_path,
)
assert os.stat(db_path).st_ino != old_inode
async def replaced():
return await catalog_columns(ds, "ext", "t") == ["id", "different"]
await wait_for(replaced)
# Cached connections were invalidated, so queries see the new file
response = await ds.client.get("/ext/t.json?_shape=array")
assert response.json() == [{"id": 1, "different": "new file"}]
ds.close()
@pytest.mark.asyncio
async def test_old_inode_connections_without_watcher(tmp_path, db_path):
# Documents the hazard the watcher fixes: with polling off, a cached
# read connection keeps reading the replaced (unlinked) file
# One SQL thread, so every request reuses the same cached connection
ds = await start_ds([db_path], interval_ms=0, settings={"num_sql_threads": 1})
response = await ds.client.get("/ext/t.json?_shape=array")
assert response.json() == [{"id": 1, "a": "one"}]
replacement = str(tmp_path / "replacement.db")
conn = sqlite3.connect(replacement)
conn.execute("create table t (id integer primary key, a text)")
conn.execute("insert into t (a) values ('new file')")
conn.commit()
conn.close()
os.replace(replacement, db_path)
response = await ds.client.get("/ext/t.json?_shape=array")
assert response.json() == [{"id": 1, "a": "one"}] # stale!
# An explicit refresh notices the inode change and fixes it
await ds.refresh_schemas(force=True)
response = await ds.client.get("/ext/t.json?_shape=array")
assert response.json() == [{"id": 1, "a": "new file"}]
ds.close()
@pytest.mark.asyncio
async def test_external_vacuum(db_path):
run_external(
db_path,
"create table big (x); insert into big select zeroblob(1000) from "
"(with recursive c(i) as (select 1 union all select i+1 from c where i < 200) select i from c);"
"drop table big;",
)
ds = await start_ds([db_path])
before = await ds.get_internal_database().execute(
"select schema_version from catalog_databases where database_name = 'ext'"
)
run_external(db_path, "vacuum")
async def bumped():
after = await ds.get_internal_database().execute(
"select schema_version from catalog_databases where database_name = 'ext'"
)
return after.first()[0] == before.first()[0] + 1
await wait_for(bumped)
ds.close()
@pytest.mark.asyncio
async def test_external_rapid_changes(db_path):
ds = await start_ds([db_path])
for i in range(10):
run_external(db_path, f"create table rapid_{i} (id)")
async def all_seen():
tables = await catalog_tables(ds, "ext")
return all(f"rapid_{i}" in tables for i in range(10))
await wait_for(all_seen)
ds.close()
def _coarse_fingerprint(real):
# Simulate a filesystem with 1 second timestamp granularity
def fingerprint(path, present=None):
parts = real(path, present)
out = []
for i, part in enumerate(parts):
if part is None:
out.append(None)
continue
part = list(part)
# main: (dev, ino, size, mtime, ctime); side files: (ino, size, mtime)
for idx in (3, 4) if i == 0 else (2,):
part[idx] = part[idx] // 10**9 * 10**9
out.append(tuple(part))
return tuple(out)
return fingerprint
@pytest.mark.parametrize("journal_mode", ("delete", "wal", "truncate"))
def test_header_schema_version_matches_pragma(tmp_path, journal_mode):
path = str(tmp_path / "h.db")
conn = sqlite3.connect(path, isolation_level=None)
conn.execute(f"PRAGMA journal_mode={journal_mode}")
for i in range(300):
conn.execute(f"create table t{i} (id integer)")
conn.execute("alter table t1 add column b")
version = conn.execute("PRAGMA schema_version").fetchone()[0]
if journal_mode == "wal":
# Pages still in the -wal: the main file header is stale, which is
# why the watcher only reads it when the -wal is empty
assert schema_watcher_module.fingerprint(path)[1] is not None
conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
assert schema_watcher_module.fingerprint(path)[1:] == (None, None)
assert schema_watcher_module.header_schema_version(path) == version == 301
conn.close()
assert (
schema_watcher_module.header_schema_version(str(tmp_path / "nope.db")) is None
)
(tmp_path / "junk.db").write_bytes(b"not a database" * 20)
assert (
schema_watcher_module.header_schema_version(str(tmp_path / "junk.db")) is None
)
@pytest.mark.asyncio
@pytest.mark.parametrize("use_header", (True, False))
@pytest.mark.parametrize("racy_rule", (True, False))
async def test_racily_clean_same_tick_change(
db_path, monkeypatch, racy_rule, use_header
):
"""Two schema changes inside one timestamp tick, same file size and
inode: only the racy-clean rule catches the second one."""
monkeypatch.setattr(schema_watcher_module, "USE_HEADER_CHECK", use_header)
monkeypatch.setattr(
schema_watcher_module,
"fingerprint",
_coarse_fingerprint(schema_watcher_module.fingerprint),
)
if not racy_rule:
monkeypatch.setattr(schema_watcher_module, "RACY_WINDOW_NS", -(10**18))
ds = Datasette(files=[db_path], settings={"schema_watch_interval_ms": 0})
await ds.invoke_startup()
# Line the first change up with the start of a fresh second
await asyncio.sleep(1 - (time.time() % 1) + 0.05)
size = os.path.getsize(db_path)
run_external(db_path, "alter table t add column b text")
await ds._refresh_schemas() # sweep: sees change 1
assert await catalog_columns(ds, "ext", "t") == ["id", "a", "b"]
run_external(db_path, "alter table t add column c text")
assert os.path.getsize(db_path) == size
assert time.time() % 1 < 0.9, "test machine too slow to stay in one tick"
await ds._refresh_schemas() # sweep: fingerprint identical to last one
columns = await catalog_columns(ds, "ext", "t")
if racy_rule:
assert columns == ["id", "a", "b", "c"]
else:
# Without the rule the change is missed
assert columns == ["id", "a", "b"]
ds.close()
@pytest.mark.asyncio
async def test_external_file_deleted_and_recreated(tmp_path, db_path):
ds = await start_ds([db_path])
assert await catalog_tables(ds, "ext") == ["t"]
run_process("-c", "import os, sys; os.unlink(sys.argv[1])", db_path)
async def gone():
return await catalog_tables(ds, "ext") == []
await wait_for(gone)
response = await ds.client.get("/ext/t.json")
assert response.status_code in (404, 500)
# Writes fail instead of silently creating a new empty file
with pytest.raises(sqlite3.OperationalError):
await ds.get_database("ext").execute_write("insert into t (a) values ('x')")
assert not os.path.exists(db_path)
make_db(db_path, sql="create table t (id integer primary key, a text, back text)")
async def back():
return await catalog_columns(ds, "ext", "t") == ["id", "a", "back"]
await wait_for(back)
response = await ds.client.get("/ext/t.json?_shape=array")
assert response.json() == [{"id": 1, "a": "one", "back": None}]
ds.close()
@pytest.mark.asyncio
async def test_remove_database_deletes_catalog_rows(tmp_path, db_path):
other = str(tmp_path / "other.db")
make_db(other)
ds = Datasette(files=[db_path, other])
await ds.invoke_startup()
assert await catalog_tables(ds, "other") == ["t"]
ds.remove_database("other")
await ds._schema_watcher.on_request()
internal = ds.get_internal_database()
for table in (
"catalog_databases",
"catalog_tables",
"catalog_columns",
"catalog_indexes",
):
rows = await internal.execute(
f"select count(*) from {table} where database_name = 'other'"
)
assert rows.first()[0] == 0
assert await catalog_tables(ds, "ext") == ["t"]
ds.close()
@pytest.mark.asyncio
async def test_add_database_visible_on_next_request(tmp_path):
ds = Datasette(settings={"schema_watch_interval_ms": 0})
await ds.invoke_startup()
await ds.client.get("/-/versions.json")
path = str(tmp_path / "late.db")
make_db(path)
ds.add_database(Database(ds, path=path), name="late")
response = await ds.client.get("/late.json")
assert response.status_code == 200
assert await catalog_tables(ds, "late") == ["t"]
ds.close()
@pytest.mark.asyncio
async def test_persisted_fingerprints_skip_unchanged_files(tmp_path, monkeypatch):
# Shrink the racy window so freshly written files count as stable
monkeypatch.setattr(schema_watcher_module, "RACY_WINDOW_NS", 1_000_000)
internal = str(tmp_path / "internal.db")
paths = []
for i in range(3):
path = str(tmp_path / f"p{i}.db")
make_db(path)
paths.append(path)
await asyncio.sleep(0.05)
ds = Datasette(files=paths, internal=internal)
await ds.invoke_startup()
assert ds._schema_watcher.counters["scans"] == 3
ds.close()
# Change one file while Datasette is "down"
run_external(paths[1], "create table added_offline (id)")
ds2 = Datasette(files=paths, internal=internal)
await ds2.invoke_startup()
counters = ds2._schema_watcher.counters
assert counters["restored_from_persisted"] == 2
assert counters["scans"] == 1
assert await catalog_tables(ds2, "p1") == ["added_offline", "t"]
assert await catalog_tables(ds2, "p0") == ["t"]
ds2.close()
@pytest.mark.asyncio
async def test_remove_database_during_catalog_scan(tmp_path):
# remove_database() closes the Database while the watcher may still be
# scanning it in a worker thread. The scan connection is not tracked by
# the Database, so close() must not pull it out from under the scan
# (that used to segfault).
path = str(tmp_path / "big.db")
conn = sqlite3.connect(path)
for i in range(150):
conn.execute(f"create table t{i} (a, b, c integer references t0(a))")
conn.execute(f"create index i{i} on t{i}(a)")
conn.commit()
conn.close()
ds = Datasette(settings={"schema_watch_interval_ms": 0})
await ds.invoke_startup()
for i in range(60):
ds.add_database(Database(ds, path=path, is_mutable=i % 2 == 0), name=f"x{i}")
await asyncio.sleep(0.0005 * (i % 5))
ds.remove_database(f"x{i}")
await asyncio.sleep(0.2)
rows = await ds.get_internal_database().execute(
"select count(*) from catalog_tables where database_name like 'x%'"
)
assert rows.first()[0] == 0
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"setting,expected", (("external", "external"), ("owned", "owned"))
)
async def test_default_schema_watch_setting(tmp_path, db_path, setting, expected):
immutable_path = str(tmp_path / "imm.db")
make_db(immutable_path)
ds = Datasette(
files=[db_path],
immutables=[immutable_path],
settings={"default_schema_watch": setting},
)
watcher = ds._schema_watcher
assert watcher.states["ext"].mode == expected
# Immutable files are always immutable
assert watcher.states["imm"].mode == "immutable"
# add_database() follows the setting too, and can ask for either mode
other = str(tmp_path / "other.db")
make_db(other)
added = ds.add_database(Database(ds, path=other), name="added")
assert added._watch_state.mode == expected
polled = ds.add_database(
Database(ds, path=other), name="polled", schema_watch="external"
)
assert polled._watch_state.mode == "external"
owned = ds.add_database(
Database(ds, path=other), name="owned", schema_watch="owned"
)
assert owned._watch_state.mode == "owned"
ds.close()
def test_default_schema_watch_config_overrides(db_path):
ds = Datasette(
files=[db_path],
settings={"default_schema_watch": "owned"},
config={"databases": {"ext": {"schema_watch": "external"}}},
)
assert ds._schema_watcher.states["ext"].mode == "external"
ds.close()
def test_default_schema_watch_invalid(db_path):
from datasette.utils import StartupError
with pytest.raises(StartupError, match="default_schema_watch"):
Datasette(files=[db_path], settings={"default_schema_watch": "immutable"})
@pytest.mark.asyncio
async def test_owned_default_is_not_polled(db_path):
ds = await start_ds([db_path], settings={"default_schema_watch": "owned"})
run_external(db_path, "create table not_polled (id)")
await asyncio.sleep(INTERVAL_MS * 4 / 1000)
assert await catalog_tables(ds, "ext") == ["t"]
# Datasette's own writes are still detected immediately
await ds.get_database("ext").execute_write("create table mine (id)")
assert await catalog_tables(ds, "ext") == ["mine", "not_polled", "t"]
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", (0, 2))
async def test_scratch_database_file_created_by_first_write(tmp_path, num_sql_threads):
# A database added before its file exists (a plugin's scratch
# database): the first write creates it. Once it has existed, deleting
# it makes writes fail rather than silently creating a new empty file.
ds = Datasette(
settings={"num_sql_threads": num_sql_threads, "schema_watch_interval_ms": 0}
)
await ds.invoke_startup()
path = str(tmp_path / "scratch.db")
db = ds.add_database(Database(ds, path=path), name="scratch")
await ds._schema_watcher.flush_pending()
assert not os.path.exists(path)
await db.execute_write("create table t (id integer primary key)")
assert os.path.exists(path)
assert await catalog_tables(ds, "scratch") == ["t"]
await db.execute_write("insert into t default values")
os.unlink(path)
await ds.refresh_schemas(force=True)
assert await catalog_tables(ds, "scratch") == []
with pytest.raises(sqlite3.OperationalError):
await db.execute_write("insert into t default values")
assert not os.path.exists(path)
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("num_sql_threads", (0, 2))
async def test_write_connection_follows_replaced_file(
tmp_path, db_path, num_sql_threads
):
ds = await start_ds(
[db_path], interval_ms=0, settings={"num_sql_threads": num_sql_threads}
)
db = ds.get_database("ext")
# Warm the write connection on the old inode
await db.execute_write("insert into t (a) values ('old')")
replacement = str(tmp_path / "replacement.db")
make_db(replacement)
os.replace(replacement, db_path)
await ds.refresh_schemas(force=True)
await db.execute_write("insert into t (a) values ('new')")
conn = sqlite3.connect(db_path)
try:
rows = conn.execute("select a from t order by id").fetchall()
finally:
conn.close()
# The write went to the new file, not the unlinked one
assert rows == [("one",), ("new",)]
assert [r["a"] for r in (await db.execute("select a from t order by id")).rows] == [
"one",
"new",
]
ds.close()

View file

@ -0,0 +1,70 @@
"""Error recovery must also work without Python 3.11's SQLite error metadata."""
import errno
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from datasette.schema_watcher import _is_transient
from datasette.utils import sqlite3
from datasette.write_budget import DatabaseResourceError, connect_with_retry
@pytest.fixture
def old_sqlite_errors(monkeypatch):
for name in ("SQLITE_CANTOPEN", "SQLITE_NOMEM"):
monkeypatch.delattr(sqlite3, name, raising=False)
@pytest.mark.parametrize(
"error,expected",
[
(sqlite3.OperationalError("unable to open database file"), True),
(sqlite3.OperationalError("out of memory"), True),
(sqlite3.OperationalError("database is locked"), True),
(sqlite3.OperationalError("no such table: missing"), False),
(OSError(errno.EMFILE, "too many open files"), True),
],
)
def test_transient_errors_without_sqlite_constants(old_sqlite_errors, error, expected):
assert _is_transient(error) is expected
@pytest.mark.parametrize("os_error", [False, True])
def test_open_recovers_without_sqlite_constants(old_sqlite_errors, tmp_path, os_error):
path = tmp_path / "db.sqlite"
path.touch()
pool = Mock()
db = SimpleNamespace(path=str(path), ds=SimpleNamespace(_read_pool_or_none=pool))
error = (
OSError(errno.EMFILE, "too many open files")
if os_error
else sqlite3.OperationalError("unable to open database file")
)
connection = object()
connect = Mock(side_effect=[error, connection])
assert connect_with_retry(db, connect) is connection
assert connect.call_count == 2
pool.evict_idle.assert_called_once_with()
@pytest.mark.parametrize("os_error", [False, True])
def test_open_failure_preserved_without_sqlite_constants(
old_sqlite_errors, tmp_path, os_error
):
path = tmp_path / "db.sqlite"
path.touch()
db = SimpleNamespace(path=str(path), ds=SimpleNamespace(_read_pool_or_none=None))
error = (
OSError(errno.EMFILE, "too many open files")
if os_error
else sqlite3.OperationalError("unable to open database file")
)
connect = Mock(side_effect=error)
with pytest.raises(
DatabaseResourceError if os_error else sqlite3.OperationalError
) as exc:
connect_with_retry(db, connect)
assert (exc.value.__cause__ if os_error else exc.value) is error
assert connect.call_count == 2

389
tests/test_write_budget.py Normal file
View file

@ -0,0 +1,389 @@
import asyncio
import sqlite3
import threading
import time
import pytest
from datasette.app import Datasette
from datasette.write_budget import (
DatabaseAdmissionTimeout,
DatabaseQueueFull,
DatabaseReentrancyError,
DatabaseResourceError,
)
def make_databases(tmp_path, count=3, **settings):
paths = []
for i in range(count):
path = tmp_path / f"budget{i}.db"
with sqlite3.connect(path) as conn:
conn.execute("create table t(id)")
conn.close()
paths.append(str(path))
ds = Datasette(paths, settings={"schema_watch_interval_ms": 0, **settings})
return ds, [ds.get_database(f"budget{i}") for i in range(count)]
async def wait_for(predicate):
deadline = time.monotonic() + 5
while not predicate():
assert time.monotonic() < deadline
await asyncio.sleep(0.005)
@pytest.mark.asyncio
async def test_many_writers_bounded_fifo_and_fairness(tmp_path):
ds, dbs = make_databases(
tmp_path, 30, max_write_connections=2, max_pending_writes=200
)
entered, release = threading.Event(), threading.Event()
order = []
def slow(conn):
entered.set()
assert release.wait(5)
first = asyncio.create_task(dbs[0].execute_write_fn(slow))
tasks = []
try:
assert await asyncio.to_thread(entered.wait, 5)
for seq in range(4):
for db in dbs:
def write(conn, name=db.name, seq=seq):
order.append((name, seq))
conn.execute("insert into t values (?)", (seq,))
tasks.append(asyncio.create_task(db.execute_write_fn(write)))
await asyncio.sleep(0.03)
release.set()
await asyncio.gather(first, *tasks)
assert ds._write_budget.snapshot()["stats"]["peak_writers"] == 2
for db in dbs:
assert [seq for name, seq in order if name == db.name] == list(range(4))
assert [
r[0] for r in (await db.execute("select id from t order by rowid")).rows
] == list(range(4))
assert order.index((dbs[-1].name, 0)) < order.index((dbs[0].name, 3))
finally:
release.set()
await asyncio.gather(first, *tasks, return_exceptions=True)
ds.close()
await wait_for(lambda: ds._write_budget.snapshot()["writers"] == 0)
@pytest.mark.asyncio
async def test_queue_full_expiration_and_internal_progress(tmp_path):
ds, dbs = make_databases(
tmp_path,
max_write_connections=1,
max_pending_writes=1,
write_queue_timeout_ms=100,
)
entered, release = threading.Event(), threading.Event()
ran = []
def slow(conn):
entered.set()
assert release.wait(5)
conn.execute("insert into t values (1)")
first = asyncio.create_task(dbs[0].execute_write_fn(slow))
queued = None
try:
assert await asyncio.to_thread(entered.wait, 5)
queued = asyncio.create_task(
dbs[1].execute_write_fn(lambda conn: ran.append(True))
)
await wait_for(lambda: ds._write_budget.snapshot()["pending"] == 1)
with pytest.raises(DatabaseQueueFull) as exc:
await dbs[2].execute_write_fn(lambda conn: ran.append(True))
assert exc.value.execution_started is False
# Internal catalog work must not compete for the single user slot.
assert (
await asyncio.wait_for(
ds._internal_database.execute_write_fn(lambda conn: 42), 2
)
== 42
)
with pytest.raises(DatabaseAdmissionTimeout):
await queued
assert not ran
assert dbs[1]._write_queue.qsize() == 0
release.set()
await first
await dbs[2].execute_write("insert into t values (2)")
assert ds._write_budget.snapshot()["stats"]["peak_pending"] == 1
finally:
release.set()
await asyncio.gather(
first, *([queued] if queued else []), return_exceptions=True
)
ds.close()
@pytest.mark.asyncio
async def test_close_waiting_database_preserves_admission_deadline(tmp_path):
ds, dbs = make_databases(
tmp_path, max_write_connections=1, write_queue_timeout_ms=50
)
entered, release = threading.Event(), threading.Event()
def slow(conn):
entered.set()
assert release.wait(5)
first = asyncio.create_task(dbs[0].execute_write_fn(slow))
try:
assert await asyncio.to_thread(entered.wait, 5)
pending = asyncio.create_task(
dbs[1].execute_write_fn(lambda conn: pytest.fail("closed callback ran"))
)
await wait_for(lambda: ds._write_budget.snapshot()["pending"] == 1)
dbs[1].close()
with pytest.raises(DatabaseAdmissionTimeout):
await pending
assert ds._write_budget.snapshot()["pending"] == 0
finally:
release.set()
await first
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("threads", [0, 3])
async def test_memory_pins_capacity_and_recovers_after_close(tmp_path, threads):
ds, dbs = make_databases(
tmp_path,
max_write_connections=1,
write_queue_timeout_ms=50,
num_sql_threads=threads,
)
memory = ds.add_memory_database("budget_memory")
try:
await memory.execute_write("create table t(id)")
with pytest.raises(DatabaseResourceError):
await dbs[0].execute_write("insert into t values (1)")
assert (
await memory.execute_write_fn(
lambda conn: conn.execute("select count(*) from t").fetchone()[0]
)
== 0
)
memory.close()
await dbs[0].execute_write("insert into t values (1)")
finally:
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("threads", [0, 3])
async def test_isolated_writer_does_not_hold_extra_file_connection(tmp_path, threads):
ds, dbs = make_databases(tmp_path, max_write_connections=1, num_sql_threads=threads)
try:
for db in dbs:
await db.execute_write("insert into t values (1)")
assert (
await db.execute_isolated_fn(
lambda conn, db=db: len(db._all_connections)
)
== 1
)
if threads == 0:
assert sum(db._write_connection is not None for db in dbs) <= 1
finally:
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["read", "write"])
async def test_reentrant_database_call_fails_without_deadlock(tmp_path, kind):
ds, dbs = make_databases(tmp_path, max_write_connections=1, max_open_connections=1)
try:
def callback(conn):
return asyncio.run(dbs[1].execute_write_fn(lambda conn: 1))
method = dbs[0].execute_fn if kind == "read" else dbs[0].execute_write_fn
with pytest.raises(DatabaseReentrancyError):
await asyncio.wait_for(method(callback), 2)
finally:
ds.close()
@pytest.mark.asyncio
async def test_admission_errors_are_http_503(tmp_path, monkeypatch):
ds, dbs = make_databases(tmp_path)
await ds.invoke_startup()
async def fail(*args, **kwargs):
raise DatabaseQueueFull("Too many queued database writes")
monkeypatch.setattr(dbs[0], "execute", fail)
try:
response = await ds.client.get("/budget0/-/query.json?sql=select+1")
assert response.status_code == 503, response.text
assert response.headers["retry-after"] == "1"
assert response.json()["code"] == "database_queue_full"
assert response.json()["execution_started"] is False
finally:
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("threads", [0, 3])
async def test_unfinished_explicit_transaction_is_rolled_back(tmp_path, threads):
from datasette.connection_pool import ConnectionLeaseError
ds, dbs = make_databases(tmp_path, num_sql_threads=threads)
try:
with pytest.raises(ConnectionLeaseError, match="rolled back"):
await dbs[0].execute_write_fn(
lambda conn: conn.execute("insert into t values (1)"), transaction=False
)
await dbs[0].execute_write("insert into t values (2)")
assert [row[0] for row in (await dbs[0].execute("select id from t")).rows] == [
2
]
finally:
ds.close()
@pytest.mark.asyncio
async def test_nonblocking_expiration_notifies_plugin(tmp_path):
from datasette import hookimpl
from datasette.plugins import pm
notifications = []
class Plugin:
@hookimpl
async def write_task_completed(self, database, task_id, exception):
notifications.append((database, task_id, exception))
plugin = Plugin()
pm.register(plugin)
ds, dbs = make_databases(
tmp_path, max_write_connections=1, write_queue_timeout_ms=50
)
entered, release = threading.Event(), threading.Event()
def slow(conn):
entered.set()
assert release.wait(5)
first = asyncio.create_task(dbs[0].execute_write_fn(slow))
try:
assert await asyncio.to_thread(entered.wait, 5)
task_id = await dbs[1].execute_write_fn(
lambda conn: pytest.fail("expired callback ran"), block=False
)
await wait_for(lambda: notifications)
assert notifications[0][:2] == (dbs[1].name, task_id)
assert isinstance(notifications[0][2], DatabaseAdmissionTimeout)
release.set()
await first
successful_id = await dbs[1].execute_write_fn(lambda conn: 42, block=False)
await wait_for(lambda: len(notifications) == 2)
assert notifications[1] == (dbs[1].name, successful_id, None)
finally:
release.set()
await first
ds.close()
pm.unregister(plugin)
@pytest.mark.asyncio
async def test_connection_open_pressure_retries_before_callback(tmp_path, monkeypatch):
import errno
from datasette.utils import sqlite3 as sqlite_module
ds, dbs = make_databases(tmp_path)
await ds.invoke_startup()
await dbs[1].execute("select 1")
original = sqlite_module.connect
calls = []
def connect(*args, **kwargs):
if "budget0.db" in str(args[0]):
calls.append(1)
if len(calls) == 1:
raise OSError(errno.EMFILE, "too many files")
return original(*args, **kwargs)
monkeypatch.setattr(sqlite_module, "connect", connect)
ran = []
try:
await dbs[0].execute_write_fn(lambda conn: ran.append(1))
assert calls == [1, 1]
assert ran == [1]
assert not dbs[1]._read_pool_state.idle
finally:
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("threads", [0, 3])
async def test_resource_error_after_execution_is_not_an_admission_error(
tmp_path, threads, monkeypatch
):
ds, dbs = make_databases(tmp_path, num_sql_threads=threads)
db = dbs[0]
await ds.invoke_startup()
try:
def committed_then_failed(conn):
conn.execute("insert into t values (1)")
conn.commit()
raise DatabaseQueueFull("A different operation could not be admitted")
with pytest.raises(RuntimeError, match="callback started") as exc:
await db.execute_write_fn(committed_then_failed, transaction=False)
assert not isinstance(exc.value, DatabaseResourceError)
assert isinstance(exc.value.__cause__, DatabaseQueueFull)
assert (await db.execute("select count(*) from t")).single_value() == 1
async def refresh_failed(*args):
raise DatabaseResourceError("Catalog connection unavailable")
monkeypatch.setattr(ds._schema_watcher, "after_write", refresh_failed)
db._watch_state.needs_scan = True
with pytest.raises(RuntimeError, match="Write completed") as exc:
await db.execute_write("insert into t values (2)")
assert not isinstance(exc.value, DatabaseResourceError)
assert (await db.execute("select count(*) from t")).single_value() == 2
finally:
ds.close()
@pytest.mark.asyncio
async def test_retiring_writer_keeps_its_slot_until_connection_closes(
tmp_path, monkeypatch
):
ds, dbs = make_databases(tmp_path, max_write_connections=1)
closing, release = threading.Event(), threading.Event()
original = dbs[0]._forget_connection
def slow_close(conn):
closing.set()
assert release.wait(5)
original(conn)
await dbs[0].execute_write("insert into t values (1)")
monkeypatch.setattr(dbs[0], "_forget_connection", slow_close)
pending = asyncio.create_task(dbs[1].execute_write("insert into t values (2)"))
try:
assert await asyncio.to_thread(closing.wait, 5)
assert not pending.done()
assert dbs[1]._write_thread is None
assert ds._write_budget.snapshot()["writers"] == 1
release.set()
await pending
assert ds._write_budget.snapshot()["stats"]["peak_writers"] == 1
finally:
release.set()
await asyncio.gather(pending, return_exceptions=True)
ds.close()

View file

@ -0,0 +1,250 @@
import asyncio
import sqlite3
import threading
import time
from pathlib import Path
import pytest
from datasette.app import Datasette
from datasette.database import Database, DatasetteClosedError
from datasette.write_budget import RETIRE_WRITER, DatabaseAdmissionTimeout
def make_databases(tmp_path, count=2, **settings):
paths = []
for i in range(count):
path = tmp_path / f"shutdown{i}.db"
with sqlite3.connect(path) as conn:
conn.execute("create table t(id)")
conn.close()
paths.append(str(path))
ds = Datasette(paths, settings={"schema_watch_interval_ms": 0, **settings})
return ds, [ds.get_database(f"shutdown{i}") for i in range(count)]
async def wait_for(predicate):
deadline = time.monotonic() + 5
while not predicate():
assert time.monotonic() < deadline
await asyncio.sleep(0.005)
def rows(db):
with sqlite3.connect(db.path) as conn:
result = conn.execute("select id from t order by rowid").fetchall()
conn.close()
return [row[0] for row in result]
@pytest.mark.asyncio
@pytest.mark.parametrize("retirement_requested", [False, True])
async def test_close_drains_writes_before_worker_admission(
tmp_path, monkeypatch, retirement_requested
):
ds, (db, _) = make_databases(tmp_path, max_write_connections=1)
admitted = threading.Event()
release = threading.Event()
run = ds._write_budget._run
def paused_scheduler():
admitted.set()
assert release.wait(5)
run()
monkeypatch.setattr(ds._write_budget, "_run", paused_scheduler)
replies = []
closing = None
try:
for i in range(20):
_, reply = await db._send_to_write_thread(
lambda conn, i=i: conn.execute("insert into t values (?)", [i]),
block=False,
)
replies.append(reply)
assert admitted.wait(1)
assert db._write_thread is None
if retirement_requested:
# A previous retirement request can precede tasks accepted later.
with db._write_queue.mutex:
db._write_queue.queue.appendleft(RETIRE_WRITER)
closing = asyncio.create_task(asyncio.to_thread(db.close))
await wait_for(lambda: db._closed)
release.set()
await asyncio.wait_for(closing, 5)
await asyncio.gather(*replies)
assert rows(db) == list(range(20))
assert ds._write_budget.snapshot()["writers"] == 0
assert ds._write_budget.snapshot()["stats"]["peak_writers"] == 1
with pytest.raises(DatasetteClosedError):
await db.execute_write("insert into t values (21)")
finally:
release.set()
if closing is not None:
await closing
await asyncio.gather(*replies, return_exceptions=True)
ds.close()
@pytest.mark.asyncio
async def test_close_waits_for_budget_capacity_then_drains(tmp_path):
ds, (first, waiting) = make_databases(tmp_path, max_write_connections=1)
entered, release = threading.Event(), threading.Event()
def slow(conn):
entered.set()
assert release.wait(5)
_, first_reply = await first._send_to_write_thread(slow, block=False)
closing = None
waiting_reply = None
try:
assert await asyncio.to_thread(entered.wait, 5)
_, waiting_reply = await waiting._send_to_write_thread(
lambda conn: conn.execute("insert into t values (1)"), block=False
)
closing = asyncio.create_task(asyncio.to_thread(waiting.close))
await wait_for(lambda: waiting._closed)
assert ds._write_budget.snapshot()["pending"] == 1
release.set()
await asyncio.wait_for(closing, 5)
await asyncio.gather(first_reply, waiting_reply)
assert rows(waiting) == [1]
assert ds._write_budget.snapshot()["stats"]["peak_writers"] == 1
finally:
release.set()
if closing is not None:
await closing
await asyncio.gather(
first_reply,
*([waiting_reply] if waiting_reply else []),
return_exceptions=True,
)
ds.close()
@pytest.mark.asyncio
async def test_close_preserves_admission_deadline_while_capacity_pinned(tmp_path):
ds, (waiting, _) = make_databases(
tmp_path, max_write_connections=1, write_queue_timeout_ms=50
)
memory = ds.add_memory_database("pinned")
ran = []
try:
await memory.execute_write("create table t(id)")
_, reply = await waiting._send_to_write_thread(
lambda conn: ran.append(True), block=False
)
await asyncio.wait_for(asyncio.to_thread(waiting.close), 2)
with pytest.raises(DatabaseAdmissionTimeout):
await reply
assert not ran
assert ds._write_budget.snapshot()["pending"] == 0
assert ds._write_budget.snapshot()["stats"]["peak_writers"] == 1
finally:
ds.close()
@pytest.mark.asyncio
async def test_datasette_shutdown_releases_later_memory_writer_before_drain(tmp_path):
ds, (waiting, _) = make_databases(
tmp_path, max_write_connections=1, write_queue_timeout_ms=1000
)
memory = ds.add_memory_database("pinned")
try:
await memory.execute_write("create table t(id)")
_, reply = await waiting._send_to_write_thread(
lambda conn: conn.execute("insert into t values (1)"), block=False
)
await asyncio.wait_for(asyncio.to_thread(ds.close), 5)
await reply
assert rows(waiting) == [1]
assert ds._write_budget.snapshot()["writers"] == 0
assert ds._write_budget.snapshot()["stats"]["peak_writers"] == 1
finally:
ds.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("database_kind", ["file", "temporary", "internal"])
async def test_close_timeout_preserves_running_connection(
tmp_path, monkeypatch, database_kind
):
import datasette.database
monkeypatch.setattr(datasette.database, "WRITE_SHUTDOWN_TIMEOUT", 0.05)
ds, (db, _) = make_databases(tmp_path, max_write_connections=1)
if database_kind == "temporary":
db = Database(ds, is_temp_disk=True)
ds.add_database(db, name="temporary")
await db.execute_write("create table t(id)")
elif database_kind == "internal":
db = ds._internal_database
await db.execute_write("create table t(id)")
entered, release = threading.Event(), threading.Event()
def slow(conn):
entered.set()
assert release.wait(5)
conn.execute("insert into t values (1)")
return conn.execute("select id from t").fetchall()[0][0]
_, running = await db._send_to_write_thread(slow, block=False)
queued = None
try:
assert await asyncio.to_thread(entered.wait, 5)
if database_kind != "internal":
_, queued = await db._send_to_write_thread(
lambda conn: pytest.fail("cancelled callback ran"), block=False
)
await asyncio.wait_for(asyncio.to_thread(db.close), 2)
if queued is not None:
with pytest.raises(DatasetteClosedError):
await queued
assert Path(db.path).exists()
assert not running.done()
release.set()
assert await running == 1
if database_kind == "file":
assert rows(db) == [1]
else:
await wait_for(lambda: not Path(db.path).exists())
await wait_for(lambda: db._write_thread is None)
finally:
release.set()
await asyncio.gather(
running, *([queued] if queued else []), return_exceptions=True
)
ds.close()
@pytest.mark.asyncio
async def test_close_timeout_during_write_connection_open(tmp_path, monkeypatch):
import datasette.database
monkeypatch.setattr(datasette.database, "WRITE_SHUTDOWN_TIMEOUT", 0.05)
ds, (db, _) = make_databases(tmp_path)
opened, release = threading.Event(), threading.Event()
connect = db.connect
def paused_connect(**kwargs):
conn = connect(**kwargs)
opened.set()
assert release.wait(5)
return conn
monkeypatch.setattr(db, "connect", paused_connect)
_, reply = await db._send_to_write_thread(
lambda conn: conn.execute("insert into t values (1)"), block=False
)
try:
assert await asyncio.to_thread(opened.wait, 5)
await asyncio.wait_for(asyncio.to_thread(db.close), 2)
release.set()
await reply
assert rows(db) == [1]
await wait_for(lambda: db._write_thread is None)
finally:
release.set()
await asyncio.gather(reply, return_exceptions=True)
ds.close()

View file

@ -0,0 +1,363 @@
"""
Tests for the connection_idle_timeout_ms setting: a database's write thread
closes its connection and exits after sitting idle, and the next write
starts a new one.
"""
import asyncio
import queue
import random
import sqlite3
import threading
import time
import pytest
from datasette.app import Datasette
from datasette.database import Database, DatasetteClosedError
def _make_db(tmp_path, name="idle", **settings):
path = tmp_path / f"{name}.db"
conn = sqlite3.connect(path)
conn.execute("create table t (id integer primary key, writer integer, seq integer)")
conn.close()
ds = Datasette([str(path)], settings=settings)
return ds, ds.get_database(name)
async def _wait_for(predicate, timeout=5):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if predicate():
return True
await asyncio.sleep(0.005)
return predicate()
def _write_threads(name):
thread_name = f"_execute_writes for database {name}"
return [t for t in threading.enumerate() if t.name == thread_name]
@pytest.mark.asyncio
async def test_write_thread_exits_when_idle_and_restarts(tmp_path):
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=50)
try:
await db.execute_write("insert into t (writer, seq) values (1, 1)")
thread = db._write_thread
assert thread is not None and thread.is_alive()
connections_with_writer = len(db._all_connections)
assert await _wait_for(lambda: db._write_thread is None)
thread.join(timeout=5)
assert not thread.is_alive()
# Its write connection was closed and is no longer tracked
assert len(db._all_connections) == connections_with_writer - 1
assert not _write_threads("idle")
# The next write starts a new thread
await db.execute_write("insert into t (writer, seq) values (1, 2)")
assert db._write_thread is not None and db._write_thread is not thread
assert db._write_threads_started == 2
count = (await db.execute("select count(*) from t")).single_value()
assert count == 2
finally:
ds.close()
@pytest.mark.asyncio
async def test_write_thread_idle_timeout_zero_keeps_thread(tmp_path):
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=0)
try:
await db.execute_write("insert into t (writer, seq) values (1, 1)")
thread = db._write_thread
await asyncio.sleep(0.2)
assert db._write_thread is thread
assert thread.is_alive()
finally:
ds.close()
assert not thread.is_alive()
@pytest.mark.asyncio
@pytest.mark.parametrize("named", (True, False))
async def test_memory_database_write_thread_never_idles(named):
# Closing the write connection of a memory database would lose its data
ds = Datasette(settings={"connection_idle_timeout_ms": 10})
if named:
db = ds.add_memory_database("idle_mem")
else:
db = ds.add_database(Database(ds, is_memory=True), name="idle_unnamed")
try:
await db.execute_write("create table t (id integer primary key)")
await db.execute_write("insert into t (id) values (1)")
thread = db._write_thread
await asyncio.sleep(0.1)
assert db._write_thread is thread and thread.is_alive()
count = await db.execute_write_fn(
lambda conn: conn.execute("select count(*) from t").fetchone()[0]
)
assert count == 1
finally:
ds.close()
@pytest.mark.asyncio
async def test_close_after_idle_exit(tmp_path):
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=10)
await db.execute_write("insert into t (writer, seq) values (1, 1)")
thread = db._write_thread
assert await _wait_for(lambda: db._write_thread is None)
ds.close()
thread.join(timeout=5)
assert not thread.is_alive()
with pytest.raises(DatasetteClosedError):
await db.execute_write("insert into t (writer, seq) values (1, 2)")
@pytest.mark.asyncio
async def test_close_drains_queued_writes_with_idle_timeout(tmp_path):
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=10)
release = threading.Event()
def slow(conn):
release.wait(5)
conn.execute("insert into t (writer, seq) values (0, 0)")
await db.execute_write_fn(slow, block=False)
futures = []
for i in range(20):
_, future = await db._send_to_write_thread(
lambda conn, i=i: conn.execute(
"insert into t (writer, seq) values (1, ?)", [i]
),
block=False,
)
futures.append(future)
release.set()
# close() is synchronous, so the queued writes complete before it returns
db.close()
await asyncio.wait_for(asyncio.gather(*futures), timeout=5)
conn = sqlite3.connect(db.path)
try:
assert conn.execute("select count(*) from t").fetchone()[0] == 21
finally:
conn.close()
ds.close()
@pytest.mark.asyncio
async def test_connect_failure_retried_after_queue_drains(tmp_path):
# With an idle timeout, a failed connect is reported to the writes that
# were queued, then the thread exits and the next write tries again
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=30000)
class ConnectError(Exception):
pass
original_connect = db.connect
def broken_connect(write=False):
if write:
raise ConnectError("no")
return original_connect(write=write)
db.connect = broken_connect
try:
with pytest.raises(ConnectError):
await db.execute_write("insert into t (writer, seq) values (1, 1)")
finally:
db.connect = original_connect
assert await _wait_for(lambda: db._write_thread is None)
await db.execute_write("insert into t (writer, seq) values (1, 2)")
assert db._write_threads_started == 2
ds.close()
class _RacyQueue(queue.Queue):
"""
The first time the write thread asks for a task while the queue is
empty, behave as if the idle timeout fired - but first pause, so the
test can enqueue a task in the window between the timeout and the
thread's exit decision.
"""
def __init__(self):
super().__init__()
self.armed = False
self.window_open = threading.Event()
self.proceed = threading.Event()
def get(self, block=True, timeout=None):
if self.armed and self.empty():
self.armed = False
self.window_open.set()
self.proceed.wait(5)
raise queue.Empty
return super().get(block, timeout)
@pytest.mark.asyncio
async def test_task_enqueued_between_timeout_and_exit_is_not_stranded(tmp_path):
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=60000)
racy = _RacyQueue()
db._write_queue = racy
try:
racy.armed = True
await db.execute_write("insert into t (writer, seq) values (1, 1)")
# The thread has finished that write and is now inside the window
assert await asyncio.to_thread(racy.window_open.wait, 5)
thread = db._write_thread
assert thread is not None
_, future = await db._send_to_write_thread(
lambda conn: conn.execute("insert into t (writer, seq) values (1, 2)"),
block=False,
)
# The thread is still registered, so no new thread was started
assert db._write_threads_started == 1
racy.proceed.set()
await asyncio.wait_for(future, timeout=5)
assert db._write_thread is thread
count = (await db.execute("select count(*) from t")).single_value()
assert count == 2
finally:
racy.proceed.set()
ds.close()
def _recorder(log, writer, seq):
# A cheap task (no disk write), so tasks finish in microseconds and the
# idle timeout keeps firing between bursts
def record(conn):
conn.execute("select 1").fetchone()
log.append((writer, seq, threading.get_ident()))
return record
@pytest.mark.asyncio
@pytest.mark.parametrize("idle_timeout_ms", (1, 5))
async def test_idle_exit_race_stress(tmp_path, idle_timeout_ms):
"""
Many concurrent writers working in rounds. Each round starts once the
previous round's tasks are done, so the write thread's idle clock starts
at about the same moment, and the writers then wait about 0.8x-1.3x of
the idle timeout before queueing a burst. Tasks therefore keep landing
on both sides of, and right on top of, the thread's exit decision.
Every task must complete, and each writer's tasks must run in the order
they were queued.
"""
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=idle_timeout_ms)
idle_timeout = idle_timeout_ms / 1000
rng = random.Random(42)
writers = 12
rounds = 150 if idle_timeout_ms == 1 else 60
seqs = [0] * writers
log = []
async def writer_round(w, delay, burst):
await asyncio.sleep(delay)
pending = []
for _ in range(burst):
seq = seqs[w]
seqs[w] += 1
fn = _recorder(log, w, seq)
if seq % 3 == 0:
# Fire-and-forget write
_, future = await db._send_to_write_thread(fn, block=False)
pending.append(future)
else:
await db.execute_write_fn(fn, transaction=bool(seq % 2))
await asyncio.gather(*pending)
async def all_rounds():
for _ in range(rounds):
# One shared delay per round close to the timeout, plus a little
# jitter per writer, so arrivals cluster around the exit decision
base = idle_timeout * rng.uniform(0.8, 1.2)
await asyncio.gather(
*(
writer_round(
w, base + idle_timeout * rng.uniform(0, 0.1), rng.randint(1, 3)
)
for w in range(writers)
)
)
try:
await asyncio.wait_for(all_rounds(), timeout=120)
assert len(log) == sum(seqs)
for w in range(writers):
assert [seq for writer_id, seq, _ in log if writer_id == w] == list(
range(seqs[w])
)
print(
f"idle_timeout_ms={idle_timeout_ms} tasks={len(log)} "
f"threads_started={db._write_threads_started} "
f"distinct_thread_idents={len({ident for _, _, ident in log})} "
f"exits_averted={db._write_thread_exits_averted}"
)
# Only meaningful if the thread really did churn
assert db._write_threads_started > rounds // 5
finally:
ds.close()
assert not _write_threads("idle")
def test_idle_exit_race_stress_many_event_loops(tmp_path):
"""
Producers on several OS threads, each with its own event loop, race
with the write thread's exit decision for real (no single-loop
ordering between producers). Rounds are synchronised with a barrier,
then each producer waits a random 0.5x-1.5x of the idle timeout.
"""
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=1)
loops = 6
rounds = 200
barrier = threading.Barrier(loops)
errors = []
log = []
def run_loop(w):
rng = random.Random(w)
async def go():
for seq in range(rounds):
barrier.wait(timeout=30)
# Force some completed retirements as well as racing the
# boundary. Scheduler latency must not decide coverage.
if seq % 4 == 0:
deadline = time.monotonic() + 5
while db._write_thread is not None:
assert time.monotonic() < deadline
await asyncio.sleep(0.001)
barrier.wait(timeout=30)
await asyncio.sleep(0.001 * rng.uniform(0.5, 1.5))
await asyncio.wait_for(
db.execute_write_fn(_recorder(log, w, seq)), timeout=10
)
try:
asyncio.run(go())
except BaseException as e: # noqa: BLE001
errors.append(e)
barrier.abort()
threads = [threading.Thread(target=run_loop, args=(w,)) for w in range(loops)]
for t in threads:
t.start()
for t in threads:
t.join(timeout=120)
try:
assert not errors, errors
assert len(log) == loops * rounds
for w in range(loops):
assert [seq for writer_id, seq, _ in log if writer_id == w] == list(
range(rounds)
)
print(
f"tasks={len(log)} threads_started={db._write_threads_started} "
f"exits_averted={db._write_thread_exits_averted}"
)
assert db._write_threads_started > rounds // 5
finally:
ds.close()