mirror of
https://github.com/simonw/datasette.git
synced 2026-11-16 07:08:34 +01:00
Merge 8bf6e19643 into d0836ddedc
This commit is contained in:
commit
5d797878b9
50 changed files with 11348 additions and 590 deletions
204
tests/connection_budget.py
Normal file
204
tests/connection_budget.py
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
"""Record which databases a block of code touches, and assert a budget.
|
||||
|
||||
With a capped, idle-reaped connection pool, anything that loops over every
|
||||
attached database opens (and evicts) a connection per database at a 0% hit
|
||||
rate. These helpers make "this request only touches the databases it is
|
||||
about" a testable property::
|
||||
|
||||
async with connection_budget() as budget:
|
||||
await ds.client.get("/db1/t1.json")
|
||||
budget.assert_only("db1") # plus _internal, always allowed
|
||||
|
||||
What counts as touching a database:
|
||||
|
||||
* ``opened``: ``Database.connect()`` calls - pooled read connections, write
|
||||
connections, isolated connections and the SchemaWatcher's short-lived
|
||||
scan/pragma connections (``track=False``)
|
||||
* ``reads``: ``Database._execute_fn()`` calls (every ``execute()``,
|
||||
``execute_fn()`` and introspection helper), whether or not the pool had an
|
||||
idle connection - so a warm pool does not hide a loop over every database
|
||||
* ``writes``: ``execute_write*()``, ``execute_isolated_fn()`` and work run on
|
||||
the write connection
|
||||
* ``attached``: databases ATTACHed to a connection while it was prepared
|
||||
(``--crossdb``)
|
||||
* ``probes``: ``sqlite3.connect()`` calls that bypass ``Database.connect()``
|
||||
(``:memory:`` feature probes and the like), keyed by filename
|
||||
|
||||
The patches are installed process-wide for the duration of the block, so
|
||||
background work (the SchemaWatcher's polling task, the read pool's reaper)
|
||||
that runs during the block is counted too. Tests that want to measure one
|
||||
request should turn polling off (``schema_watch_interval_ms=0``) or measure
|
||||
it separately.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import contextlib
|
||||
import threading
|
||||
|
||||
from datasette import app as app_module
|
||||
from datasette.database import Database
|
||||
from datasette.utils import sqlite3
|
||||
|
||||
INTERNAL = "__INTERNAL__"
|
||||
|
||||
_local = threading.local()
|
||||
|
||||
|
||||
class ConnectionBudget:
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self.opened = collections.Counter()
|
||||
self.reads = collections.Counter()
|
||||
self.writes = collections.Counter()
|
||||
self.attached = collections.Counter()
|
||||
self.probes = collections.Counter()
|
||||
|
||||
def _add(self, counter, key):
|
||||
with self._lock:
|
||||
counter[key] += 1
|
||||
|
||||
def touched(self):
|
||||
"""Names of every database opened, read, written or ATTACHed."""
|
||||
names = set()
|
||||
for counter in (self.opened, self.reads, self.writes, self.attached):
|
||||
names.update(counter)
|
||||
return names
|
||||
|
||||
def summary(self):
|
||||
return {
|
||||
"opened": dict(self.opened),
|
||||
"reads": dict(self.reads),
|
||||
"writes": dict(self.writes),
|
||||
"attached": dict(self.attached),
|
||||
"probes": dict(self.probes),
|
||||
}
|
||||
|
||||
def assert_only(self, *names, internal=True, probes=0):
|
||||
"""Assert that nothing outside ``names`` was touched.
|
||||
|
||||
``_internal`` is allowed unless ``internal=False``. ``probes`` is the
|
||||
number of direct ``sqlite3.connect()`` calls allowed.
|
||||
"""
|
||||
allowed = set(names)
|
||||
if internal:
|
||||
allowed.add(INTERNAL)
|
||||
extra = self.touched() - allowed
|
||||
assert not extra, (
|
||||
f"touched {len(extra)} database(s) outside the budget "
|
||||
f"{sorted(allowed)}: {sorted(extra)[:10]}"
|
||||
f"{' ...' if len(extra) > 10 else ''}\n{self.summary()}"
|
||||
)
|
||||
total_probes = sum(self.probes.values())
|
||||
assert (
|
||||
total_probes <= probes
|
||||
), f"{total_probes} direct sqlite3.connect() call(s): {dict(self.probes)}"
|
||||
|
||||
def assert_at_most(self, n, *, internal=True):
|
||||
"""Assert that at most ``n`` user databases were touched."""
|
||||
touched = self.touched()
|
||||
if internal:
|
||||
touched.discard(INTERNAL)
|
||||
assert len(touched) <= n, (
|
||||
f"touched {len(touched)} databases, budget {n}: "
|
||||
f"{sorted(touched)[:10]}\n{self.summary()}"
|
||||
)
|
||||
|
||||
def assert_no_connections(self, internal=True):
|
||||
"""Assert that no connection was opened at all (``_internal`` aside)."""
|
||||
opened = {
|
||||
k: v for k, v in self.opened.items() if not (internal and k == INTERNAL)
|
||||
}
|
||||
assert not opened, f"opened connections: {opened}"
|
||||
assert not self.attached, f"attached: {dict(self.attached)}"
|
||||
assert not self.probes, f"probes: {dict(self.probes)}"
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _patched(budget):
|
||||
original_connect = Database.connect
|
||||
original_execute_fn = Database._execute_fn
|
||||
original_execute_write_fn = Database._execute_write_fn
|
||||
original_isolated = Database.execute_isolated_fn
|
||||
original_on_write_conn = Database._execute_on_write_connection
|
||||
original_prepare = app_module.Datasette._prepare_connection
|
||||
original_sqlite_connect = sqlite3.connect
|
||||
|
||||
def connect(self, *args, **kwargs):
|
||||
budget._add(budget.opened, self.name)
|
||||
_local.depth = getattr(_local, "depth", 0) + 1
|
||||
try:
|
||||
return original_connect(self, *args, **kwargs)
|
||||
finally:
|
||||
_local.depth -= 1
|
||||
|
||||
async def _execute_fn(self, fn):
|
||||
budget._add(budget.reads, self.name)
|
||||
return await original_execute_fn(self, fn)
|
||||
|
||||
async def _execute_write_fn(self, fn, *args, **kwargs):
|
||||
budget._add(budget.writes, self.name)
|
||||
return await original_execute_write_fn(self, fn, *args, **kwargs)
|
||||
|
||||
async def execute_isolated_fn(self, fn):
|
||||
budget._add(budget.writes, self.name)
|
||||
return await original_isolated(self, fn)
|
||||
|
||||
async def _execute_on_write_connection(self, fn):
|
||||
budget._add(budget.writes, self.name)
|
||||
return await original_on_write_conn(self, fn)
|
||||
|
||||
def _prepare_connection(self, conn, database, **kwargs):
|
||||
result = original_prepare(self, conn, database, **kwargs)
|
||||
try:
|
||||
rows = conn.execute("PRAGMA database_list").fetchall()
|
||||
except Exception: # noqa: BLE001
|
||||
rows = []
|
||||
for row in rows:
|
||||
name = row[1]
|
||||
if name not in ("main", "temp"):
|
||||
budget._add(budget.attached, name)
|
||||
return result
|
||||
|
||||
def sqlite_connect(database, *args, **kwargs):
|
||||
if not getattr(_local, "depth", 0):
|
||||
budget._add(budget.probes, str(database))
|
||||
return original_sqlite_connect(database, *args, **kwargs)
|
||||
|
||||
Database.connect = connect
|
||||
Database._execute_fn = _execute_fn
|
||||
Database._execute_write_fn = _execute_write_fn
|
||||
Database.execute_isolated_fn = execute_isolated_fn
|
||||
Database._execute_on_write_connection = _execute_on_write_connection
|
||||
app_module.Datasette._prepare_connection = _prepare_connection
|
||||
sqlite3.connect = sqlite_connect
|
||||
try:
|
||||
yield budget
|
||||
finally:
|
||||
Database.connect = original_connect
|
||||
Database._execute_fn = original_execute_fn
|
||||
Database._execute_write_fn = original_execute_write_fn
|
||||
Database.execute_isolated_fn = original_isolated
|
||||
Database._execute_on_write_connection = original_on_write_conn
|
||||
app_module.Datasette._prepare_connection = original_prepare
|
||||
sqlite3.connect = original_sqlite_connect
|
||||
|
||||
|
||||
class connection_budget:
|
||||
"""Context manager (sync or async) yielding a ConnectionBudget."""
|
||||
|
||||
def __init__(self):
|
||||
self.budget = ConnectionBudget()
|
||||
self._cm = None
|
||||
|
||||
def __enter__(self):
|
||||
self._cm = _patched(self.budget)
|
||||
return self._cm.__enter__()
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return self._cm.__exit__(*exc)
|
||||
|
||||
async def __aenter__(self):
|
||||
return self.__enter__()
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
return self.__exit__(*exc)
|
||||
1251
tests/connection_torture.py
Normal file
1251
tests/connection_torture.py
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -759,6 +759,13 @@ async def test_settings_json(ds_client):
|
|||
"suggest_facets": True,
|
||||
"default_cache_ttl": 5,
|
||||
"num_sql_threads": 1,
|
||||
"max_open_connections": 32,
|
||||
"max_write_connections": 8,
|
||||
"max_pending_writes": 256,
|
||||
"write_queue_timeout_ms": 5000,
|
||||
"connection_idle_timeout_ms": 30000,
|
||||
"schema_watch_interval_ms": 1000,
|
||||
"default_schema_watch": "external",
|
||||
"cache_size_kb": 0,
|
||||
"allow_csv_stream": True,
|
||||
"max_csv_mb": 100,
|
||||
|
|
|
|||
652
tests/test_connection_budget.py
Normal file
652
tests/test_connection_budget.py
Normal file
|
|
@ -0,0 +1,652 @@
|
|||
"""Requests, startup and background work only touch the databases they need.
|
||||
|
||||
With a capped, idle-reaped connection pool, a loop over every attached
|
||||
database opens (and evicts) one connection per database. These tests attach
|
||||
50 databases and use tests/connection_budget.py to assert which databases
|
||||
each endpoint opens, reads or writes. Every test uses a fresh instance, so
|
||||
caches that are only warm after a first request do not hide a sweep.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from datasette.app import Datasette
|
||||
from datasette.utils.catalog import (
|
||||
catalog_relationship_counts,
|
||||
catalog_summaries,
|
||||
catalog_table_details,
|
||||
)
|
||||
from datasette.utils.sqlite import (
|
||||
sqlite_derived_table_dependencies,
|
||||
supports_table_list,
|
||||
)
|
||||
|
||||
from .connection_budget import INTERNAL, connection_budget
|
||||
|
||||
N = 50
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def only_default_plugins():
|
||||
"""Plugins other tests leave registered (tests/plugins/my_plugin.py and
|
||||
friends query datasette.get_database() from their hooks) are not part
|
||||
of what these tests measure: unregister them for the duration."""
|
||||
from datasette.plugins import DEFAULT_PLUGINS, pm
|
||||
|
||||
removed = [
|
||||
(name, plugin)
|
||||
for name, plugin in pm.list_name_plugin()
|
||||
if name not in DEFAULT_PLUGINS
|
||||
]
|
||||
for name, _ in removed:
|
||||
pm.unregister(name=name)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for name, plugin in removed:
|
||||
if pm.get_plugin(name) is None:
|
||||
pm.register(plugin, name=name)
|
||||
|
||||
|
||||
def _make_db(path, i):
|
||||
conn = sqlite3.connect(path)
|
||||
conn.executescript("""
|
||||
create table t2 (id integer primary key, title text);
|
||||
create table t1 (
|
||||
id integer primary key, name text, other_id integer references t2(id)
|
||||
);
|
||||
""")
|
||||
conn.executemany(
|
||||
"insert into t1 (name, other_id) values (?, ?)",
|
||||
[(f"n{j}", j % 3) for j in range(20)],
|
||||
)
|
||||
conn.executemany(
|
||||
"insert into t2 (title) values (?)", [(f"t{j}",) for j in range(3)]
|
||||
)
|
||||
if i % 5 == 0:
|
||||
conn.executescript("""
|
||||
create virtual table t1_fts using fts5(
|
||||
name, content="t1", content_rowid='id'
|
||||
);
|
||||
insert into t1_fts(t1_fts) values('rebuild');
|
||||
""")
|
||||
if i % 3 == 0:
|
||||
conn.execute("create view v1 as select * from t1")
|
||||
if i % 7 == 0:
|
||||
conn.execute("create table _hidden (x)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def db_files(tmp_path_factory):
|
||||
directory = tmp_path_factory.mktemp("budget")
|
||||
paths = []
|
||||
for i in range(N):
|
||||
path = str(directory / f"db{i:03d}.db")
|
||||
_make_db(path, i)
|
||||
paths.append(path)
|
||||
# Older than the SchemaWatcher's racy window, so a sweep is stat-only
|
||||
old = time.time() - 60
|
||||
for path in paths:
|
||||
os.utime(path, (old, old))
|
||||
return paths
|
||||
|
||||
|
||||
def _ds(paths, **kwargs):
|
||||
settings = {"schema_watch_interval_ms": 0}
|
||||
settings.update(kwargs.pop("settings", {}))
|
||||
ds = Datasette(paths, settings=settings, **kwargs)
|
||||
ds.root_enabled = True
|
||||
return ds
|
||||
|
||||
|
||||
def _root(ds):
|
||||
return {"ds_actor": ds.client.actor_cookie({"id": "root"})}
|
||||
|
||||
|
||||
async def _get(ds, path, cookies=None):
|
||||
async with connection_budget() as budget:
|
||||
response = await ds.client.get(path, cookies=cookies or {})
|
||||
assert response.status_code == 200, (path, response.status_code, response.text)
|
||||
return response, budget
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ["/", "/.json", "/-/", "/-/.json"])
|
||||
@pytest.mark.parametrize("as_root", [False, True])
|
||||
async def test_index_page_opens_no_database(db_files, path, as_root):
|
||||
ds = _ds(db_files)
|
||||
await ds.invoke_startup()
|
||||
response, budget = await _get(ds, path, _root(ds) if as_root else None)
|
||||
budget.assert_only()
|
||||
budget.assert_no_connections()
|
||||
if path.endswith(".json"):
|
||||
data = response.json()
|
||||
assert len(data["databases"]) == N
|
||||
# More databases than COUNT_MAX_DATABASES: no row counts
|
||||
assert not any(d["show_table_row_counts"] for d in data["databases"])
|
||||
db000 = next(d for d in data["databases"] if d["name"] == "db000")
|
||||
assert db000["tables_count"] == 2 # t1, t2
|
||||
# _hidden, and t1_fts (an external content FTS table is hidden).
|
||||
# Its fts5 shadow tables are not listed at all: they derive from
|
||||
# t1_fts, which is itself derived (the one-hop permission policy)
|
||||
assert db000["hidden_tables_count"] == 2
|
||||
assert db000["views_count"] == 1
|
||||
shown = {t["name"]: t for t in db000["tables_and_views_truncated"]}
|
||||
assert shown["t1"]["columns"] == ["id", "name", "other_id"]
|
||||
assert shown["t1"]["primary_keys"] == ["id"]
|
||||
assert shown["t1"]["fts_table"] == "t1_fts"
|
||||
# Sorted by foreign key relationships when there are no counts
|
||||
assert shown["t1"]["num_relationships_for_sorting"] == 1
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_page_counts_rows_for_few_databases(db_files):
|
||||
# At most COUNT_MAX_DATABASES databases: rows are counted, and only the
|
||||
# databases listed are touched
|
||||
ds = _ds(db_files[:3])
|
||||
await ds.invoke_startup()
|
||||
response, budget = await _get(ds, "/.json")
|
||||
budget.assert_only("db000", "db001", "db002")
|
||||
data = response.json()
|
||||
assert all(d["show_table_row_counts"] for d in data["databases"])
|
||||
assert data["databases"][0]["table_rows_sum"] == 23
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_page_uses_cached_immutable_counts(db_files):
|
||||
# Immutable databases' counts are computed once, when first needed, and
|
||||
# reused even when there are too many databases to count
|
||||
ds = Datasette(
|
||||
db_files[3:],
|
||||
immutables=db_files[:3],
|
||||
settings={"schema_watch_interval_ms": 0},
|
||||
)
|
||||
await ds.invoke_startup()
|
||||
for name in ("db000", "db001", "db002"):
|
||||
await ds.get_database(name).table_counts()
|
||||
response, budget = await _get(ds, "/.json")
|
||||
budget.assert_no_connections()
|
||||
budget.assert_only()
|
||||
by_name = {d["name"]: d for d in response.json()["databases"]}
|
||||
assert by_name["db000"]["show_table_row_counts"]
|
||||
assert not by_name["db010"]["show_table_row_counts"]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("path", ["/-/databases", "/-/databases.json"])
|
||||
async def test_databases_endpoint(db_files, path):
|
||||
ds = _ds(db_files)
|
||||
await ds.invoke_startup()
|
||||
_, budget = await _get(ds, path)
|
||||
budget.assert_only()
|
||||
budget.assert_no_connections()
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
[
|
||||
"/db001",
|
||||
"/db001.json",
|
||||
"/db000.json?_search=n",
|
||||
"/db001/t1",
|
||||
"/db001/t1.json",
|
||||
"/db001/t1?_facet=name",
|
||||
"/db001/t1.json?_facet=name",
|
||||
"/db000/t1_fts.json?_search=n1",
|
||||
"/db001/-/query?sql=select+1",
|
||||
"/db001/-/query.json?sql=select+1",
|
||||
"/db001/-/query",
|
||||
"/db001/t1/1",
|
||||
"/db001/t1/1.json",
|
||||
"/db001/-/schema.json",
|
||||
"/db001/t1/-/autocomplete?q=n",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("as_root", [False, True])
|
||||
async def test_database_scoped_pages_touch_only_their_database(db_files, path, as_root):
|
||||
ds = _ds(db_files)
|
||||
await ds.invoke_startup()
|
||||
database = path.split("/")[1].split(".")[0]
|
||||
_, budget = await _get(ds, path, _root(ds) if as_root else None)
|
||||
budget.assert_only(database)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"path,as_root",
|
||||
[
|
||||
("/-/versions", False),
|
||||
("/-/versions.json", False),
|
||||
("/-/plugins.json", False),
|
||||
("/-/settings.json", False),
|
||||
("/-/config.json", False),
|
||||
("/-/actor.json", False),
|
||||
("/-/threads", True),
|
||||
("/-/threads.json", True),
|
||||
("/-/tasks.json", True),
|
||||
("/-/actions.json", True),
|
||||
("/-/jump?q=t1", False),
|
||||
("/-/jump.json?q=t1", False),
|
||||
("/-/jump.json", False),
|
||||
("/-/allowed.json?action=view-table", False),
|
||||
("/-/allowed.json?action=view-table", True),
|
||||
("/-/allowed.json?action=view-database", False),
|
||||
("/-/allowed?action=view-table", True),
|
||||
("/-/rules.json?action=view-table", True),
|
||||
("/-/check.json?action=view-table&parent=db001&child=t1", True),
|
||||
("/-/permissions", True),
|
||||
("/-/allow-debug", False),
|
||||
("/-/api", False),
|
||||
("/-/api", True),
|
||||
("/-/create-token", True),
|
||||
("/-/queries.json", False),
|
||||
("/-/patterns", False),
|
||||
],
|
||||
)
|
||||
async def test_instance_pages_open_no_user_database(db_files, path, as_root):
|
||||
ds = _ds(db_files)
|
||||
await ds.invoke_startup()
|
||||
# /-/versions probes the SQLite library once per instance
|
||||
await ds.client.get("/-/versions.json")
|
||||
_, budget = await _get(ds, path, _root(ds) if as_root else None)
|
||||
budget.assert_only()
|
||||
budget.assert_no_connections()
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_versions_probes_sqlite_once(db_files):
|
||||
ds = _ds(db_files[:12], crossdb=True)
|
||||
await ds.invoke_startup()
|
||||
async with connection_budget() as budget:
|
||||
for _ in range(3):
|
||||
assert (await ds.client.get("/-/versions.json")).status_code == 200
|
||||
# One :memory: connection for the first request, none after; it is not
|
||||
# given the --crossdb ATTACHes
|
||||
assert sum(budget.probes.values()) == 1
|
||||
assert not budget.attached
|
||||
budget.assert_only(probes=1)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detect_json1_is_cached(db_files):
|
||||
ds = _ds(db_files[:2])
|
||||
await ds.invoke_startup()
|
||||
await ds.client.get("/db001/t1")
|
||||
_, budget = await _get(ds, "/db001/t1?_facet=name")
|
||||
assert not budget.probes
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_instance_schema_reads_each_visible_database_once(db_files):
|
||||
# /-/schema dumps every visible database's sqlite_master: one read each,
|
||||
# nothing for databases the actor cannot see
|
||||
ds = _ds(
|
||||
db_files[:10],
|
||||
config={"databases": {"db003": {"allow": False}}},
|
||||
)
|
||||
await ds.invoke_startup()
|
||||
_, budget = await _get(ds, "/-/schema.json")
|
||||
expected = {f"db{i:03d}" for i in range(10)} - {"db003"}
|
||||
assert budget.touched() - {INTERNAL} == expected
|
||||
assert all(budget.reads[name] == 1 for name in expected)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_autocomplete_debug_is_bounded(db_files):
|
||||
ds = _ds(db_files)
|
||||
await ds.invoke_startup()
|
||||
_, budget = await _get(ds, "/-/debug/autocomplete")
|
||||
# Only databases holding one of the (at most 5) suggestions
|
||||
budget.assert_at_most(5)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_write_touches_only_its_database(db_files, tmp_path):
|
||||
paths = []
|
||||
for path in db_files[:20]:
|
||||
target = tmp_path / os.path.basename(path)
|
||||
shutil.copy(path, target)
|
||||
paths.append(str(target))
|
||||
ds = _ds(paths)
|
||||
await ds.invoke_startup()
|
||||
async with connection_budget() as budget:
|
||||
response = await ds.client.post(
|
||||
"/db004/-/execute-write",
|
||||
actor={"id": "root"},
|
||||
data={"sql": "create table new_table (id integer primary key)"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert "Query executed" in response.text
|
||||
budget.assert_only("db004")
|
||||
# The catalog already has the new table
|
||||
rows = await ds.get_internal_database().execute(
|
||||
"select 1 from catalog_tables where database_name = 'db004' and table_name = 'new_table'"
|
||||
)
|
||||
assert rows.rows
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telemetry_metric_callbacks_open_nothing(db_files):
|
||||
from datasette import telemetry
|
||||
|
||||
ds = _ds(db_files)
|
||||
await ds.invoke_startup()
|
||||
telemetry.register_datasette(ds)
|
||||
try:
|
||||
with connection_budget() as budget:
|
||||
observations = 0
|
||||
for callback in (
|
||||
telemetry.observe_sql_thread_limit,
|
||||
telemetry.observe_sql_thread_queue_depth,
|
||||
telemetry.observe_pending_queries,
|
||||
telemetry.observe_write_queue_depth,
|
||||
telemetry.observe_open_connections,
|
||||
):
|
||||
observations += len(list(callback()))
|
||||
assert observations
|
||||
budget.assert_only(internal=False)
|
||||
budget.assert_no_connections(internal=False)
|
||||
finally:
|
||||
telemetry.unregister_datasette(ds)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 3])
|
||||
async def test_startup_opens_each_database_at_most_once(db_files, num_sql_threads):
|
||||
imm = db_files[:5]
|
||||
with connection_budget() as budget:
|
||||
ds = Datasette(
|
||||
db_files[5:],
|
||||
immutables=imm,
|
||||
settings={
|
||||
"schema_watch_interval_ms": 0,
|
||||
"num_sql_threads": num_sql_threads,
|
||||
},
|
||||
)
|
||||
await ds._startup_sequence()
|
||||
# One catalog scan connection per database, no pooled reads or writes
|
||||
# (immutable table counts are no longer precomputed at startup)
|
||||
assert set(budget.opened) - {INTERNAL, "_memory"} == {
|
||||
f"db{i:03d}" for i in range(N)
|
||||
}
|
||||
assert all(
|
||||
count == 1 for name, count in budget.opened.items() if name != INTERNAL
|
||||
), budget.opened
|
||||
assert set(budget.reads) <= {INTERNAL}, budget.reads
|
||||
assert set(budget.writes) <= {INTERNAL}, budget.writes
|
||||
assert not budget.attached
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_startup_with_crossdb_attaches_nothing(db_files):
|
||||
with connection_budget() as budget:
|
||||
ds = Datasette(
|
||||
db_files[:12],
|
||||
settings={"schema_watch_interval_ms": 0},
|
||||
crossdb=True,
|
||||
)
|
||||
await ds._startup_sequence()
|
||||
assert not budget.attached
|
||||
# A crossdb query still sees the attached databases
|
||||
response = await ds.client.get(
|
||||
"/_memory/-/query.json?sql=select+count(*)+from+db001.t1&_shape=array"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_restart_with_persisted_catalog_opens_nothing(
|
||||
db_files, tmp_path, monkeypatch
|
||||
):
|
||||
from datasette import schema_watcher
|
||||
|
||||
# The files' ctimes are recent (os.utime() in the fixture): without this
|
||||
# their stored fingerprints would be racy, and so rescanned
|
||||
monkeypatch.setattr(schema_watcher, "RACY_WINDOW_NS", 0)
|
||||
internal = str(tmp_path / "internal.db")
|
||||
ds = Datasette(db_files, internal=internal)
|
||||
await ds.invoke_startup()
|
||||
ds.close()
|
||||
with connection_budget() as budget:
|
||||
ds = Datasette(
|
||||
db_files, internal=internal, settings={"schema_watch_interval_ms": 0}
|
||||
)
|
||||
await ds._startup_sequence()
|
||||
response = await ds.client.get("/.json")
|
||||
assert response.status_code == 200
|
||||
budget.assert_only()
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idle_sweeps_open_nothing(db_files):
|
||||
ds = _ds(db_files)
|
||||
await ds.invoke_startup()
|
||||
with connection_budget() as budget:
|
||||
for _ in range(3):
|
||||
await ds._refresh_schemas(background=True)
|
||||
await ds.refresh_schemas(force=True)
|
||||
budget.assert_only(internal=False)
|
||||
budget.assert_no_connections(internal=False)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_scan_is_not_retried_by_idle_sweeps(
|
||||
db_files, tmp_path, monkeypatch
|
||||
):
|
||||
from datasette import schema_watcher
|
||||
|
||||
# The file was just written: do not wait out the racy window
|
||||
monkeypatch.setattr(schema_watcher, "RACY_WINDOW_NS", 0)
|
||||
bad = tmp_path / "bad.db"
|
||||
bad.write_bytes(b"this is not a SQLite database" * 10)
|
||||
old = time.time() - 60
|
||||
os.utime(bad, (old, old))
|
||||
ds = _ds(db_files[:3] + [str(bad)])
|
||||
await ds.invoke_startup()
|
||||
assert ds.get_database("bad")._watch_state.error is not None
|
||||
with connection_budget() as budget:
|
||||
for _ in range(3):
|
||||
await ds._refresh_schemas(background=True)
|
||||
assert "bad" not in budget.touched()
|
||||
# ...until the file changes
|
||||
good = sqlite3.connect(tmp_path / "good.db")
|
||||
good.execute("create table fixed (id integer primary key)")
|
||||
good.commit()
|
||||
good.close()
|
||||
os.replace(tmp_path / "good.db", bad)
|
||||
await ds._refresh_schemas(background=True)
|
||||
rows = await ds.get_internal_database().execute(
|
||||
"select table_name from catalog_tables where database_name = 'bad'"
|
||||
)
|
||||
assert [r[0] for r in rows.rows] == ["fixed"]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_databases_reuses_catalog_scan(db_files, tmp_path):
|
||||
from datasette.cli import check_databases
|
||||
from datasette.utils import ConnectionProblem
|
||||
|
||||
ds = _ds(db_files)
|
||||
with connection_budget() as budget:
|
||||
await check_databases(ds)
|
||||
# The catalog scan opened each database once; no second pass
|
||||
assert all(count == 1 for name, count in budget.opened.items() if name != INTERNAL)
|
||||
assert set(budget.reads) <= {INTERNAL, "_memory"}, budget.reads
|
||||
ds.close()
|
||||
|
||||
bad = tmp_path / "bad.db"
|
||||
bad.write_bytes(b"this is not a SQLite database" * 10)
|
||||
ds = _ds(db_files[:3] + [str(bad)])
|
||||
with pytest.raises(Exception) as excinfo:
|
||||
await check_databases(ds)
|
||||
assert "bad.db" in str(excinfo.value)
|
||||
assert "file is not a database" in str(excinfo.value)
|
||||
assert not isinstance(excinfo.value, ConnectionProblem)
|
||||
ds.close()
|
||||
|
||||
|
||||
# Parity: the catalog helpers answer exactly what the live helpers do
|
||||
|
||||
|
||||
MISC_SQL = """
|
||||
create table docs (id integer primary key, title text, body text);
|
||||
create virtual table docs_fts using fts4(title, body, content="docs");
|
||||
create virtual table docs_vocab using fts4aux(docs_fts);
|
||||
create virtual table plain using fts5(x);
|
||||
create virtual table geo using rtree(id, minx, maxx);
|
||||
create table _private (x);
|
||||
create table geometry_columns (f_table_name text);
|
||||
create table idx_foo_bar (x);
|
||||
create table IDXA (x);
|
||||
create table parent (id integer primary key);
|
||||
create table child (
|
||||
id integer primary key,
|
||||
p integer references parent(id),
|
||||
q integer references missing(id)
|
||||
);
|
||||
create table comp (a, b, foreign key (a, b) references parent(id, id));
|
||||
create table self_ref (id integer primary key, up integer references self_ref(id));
|
||||
create table wide (b text, a integer, c, primary key (a, b));
|
||||
create view v_docs as select * from docs;
|
||||
"""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("name", ["misc", "fixtures"])
|
||||
async def test_catalog_helpers_match_live_introspection(tmp_path, name):
|
||||
from datasette.fixtures import write_fixture_database
|
||||
from datasette.utils.catalog import (
|
||||
all_derived_table_dependencies,
|
||||
catalog_all_foreign_keys,
|
||||
)
|
||||
|
||||
path = str(tmp_path / f"{name}.db")
|
||||
if name == "fixtures":
|
||||
write_fixture_database(path)
|
||||
config_hidden = set()
|
||||
config = {}
|
||||
else:
|
||||
conn = sqlite3.connect(path)
|
||||
conn.executescript(MISC_SQL)
|
||||
conn.close()
|
||||
config_hidden = {"docs"}
|
||||
config = {"databases": {"misc": {"tables": {"docs": {"hidden": True}}}}}
|
||||
ds = Datasette([path], config=config, settings={"schema_watch_interval_ms": 0})
|
||||
await ds.invoke_startup()
|
||||
db = ds.get_database(name)
|
||||
tables = await db.table_names()
|
||||
summary = (await catalog_summaries(ds, [name]))[name]
|
||||
# hidden_table_names() also lists SpatiaLite names that do not exist
|
||||
hidden = set(await db.hidden_table_names()) & set(tables)
|
||||
assert {t for t in tables if summary.is_hidden(t, config_hidden)} == hidden
|
||||
assert summary.views == set(await db.view_names())
|
||||
|
||||
details = await catalog_table_details(ds, [(name, t) for t in tables])
|
||||
for table in tables:
|
||||
assert details[(name, table)]["columns"] == await db.table_columns(table)
|
||||
assert details[(name, table)]["primary_keys"] == await db.primary_keys(table)
|
||||
assert details[(name, table)]["fts_table"] == await db.fts_table(table)
|
||||
|
||||
all_foreign_keys = await db.get_all_foreign_keys()
|
||||
assert await catalog_all_foreign_keys(ds, name) == all_foreign_keys
|
||||
relationships = (await catalog_relationship_counts(ds, [name])).get(name, {})
|
||||
for table, fks in all_foreign_keys.items():
|
||||
assert relationships.get(table, 0) == len(fks["incoming"] + fks["outgoing"])
|
||||
|
||||
derived = await all_derived_table_dependencies(ds)
|
||||
live = await db.execute_fn(sqlite_derived_table_dependencies)
|
||||
assert derived.get(name, {}) == live
|
||||
# Restored from the catalog rather than remembered from the scan
|
||||
db._cached_derived_table_dependencies = None
|
||||
assert await db.derived_table_dependencies() == live
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(not supports_table_list(), reason="needs PRAGMA table_list")
|
||||
async def test_catalog_records_table_types(tmp_path):
|
||||
path = str(tmp_path / "types.db")
|
||||
conn = sqlite3.connect(path)
|
||||
conn.executescript("""
|
||||
create table plain (id integer primary key);
|
||||
create virtual table f using fts5(x);
|
||||
""")
|
||||
conn.close()
|
||||
ds = Datasette([path], settings={"schema_watch_interval_ms": 0})
|
||||
await ds.invoke_startup()
|
||||
rows = await ds.get_internal_database().execute(
|
||||
"select table_name, type from catalog_tables where database_name = 'types'"
|
||||
)
|
||||
types = dict(rows.rows)
|
||||
assert types["plain"] == "table"
|
||||
assert types["f"] == "virtual"
|
||||
assert types["f_data"] == "shadow"
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_catalog_derived_dependencies_follow_catalog_changes(tmp_path):
|
||||
path = str(tmp_path / "live.db")
|
||||
sqlite3.connect(path).execute("create table docs (body text)").connection.close()
|
||||
ds = Datasette([path], settings={"schema_watch_interval_ms": 0})
|
||||
await ds.invoke_startup()
|
||||
from datasette.utils.catalog import all_derived_table_dependencies
|
||||
|
||||
assert "live" not in await all_derived_table_dependencies(ds)
|
||||
db = ds.get_database("live")
|
||||
await db.execute_write(
|
||||
"create virtual table docs_fts using fts5(body, content='docs')"
|
||||
)
|
||||
derived = await all_derived_table_dependencies(ds)
|
||||
assert derived["live"]["docs_fts"] == "docs"
|
||||
assert derived["live"]["docs_fts_data"] == "docs_fts"
|
||||
ds.close()
|
||||
|
||||
|
||||
def test_connection_budget_helper_counts(db_files):
|
||||
# The helper itself: a direct execute_fn is a read, connect() an open
|
||||
import asyncio
|
||||
|
||||
async def run():
|
||||
ds = _ds(db_files[:2])
|
||||
await ds.invoke_startup()
|
||||
db = ds.get_database("db001")
|
||||
async with connection_budget() as budget:
|
||||
await db.execute("select 1")
|
||||
conn = sqlite3.connect(":memory:")
|
||||
conn.close()
|
||||
assert budget.reads["db001"] == 1
|
||||
assert budget.touched() == {"db001"}
|
||||
with pytest.raises(AssertionError, match="db001"):
|
||||
budget.assert_only("db000")
|
||||
ds.close()
|
||||
return budget
|
||||
|
||||
budget = asyncio.run(run())
|
||||
assert sum(budget.probes.values()) == 1
|
||||
assert json.dumps(budget.summary())
|
||||
283
tests/test_connection_leases.py
Normal file
283
tests/test_connection_leases.py
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
"""
|
||||
Connections handed to execute_write_fn(), the SQL write methods and
|
||||
execute_isolated_fn() callbacks are lease proxies, like read connections:
|
||||
they stop working once the callback returns.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import sqlite_utils
|
||||
|
||||
from datasette.app import Datasette
|
||||
from datasette.database import ConnectionLeaseError
|
||||
|
||||
|
||||
def _make_db(tmp_path, name="leases"):
|
||||
path = str(tmp_path / f"{name}.db")
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute("create table t (id integer primary key, v text)")
|
||||
conn.execute("insert into t (v) values ('x')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return path
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 2])
|
||||
@pytest.mark.parametrize("transaction", [True, False])
|
||||
async def test_write_callback_connection_expires(
|
||||
tmp_path, num_sql_threads, transaction
|
||||
):
|
||||
ds = Datasette([_make_db(tmp_path)], settings={"num_sql_threads": num_sql_threads})
|
||||
db = ds.get_database("leases")
|
||||
stash = {}
|
||||
|
||||
def write(conn):
|
||||
stash["conn"] = conn
|
||||
assert isinstance(conn, sqlite3.Connection)
|
||||
conn.execute("insert into t (v) values ('y')")
|
||||
if not transaction:
|
||||
# transaction=False DML is not committed for you (unchanged)
|
||||
conn.commit()
|
||||
return conn.execute("select count(*) from t").fetchone()[0]
|
||||
|
||||
assert await db.execute_write_fn(write, transaction=transaction) == 2
|
||||
with pytest.raises(ConnectionLeaseError, match="execute_write_fn"):
|
||||
stash["conn"].execute("select 1")
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
stash["conn"].row_factory = None
|
||||
# The write connection itself is fine for the next write
|
||||
await db.execute_write("insert into t (v) values ('z')")
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 3
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 2])
|
||||
async def test_write_callback_may_return_cursor(tmp_path, num_sql_threads):
|
||||
# A common plugin pattern: lambda conn: conn.execute(...). The returned
|
||||
# cursor's attributes stay readable; only the connection is leased.
|
||||
ds = Datasette([_make_db(tmp_path)], settings={"num_sql_threads": num_sql_threads})
|
||||
db = ds.get_database("leases")
|
||||
cursor = await db.execute_write_fn(
|
||||
lambda conn: conn.execute("insert into t (v) values ('y')")
|
||||
)
|
||||
assert cursor.lastrowid == 2
|
||||
assert cursor.rowcount == 1
|
||||
many = await db.execute_write_many("insert into t (v) values (?)", [["a"], ["b"]])
|
||||
assert many.rowcount == 2
|
||||
await db.execute_write_script("insert into t (v) values ('c');")
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 5
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 2])
|
||||
async def test_with_conn_inside_write_callback_yields_proxy(tmp_path, num_sql_threads):
|
||||
ds = Datasette([_make_db(tmp_path)], settings={"num_sql_threads": num_sql_threads})
|
||||
db = ds.get_database("leases")
|
||||
stash = {}
|
||||
|
||||
def write(conn):
|
||||
with conn as inner:
|
||||
stash["inner"] = inner
|
||||
inner.execute("insert into t (v) values ('y')")
|
||||
|
||||
await db.execute_write_fn(write, transaction=False)
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
stash["inner"].execute("select 1")
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 2
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 2])
|
||||
async def test_sqlite_utils_on_write_lease(tmp_path, num_sql_threads):
|
||||
ds = Datasette([_make_db(tmp_path)], settings={"num_sql_threads": num_sql_threads})
|
||||
db = ds.get_database("leases")
|
||||
|
||||
def write(conn):
|
||||
sdb = sqlite_utils.Database(conn)
|
||||
sdb["dogs"].insert_all([{"id": 1, "name": "Cleo"}], pk="id")
|
||||
sdb["dogs"].transform(rename={"name": "dog_name"})
|
||||
return sdb["dogs"].columns_dict
|
||||
|
||||
assert await db.execute_write_fn(write) == {"id": int, "dog_name": str}
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 2])
|
||||
@pytest.mark.parametrize("immutable", [False, True])
|
||||
async def test_isolated_callback_connection_expires(
|
||||
tmp_path, num_sql_threads, immutable
|
||||
):
|
||||
path = _make_db(tmp_path)
|
||||
if immutable:
|
||||
ds = Datasette(immutables=[path], settings={"num_sql_threads": num_sql_threads})
|
||||
else:
|
||||
ds = Datasette([path], settings={"num_sql_threads": num_sql_threads})
|
||||
db = ds.get_database("leases")
|
||||
stash = {}
|
||||
|
||||
def isolated(conn):
|
||||
stash["conn"] = conn
|
||||
return conn.execute("select count(*) from t").fetchone()[0]
|
||||
|
||||
assert await db.execute_isolated_fn(isolated) == 1
|
||||
with pytest.raises(ConnectionLeaseError, match="execute_isolated_fn"):
|
||||
stash["conn"].execute("select 1")
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_wrapper_receives_lease(tmp_path):
|
||||
from datasette import hookimpl
|
||||
from datasette.plugins import pm
|
||||
|
||||
stash = {}
|
||||
|
||||
class WrapperPlugin:
|
||||
__name__ = "WrapperPlugin"
|
||||
|
||||
@hookimpl
|
||||
def write_wrapper(self, datasette, database, request, transaction):
|
||||
if database != "leases":
|
||||
return None
|
||||
|
||||
def wrapper(conn):
|
||||
stash["conn"] = conn
|
||||
yield
|
||||
|
||||
return wrapper
|
||||
|
||||
pm.register(WrapperPlugin(), name="test_write_wrapper_receives_lease")
|
||||
try:
|
||||
ds = Datasette([_make_db(tmp_path)])
|
||||
db = ds.get_database("leases")
|
||||
await db.execute_write("insert into t (v) values ('y')")
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
stash["conn"].execute("select 1")
|
||||
ds.close()
|
||||
finally:
|
||||
pm.unregister(name="test_write_wrapper_receives_lease")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_waits_for_immutable_isolated_fn(tmp_path):
|
||||
# On main this ran on the executor untracked, so close() could close
|
||||
# the connection while the callback was still stepping it (a segfault
|
||||
# with a progress handler installed)
|
||||
ds = Datasette(immutables=[_make_db(tmp_path)], settings={"num_sql_threads": 2})
|
||||
db = ds.get_database("leases")
|
||||
started = threading.Event()
|
||||
|
||||
def slow(conn):
|
||||
started.set()
|
||||
time.sleep(0.3)
|
||||
return conn.execute("select count(*) from t").fetchone()[0]
|
||||
|
||||
task = asyncio.ensure_future(db.execute_isolated_fn(slow))
|
||||
assert await asyncio.to_thread(started.wait, 5)
|
||||
assert len(db._pending_execute_futures) == 1
|
||||
db.close()
|
||||
assert await task == 1
|
||||
ds.close()
|
||||
|
||||
|
||||
NO_THREADS_SCRIPT = r"""
|
||||
import asyncio, os, sqlite3, sys, threading
|
||||
|
||||
def no_threads(self):
|
||||
raise RuntimeError("can't start new thread")
|
||||
|
||||
threading.Thread.start = no_threads
|
||||
|
||||
from datasette.app import Datasette
|
||||
from datasette.database import Database
|
||||
|
||||
path, scratch = sys.argv[1], sys.argv[2]
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute("create table t (id integer primary key, v text)")
|
||||
conn.execute("insert into t (v) values ('x')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
async def main():
|
||||
ds = Datasette(
|
||||
[path],
|
||||
settings={
|
||||
"num_sql_threads": 0,
|
||||
"connection_idle_timeout_ms": 50,
|
||||
"schema_watch_interval_ms": 50,
|
||||
},
|
||||
)
|
||||
await ds.invoke_startup()
|
||||
db = ds.get_database("nothreads")
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 1
|
||||
await db.execute_write("insert into t (v) values ('y')")
|
||||
await db.execute_write("create table t2 (id)")
|
||||
tables = await ds.get_internal_database().execute(
|
||||
"select table_name from catalog_tables where database_name = 'nothreads' "
|
||||
"order by table_name"
|
||||
)
|
||||
assert [r[0] for r in tables.rows] == ["t", "t2"], tables.rows
|
||||
assert await db.execute_isolated_fn(
|
||||
lambda conn: conn.execute("select count(*) from t").fetchone()[0]
|
||||
) == 2
|
||||
await db.execute_write("insert into t (v) values ('z')", block=False)
|
||||
response = await ds.client.get("/nothreads/t.json?_shape=array")
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(response.json()) == 3
|
||||
# External change detected by the polling task, inline on the loop
|
||||
other = sqlite3.connect(path)
|
||||
other.execute("create table external_t (id)")
|
||||
other.commit()
|
||||
other.close()
|
||||
for _ in range(100):
|
||||
await asyncio.sleep(0.02)
|
||||
tables = await ds.get_internal_database().execute(
|
||||
"select 1 from catalog_tables where table_name = 'external_t'"
|
||||
)
|
||||
if tables.rows:
|
||||
break
|
||||
else:
|
||||
raise AssertionError("external change not seen")
|
||||
# Scratch database added later, created by its first write
|
||||
sdb = ds.add_database(Database(ds, path=scratch), name="scratch")
|
||||
await sdb.execute_write("create table s (id)")
|
||||
assert os.path.exists(scratch)
|
||||
ds.close()
|
||||
print("ok", threading.active_count())
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
"""
|
||||
|
||||
|
||||
def test_num_sql_threads_zero_starts_no_threads(tmp_path):
|
||||
# Simulates Pyodide, where no thread can be started: nothing in the
|
||||
# read pool, write path or schema watcher may need one
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
NO_THREADS_SCRIPT,
|
||||
str(tmp_path / "nothreads.db"),
|
||||
str(tmp_path / "scratch.db"),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert result.stdout.strip() == "ok 1"
|
||||
92
tests/test_connection_limits.py
Normal file
92
tests/test_connection_limits.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
"""Exercise real HTTP sockets and WAL files under a small OS descriptor limit."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Requires RLIMIT_NOFILE")
|
||||
def test_500_wal_databases_with_http_and_writes_under_256_fds():
|
||||
script = r"""
|
||||
import asyncio, concurrent.futures, json, resource, socket, sqlite3, tempfile, threading, time, urllib.request
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
import uvicorn
|
||||
from datasette.app import Datasette
|
||||
|
||||
async def main():
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
paths = []
|
||||
for i in range(500):
|
||||
path = str(Path(directory) / f"db{i}.db")
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute("pragma journal_mode=wal")
|
||||
conn.execute("create table t(id integer primary key)")
|
||||
conn.close()
|
||||
paths.append(path)
|
||||
_, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
||||
resource.setrlimit(resource.RLIMIT_NOFILE, (256, hard))
|
||||
ds = Datasette(paths)
|
||||
await ds.invoke_startup()
|
||||
process = psutil.Process()
|
||||
baseline_rss = process.memory_info().rss
|
||||
peak = {"fds": 0, "rss": 0, "writers": 0}
|
||||
stopped = threading.Event()
|
||||
def monitor():
|
||||
while not stopped.wait(0.002):
|
||||
peak["fds"] = max(peak["fds"], process.num_fds())
|
||||
peak["rss"] = max(peak["rss"], process.memory_info().rss)
|
||||
peak["writers"] = max(peak["writers"], ds._write_budget.snapshot()["writers"])
|
||||
watcher = threading.Thread(target=monitor)
|
||||
watcher.start()
|
||||
sock = socket.socket()
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
server = uvicorn.Server(uvicorn.Config(ds.app(), log_level="error"))
|
||||
thread = threading.Thread(target=lambda: server.run(sockets=[sock]))
|
||||
thread.start()
|
||||
try:
|
||||
deadline = time.monotonic() + 10
|
||||
while not server.started:
|
||||
assert time.monotonic() < deadline
|
||||
await asyncio.sleep(0.01)
|
||||
def request(i):
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{port}/db{i}/-/query.json?sql=select+count(*)+from+t", timeout=20) as response:
|
||||
assert response.status == 200
|
||||
assert json.load(response)["ok"]
|
||||
def reads():
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as clients:
|
||||
list(clients.map(request, range(500)))
|
||||
async def writes():
|
||||
for start in range(0, 500, 16):
|
||||
await asyncio.gather(*(ds.get_database(f"db{i}").execute_write("insert into t values (1)") for i in range(start, min(start + 16, 500))))
|
||||
await asyncio.gather(asyncio.to_thread(reads), writes())
|
||||
for i in range(500):
|
||||
assert (await ds.get_database(f"db{i}").execute("select count(*) from t")).single_value() == 1
|
||||
assert ds._read_pool.stats["peak_open"] <= 32
|
||||
assert ds._write_budget.snapshot()["stats"]["peak_writers"] <= 8
|
||||
print(json.dumps({"peak": peak, "baseline_rss": baseline_rss, "writes": 500, "http_reads": 500}), flush=True)
|
||||
finally:
|
||||
server.should_exit = True
|
||||
await asyncio.to_thread(thread.join, 15)
|
||||
assert not thread.is_alive()
|
||||
stopped.set()
|
||||
watcher.join()
|
||||
sock.close()
|
||||
ds.close()
|
||||
asyncio.run(main())
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=90,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
measurements = json.loads(result.stdout.strip().splitlines()[-1])
|
||||
assert measurements["peak"]["fds"] < 256
|
||||
assert measurements["peak"]["writers"] <= 8
|
||||
print(measurements)
|
||||
112
tests/test_connection_recovery.py
Normal file
112
tests/test_connection_recovery.py
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
import errno
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from datasette.app import Datasette
|
||||
from datasette.connection_pool import ConnectionLeaseError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def database_file(tmp_path):
|
||||
path = tmp_path / "data.db"
|
||||
conn = sqlite3.connect(path)
|
||||
conn.executescript("create table t(id); insert into t values (1), (2)")
|
||||
conn.close()
|
||||
return str(path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", ["read", "write", "isolated"])
|
||||
async def test_saved_methods_and_cursors_cannot_escape_callback(database_file, kind):
|
||||
ds = Datasette([database_file], settings={"schema_watch_interval_ms": 0})
|
||||
db = ds.get_database("data")
|
||||
saved = {}
|
||||
|
||||
def callback(conn):
|
||||
saved["commit"] = conn.commit
|
||||
saved["cursor"] = conn.execute("select * from t")
|
||||
saved["fetch"] = saved["cursor"].fetchone
|
||||
saved["connection"] = saved["cursor"].connection
|
||||
assert saved["fetch"]()[0] == 1
|
||||
|
||||
method = {
|
||||
"read": db.execute_fn,
|
||||
"write": db.execute_write_fn,
|
||||
"isolated": db.execute_isolated_fn,
|
||||
}[kind]
|
||||
try:
|
||||
await method(callback)
|
||||
for call in (saved["commit"], lambda: saved["connection"].execute("select 1")):
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
call()
|
||||
for call in (saved["fetch"], lambda: next(saved["cursor"])):
|
||||
with pytest.raises((ConnectionLeaseError, sqlite3.ProgrammingError)):
|
||||
call()
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 2
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_result_containing_cursor_is_rejected(database_file):
|
||||
ds = Datasette([database_file])
|
||||
db = ds.get_database("data")
|
||||
try:
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
await db.execute_fn(lambda c: {"nested": [c.execute("select * from t")]})
|
||||
result = await db.execute_write_fn(
|
||||
lambda c: {"nested": [c.execute("insert into t values (3)")]}
|
||||
)
|
||||
assert result["nested"][0].rowcount == 1
|
||||
with pytest.raises((ConnectionLeaseError, sqlite3.ProgrammingError)):
|
||||
result["nested"][0].fetchone()
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 3
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"failure",
|
||||
[
|
||||
sqlite3.OperationalError("unable to open database file"),
|
||||
OSError(errno.EMFILE, "Too many open files"),
|
||||
],
|
||||
)
|
||||
async def test_schema_recovers_after_resource_error_without_file_change(
|
||||
database_file, monkeypatch, failure
|
||||
):
|
||||
ds = Datasette([database_file], settings={"schema_watch_interval_ms": 0})
|
||||
try:
|
||||
await ds.invoke_startup()
|
||||
db = ds.get_database("data")
|
||||
conn = sqlite3.connect(database_file)
|
||||
conn.execute("create table added_externally(id)")
|
||||
conn.close()
|
||||
watcher = ds._schema_watcher
|
||||
original = watcher._connect
|
||||
|
||||
def fail(database, prepare):
|
||||
if database is db and prepare:
|
||||
raise failure
|
||||
return original(database, prepare)
|
||||
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(watcher, "_connect", fail)
|
||||
await ds.refresh_schemas()
|
||||
state = db._watch_state
|
||||
assert state.needs_scan
|
||||
# Move past the retry backoff and fingerprint race window without
|
||||
# touching the file or relying on timing-sensitive sleeps.
|
||||
state.retry_at = 0
|
||||
state.fp_racy = False
|
||||
await watcher.sweep(background=True)
|
||||
tables = await ds.get_internal_database().execute(
|
||||
"select table_name from catalog_tables where database_name=?", ["data"]
|
||||
)
|
||||
assert {row[0] for row in tables.rows} == {"t", "added_externally"}
|
||||
assert state.error is None
|
||||
assert not state.needs_scan
|
||||
finally:
|
||||
ds.close()
|
||||
326
tests/test_connection_resource_leases.py
Normal file
326
tests/test_connection_resource_leases.py
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
"""SQLite resources must not retain locks or run after their callback ends."""
|
||||
|
||||
import operator
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from datasette.app import Datasette
|
||||
from datasette.connection_pool import ConnectionLeaseError
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def resource_db(tmp_path):
|
||||
path = tmp_path / "resources.db"
|
||||
conn = sqlite3.connect(path)
|
||||
conn.executescript(
|
||||
"create table t(id integer primary key, data blob);"
|
||||
"insert into t(data) values (x'616263646566'), (zeroblob(6)), (zeroblob(6));"
|
||||
"create table another(id);"
|
||||
)
|
||||
conn.close()
|
||||
return str(path)
|
||||
|
||||
|
||||
def callback_method(db, kind):
|
||||
return {
|
||||
"read": db.execute_fn,
|
||||
"write": db.execute_write_fn,
|
||||
"isolated": db.execute_isolated_fn,
|
||||
}[kind]
|
||||
|
||||
|
||||
def check_no_read_lock(path):
|
||||
# A retained cursor/blob would block this commit in rollback journal mode.
|
||||
conn = sqlite3.connect(path, timeout=0)
|
||||
try:
|
||||
conn.execute("insert into another values (1)")
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 2])
|
||||
@pytest.mark.parametrize("kind", ["read", "write", "isolated", "immutable_isolated"])
|
||||
@pytest.mark.parametrize("started", [False, True])
|
||||
async def test_stashed_dump_iterator_expires_and_releases_lock(
|
||||
resource_db, num_sql_threads, kind, started
|
||||
):
|
||||
ds = Datasette(
|
||||
**{("immutables" if kind == "immutable_isolated" else "files"): [resource_db]},
|
||||
settings={"num_sql_threads": num_sql_threads},
|
||||
)
|
||||
db = ds.get_database("resources")
|
||||
saved = {}
|
||||
|
||||
def callback(conn):
|
||||
saved["dump"] = dump = conn.iterdump()
|
||||
saved["next"] = dump.__next__
|
||||
saved["send"] = dump.send
|
||||
saved["close"] = dump.close
|
||||
assert iter(dump) is dump
|
||||
if started:
|
||||
for statement in dump:
|
||||
if statement.startswith('INSERT INTO "t"'):
|
||||
break
|
||||
else:
|
||||
pytest.fail("dump did not reach a data row")
|
||||
return "done"
|
||||
|
||||
try:
|
||||
assert (
|
||||
await callback_method(db, kind.replace("immutable_", ""))(callback)
|
||||
== "done"
|
||||
)
|
||||
for call in (
|
||||
saved["next"],
|
||||
lambda: iter(saved["dump"]),
|
||||
lambda: saved["send"](None),
|
||||
lambda: saved["dump"].throw(ValueError),
|
||||
saved["close"],
|
||||
):
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
call()
|
||||
if kind != "immutable_isolated":
|
||||
check_no_read_lock(resource_db)
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 3
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 2])
|
||||
@pytest.mark.parametrize("kind", ["read", "write", "isolated"])
|
||||
async def test_dump_can_be_consumed_inside_callback(resource_db, num_sql_threads, kind):
|
||||
ds = Datasette([resource_db], settings={"num_sql_threads": num_sql_threads})
|
||||
try:
|
||||
statements = await callback_method(ds.get_database("resources"), kind)(
|
||||
lambda conn: list(conn.iterdump())
|
||||
)
|
||||
restored = sqlite3.connect(":memory:")
|
||||
try:
|
||||
restored.executescript("\n".join(statements))
|
||||
assert (
|
||||
restored.execute("select data from t where id=1").fetchone()[0]
|
||||
== b"abcdef"
|
||||
)
|
||||
finally:
|
||||
restored.close()
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("nested", [False, True])
|
||||
async def test_returning_dump_from_read_callback_is_rejected(resource_db, nested):
|
||||
ds = Datasette([resource_db])
|
||||
|
||||
def callback(conn):
|
||||
dump = conn.iterdump()
|
||||
return {"dump": [dump]} if nested else dump
|
||||
|
||||
try:
|
||||
with pytest.raises(ConnectionLeaseError, match="database resources"):
|
||||
await ds.get_database("resources").execute_fn(callback)
|
||||
check_no_read_lock(resource_db)
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generator_over_dump_from_read_callback_is_rejected(resource_db):
|
||||
ds = Datasette([resource_db])
|
||||
try:
|
||||
with pytest.raises(ConnectionLeaseError, match="database resources"):
|
||||
await ds.get_database("resources").execute_fn(
|
||||
lambda conn: (line for line in conn.iterdump())
|
||||
)
|
||||
check_no_read_lock(resource_db)
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
requires_blob = pytest.mark.skipif(
|
||||
not hasattr(sqlite3.Connection, "blobopen"),
|
||||
reason="Incremental BLOB access requires Python 3.11",
|
||||
)
|
||||
|
||||
|
||||
@requires_blob
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 2])
|
||||
@pytest.mark.parametrize("kind", ["read", "write", "isolated", "immutable_isolated"])
|
||||
async def test_stashed_blob_expires_and_releases_lock(
|
||||
resource_db, num_sql_threads, kind
|
||||
):
|
||||
ds = Datasette(
|
||||
**{("immutables" if kind == "immutable_isolated" else "files"): [resource_db]},
|
||||
settings={"num_sql_threads": num_sql_threads},
|
||||
)
|
||||
saved = {}
|
||||
|
||||
def callback(conn):
|
||||
saved["blob"] = blob = conn.blobopen("t", "data", 1, readonly=True)
|
||||
saved["read"] = blob.read
|
||||
saved["close"] = blob.close
|
||||
assert isinstance(blob, sqlite3.Blob)
|
||||
assert len(blob) == 6
|
||||
assert blob[0] == ord("a")
|
||||
assert blob[1:3] == b"bc"
|
||||
assert blob.read(2) == b"ab"
|
||||
assert blob.tell() == 2
|
||||
blob.seek(-1, 2)
|
||||
assert saved["read"]() == b"f"
|
||||
|
||||
try:
|
||||
await callback_method(
|
||||
ds.get_database("resources"), kind.replace("immutable_", "")
|
||||
)(callback)
|
||||
blob = saved["blob"]
|
||||
for call in (
|
||||
saved["read"],
|
||||
saved["close"],
|
||||
lambda: blob.write(b"x"),
|
||||
lambda: blob.tell(),
|
||||
lambda: blob.seek(0),
|
||||
lambda: len(blob),
|
||||
lambda: blob[0],
|
||||
lambda: blob[:],
|
||||
lambda: operator.setitem(blob, 0, 120),
|
||||
lambda: operator.setitem(blob, slice(0, 1), b"x"),
|
||||
blob.__enter__,
|
||||
lambda: blob.__exit__(None, None, None),
|
||||
):
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
call()
|
||||
if kind != "immutable_isolated":
|
||||
check_no_read_lock(resource_db)
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", ["write", "isolated"])
|
||||
@pytest.mark.parametrize(
|
||||
"resource", ["dump", pytest.param("blob", marks=requires_blob)]
|
||||
)
|
||||
async def test_returned_write_resource_is_expired(resource_db, kind, resource):
|
||||
ds = Datasette([resource_db])
|
||||
|
||||
def callback(conn):
|
||||
if resource == "dump":
|
||||
return conn.iterdump()
|
||||
return conn.blobopen("t", "data", 1, readonly=True)
|
||||
|
||||
try:
|
||||
result = await callback_method(ds.get_database("resources"), kind)(callback)
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
if resource == "dump":
|
||||
next(result)
|
||||
else:
|
||||
result.read()
|
||||
check_no_read_lock(resource_db)
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@requires_blob
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 2])
|
||||
async def test_writable_blob_closes_before_transaction_commits(
|
||||
resource_db, num_sql_threads
|
||||
):
|
||||
ds = Datasette([resource_db], settings={"num_sql_threads": num_sql_threads})
|
||||
db = ds.get_database("resources")
|
||||
saved = {}
|
||||
|
||||
def write(conn):
|
||||
saved["blob"] = blob = conn.blobopen("t", "data", 1)
|
||||
blob.write(b"xy")
|
||||
blob[2] = ord("z")
|
||||
blob[3:] = b"123"
|
||||
# Leaving this BLOB open on SQLite makes the enclosing COMMIT fail
|
||||
# with "SQL statements in progress"; expiry must close it first.
|
||||
return "written"
|
||||
|
||||
try:
|
||||
assert await db.execute_write_fn(write) == "written"
|
||||
assert (
|
||||
await db.execute("select data from t where id=1")
|
||||
).single_value() == b"xyz123"
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
saved["blob"].read()
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@requires_blob
|
||||
@pytest.mark.asyncio
|
||||
async def test_blob_context_manager_returns_lease(resource_db):
|
||||
ds = Datasette([resource_db])
|
||||
saved = {}
|
||||
|
||||
def write(conn):
|
||||
with conn.blobopen("t", "data", 1) as blob:
|
||||
saved["blob"] = blob
|
||||
blob.write(b"xyz")
|
||||
with pytest.raises(sqlite3.ProgrammingError):
|
||||
blob.read()
|
||||
|
||||
try:
|
||||
await ds.get_database("resources").execute_write_fn(write)
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
saved["blob"].read()
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@requires_blob
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("nested", [False, True])
|
||||
async def test_returning_blob_from_read_callback_is_rejected(resource_db, nested):
|
||||
ds = Datasette([resource_db])
|
||||
|
||||
def callback(conn):
|
||||
blob = conn.blobopen("t", "data", 1, readonly=True)
|
||||
return {"blob": [blob]} if nested else blob
|
||||
|
||||
try:
|
||||
with pytest.raises(ConnectionLeaseError, match="database resources"):
|
||||
await ds.get_database("resources").execute_fn(callback)
|
||||
check_no_read_lock(resource_db)
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", ["read", "write", "isolated"])
|
||||
@pytest.mark.parametrize(
|
||||
"resource", ["dump", pytest.param("blob", marks=requires_blob)]
|
||||
)
|
||||
async def test_resources_close_when_callback_raises(resource_db, kind, resource):
|
||||
ds = Datasette([resource_db])
|
||||
saved = {}
|
||||
|
||||
def fail(conn):
|
||||
if resource == "dump":
|
||||
saved["resource"] = value = conn.iterdump()
|
||||
for statement in value:
|
||||
if statement.startswith('INSERT INTO "t"'):
|
||||
break
|
||||
else:
|
||||
saved["resource"] = conn.blobopen("t", "data", 1, readonly=True)
|
||||
raise ValueError("callback failed")
|
||||
|
||||
try:
|
||||
with pytest.raises(ValueError, match="callback failed"):
|
||||
await callback_method(ds.get_database("resources"), kind)(fail)
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
if resource == "dump":
|
||||
next(saved["resource"])
|
||||
else:
|
||||
saved["resource"].read()
|
||||
check_no_read_lock(resource_db)
|
||||
finally:
|
||||
ds.close()
|
||||
862
tests/test_connection_review_fixes.py
Normal file
862
tests/test_connection_review_fixes.py
Normal file
|
|
@ -0,0 +1,862 @@
|
|||
"""
|
||||
Regression tests for problems found by the second review of the connection
|
||||
management redesign (concurrency, lifecycle and public-contract findings).
|
||||
Each test names the problem it pins down.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from datasette import schema_watcher as schema_watcher_module
|
||||
from datasette.app import Datasette
|
||||
from datasette.connection_pool import ConnectionLeaseError
|
||||
from datasette.database import Database
|
||||
|
||||
|
||||
def make_db(path, sql="create table t (id integer primary key, v text)"):
|
||||
conn = sqlite3.connect(path)
|
||||
conn.executescript(sql)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
async def catalog_tables(ds, name):
|
||||
rows = await ds.get_internal_database().execute(
|
||||
"select table_name from catalog_tables where database_name = ? "
|
||||
"order by table_name",
|
||||
[name],
|
||||
)
|
||||
return [r[0] for r in rows.rows]
|
||||
|
||||
|
||||
def live_tables(path):
|
||||
conn = sqlite3.connect(path)
|
||||
try:
|
||||
return sorted(
|
||||
r[0]
|
||||
for r in conn.execute(
|
||||
"select name from sqlite_master where type = 'table' "
|
||||
"and name not like 'sqlite_%'"
|
||||
)
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def run_in_threads(*coroutine_functions, timeout=20):
|
||||
"""Run each coroutine function with asyncio.run() in its own thread:
|
||||
several event loops driving one Datasette at the same time."""
|
||||
results = [None] * len(coroutine_functions)
|
||||
|
||||
def runner(i, fn):
|
||||
try:
|
||||
results[i] = ("ok", asyncio.run(fn()))
|
||||
except BaseException as e: # noqa: BLE001
|
||||
results[i] = ("error", e)
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=runner, args=(i, fn), daemon=True)
|
||||
for i, fn in enumerate(coroutine_functions)
|
||||
]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
deadline = time.monotonic() + timeout
|
||||
for thread in threads:
|
||||
thread.join(max(0.0, deadline - time.monotonic()))
|
||||
hung = [i for i, thread in enumerate(threads) if thread.is_alive()]
|
||||
assert not hung, f"event loop threads {hung} did not finish within {timeout}s"
|
||||
for status, value in results:
|
||||
if status == "error":
|
||||
raise value
|
||||
return [value for _, value in results]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def owned_db(tmp_path):
|
||||
path = str(tmp_path / "data.db")
|
||||
make_db(path)
|
||||
ds = Datasette(settings={"schema_watch_interval_ms": 0})
|
||||
db = ds.add_database(Database(ds, path=path), name="data", schema_watch="owned")
|
||||
asyncio.run(ds.invoke_startup())
|
||||
yield ds, db, path
|
||||
ds.close()
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Concurrency and lifecycle
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cross_loop_ddl_writes_do_not_hang(owned_db, monkeypatch):
|
||||
# C1: refresh() treated a scan started on another *live* event loop as
|
||||
# abandoned, stole it, and each loop then resolved or cleared the
|
||||
# other's future: blocking writes on one loop never returned
|
||||
ds, db, path = owned_db
|
||||
original = schema_watcher_module.SchemaWatcher._scan_chunk_sync
|
||||
|
||||
def slow_scan(self, states):
|
||||
time.sleep(0.2)
|
||||
return original(self, states)
|
||||
|
||||
monkeypatch.setattr(
|
||||
schema_watcher_module.SchemaWatcher, "_scan_chunk_sync", slow_scan
|
||||
)
|
||||
|
||||
def writer(prefix, count):
|
||||
async def run():
|
||||
for i in range(count):
|
||||
await asyncio.wait_for(
|
||||
db.execute_write(f"create table {prefix}{i} (id integer)"), 10
|
||||
)
|
||||
|
||||
return run
|
||||
|
||||
def inserter():
|
||||
async def run():
|
||||
async def worker():
|
||||
for _ in range(20):
|
||||
await asyncio.wait_for(
|
||||
db.execute_write("insert into t (v) values ('x')"), 10
|
||||
)
|
||||
|
||||
await asyncio.gather(*(worker() for _ in range(3)))
|
||||
|
||||
return run
|
||||
|
||||
run_in_threads(writer("a", 3), writer("b", 3), inserter())
|
||||
|
||||
async def check():
|
||||
return await catalog_tables(ds, "data")
|
||||
|
||||
assert asyncio.run(check()) == live_tables(path)
|
||||
state = db._watch_state
|
||||
assert state.scan_future is None
|
||||
assert not state.needs_scan
|
||||
|
||||
|
||||
def test_scan_from_stopped_loop_is_taken_over(owned_db):
|
||||
# The rule for taking over a scan: its loop has closed or stopped, so
|
||||
# nothing will finish it. A loop stopped by run_until_complete()
|
||||
# returning while a scan task is pending must not hang other loops.
|
||||
ds, db, _path = owned_db
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
gate = threading.Event()
|
||||
original = schema_watcher_module.SchemaWatcher._scan_chunk_sync
|
||||
|
||||
def gated_scan(self, states):
|
||||
gate.wait(5)
|
||||
return original(self, states)
|
||||
|
||||
schema_watcher_module.SchemaWatcher._scan_chunk_sync = gated_scan
|
||||
try:
|
||||
|
||||
async def start_scan():
|
||||
# block=False: the scan task is spawned on this loop and
|
||||
# left pending when run_until_complete() returns
|
||||
await db.execute_write(
|
||||
"create table stopped1 (id integer)", block=False
|
||||
)
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
loop.run_until_complete(start_scan())
|
||||
finally:
|
||||
schema_watcher_module.SchemaWatcher._scan_chunk_sync = original
|
||||
gate.set()
|
||||
|
||||
async def other_loop():
|
||||
await asyncio.wait_for(
|
||||
db.execute_write("create table other1 (id integer)"), 10
|
||||
)
|
||||
return await catalog_tables(ds, "data")
|
||||
|
||||
tables = run_in_threads(other_loop)[0]
|
||||
assert {"stopped1", "other1"} <= set(tables)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
def test_one_poller_across_live_event_loops(tmp_path):
|
||||
# C2: every switch between two live event loops started another
|
||||
# poller task, each sweeping every interval
|
||||
path = str(tmp_path / "ext.db")
|
||||
make_db(path)
|
||||
ds = Datasette([path], settings={"schema_watch_interval_ms": 100})
|
||||
asyncio.run(ds.invoke_startup())
|
||||
watcher = ds._schema_watcher
|
||||
turns = [threading.Event(), threading.Event()]
|
||||
done = threading.Event()
|
||||
counts = {}
|
||||
|
||||
def client(me, other, rounds):
|
||||
async def run():
|
||||
for _ in range(rounds):
|
||||
turns[me].wait(5)
|
||||
turns[me].clear()
|
||||
assert (await ds.client.get("/ext.json")).status_code == 200
|
||||
turns[other].set()
|
||||
if me == 0:
|
||||
sweeps = watcher.counters["sweeps"]
|
||||
await asyncio.sleep(1.0)
|
||||
counts["sweeps_per_s"] = watcher.counters["sweeps"] - sweeps
|
||||
counts["pollers"] = len([t for t in watcher._pollers if not t.done()])
|
||||
done.set()
|
||||
else:
|
||||
done.wait(10)
|
||||
|
||||
return run
|
||||
|
||||
turns[0].set()
|
||||
run_in_threads(client(0, 1, 10), client(1, 0, 10))
|
||||
assert counts["pollers"] == 1
|
||||
# One poller at a 100ms interval: about 10 sweeps a second, not 10 per
|
||||
# loop switch
|
||||
assert counts["sweeps_per_s"] <= 15
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_catalog_write_is_retried(owned_db, monkeypatch):
|
||||
# C3: refresh() cleared needs_scan before scanning and never restored
|
||||
# it, so an owned database lost a catalog update for good if the
|
||||
# catalog write failed (internal database locked)
|
||||
ds, db, _path = owned_db
|
||||
original = schema_watcher_module.write_catalog_entries
|
||||
locked = threading.Event()
|
||||
locked.set()
|
||||
|
||||
def flaky(conn, entries):
|
||||
if locked.is_set():
|
||||
raise sqlite3.OperationalError("database is locked")
|
||||
return original(conn, entries)
|
||||
|
||||
monkeypatch.setattr(schema_watcher_module, "write_catalog_entries", flaky)
|
||||
await db.execute_write("create table added_while_locked (id integer)")
|
||||
# Let the refresh task spawned by the write's notification fail too
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0.01)
|
||||
state = db._watch_state
|
||||
assert state.needs_scan
|
||||
assert state in ds._schema_watcher._pending
|
||||
assert await catalog_tables(ds, "data") == ["t"]
|
||||
# While still locked, a request is not failed by the catalog update
|
||||
state.retry_at = 0.0
|
||||
assert (await ds.client.get("/data.json")).status_code == 200
|
||||
assert state.retry_at > time.monotonic()
|
||||
# Once unlocked, the next request retries (after the back-off)
|
||||
locked.clear()
|
||||
state.retry_at = 0.0
|
||||
await ds.client.get("/data.json")
|
||||
assert await catalog_tables(ds, "data") == ["added_while_locked", "t"]
|
||||
response = await ds.client.get("/data.json")
|
||||
assert {t["name"] for t in response.json()["tables"]} == {
|
||||
"added_while_locked",
|
||||
"t",
|
||||
}
|
||||
|
||||
|
||||
def test_block_false_write_from_closed_loop_reaches_catalog(owned_db):
|
||||
# C3: the schema-change notification went to the caller's loop, which
|
||||
# had closed: notified_version was recorded anyway and the change was
|
||||
# never picked up
|
||||
ds, db, path = owned_db
|
||||
|
||||
async def write():
|
||||
await db.execute_write(
|
||||
"create table added_nonblocking (id integer)", block=False
|
||||
)
|
||||
|
||||
asyncio.run(write())
|
||||
# Let the write thread finish (it delivers to a closed loop)
|
||||
deadline = time.monotonic() + 5
|
||||
while "added_nonblocking" not in live_tables(path):
|
||||
assert time.monotonic() < deadline
|
||||
time.sleep(0.01)
|
||||
time.sleep(0.05)
|
||||
|
||||
async def later():
|
||||
await ds.client.get("/data.json")
|
||||
return await catalog_tables(ds, "data")
|
||||
|
||||
assert asyncio.run(later()) == ["added_nonblocking", "t"]
|
||||
|
||||
|
||||
def test_unclosed_datasette_is_collected():
|
||||
# C4: the temp internal database's atexit.register(bound method) pinned
|
||||
# every Datasette even after the caller released its last reference.
|
||||
ds = Datasette()
|
||||
ref = weakref.ref(ds)
|
||||
del ds
|
||||
gc.collect()
|
||||
assert ref() is None
|
||||
|
||||
|
||||
def test_closed_datasette_is_collected(tmp_path):
|
||||
# C4 + C5: atexit and the read-pool reaper's loop variable both kept a
|
||||
# closed Datasette (and every Database, cache and plugin state) alive
|
||||
path = str(tmp_path / "data.db")
|
||||
make_db(path)
|
||||
ds = Datasette([path], settings={"connection_idle_timeout_ms": 20})
|
||||
|
||||
async def use(ds):
|
||||
await ds.invoke_startup()
|
||||
await ds.get_database("data").execute("select 1")
|
||||
await ds.get_database("data").execute_write("insert into t (v) values (1)")
|
||||
|
||||
asyncio.run(use(ds))
|
||||
# Let the reaper scan (and close) the idle connection
|
||||
time.sleep(0.3)
|
||||
ds.close()
|
||||
ref = weakref.ref(ds)
|
||||
del ds
|
||||
gc.collect()
|
||||
assert ref() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_callback_cannot_close_write_connection(owned_db):
|
||||
# C6: closing the write lease closed the write thread's connection;
|
||||
# every later write failed until the thread idled out
|
||||
_ds, db, _path = owned_db
|
||||
|
||||
def close_it(conn):
|
||||
conn.close()
|
||||
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
await db.execute_write_fn(close_it)
|
||||
await db.execute_write("insert into t (v) values ('after')")
|
||||
|
||||
# cursor.connection has the same lease protections as the connection.
|
||||
def close_raw(conn):
|
||||
conn.execute("select 1").connection.close()
|
||||
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
await db.execute_write_fn(close_raw)
|
||||
for _ in range(3):
|
||||
await db.execute_write("insert into t (v) values ('again')")
|
||||
rows = await db.execute("select count(*) from t")
|
||||
assert rows.first()[0] == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", (3, 0))
|
||||
async def test_isolated_lease_close_is_allowed(tmp_path, num_sql_threads):
|
||||
path = str(tmp_path / "data.db")
|
||||
make_db(path)
|
||||
ds = Datasette([path], settings={"num_sql_threads": num_sql_threads})
|
||||
db = ds.get_database("data")
|
||||
await db.execute_isolated_fn(lambda conn: conn.close())
|
||||
await db.execute_write("insert into t (v) values (1)")
|
||||
ds.close()
|
||||
|
||||
|
||||
class Abort(BaseException):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("exception_class", (Abort, SystemExit))
|
||||
async def test_write_callback_base_exception_is_delivered(owned_db, exception_class):
|
||||
# C7: a BaseException from a write callback killed the write thread
|
||||
# without answering the caller, who waited forever
|
||||
_ds, db, _path = owned_db
|
||||
|
||||
def boom(conn):
|
||||
raise exception_class("stop")
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
await asyncio.wait_for(db.execute_write_fn(boom), 5)
|
||||
assert isinstance(excinfo.value.__cause__, exception_class)
|
||||
# Same for an isolated function
|
||||
with pytest.raises(RuntimeError):
|
||||
await asyncio.wait_for(db.execute_isolated_fn(boom), 5)
|
||||
await asyncio.wait_for(db.execute_write("insert into t (v) values (1)"), 5)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Public contract
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def _fts_setup(path):
|
||||
make_db(path, "create table secret (id integer primary key, body text)")
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute("insert into secret values (1, 'TOP SECRET PAYLOAD')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def _add_fts(path):
|
||||
conn = sqlite3.connect(path)
|
||||
conn.executescript("""
|
||||
create virtual table secret_fts using fts5(body, content='secret', content_rowid='id');
|
||||
insert into secret_fts(secret_fts) values ('rebuild');
|
||||
""")
|
||||
conn.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("mode", ("external", "owned"))
|
||||
async def test_derived_table_rule_uses_current_schema(tmp_path, mode):
|
||||
# Contract C1: the "external-content FTS tables require access to their
|
||||
# content table" rule was decided from a catalog that had not seen an
|
||||
# FTS table created outside Datasette: its rows were served to
|
||||
# anonymous users (until the next poll, or for good for owned DBs)
|
||||
path = str(tmp_path / "data.db")
|
||||
_fts_setup(path)
|
||||
ds = Datasette(
|
||||
config={
|
||||
"databases": {"data": {"tables": {"secret": {"allow": {"id": "root"}}}}}
|
||||
},
|
||||
settings={"default_allow_sql": False, "schema_watch_interval_ms": 0},
|
||||
)
|
||||
ds.add_database(Database(ds, path=path), name="data", schema_watch=mode)
|
||||
await ds.invoke_startup()
|
||||
assert (await ds.client.get("/data/secret.json")).status_code == 403
|
||||
_add_fts(path)
|
||||
response = await ds.client.get("/data/secret_fts.json")
|
||||
assert response.status_code == 403
|
||||
assert "TOP SECRET" not in response.text
|
||||
# Finding the catalog behind schedules a rescan
|
||||
await ds.client.get("/data.json")
|
||||
assert "secret_fts" in await catalog_tables(ds, "data")
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_database_file_changes_are_picked_up(tmp_path):
|
||||
# Contract C2: add_database() defaulted to "owned" (never polled), so a
|
||||
# table created by another process never appeared in listings
|
||||
path = str(tmp_path / "added.db")
|
||||
make_db(path)
|
||||
ds = Datasette(settings={"schema_watch_interval_ms": 50})
|
||||
await ds.invoke_startup()
|
||||
ds.add_database(Database(ds, path=path), name="added")
|
||||
await ds.client.get("/added.json")
|
||||
make_db(path, "create table t_ext (id integer)")
|
||||
|
||||
async def listed():
|
||||
response = await ds.client.get("/added.json")
|
||||
return {t["name"] for t in response.json()["tables"]}
|
||||
|
||||
deadline = time.monotonic() + 5
|
||||
while "t_ext" not in await listed():
|
||||
assert time.monotonic() < deadline, "t_ext never appeared"
|
||||
await asyncio.sleep(0.05)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_named_memory_database_out_of_band_changes(tmp_path):
|
||||
# Contract C2: named in-memory databases could not be watched at all;
|
||||
# datasette-app-support restores one with backup() into
|
||||
# db.connect(write=True) and then shows it
|
||||
ds = Datasette(settings={"schema_watch_interval_ms": 50})
|
||||
await ds.invoke_startup()
|
||||
memory = ds.add_memory_database("review_fixes_temporary", name="temporary")
|
||||
await memory.execute_write("create table m0 (id integer)")
|
||||
source_path = str(tmp_path / "backup.db")
|
||||
make_db(source_path, "create table restored_table (id integer)")
|
||||
source = sqlite3.connect(source_path)
|
||||
target = memory.connect(write=True)
|
||||
source.backup(target)
|
||||
source.close()
|
||||
target.close()
|
||||
|
||||
async def listed():
|
||||
response = await ds.client.get("/temporary.json")
|
||||
return {t["name"] for t in response.json()["tables"]}
|
||||
|
||||
deadline = time.monotonic() + 5
|
||||
while "restored_table" not in await listed():
|
||||
assert time.monotonic() < deadline, "restored_table never appeared"
|
||||
await asyncio.sleep(0.05)
|
||||
ds.close()
|
||||
|
||||
|
||||
FTS_SETUP = """
|
||||
create table docs (id integer primary key, body text);
|
||||
create virtual table docs_fts using fts5(body, content='docs', content_rowid='id');
|
||||
create trigger docs_ai after insert on docs begin
|
||||
insert into docs_fts (rowid, body) values (new.id, new.body);
|
||||
end;
|
||||
create trigger docs_ad after delete on docs begin
|
||||
insert into docs_fts (docs_fts, rowid, body) values ('delete', old.id, old.body);
|
||||
end;
|
||||
insert into docs (id, body) values (1, 'oldword');
|
||||
"""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", (3, 0))
|
||||
async def test_write_connection_pins_recursive_triggers(tmp_path, num_sql_threads):
|
||||
# Contract C4: INSERT OR REPLACE on a sqlite-utils style FTS table only
|
||||
# cleaned up the old index entry once some code had wrapped the write
|
||||
# connection in sqlite_utils.Database (recursive_triggers drift) - and
|
||||
# the idle write thread's fresh connections lost the drift again
|
||||
path = str(tmp_path / "fts.db")
|
||||
make_db(path, FTS_SETUP)
|
||||
ds = Datasette(
|
||||
[path],
|
||||
settings={
|
||||
"num_sql_threads": num_sql_threads,
|
||||
"connection_idle_timeout_ms": 50,
|
||||
},
|
||||
)
|
||||
db = ds.get_database("fts")
|
||||
for word in ("newword", "newerword"):
|
||||
flag = await db.execute_write_fn(
|
||||
lambda conn: conn.execute("PRAGMA recursive_triggers").fetchone()[0]
|
||||
)
|
||||
assert flag == 1
|
||||
await db.execute_write(
|
||||
"insert or replace into docs (id, body) values (1, ?)", [word]
|
||||
)
|
||||
# Let the idle write thread exit: the next write opens a new
|
||||
# connection
|
||||
await asyncio.sleep(0.3)
|
||||
stale = await db.execute(
|
||||
"select rowid from docs_fts where docs_fts match 'oldword OR newword'"
|
||||
)
|
||||
assert stale.rows == []
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_memory_database_is_not_scanned_at_registration():
|
||||
# Contract C5: add_memory_database() after startup scanned the
|
||||
# shared-cache database straight away, racing plugins that fill it from
|
||||
# their own connection (datasette-copy-to-memory's VACUUM INTO failed
|
||||
# with "database table is locked")
|
||||
ds = Datasette(settings={"schema_watch_interval_ms": 0})
|
||||
await ds.invoke_startup()
|
||||
watcher = ds._schema_watcher
|
||||
scans = watcher.counters["scans"]
|
||||
memory = ds.add_memory_database("review_fixes_copy", name="copy")
|
||||
await memory.execute("select 1 + 1")
|
||||
await asyncio.sleep(0.05)
|
||||
assert watcher.counters["scans"] == scans
|
||||
state = memory._watch_state
|
||||
assert state.scan_future is None
|
||||
assert state in watcher._pending
|
||||
# Fill it from a connection of its own, as the plugin does. (The plugin
|
||||
# uses VACUUM INTO from a connection opened with uri=True; a plain
|
||||
# connection is simpler and does not depend on whether the SQLite build
|
||||
# treats URI file names as URIs without that flag)
|
||||
conn = sqlite3.connect("file:review_fixes_copy?mode=memory&cache=shared", uri=True)
|
||||
conn.execute("create table copied (id integer)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
# The next request scans it
|
||||
response = await ds.client.get("/copy.json")
|
||||
assert {t["name"] for t in response.json()["tables"]} == {"copied"}
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_locked_memory_scan_is_retried(monkeypatch):
|
||||
ds = Datasette(settings={"schema_watch_interval_ms": 0})
|
||||
await ds.invoke_startup()
|
||||
memory = ds.add_memory_database("review_fixes_locked", name="locked")
|
||||
original = schema_watcher_module.collect_schema
|
||||
locked = threading.Event()
|
||||
locked.set()
|
||||
|
||||
def locked_scan(conn, name):
|
||||
if locked.is_set():
|
||||
raise sqlite3.OperationalError("database table is locked: sqlite_master")
|
||||
return original(conn, name)
|
||||
|
||||
monkeypatch.setattr(schema_watcher_module, "collect_schema", locked_scan)
|
||||
await memory.execute_write("create table m (id integer)")
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0.01)
|
||||
state = memory._watch_state
|
||||
# Transient: not recorded as a broken database
|
||||
assert state.error is None
|
||||
assert state.needs_scan
|
||||
locked.clear()
|
||||
state.retry_at = 0.0
|
||||
await ds.client.get("/locked.json")
|
||||
assert await catalog_tables(ds, "locked") == ["m"]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stale_owned_catalog_is_not_persisted_at_close(tmp_path):
|
||||
# Contract C6: close() stored a fresh fingerprint for an owned database
|
||||
# whose catalog was stale, so a restart restored the stale catalog
|
||||
path = str(tmp_path / "data.db")
|
||||
internal = str(tmp_path / "internal.db")
|
||||
make_db(path)
|
||||
config = {"databases": {"data": {"schema_watch": "owned"}}}
|
||||
|
||||
async def tables_after_start(change_before_close=None):
|
||||
ds = Datasette([path], internal=internal, config=config)
|
||||
await ds.invoke_startup()
|
||||
response = await ds.client.get("/data.json")
|
||||
if change_before_close:
|
||||
# Behind Datasette's back, after the last request: an owned
|
||||
# database is not polled, so the catalog is now stale
|
||||
change_before_close()
|
||||
ds.close()
|
||||
return [t["name"] for t in response.json()["tables"]]
|
||||
|
||||
assert await tables_after_start() == ["t"]
|
||||
assert await tables_after_start(
|
||||
lambda: make_db(path, "create table t2 (id integer)")
|
||||
) == ["t"]
|
||||
# The stale catalog was not persisted as current: the restart rescans
|
||||
assert await tables_after_start() == ["t", "t2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crossdb_memory_connections_are_reaped_and_refreshed(tmp_path):
|
||||
# Contract C7: --crossdb _memory connections ATTACH up to ten files but
|
||||
# were never counted, reaped or refreshed: the fds stayed open and a
|
||||
# removed database stayed queryable
|
||||
paths = []
|
||||
for i in range(3):
|
||||
path = str(tmp_path / f"db{i}.db")
|
||||
make_db(path)
|
||||
paths.append(path)
|
||||
ds = Datasette(paths[:2], crossdb=True, settings={"connection_idle_timeout_ms": 50})
|
||||
await ds.invoke_startup()
|
||||
memory = ds.get_database("_memory")
|
||||
assert (await memory.execute("select count(*) from db0.t")).first()[0] == 0
|
||||
pool = ds._read_pool
|
||||
assert pool.snapshot()["open"] >= 1
|
||||
deadline = time.monotonic() + 5
|
||||
while pool.snapshot()["open"]:
|
||||
assert time.monotonic() < deadline, "crossdb connections were not reaped"
|
||||
await asyncio.sleep(0.05)
|
||||
# Added and removed databases are reflected immediately
|
||||
await memory.execute("select 1")
|
||||
ds.add_database(Database(ds, path=paths[2]), name="db2")
|
||||
assert (await memory.execute("select count(*) from db2.t")).first()[0] == 0
|
||||
ds.remove_database("db0")
|
||||
with pytest.raises(sqlite3.OperationalError):
|
||||
await memory.execute("select count(*) from db0.t")
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_fn_may_return_generator_over_fetched_rows():
|
||||
# Contract C8: every generator was rejected, including harmless ones
|
||||
ds = Datasette(memory=True)
|
||||
db = ds.get_database("_memory")
|
||||
|
||||
def fetched(conn):
|
||||
rows = conn.execute("select 1 union all select 2").fetchall()
|
||||
return (row[0] for row in rows)
|
||||
|
||||
assert list(await db.execute_fn(fetched)) == [1, 2]
|
||||
|
||||
def live_cursor(conn):
|
||||
return (row[0] for row in conn.execute("select 1"))
|
||||
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
await db.execute_fn(live_cursor)
|
||||
|
||||
def closure(conn):
|
||||
def rows():
|
||||
yield from conn.execute("select 1")
|
||||
|
||||
return rows()
|
||||
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
await db.execute_fn(closure)
|
||||
ds.close()
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Found by the torture harness (tests/connection_torture.py)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_survives_database_removed_mid_request(tmp_path, monkeypatch):
|
||||
# The index page looked databases up by name again after listing them:
|
||||
# one removed meanwhile (remove_database())
|
||||
# failed the whole page with a KeyError
|
||||
from datasette.views import index as index_module
|
||||
|
||||
paths = []
|
||||
for name in ("keep", "gone"):
|
||||
path = str(tmp_path / f"{name}.db")
|
||||
make_db(path)
|
||||
paths.append(path)
|
||||
ds = Datasette(paths)
|
||||
await ds.invoke_startup()
|
||||
original = index_module.catalog_summaries
|
||||
|
||||
async def remove_during(datasette, names):
|
||||
result = await original(datasette, names)
|
||||
if "gone" in datasette.databases:
|
||||
datasette.remove_database("gone")
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(index_module, "catalog_summaries", remove_during)
|
||||
response = await ds.client.get("/.json")
|
||||
assert response.status_code == 200
|
||||
assert [d["name"] for d in response.json()["databases"]] == ["keep"]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_and_databases_with_closed_or_deleted_database(tmp_path):
|
||||
paths = []
|
||||
for name in ("keep", "closed", "deleted"):
|
||||
path = str(tmp_path / f"{name}.db")
|
||||
make_db(path)
|
||||
paths.append(path)
|
||||
ds = Datasette(paths, settings={"schema_watch_interval_ms": 0})
|
||||
await ds.invoke_startup()
|
||||
# Closed by a plugin but still attached
|
||||
ds.get_database("closed").close()
|
||||
# Deleted from under Datasette before any sweep noticed
|
||||
os.unlink(paths[2])
|
||||
for path in ("/.json", "/", "/-/databases.json"):
|
||||
response = await ds.client.get(path)
|
||||
assert response.status_code == 200, (path, response.text[:300])
|
||||
databases = (await ds.client.get("/-/databases.json")).json()
|
||||
if isinstance(databases, dict):
|
||||
databases = databases["databases"]
|
||||
sizes = {d["name"]: d["size"] for d in databases}
|
||||
assert sizes["deleted"] == 0
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cursors_are_closed_before_leaving_their_thread(owned_db):
|
||||
# A cursor still references its connection's cached statement, and on
|
||||
# Python < 3.12 freeing it resets that statement. execute_write_many()
|
||||
# handed its cursor to the caller's event loop thread; freed there while
|
||||
# the write thread ran the same SQL again, the write failed with
|
||||
# "bad parameter or other API misuse"
|
||||
_ds, db, _path = owned_db
|
||||
cursor = await db.execute_write_many(
|
||||
"insert into t (v) values (?)", [("a",), ("b",)]
|
||||
)
|
||||
assert cursor.rowcount == 2
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
cursor.fetchall()
|
||||
cursor = await db.execute_write_fn(
|
||||
lambda conn: conn.execute("insert into t (v) values ('c')")
|
||||
)
|
||||
assert cursor.lastrowid == 3
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
cursor.fetchone()
|
||||
|
||||
# Cursors held by the frames of an exception raised by a callback are
|
||||
# closed before the connection goes back to the pool (or write thread)
|
||||
captured = []
|
||||
|
||||
def read_then_fail(conn):
|
||||
cursor = conn.execute("select * from t")
|
||||
captured.append(cursor)
|
||||
raise ValueError("boom")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
await db.execute_fn(read_then_fail)
|
||||
with pytest.raises(ValueError):
|
||||
await db.execute_write_fn(read_then_fail)
|
||||
for cursor in captured:
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
cursor.fetchone()
|
||||
|
||||
|
||||
def test_startup_from_several_event_loops_runs_once(monkeypatch):
|
||||
# _startup_sequence() serialized startup with an asyncio.Lock: callers
|
||||
# on a second event loop got "is bound to a different event loop" (or,
|
||||
# free-threaded, waited forever), and invoke_startup() itself let two
|
||||
# loops run every startup hook twice
|
||||
ds = Datasette(memory=True)
|
||||
calls = []
|
||||
original = Datasette._apply_column_types_config
|
||||
|
||||
async def slow(self):
|
||||
calls.append(threading.current_thread().name)
|
||||
await asyncio.sleep(0.3)
|
||||
return await original(self)
|
||||
|
||||
monkeypatch.setattr(Datasette, "_apply_column_types_config", slow)
|
||||
|
||||
def starter(use_sequence):
|
||||
async def run():
|
||||
if use_sequence:
|
||||
await ds._startup_sequence()
|
||||
else:
|
||||
await ds.invoke_startup()
|
||||
assert ds._startup_invoked
|
||||
return (await ds.client.get("/_memory.json")).status_code
|
||||
|
||||
return run
|
||||
|
||||
statuses = run_in_threads(starter(True), starter(True), starter(False))
|
||||
assert statuses == [200, 200, 200]
|
||||
assert len(calls) == 1
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_startup_hook_can_make_requests():
|
||||
# A request made from inside startup (same task) must not wait for
|
||||
# startup to finish - that would wait forever
|
||||
from datasette import hookimpl
|
||||
from datasette.plugins import pm
|
||||
|
||||
statuses = []
|
||||
|
||||
class Plugin:
|
||||
__name__ = "StartupRequestPlugin"
|
||||
|
||||
@hookimpl
|
||||
def startup(self, datasette):
|
||||
async def inner():
|
||||
statuses.append(
|
||||
(await datasette.client.get("/-/versions.json")).status_code
|
||||
)
|
||||
|
||||
return inner
|
||||
|
||||
pm.register(Plugin(), name="startup_request_plugin")
|
||||
try:
|
||||
ds = Datasette(memory=True)
|
||||
await asyncio.wait_for(ds.invoke_startup(), 10)
|
||||
assert statuses == [200]
|
||||
ds.close()
|
||||
finally:
|
||||
pm.unregister(name="startup_request_plugin")
|
||||
|
||||
|
||||
def test_execute_write_many_from_several_event_loops(owned_db):
|
||||
# T1 end to end: on Python 3.11 before the fix, dozens of these calls
|
||||
# failed with "bad parameter or other API misuse" (3 of 3 runs)
|
||||
_ds, db, _path = owned_db
|
||||
errors = []
|
||||
|
||||
def loop(i):
|
||||
async def run():
|
||||
async def worker(t):
|
||||
for seq in range(60):
|
||||
try:
|
||||
await db.execute_write_many(
|
||||
"insert into t (v) values (?)", [(f"{i}-{t}-{seq}",)]
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
errors.append(repr(e))
|
||||
|
||||
await asyncio.gather(*(worker(t) for t in range(5)))
|
||||
|
||||
return run
|
||||
|
||||
run_in_threads(loop(0), loop(1), loop(2))
|
||||
assert errors == []
|
||||
75
tests/test_connection_torture.py
Normal file
75
tests/test_connection_torture.py
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
"""
|
||||
A fast, fixed-seed subset of the connection torture harness
|
||||
(tests/connection_torture.py). Each scenario runs in its own process, so a
|
||||
segfault or a hang fails one test instead of killing the test run.
|
||||
|
||||
Several event loops (threads) drive one Datasette with extreme settings
|
||||
(smallest pool, 1ms idle timeout, 10ms schema polling) through reads,
|
||||
writes of every kind, DDL, isolated functions, analyze_sql, cancellation,
|
||||
add/remove_database, external schema changes, files
|
||||
replaced and deleted, Database.close() and Datasette.close() mid-flight,
|
||||
and non-threaded mode. See the harness docstring for what is checked.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
HARNESS = os.path.join(os.path.dirname(__file__), "connection_torture.py")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scenario,seed",
|
||||
[
|
||||
("mixed", 101),
|
||||
("lifecycle", 102),
|
||||
("close", 103),
|
||||
("cancel", 104),
|
||||
("pool", 105),
|
||||
("nothreads", 106),
|
||||
("writes", 107),
|
||||
],
|
||||
)
|
||||
def test_connection_torture(scenario, seed):
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
HARNESS,
|
||||
scenario,
|
||||
"--seed",
|
||||
str(seed),
|
||||
"--duration",
|
||||
"1.5",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=150,
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired as e: # pragma: no cover
|
||||
pytest.fail(f"{scenario} hung:\n{(e.stderr or '')[-4000:]}")
|
||||
if proc.returncode < 0: # pragma: no cover
|
||||
pytest.fail(
|
||||
f"{scenario} crashed with {signal.Signals(-proc.returncode).name}:\n"
|
||||
f"{proc.stderr[-4000:]}"
|
||||
)
|
||||
lines = proc.stdout.strip().splitlines()
|
||||
if not lines:
|
||||
pytest.fail(
|
||||
f"{scenario} exited {proc.returncode} without a summary:\n"
|
||||
f"{proc.stderr[-4000:]}"
|
||||
)
|
||||
summary = json.loads(lines[-1])
|
||||
problems = "\n".join(
|
||||
f"[{p['kind']}] {p['message'][:600]}" for p in summary["problems"]
|
||||
)
|
||||
assert proc.returncode == 0, f"{scenario} seed {seed}:\n{problems}"
|
||||
assert summary["ok"]
|
||||
# It did something (the slow-query scenarios do a few dozen operations
|
||||
# in 1.5s on a busy machine, the others hundreds)
|
||||
assert sum(summary["counts"].values()) > 10
|
||||
|
|
@ -95,6 +95,11 @@ async def test_derived_permission_discovery_error_is_retried(monkeypatch, listin
|
|||
action="view-table", resource=TableResource("data", "documents")
|
||||
)
|
||||
|
||||
# While the catalog is current, dependencies come from its rows and
|
||||
# nothing is discovered live. Live discovery - the path that can fail -
|
||||
# happens while a rescan of the database is pending.
|
||||
db._watch_state.needs_scan = True
|
||||
db._cached_derived_table_dependencies = None
|
||||
token = _permission_check_cache.set({})
|
||||
try:
|
||||
with monkeypatch.context() as patch:
|
||||
|
|
|
|||
|
|
@ -29,7 +29,9 @@ async def test_internal_migrations_recorded(ds_client):
|
|||
order by id
|
||||
""")
|
||||
assert [tuple(row) for row in migrations.rows] == [
|
||||
("datasette_internal", "0001_initial")
|
||||
("datasette_internal", "0001_initial"),
|
||||
("datasette_internal", "0002_catalog_fingerprint"),
|
||||
("datasette_internal", "0003_catalog_table_type"),
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -61,7 +63,9 @@ async def test_internal_migrations_adopt_existing_internal_db(tmp_path):
|
|||
order by id
|
||||
""")
|
||||
assert [tuple(row) for row in migrations.rows] == [
|
||||
("datasette_internal", "0001_initial")
|
||||
("datasette_internal", "0001_initial"),
|
||||
("datasette_internal", "0002_catalog_fingerprint"),
|
||||
("datasette_internal", "0003_catalog_table_type"),
|
||||
]
|
||||
|
||||
ds.close()
|
||||
|
|
@ -73,7 +77,13 @@ async def test_internal_tables(ds_client):
|
|||
tables = await internal_db.execute("select * from catalog_tables")
|
||||
assert len(tables) > 5
|
||||
table = tables.rows[0]
|
||||
assert set(table.keys()) == {"rootpage", "table_name", "database_name", "sql"}
|
||||
assert set(table.keys()) == {
|
||||
"rootpage",
|
||||
"table_name",
|
||||
"database_name",
|
||||
"sql",
|
||||
"type",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from opentelemetry import context as otel_context_api
|
|||
|
||||
from datasette.app import Datasette
|
||||
from datasette.database import (
|
||||
ConnectionLeaseError,
|
||||
Database,
|
||||
DatasetteClosedError,
|
||||
ExecuteWriteResult,
|
||||
|
|
@ -1263,10 +1264,11 @@ async def test_database_close_shuts_down_write_thread(tmpdir):
|
|||
await db.execute_write("insert into t (id) values (1)")
|
||||
assert db._write_thread is not None
|
||||
assert db._write_thread.is_alive()
|
||||
thread = db._write_thread
|
||||
db.close()
|
||||
# Wait briefly for the thread to exit — the sentinel should cause it to return.
|
||||
db._write_thread.join(timeout=5)
|
||||
assert not db._write_thread.is_alive()
|
||||
thread.join(timeout=5)
|
||||
assert not thread.is_alive()
|
||||
assert db._write_thread is None
|
||||
ds._internal_database.close()
|
||||
|
||||
|
||||
|
|
@ -1312,10 +1314,18 @@ async def test_database_close_is_idempotent(tmpdir):
|
|||
async def test_close_releases_memory_connections(num_sql_threads, named):
|
||||
ds = Datasette(memory=True, settings={"num_sql_threads": num_sql_threads})
|
||||
db = ds.add_memory_database(uuid.uuid4().hex) if named else ds.get_database()
|
||||
read_connection = await db.execute_fn(lambda conn: conn)
|
||||
write_connection = await db.execute_write_fn(lambda conn: conn)
|
||||
leased = await db.execute_fn(lambda conn: conn)
|
||||
leased_write = await db.execute_write_fn(lambda conn: conn)
|
||||
# Inspect the owned raw connections to verify physical closure as well
|
||||
# as lease expiry. cursor.connection intentionally no longer leaks them.
|
||||
connections = list(db._all_connections)
|
||||
assert len(connections) >= 2
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
leased.execute("select 1")
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
leased_write.execute("select 1")
|
||||
ds.close()
|
||||
for conn in (read_connection, write_connection):
|
||||
for conn in connections:
|
||||
with pytest.raises(sqlite3.ProgrammingError, match="closed"):
|
||||
conn.execute("select 1")
|
||||
|
||||
|
|
|
|||
116
tests/test_isolated_memory_keeper.py
Normal file
116
tests/test_isolated_memory_keeper.py
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
import sqlite3
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from datasette.app import Datasette
|
||||
from datasette.write_budget import DatabaseResourceError
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("threads", [0, 2])
|
||||
@pytest.mark.parametrize("raise_after_commit", [False, True])
|
||||
async def test_first_isolated_memory_write_keeps_committed_data(
|
||||
threads, raise_after_commit
|
||||
):
|
||||
ds = Datasette(settings={"num_sql_threads": threads, "schema_watch_interval_ms": 0})
|
||||
db = ds.add_memory_database(uuid.uuid4().hex)
|
||||
try:
|
||||
|
||||
def create(conn):
|
||||
conn.execute("create table t(id)")
|
||||
conn.execute("insert into t values (123)")
|
||||
conn.commit()
|
||||
if raise_after_commit:
|
||||
raise ValueError("after commit")
|
||||
return conn.execute("select id from t").fetchone()[0]
|
||||
|
||||
if raise_after_commit:
|
||||
with pytest.raises(ValueError, match="after commit"):
|
||||
await db.execute_isolated_fn(create)
|
||||
else:
|
||||
assert await db.execute_isolated_fn(create) == 123
|
||||
# Only the keeper remains after the isolated callback, on both
|
||||
# success and failure. Do not open a reader before checking this.
|
||||
assert len(db._all_connections) == 1
|
||||
assert (await db.execute("select id from t")).single_value() == 123
|
||||
assert (
|
||||
await db.execute_isolated_fn(
|
||||
lambda conn: conn.execute("select id from t").fetchone()[0]
|
||||
)
|
||||
== 123
|
||||
)
|
||||
await db.execute_write("insert into t values (456)")
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 2
|
||||
connections = list(db._all_connections)
|
||||
finally:
|
||||
ds.close()
|
||||
for conn in connections:
|
||||
with pytest.raises(sqlite3.ProgrammingError):
|
||||
conn.execute("select 1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("threads", [0, 2])
|
||||
async def test_isolated_memory_keeper_occupies_writer_slot(threads, tmp_path):
|
||||
path = tmp_path / "file.db"
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute("create table t(id)")
|
||||
conn.close()
|
||||
ds = Datasette(
|
||||
[str(path)],
|
||||
settings={
|
||||
"num_sql_threads": threads,
|
||||
"max_write_connections": 1,
|
||||
"write_queue_timeout_ms": 50,
|
||||
"schema_watch_interval_ms": 0,
|
||||
},
|
||||
)
|
||||
memory = ds.add_memory_database(uuid.uuid4().hex)
|
||||
file_db = ds.get_database("file")
|
||||
try:
|
||||
await memory.execute_isolated_fn(
|
||||
lambda conn: conn.execute("create table t(id)")
|
||||
)
|
||||
with pytest.raises(DatabaseResourceError):
|
||||
await file_db.execute_write("insert into t values (1)")
|
||||
assert "t" in await memory.table_names()
|
||||
memory.close()
|
||||
await file_db.execute_write("insert into t values (1)")
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("threads", [0, 2])
|
||||
async def test_keeper_preparation_failure_does_not_run_isolated_callback(
|
||||
threads, monkeypatch
|
||||
):
|
||||
ds = Datasette(
|
||||
settings={
|
||||
"num_sql_threads": threads,
|
||||
"max_write_connections": 1,
|
||||
"schema_watch_interval_ms": 0,
|
||||
}
|
||||
)
|
||||
db = ds.add_memory_database(uuid.uuid4().hex)
|
||||
original = ds._prepare_connection
|
||||
ran = []
|
||||
|
||||
def prepare(conn, name):
|
||||
if name == db.name:
|
||||
raise ValueError("keeper failed")
|
||||
return original(conn, name)
|
||||
|
||||
try:
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(ds, "_prepare_connection", prepare)
|
||||
with pytest.raises(ValueError, match="keeper failed"):
|
||||
await db.execute_isolated_fn(lambda conn: ran.append(True))
|
||||
assert not ran
|
||||
assert not db._all_connections
|
||||
db.close()
|
||||
another = ds.add_memory_database(uuid.uuid4().hex)
|
||||
await another.execute_write("create table t(id)")
|
||||
finally:
|
||||
ds.close()
|
||||
|
|
@ -215,18 +215,15 @@ async def test_concurrent_first_requests_all_wait_for_slow_startup():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_setup_db_still_runs_when_invoke_startup_ran_first(tmp_path, monkeypatch):
|
||||
# Regression test: `datasette serve` (cli.py _serve_async) calls
|
||||
# ds.invoke_startup() directly, before uvicorn ever sends a
|
||||
# lifespan.startup event that drives _startup_sequence(). If
|
||||
# _startup_sequence()'s fast path only checked `_startup_invoked`, it
|
||||
# would see startup already done and skip the immutable-database
|
||||
# table-count precompute (setup_db) entirely - a silent regression
|
||||
# versus main, where AsgiRunOnFirstRequest ran setup_db unconditionally
|
||||
# on request #1.
|
||||
async def test_immutable_table_counts_computed_when_first_needed(tmp_path, monkeypatch):
|
||||
# Immutable databases used to have their table counts precomputed by
|
||||
# _startup_sequence(), opening every immutable database at startup.
|
||||
# They are now computed the first time a page needs them, with the same
|
||||
# generous time limit, and cached.
|
||||
db_path = tmp_path / "immutable.db"
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.execute("create table t (id integer primary key)")
|
||||
conn.execute("insert into t default values")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
|
@ -241,22 +238,22 @@ async def test_setup_db_still_runs_when_invoke_startup_ran_first(tmp_path, monke
|
|||
|
||||
monkeypatch.setattr(Database, "table_counts", counting_table_counts)
|
||||
|
||||
# Simulate the CLI path: invoke_startup() runs directly and completes
|
||||
# BEFORE _startup_sequence() ever gets a chance to run setup_db.
|
||||
await ds.invoke_startup()
|
||||
await ds._startup_sequence()
|
||||
assert ds._startup_invoked is True
|
||||
assert call_count["n"] == 0
|
||||
|
||||
# The lifespan/first-request path (or the CLI itself, per the fix)
|
||||
# calling the shared entry point afterwards must still precompute
|
||||
# table counts for immutable databases.
|
||||
await ds._startup_sequence()
|
||||
assert call_count["n"] == 1
|
||||
assert ds._setup_db_done is True
|
||||
assert call_count["n"] == 0
|
||||
assert ds.get_database("immutable").cached_table_counts is None
|
||||
|
||||
# Idempotency: a second call must not recompute.
|
||||
await ds._startup_sequence()
|
||||
response = await ds.client.get("/.json")
|
||||
assert response.json()["databases"][0]["table_rows_sum"] == 1
|
||||
assert call_count["n"] == 1
|
||||
assert ds.get_database("immutable").cached_table_counts == {"t": 1}
|
||||
|
||||
# Cached from then on
|
||||
await ds.client.get("/immutable.json")
|
||||
assert ds.get_database("immutable").cached_table_counts == {"t": 1}
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -48,6 +48,11 @@ def test_plugin_hooks_have_tests(plugin_hook):
|
|||
assert ok, f"Plugin hook is missing tests: {plugin_hook}"
|
||||
|
||||
|
||||
def test_hook_write_task_completed():
|
||||
# Success and expiry delivery are exercised in tests/test_write_budget.py.
|
||||
assert "write_task_completed" in dir(pm.hook)
|
||||
|
||||
|
||||
def test_hook_jump_items_sql():
|
||||
# Detailed behavior is covered in tests/test_jump.py.
|
||||
assert "jump_items_sql" in dir(pm.hook)
|
||||
|
|
|
|||
378
tests/test_read_pool.py
Normal file
378
tests/test_read_pool.py
Normal file
|
|
@ -0,0 +1,378 @@
|
|||
import asyncio
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import sqlite_utils
|
||||
|
||||
from datasette.app import Datasette
|
||||
from datasette.connection_pool import ConnectionLeaseError
|
||||
|
||||
|
||||
def _make_dbs(tmp_path, n):
|
||||
paths = []
|
||||
for i in range(n):
|
||||
path = tmp_path / f"db{i}.db"
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute("create table t (id integer primary key, v text)")
|
||||
conn.execute("insert into t (v) values ('x')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
paths.append(str(path))
|
||||
return paths
|
||||
|
||||
|
||||
def _file_conns(db):
|
||||
return [c for c in db._all_connections]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 2])
|
||||
async def test_lease_expires_after_callback(tmp_path, num_sql_threads):
|
||||
(path,) = _make_dbs(tmp_path, 1)
|
||||
ds = Datasette([path], settings={"num_sql_threads": num_sql_threads})
|
||||
db = ds.get_database("db0")
|
||||
stash = {}
|
||||
|
||||
def callback(conn):
|
||||
stash["conn"] = conn
|
||||
assert isinstance(conn, sqlite3.Connection)
|
||||
return conn.execute("select count(*) from t").fetchone()[0]
|
||||
|
||||
assert await db.execute_fn(callback) == 1
|
||||
with pytest.raises(ConnectionLeaseError, match="db0"):
|
||||
stash["conn"].execute("select 1")
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
stash["conn"].row_factory = None
|
||||
# A second lease of the same underlying connection gets a new proxy, so
|
||||
# the stale one still fails while another callback is using it
|
||||
seen = {}
|
||||
|
||||
def callback2(conn):
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
stash["conn"].execute("select 1")
|
||||
seen["ok"] = True
|
||||
|
||||
await db.execute_fn(callback2)
|
||||
assert seen["ok"]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_returning_cursor_raises(tmp_path):
|
||||
(path,) = _make_dbs(tmp_path, 1)
|
||||
ds = Datasette([path])
|
||||
db = ds.get_database("db0")
|
||||
with pytest.raises(ConnectionLeaseError, match="returned a Cursor"):
|
||||
await db.execute_fn(lambda conn: conn.execute("select * from t"))
|
||||
# Pool is still usable afterwards
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 1
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_leased_connection_compatibility(tmp_path):
|
||||
(path,) = _make_dbs(tmp_path, 1)
|
||||
ds = Datasette([path])
|
||||
db = ds.get_database("db0")
|
||||
|
||||
def callback(conn):
|
||||
out = {}
|
||||
out["tables"] = sqlite_utils.Database(conn).table_names()
|
||||
out["columns"] = list(sqlite_utils.Database(conn)["t"].columns_dict)
|
||||
target = sqlite3.connect(":memory:")
|
||||
conn.backup(target)
|
||||
out["backup"] = target.execute("select count(*) from t").fetchone()[0]
|
||||
conn.create_function("double", 1, lambda x: x * 2)
|
||||
out["fn"] = conn.execute("select double(21)").fetchone()[0]
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("select v from t")
|
||||
out["cursor"] = cursor.fetchall()[0][0]
|
||||
return out
|
||||
|
||||
assert await db.execute_fn(callback) == {
|
||||
"tables": ["t"],
|
||||
"columns": ["id", "v"],
|
||||
"backup": 1,
|
||||
"fn": 42,
|
||||
"cursor": "x",
|
||||
}
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 3])
|
||||
async def test_global_cap_evicts_least_recently_used(tmp_path, num_sql_threads):
|
||||
paths = _make_dbs(tmp_path, 6)
|
||||
ds = Datasette(paths, settings={"num_sql_threads": num_sql_threads})
|
||||
pool = ds._read_pool
|
||||
# Below the 4 x num_sql_threads clamp, so set the private attribute
|
||||
pool.max_open = 2
|
||||
for i in range(6):
|
||||
await ds.get_database(f"db{i}").execute("select 1")
|
||||
assert pool.snapshot()["open"] <= 2
|
||||
# The two most recently used databases are the ones still open
|
||||
open_dbs = {
|
||||
name
|
||||
for name in (f"db{i}" for i in range(6))
|
||||
if ds.get_database(name)._read_pool_state.open
|
||||
}
|
||||
assert open_dbs == {"db4", "db5"}
|
||||
assert pool.stats["evicted_lru"] >= 4
|
||||
for i in range(6):
|
||||
db = ds.get_database(f"db{i}")
|
||||
assert len(_file_conns(db)) == db._read_pool_state.open
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepare_connection_runs_for_every_pooled_connection(tmp_path):
|
||||
paths = _make_dbs(tmp_path, 3)
|
||||
ds = Datasette(paths)
|
||||
ds._read_pool.max_open = 1
|
||||
calls = []
|
||||
original = ds._prepare_connection
|
||||
|
||||
def counting(conn, database):
|
||||
calls.append(database)
|
||||
return original(conn, database)
|
||||
|
||||
ds._prepare_connection = counting
|
||||
# Pool may have been created already with the original bound method
|
||||
ds._read_pool._prepare_connection = counting
|
||||
for _ in range(2):
|
||||
for i in range(3):
|
||||
assert (
|
||||
await ds.get_database(f"db{i}").execute("select v from t")
|
||||
).single_value() == "x"
|
||||
# Cap of 1 means every switch of database reopens and re-prepares
|
||||
assert calls == ["db0", "db1", "db2"] * 2
|
||||
ds.close()
|
||||
|
||||
|
||||
def test_idle_reaper_closes_connections_without_event_loop(tmp_path):
|
||||
paths = _make_dbs(tmp_path, 3)
|
||||
ds = Datasette(paths, settings={"num_sql_threads": 0})
|
||||
pool = ds._read_pool
|
||||
pool.idle_timeout = 0.2
|
||||
|
||||
async def query_all():
|
||||
for i in range(3):
|
||||
await ds.get_database(f"db{i}").execute("select 1")
|
||||
|
||||
asyncio.run(query_all())
|
||||
assert pool.snapshot()["open"] == 3
|
||||
# No event loop is running now - the reaper thread does the work
|
||||
deadline = time.monotonic() + 5
|
||||
while pool.snapshot()["open"] and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
assert pool.snapshot()["open"] == 0
|
||||
assert pool.stats["expired"] == 3
|
||||
for i in range(3):
|
||||
assert _file_conns(ds.get_database(f"db{i}")) == []
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_sql_threads,configured,expected",
|
||||
[
|
||||
(3, 128, 128),
|
||||
(3, 5, 5),
|
||||
(8, 20, 20),
|
||||
(0, 1, 1),
|
||||
(3, 0, 0),
|
||||
],
|
||||
)
|
||||
def test_max_open_connections_respects_configured_limit(
|
||||
num_sql_threads, configured, expected
|
||||
):
|
||||
ds = Datasette(
|
||||
settings={
|
||||
"num_sql_threads": num_sql_threads,
|
||||
"max_open_connections": configured,
|
||||
}
|
||||
)
|
||||
assert ds._read_pool.max_open == expected
|
||||
# /-/settings.json reports the configured value
|
||||
assert ds.setting("max_open_connections") == configured
|
||||
ds.close()
|
||||
|
||||
|
||||
def test_connection_idle_timeout_ms_setting():
|
||||
ds = Datasette(settings={"connection_idle_timeout_ms": 1500})
|
||||
assert ds._read_pool.idle_timeout == 1.5
|
||||
ds.close()
|
||||
ds = Datasette(settings={"connection_idle_timeout_ms": 0})
|
||||
assert ds._read_pool.idle_timeout == 0
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hard_cap_when_every_connection_is_leased(tmp_path):
|
||||
paths = _make_dbs(tmp_path, 3)
|
||||
ds = Datasette(paths, settings={"num_sql_threads": 3, "max_open_connections": 1})
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def slow(conn):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
return conn.execute("select 1").fetchone()[0]
|
||||
|
||||
tasks = [
|
||||
asyncio.create_task(ds.get_database(f"db{i}").execute_fn(slow))
|
||||
for i in range(3)
|
||||
]
|
||||
try:
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
await asyncio.sleep(0.05)
|
||||
assert ds._read_pool.snapshot()["open"] == 1
|
||||
assert not any(task.done() for task in tasks)
|
||||
release.set()
|
||||
assert await asyncio.gather(*tasks) == [1, 1, 1]
|
||||
assert ds._read_pool.stats["peak_open"] == 1
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_memory_databases_not_counted(tmp_path):
|
||||
ds = Datasette(memory=True, settings={"max_open_connections": 1})
|
||||
db = ds.add_memory_database("pool_mem_test")
|
||||
await db.execute_write("create table t (id integer)")
|
||||
await db.execute_write("insert into t values (1)")
|
||||
await ds.get_database("_memory").execute("select 1")
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 1
|
||||
assert ds._read_pool.snapshot()["open"] == 0
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_database_close_with_leased_connection(tmp_path):
|
||||
(path,) = _make_dbs(tmp_path, 1)
|
||||
ds = Datasette([path], settings={"num_sql_threads": 0})
|
||||
db = ds.get_database("db0")
|
||||
await db.execute("select 1")
|
||||
|
||||
def closes_database(conn):
|
||||
conn.execute("select 1")
|
||||
db.close()
|
||||
return "done"
|
||||
|
||||
assert await db.execute_fn(closes_database) == "done"
|
||||
assert _file_conns(db) == []
|
||||
assert ds._read_pool.snapshot()["open"] == 0
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_callback_that_closes_connection_is_discarded(tmp_path):
|
||||
(path,) = _make_dbs(tmp_path, 1)
|
||||
ds = Datasette([path])
|
||||
db = ds.get_database("db0")
|
||||
await db.execute_fn(lambda conn: conn.close())
|
||||
assert ds._read_pool.stats["discarded_broken"] == 1
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 1
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_transaction_rolled_back_on_release(tmp_path):
|
||||
(path,) = _make_dbs(tmp_path, 1)
|
||||
ds = Datasette([path], settings={"num_sql_threads": 1})
|
||||
db = ds.get_database("db0")
|
||||
|
||||
def begin(conn):
|
||||
conn.execute("begin")
|
||||
conn.execute("select * from t").fetchall()
|
||||
|
||||
await db.execute_fn(begin)
|
||||
assert ds._read_pool.stats["rolled_back"] == 1
|
||||
assert not await db.execute_fn(lambda conn: conn.in_transaction)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", [0, 2])
|
||||
async def test_invalidate_database_discards_stale_connections(
|
||||
tmp_path, num_sql_threads
|
||||
):
|
||||
paths = _make_dbs(tmp_path, 2)
|
||||
ds = Datasette(paths, settings={"num_sql_threads": num_sql_threads})
|
||||
db0, db1 = ds.get_database("db0"), ds.get_database("db1")
|
||||
pool = ds._read_pool
|
||||
await db0.execute("select 1")
|
||||
await db1.execute("select 1")
|
||||
old = db0._read_pool_state.idle[-1].conn
|
||||
assert db0._read_pool_state.open == 1
|
||||
# What the SchemaWatcher does when it sees db0's file replaced
|
||||
db0._invalidate_connections()
|
||||
# Idle connections for db0 are closed at once; db1 is untouched
|
||||
assert db0._read_pool_state.open == 0
|
||||
assert db1._read_pool_state.open == 1
|
||||
assert old not in db0._all_connections
|
||||
with pytest.raises(sqlite3.ProgrammingError):
|
||||
old.execute("select 1")
|
||||
# The next read opens a new connection
|
||||
await db0.execute("select 1")
|
||||
assert db0._read_pool_state.idle[-1].conn is not old
|
||||
assert pool.stats["discarded_stale"] == 1
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_leased_connection_discarded_on_release_after_invalidation(tmp_path):
|
||||
(path,) = _make_dbs(tmp_path, 1)
|
||||
ds = Datasette([path], settings={"num_sql_threads": 2})
|
||||
db = ds.get_database("db0")
|
||||
pool = ds._read_pool
|
||||
seen = {}
|
||||
|
||||
def callback(conn):
|
||||
seen["raw"] = conn.execute("select 1").connection
|
||||
# The file is replaced while this callback holds the connection:
|
||||
# it must not be closed under us, only discarded on release
|
||||
db._invalidate_connections()
|
||||
return conn.execute("select count(*) from t").fetchone()[0]
|
||||
|
||||
assert await db.execute_fn(callback) == 1
|
||||
assert db._read_pool_state.open == 0
|
||||
assert pool.stats["discarded_stale"] == 1
|
||||
with pytest.raises(ConnectionLeaseError):
|
||||
seen["raw"].execute("select 1")
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 1
|
||||
ds.close()
|
||||
|
||||
|
||||
def test_reaper_unavailable_falls_back_to_checkout_reaping(tmp_path, monkeypatch):
|
||||
# Pyodide cannot start threads: idle connections must then be reaped by
|
||||
# the next checkout instead, and reads must keep working
|
||||
from datasette import connection_pool
|
||||
|
||||
reaper = connection_pool._Reaper()
|
||||
monkeypatch.setattr(connection_pool, "_reaper", reaper)
|
||||
|
||||
def no_threads(self):
|
||||
raise RuntimeError("can't start new thread")
|
||||
|
||||
monkeypatch.setattr(threading.Thread, "start", no_threads)
|
||||
paths = _make_dbs(tmp_path, 2)
|
||||
ds = Datasette(paths, settings={"num_sql_threads": 0})
|
||||
pool = ds._read_pool
|
||||
pool.idle_timeout = 0.05
|
||||
|
||||
async def read(name):
|
||||
return (await ds.get_database(name).execute("select 1")).single_value()
|
||||
|
||||
assert asyncio.run(read("db0")) == 1
|
||||
assert reaper.unavailable
|
||||
assert pool.snapshot()["open"] == 1
|
||||
time.sleep(0.1)
|
||||
assert asyncio.run(read("db1")) == 1
|
||||
# db0's expired connection was closed during db1's checkout
|
||||
assert pool.stats["expired"] == 1
|
||||
assert ds.get_database("db0")._read_pool_state.open == 0
|
||||
ds.close()
|
||||
718
tests/test_schema_watcher.py
Normal file
718
tests/test_schema_watcher.py
Normal file
|
|
@ -0,0 +1,718 @@
|
|||
"""
|
||||
SchemaWatcher tests. Changes to "external" databases are made from a
|
||||
separate OS process (a child Python interpreter using plain sqlite3, or the
|
||||
sqlite-utils CLI), the way other tools change files Datasette is serving.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from datasette import schema_watcher as schema_watcher_module
|
||||
from datasette.app import Datasette
|
||||
from datasette.database import Database
|
||||
|
||||
INTERVAL_MS = 50
|
||||
|
||||
|
||||
def run_external(path, sql, journal_mode=None, keep_wal=False):
|
||||
"""Run sql against path in a separate Python process; returns the
|
||||
child's time.monotonic() right after COMMIT."""
|
||||
script = textwrap.dedent("""
|
||||
import sqlite3, sys, time
|
||||
path, sql, journal_mode, keep_wal = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
|
||||
conn = sqlite3.connect(path, isolation_level=None, timeout=10)
|
||||
if journal_mode:
|
||||
conn.execute("PRAGMA journal_mode=" + journal_mode)
|
||||
if keep_wal == "1":
|
||||
# Never checkpoint: changes stay in the -wal file
|
||||
conn.execute("PRAGMA wal_autocheckpoint=0")
|
||||
conn.executescript(sql)
|
||||
print(time.monotonic(), flush=True)
|
||||
if keep_wal == "1":
|
||||
# Keep the connection open so close() does not checkpoint
|
||||
sys.stdin.read()
|
||||
conn.close()
|
||||
""")
|
||||
if keep_wal:
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-c", script, path, sql, journal_mode or "", "1"],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
line = proc.stdout.readline()
|
||||
return float(line), proc
|
||||
out = subprocess.run(
|
||||
[sys.executable, "-c", script, path, sql, journal_mode or "", "0"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return float(out.stdout.strip())
|
||||
|
||||
|
||||
def run_process(*args):
|
||||
subprocess.run([sys.executable, *args], check=True)
|
||||
|
||||
|
||||
def make_db(path, sql="create table t (id integer primary key, a text)", wal=False):
|
||||
conn = sqlite3.connect(path)
|
||||
if wal:
|
||||
conn.execute("PRAGMA journal_mode=wal")
|
||||
conn.executescript(sql)
|
||||
conn.execute("insert into t (a) values ('one')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
async def catalog_columns(ds, db_name, table):
|
||||
rows = await ds.get_internal_database().execute(
|
||||
"select name from catalog_columns where database_name = ? and table_name = ? order by cid",
|
||||
[db_name, table],
|
||||
)
|
||||
return [r[0] for r in rows.rows]
|
||||
|
||||
|
||||
async def catalog_tables(ds, db_name):
|
||||
rows = await ds.get_internal_database().execute(
|
||||
"select table_name from catalog_tables where database_name = ? order by table_name",
|
||||
[db_name],
|
||||
)
|
||||
return [r[0] for r in rows.rows]
|
||||
|
||||
|
||||
async def wait_for(check, timeout=5.0):
|
||||
t0 = time.monotonic()
|
||||
while True:
|
||||
result = await check()
|
||||
if result:
|
||||
return time.monotonic() - t0
|
||||
if time.monotonic() - t0 > timeout:
|
||||
raise AssertionError(f"timed out after {timeout}s")
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db_path(tmp_path):
|
||||
path = str(tmp_path / "ext.db")
|
||||
make_db(path)
|
||||
return path
|
||||
|
||||
|
||||
async def start_ds(paths, interval_ms=INTERVAL_MS, settings=None, **kwargs):
|
||||
ds = Datasette(
|
||||
files=paths,
|
||||
settings={"schema_watch_interval_ms": interval_ms, **(settings or {})},
|
||||
**kwargs,
|
||||
)
|
||||
await ds.invoke_startup()
|
||||
await ds.start_background_tasks()
|
||||
return ds
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_modes_and_defaults(tmp_path, db_path):
|
||||
immutable_path = str(tmp_path / "imm.db")
|
||||
make_db(immutable_path)
|
||||
ds = Datasette(files=[db_path], immutables=[immutable_path])
|
||||
await ds.invoke_startup()
|
||||
watcher = ds._schema_watcher
|
||||
assert watcher.states["ext"].mode == "external"
|
||||
assert watcher.states["imm"].mode == "immutable"
|
||||
# add_database(): files and named in-memory databases can be changed
|
||||
# by code other than Datasette's write methods (a plugin's own
|
||||
# connection, a backup() into a named in-memory database), so they get
|
||||
# the default_schema_watch mode too unless the caller says otherwise
|
||||
added = ds.add_database(Database(ds, memory_name="sw_owned_mem"), name="added")
|
||||
assert watcher.states["added"].mode == "external"
|
||||
assert added._watch_state is watcher.states["added"]
|
||||
added_file = str(tmp_path / "added_file.db")
|
||||
make_db(added_file)
|
||||
ds.add_database(Database(ds, path=added_file), name="added_file")
|
||||
assert watcher.states["added_file"].mode == "external"
|
||||
ds.add_database(
|
||||
Database(ds, memory_name="sw_owned_mem2"), name="owned", schema_watch="owned"
|
||||
)
|
||||
assert watcher.states["owned"].mode == "owned"
|
||||
# A private ":memory:" database: nothing else can change it
|
||||
ds.add_database(Database(ds, is_memory=True), name="private")
|
||||
assert watcher.states["private"].mode == "owned"
|
||||
ds.close()
|
||||
# default_schema_watch=owned applies to add_database() too
|
||||
ds = Datasette(files=[db_path], settings={"default_schema_watch": "owned"})
|
||||
ds.add_database(Database(ds, memory_name="sw_owned_mem3"), name="added")
|
||||
assert ds._schema_watcher.states["added"].mode == "owned"
|
||||
assert ds._schema_watcher.states["ext"].mode == "owned"
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mode_from_config(tmp_path, db_path):
|
||||
ds = Datasette(
|
||||
files=[db_path], config={"databases": {"ext": {"schema_watch": "owned"}}}
|
||||
)
|
||||
assert ds._schema_watcher.states["ext"].mode == "owned"
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_request_path_sweep(db_path):
|
||||
ds = await start_ds([db_path], interval_ms=0)
|
||||
watcher = ds._schema_watcher
|
||||
sweeps = watcher.counters["sweeps"]
|
||||
for _ in range(5):
|
||||
response = await ds.client.get("/ext/t.json")
|
||||
assert response.status_code == 200
|
||||
assert watcher.counters["sweeps"] == sweeps
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_owned_write_path_detects_schema_change(tmp_path):
|
||||
ds = Datasette(settings={"schema_watch_interval_ms": 0})
|
||||
await ds.invoke_startup()
|
||||
db = ds.add_database(Database(ds, memory_name="sw_owned"), name="owned")
|
||||
await ds.refresh_schemas(force=True)
|
||||
watcher = ds._schema_watcher
|
||||
scans = watcher.counters["scans"]
|
||||
# Data-only writes: no catalog rebuild
|
||||
await db.execute_write("create table if not exists t (id integer primary key)")
|
||||
assert await catalog_tables(ds, "owned") == ["t"]
|
||||
scans = watcher.counters["scans"]
|
||||
for i in range(5):
|
||||
await db.execute_write("insert into t default values")
|
||||
assert watcher.counters["scans"] == scans
|
||||
# Each write API: catalog is current as soon as the call returns
|
||||
await db.execute_write("create table t2 (id integer primary key)")
|
||||
assert "t2" in await catalog_tables(ds, "owned")
|
||||
await db.execute_write_script("create table t3 (id integer); create table t4 (id);")
|
||||
assert {"t3", "t4"} <= set(await catalog_tables(ds, "owned"))
|
||||
scans = watcher.counters["scans"]
|
||||
await db.execute_write_many("insert into t2 (id) values (?)", [[1], [2]])
|
||||
assert watcher.counters["scans"] == scans
|
||||
|
||||
def plugin_write(conn):
|
||||
conn.execute("alter table t add column extra text")
|
||||
|
||||
await db.execute_write_fn(plugin_write)
|
||||
assert await catalog_columns(ds, "owned", "t") == ["id", "extra"]
|
||||
|
||||
def isolated(conn):
|
||||
conn.execute("create table t6 (id)")
|
||||
|
||||
await db.execute_isolated_fn(isolated)
|
||||
assert "t6" in await catalog_tables(ds, "owned")
|
||||
# block=False: refreshed in the background
|
||||
await db.execute_write("create table t7 (id)", block=False)
|
||||
await wait_for(lambda: _contains(ds, "owned", "t7"))
|
||||
ds.close()
|
||||
|
||||
|
||||
async def _contains(ds, db_name, table):
|
||||
return table in await catalog_tables(ds, db_name)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_owned_write_path_non_threaded(tmp_path):
|
||||
path = str(tmp_path / "nt.db")
|
||||
make_db(path)
|
||||
ds = Datasette(settings={"num_sql_threads": 0, "schema_watch_interval_ms": 0})
|
||||
await ds.invoke_startup()
|
||||
db = ds.add_database(Database(ds, path=path), name="nt")
|
||||
await ds._schema_watcher.flush_pending()
|
||||
await db.execute_write("create table t2 (id)")
|
||||
assert await catalog_tables(ds, "nt") == ["t", "t2"]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_data_only_insert_does_not_rescan(db_path):
|
||||
ds = await start_ds([db_path])
|
||||
watcher = ds._schema_watcher
|
||||
state = watcher.states["ext"]
|
||||
scans = state.stats["scans"]
|
||||
checks = state.stats["checks"]
|
||||
for i in range(3):
|
||||
run_external(db_path, f"insert into t (a) values ('x{i}')")
|
||||
await asyncio.sleep(INTERVAL_MS * 3 / 1000)
|
||||
# The fingerprint changed, so schema_version was checked...
|
||||
assert state.stats["checks"] > checks
|
||||
# ... but the catalog was not rebuilt
|
||||
assert state.stats["scans"] == scans
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_header", (True, False))
|
||||
async def test_external_add_column(db_path, monkeypatch, use_header):
|
||||
monkeypatch.setattr(schema_watcher_module, "USE_HEADER_CHECK", use_header)
|
||||
ds = await start_ds([db_path])
|
||||
run_external(db_path, "alter table t add column b text")
|
||||
|
||||
async def has_b():
|
||||
return await catalog_columns(ds, "ext", "t") == ["id", "a", "b"]
|
||||
|
||||
await wait_for(has_b)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_sqlite_utils_transform(db_path):
|
||||
ds = await start_ds([db_path])
|
||||
run_process(
|
||||
"-m", "sqlite_utils", "transform", db_path, "t", "--rename", "a", "renamed"
|
||||
)
|
||||
|
||||
async def renamed():
|
||||
return await catalog_columns(ds, "ext", "t") == ["id", "renamed"]
|
||||
|
||||
await wait_for(renamed)
|
||||
# And the table page uses the new column too
|
||||
response = await ds.client.get("/ext/t.json?_shape=array")
|
||||
assert response.json() == [{"id": 1, "renamed": "one"}]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_create_table_in_wal_before_checkpoint(tmp_path):
|
||||
path = str(tmp_path / "walled.db")
|
||||
make_db(path, wal=True)
|
||||
ds = await start_ds([path])
|
||||
_, proc = run_external(
|
||||
path, "create table in_wal (id integer)", journal_mode="wal", keep_wal=True
|
||||
)
|
||||
try:
|
||||
assert os.path.getsize(path + "-wal") > 0
|
||||
|
||||
async def seen():
|
||||
return "in_wal" in await catalog_tables(ds, "walled")
|
||||
|
||||
await wait_for(seen)
|
||||
# Main file still does not contain it: it was only in the WAL
|
||||
assert os.path.exists(path + "-wal")
|
||||
finally:
|
||||
proc.stdin.close()
|
||||
proc.wait()
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_atomic_replace_by_rename(tmp_path, db_path):
|
||||
ds = await start_ds([db_path])
|
||||
# Warm a cached read connection on the old inode
|
||||
response = await ds.client.get("/ext/t.json?_shape=array")
|
||||
assert response.json() == [{"id": 1, "a": "one"}]
|
||||
replacement = str(tmp_path / "replacement.db")
|
||||
conn = sqlite3.connect(replacement)
|
||||
# Same schema_version as the original file (1 CREATE TABLE) but a
|
||||
# different table: only the inode change gives it away
|
||||
conn.execute("create table t (id integer primary key, different text)")
|
||||
conn.execute("insert into t (different) values ('new file')")
|
||||
conn.commit()
|
||||
assert conn.execute("pragma schema_version").fetchone()[0] == 1
|
||||
conn.close()
|
||||
old_inode = os.stat(db_path).st_ino
|
||||
run_process(
|
||||
"-c",
|
||||
"import os, sys; os.replace(sys.argv[1], sys.argv[2])",
|
||||
replacement,
|
||||
db_path,
|
||||
)
|
||||
assert os.stat(db_path).st_ino != old_inode
|
||||
|
||||
async def replaced():
|
||||
return await catalog_columns(ds, "ext", "t") == ["id", "different"]
|
||||
|
||||
await wait_for(replaced)
|
||||
# Cached connections were invalidated, so queries see the new file
|
||||
response = await ds.client.get("/ext/t.json?_shape=array")
|
||||
assert response.json() == [{"id": 1, "different": "new file"}]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_old_inode_connections_without_watcher(tmp_path, db_path):
|
||||
# Documents the hazard the watcher fixes: with polling off, a cached
|
||||
# read connection keeps reading the replaced (unlinked) file
|
||||
# One SQL thread, so every request reuses the same cached connection
|
||||
ds = await start_ds([db_path], interval_ms=0, settings={"num_sql_threads": 1})
|
||||
response = await ds.client.get("/ext/t.json?_shape=array")
|
||||
assert response.json() == [{"id": 1, "a": "one"}]
|
||||
replacement = str(tmp_path / "replacement.db")
|
||||
conn = sqlite3.connect(replacement)
|
||||
conn.execute("create table t (id integer primary key, a text)")
|
||||
conn.execute("insert into t (a) values ('new file')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
os.replace(replacement, db_path)
|
||||
response = await ds.client.get("/ext/t.json?_shape=array")
|
||||
assert response.json() == [{"id": 1, "a": "one"}] # stale!
|
||||
# An explicit refresh notices the inode change and fixes it
|
||||
await ds.refresh_schemas(force=True)
|
||||
response = await ds.client.get("/ext/t.json?_shape=array")
|
||||
assert response.json() == [{"id": 1, "a": "new file"}]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_vacuum(db_path):
|
||||
run_external(
|
||||
db_path,
|
||||
"create table big (x); insert into big select zeroblob(1000) from "
|
||||
"(with recursive c(i) as (select 1 union all select i+1 from c where i < 200) select i from c);"
|
||||
"drop table big;",
|
||||
)
|
||||
ds = await start_ds([db_path])
|
||||
before = await ds.get_internal_database().execute(
|
||||
"select schema_version from catalog_databases where database_name = 'ext'"
|
||||
)
|
||||
run_external(db_path, "vacuum")
|
||||
|
||||
async def bumped():
|
||||
after = await ds.get_internal_database().execute(
|
||||
"select schema_version from catalog_databases where database_name = 'ext'"
|
||||
)
|
||||
return after.first()[0] == before.first()[0] + 1
|
||||
|
||||
await wait_for(bumped)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_rapid_changes(db_path):
|
||||
ds = await start_ds([db_path])
|
||||
for i in range(10):
|
||||
run_external(db_path, f"create table rapid_{i} (id)")
|
||||
|
||||
async def all_seen():
|
||||
tables = await catalog_tables(ds, "ext")
|
||||
return all(f"rapid_{i}" in tables for i in range(10))
|
||||
|
||||
await wait_for(all_seen)
|
||||
ds.close()
|
||||
|
||||
|
||||
def _coarse_fingerprint(real):
|
||||
# Simulate a filesystem with 1 second timestamp granularity
|
||||
def fingerprint(path, present=None):
|
||||
parts = real(path, present)
|
||||
out = []
|
||||
for i, part in enumerate(parts):
|
||||
if part is None:
|
||||
out.append(None)
|
||||
continue
|
||||
part = list(part)
|
||||
# main: (dev, ino, size, mtime, ctime); side files: (ino, size, mtime)
|
||||
for idx in (3, 4) if i == 0 else (2,):
|
||||
part[idx] = part[idx] // 10**9 * 10**9
|
||||
out.append(tuple(part))
|
||||
return tuple(out)
|
||||
|
||||
return fingerprint
|
||||
|
||||
|
||||
@pytest.mark.parametrize("journal_mode", ("delete", "wal", "truncate"))
|
||||
def test_header_schema_version_matches_pragma(tmp_path, journal_mode):
|
||||
path = str(tmp_path / "h.db")
|
||||
conn = sqlite3.connect(path, isolation_level=None)
|
||||
conn.execute(f"PRAGMA journal_mode={journal_mode}")
|
||||
for i in range(300):
|
||||
conn.execute(f"create table t{i} (id integer)")
|
||||
conn.execute("alter table t1 add column b")
|
||||
version = conn.execute("PRAGMA schema_version").fetchone()[0]
|
||||
if journal_mode == "wal":
|
||||
# Pages still in the -wal: the main file header is stale, which is
|
||||
# why the watcher only reads it when the -wal is empty
|
||||
assert schema_watcher_module.fingerprint(path)[1] is not None
|
||||
conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
assert schema_watcher_module.fingerprint(path)[1:] == (None, None)
|
||||
assert schema_watcher_module.header_schema_version(path) == version == 301
|
||||
conn.close()
|
||||
assert (
|
||||
schema_watcher_module.header_schema_version(str(tmp_path / "nope.db")) is None
|
||||
)
|
||||
(tmp_path / "junk.db").write_bytes(b"not a database" * 20)
|
||||
assert (
|
||||
schema_watcher_module.header_schema_version(str(tmp_path / "junk.db")) is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_header", (True, False))
|
||||
@pytest.mark.parametrize("racy_rule", (True, False))
|
||||
async def test_racily_clean_same_tick_change(
|
||||
db_path, monkeypatch, racy_rule, use_header
|
||||
):
|
||||
"""Two schema changes inside one timestamp tick, same file size and
|
||||
inode: only the racy-clean rule catches the second one."""
|
||||
monkeypatch.setattr(schema_watcher_module, "USE_HEADER_CHECK", use_header)
|
||||
monkeypatch.setattr(
|
||||
schema_watcher_module,
|
||||
"fingerprint",
|
||||
_coarse_fingerprint(schema_watcher_module.fingerprint),
|
||||
)
|
||||
if not racy_rule:
|
||||
monkeypatch.setattr(schema_watcher_module, "RACY_WINDOW_NS", -(10**18))
|
||||
ds = Datasette(files=[db_path], settings={"schema_watch_interval_ms": 0})
|
||||
await ds.invoke_startup()
|
||||
# Line the first change up with the start of a fresh second
|
||||
await asyncio.sleep(1 - (time.time() % 1) + 0.05)
|
||||
size = os.path.getsize(db_path)
|
||||
run_external(db_path, "alter table t add column b text")
|
||||
await ds._refresh_schemas() # sweep: sees change 1
|
||||
assert await catalog_columns(ds, "ext", "t") == ["id", "a", "b"]
|
||||
run_external(db_path, "alter table t add column c text")
|
||||
assert os.path.getsize(db_path) == size
|
||||
assert time.time() % 1 < 0.9, "test machine too slow to stay in one tick"
|
||||
await ds._refresh_schemas() # sweep: fingerprint identical to last one
|
||||
columns = await catalog_columns(ds, "ext", "t")
|
||||
if racy_rule:
|
||||
assert columns == ["id", "a", "b", "c"]
|
||||
else:
|
||||
# Without the rule the change is missed
|
||||
assert columns == ["id", "a", "b"]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_file_deleted_and_recreated(tmp_path, db_path):
|
||||
ds = await start_ds([db_path])
|
||||
assert await catalog_tables(ds, "ext") == ["t"]
|
||||
run_process("-c", "import os, sys; os.unlink(sys.argv[1])", db_path)
|
||||
|
||||
async def gone():
|
||||
return await catalog_tables(ds, "ext") == []
|
||||
|
||||
await wait_for(gone)
|
||||
response = await ds.client.get("/ext/t.json")
|
||||
assert response.status_code in (404, 500)
|
||||
# Writes fail instead of silently creating a new empty file
|
||||
with pytest.raises(sqlite3.OperationalError):
|
||||
await ds.get_database("ext").execute_write("insert into t (a) values ('x')")
|
||||
assert not os.path.exists(db_path)
|
||||
make_db(db_path, sql="create table t (id integer primary key, a text, back text)")
|
||||
|
||||
async def back():
|
||||
return await catalog_columns(ds, "ext", "t") == ["id", "a", "back"]
|
||||
|
||||
await wait_for(back)
|
||||
response = await ds.client.get("/ext/t.json?_shape=array")
|
||||
assert response.json() == [{"id": 1, "a": "one", "back": None}]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_remove_database_deletes_catalog_rows(tmp_path, db_path):
|
||||
other = str(tmp_path / "other.db")
|
||||
make_db(other)
|
||||
ds = Datasette(files=[db_path, other])
|
||||
await ds.invoke_startup()
|
||||
assert await catalog_tables(ds, "other") == ["t"]
|
||||
ds.remove_database("other")
|
||||
await ds._schema_watcher.on_request()
|
||||
internal = ds.get_internal_database()
|
||||
for table in (
|
||||
"catalog_databases",
|
||||
"catalog_tables",
|
||||
"catalog_columns",
|
||||
"catalog_indexes",
|
||||
):
|
||||
rows = await internal.execute(
|
||||
f"select count(*) from {table} where database_name = 'other'"
|
||||
)
|
||||
assert rows.first()[0] == 0
|
||||
assert await catalog_tables(ds, "ext") == ["t"]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_database_visible_on_next_request(tmp_path):
|
||||
ds = Datasette(settings={"schema_watch_interval_ms": 0})
|
||||
await ds.invoke_startup()
|
||||
await ds.client.get("/-/versions.json")
|
||||
path = str(tmp_path / "late.db")
|
||||
make_db(path)
|
||||
ds.add_database(Database(ds, path=path), name="late")
|
||||
response = await ds.client.get("/late.json")
|
||||
assert response.status_code == 200
|
||||
assert await catalog_tables(ds, "late") == ["t"]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_persisted_fingerprints_skip_unchanged_files(tmp_path, monkeypatch):
|
||||
# Shrink the racy window so freshly written files count as stable
|
||||
monkeypatch.setattr(schema_watcher_module, "RACY_WINDOW_NS", 1_000_000)
|
||||
internal = str(tmp_path / "internal.db")
|
||||
paths = []
|
||||
for i in range(3):
|
||||
path = str(tmp_path / f"p{i}.db")
|
||||
make_db(path)
|
||||
paths.append(path)
|
||||
await asyncio.sleep(0.05)
|
||||
ds = Datasette(files=paths, internal=internal)
|
||||
await ds.invoke_startup()
|
||||
assert ds._schema_watcher.counters["scans"] == 3
|
||||
ds.close()
|
||||
# Change one file while Datasette is "down"
|
||||
run_external(paths[1], "create table added_offline (id)")
|
||||
ds2 = Datasette(files=paths, internal=internal)
|
||||
await ds2.invoke_startup()
|
||||
counters = ds2._schema_watcher.counters
|
||||
assert counters["restored_from_persisted"] == 2
|
||||
assert counters["scans"] == 1
|
||||
assert await catalog_tables(ds2, "p1") == ["added_offline", "t"]
|
||||
assert await catalog_tables(ds2, "p0") == ["t"]
|
||||
ds2.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_remove_database_during_catalog_scan(tmp_path):
|
||||
# remove_database() closes the Database while the watcher may still be
|
||||
# scanning it in a worker thread. The scan connection is not tracked by
|
||||
# the Database, so close() must not pull it out from under the scan
|
||||
# (that used to segfault).
|
||||
path = str(tmp_path / "big.db")
|
||||
conn = sqlite3.connect(path)
|
||||
for i in range(150):
|
||||
conn.execute(f"create table t{i} (a, b, c integer references t0(a))")
|
||||
conn.execute(f"create index i{i} on t{i}(a)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
ds = Datasette(settings={"schema_watch_interval_ms": 0})
|
||||
await ds.invoke_startup()
|
||||
for i in range(60):
|
||||
ds.add_database(Database(ds, path=path, is_mutable=i % 2 == 0), name=f"x{i}")
|
||||
await asyncio.sleep(0.0005 * (i % 5))
|
||||
ds.remove_database(f"x{i}")
|
||||
await asyncio.sleep(0.2)
|
||||
rows = await ds.get_internal_database().execute(
|
||||
"select count(*) from catalog_tables where database_name like 'x%'"
|
||||
)
|
||||
assert rows.first()[0] == 0
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"setting,expected", (("external", "external"), ("owned", "owned"))
|
||||
)
|
||||
async def test_default_schema_watch_setting(tmp_path, db_path, setting, expected):
|
||||
immutable_path = str(tmp_path / "imm.db")
|
||||
make_db(immutable_path)
|
||||
ds = Datasette(
|
||||
files=[db_path],
|
||||
immutables=[immutable_path],
|
||||
settings={"default_schema_watch": setting},
|
||||
)
|
||||
watcher = ds._schema_watcher
|
||||
assert watcher.states["ext"].mode == expected
|
||||
# Immutable files are always immutable
|
||||
assert watcher.states["imm"].mode == "immutable"
|
||||
# add_database() follows the setting too, and can ask for either mode
|
||||
other = str(tmp_path / "other.db")
|
||||
make_db(other)
|
||||
added = ds.add_database(Database(ds, path=other), name="added")
|
||||
assert added._watch_state.mode == expected
|
||||
polled = ds.add_database(
|
||||
Database(ds, path=other), name="polled", schema_watch="external"
|
||||
)
|
||||
assert polled._watch_state.mode == "external"
|
||||
owned = ds.add_database(
|
||||
Database(ds, path=other), name="owned", schema_watch="owned"
|
||||
)
|
||||
assert owned._watch_state.mode == "owned"
|
||||
ds.close()
|
||||
|
||||
|
||||
def test_default_schema_watch_config_overrides(db_path):
|
||||
ds = Datasette(
|
||||
files=[db_path],
|
||||
settings={"default_schema_watch": "owned"},
|
||||
config={"databases": {"ext": {"schema_watch": "external"}}},
|
||||
)
|
||||
assert ds._schema_watcher.states["ext"].mode == "external"
|
||||
ds.close()
|
||||
|
||||
|
||||
def test_default_schema_watch_invalid(db_path):
|
||||
from datasette.utils import StartupError
|
||||
|
||||
with pytest.raises(StartupError, match="default_schema_watch"):
|
||||
Datasette(files=[db_path], settings={"default_schema_watch": "immutable"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_owned_default_is_not_polled(db_path):
|
||||
ds = await start_ds([db_path], settings={"default_schema_watch": "owned"})
|
||||
run_external(db_path, "create table not_polled (id)")
|
||||
await asyncio.sleep(INTERVAL_MS * 4 / 1000)
|
||||
assert await catalog_tables(ds, "ext") == ["t"]
|
||||
# Datasette's own writes are still detected immediately
|
||||
await ds.get_database("ext").execute_write("create table mine (id)")
|
||||
assert await catalog_tables(ds, "ext") == ["mine", "not_polled", "t"]
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", (0, 2))
|
||||
async def test_scratch_database_file_created_by_first_write(tmp_path, num_sql_threads):
|
||||
# A database added before its file exists (a plugin's scratch
|
||||
# database): the first write creates it. Once it has existed, deleting
|
||||
# it makes writes fail rather than silently creating a new empty file.
|
||||
ds = Datasette(
|
||||
settings={"num_sql_threads": num_sql_threads, "schema_watch_interval_ms": 0}
|
||||
)
|
||||
await ds.invoke_startup()
|
||||
path = str(tmp_path / "scratch.db")
|
||||
db = ds.add_database(Database(ds, path=path), name="scratch")
|
||||
await ds._schema_watcher.flush_pending()
|
||||
assert not os.path.exists(path)
|
||||
await db.execute_write("create table t (id integer primary key)")
|
||||
assert os.path.exists(path)
|
||||
assert await catalog_tables(ds, "scratch") == ["t"]
|
||||
await db.execute_write("insert into t default values")
|
||||
os.unlink(path)
|
||||
await ds.refresh_schemas(force=True)
|
||||
assert await catalog_tables(ds, "scratch") == []
|
||||
with pytest.raises(sqlite3.OperationalError):
|
||||
await db.execute_write("insert into t default values")
|
||||
assert not os.path.exists(path)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("num_sql_threads", (0, 2))
|
||||
async def test_write_connection_follows_replaced_file(
|
||||
tmp_path, db_path, num_sql_threads
|
||||
):
|
||||
ds = await start_ds(
|
||||
[db_path], interval_ms=0, settings={"num_sql_threads": num_sql_threads}
|
||||
)
|
||||
db = ds.get_database("ext")
|
||||
# Warm the write connection on the old inode
|
||||
await db.execute_write("insert into t (a) values ('old')")
|
||||
replacement = str(tmp_path / "replacement.db")
|
||||
make_db(replacement)
|
||||
os.replace(replacement, db_path)
|
||||
await ds.refresh_schemas(force=True)
|
||||
await db.execute_write("insert into t (a) values ('new')")
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
rows = conn.execute("select a from t order by id").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
# The write went to the new file, not the unlinked one
|
||||
assert rows == [("one",), ("new",)]
|
||||
assert [r["a"] for r in (await db.execute("select a from t order by id")).rows] == [
|
||||
"one",
|
||||
"new",
|
||||
]
|
||||
ds.close()
|
||||
70
tests/test_sqlite_error_compatibility.py
Normal file
70
tests/test_sqlite_error_compatibility.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
"""Error recovery must also work without Python 3.11's SQLite error metadata."""
|
||||
|
||||
import errno
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from datasette.schema_watcher import _is_transient
|
||||
from datasette.utils import sqlite3
|
||||
from datasette.write_budget import DatabaseResourceError, connect_with_retry
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def old_sqlite_errors(monkeypatch):
|
||||
for name in ("SQLITE_CANTOPEN", "SQLITE_NOMEM"):
|
||||
monkeypatch.delattr(sqlite3, name, raising=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error,expected",
|
||||
[
|
||||
(sqlite3.OperationalError("unable to open database file"), True),
|
||||
(sqlite3.OperationalError("out of memory"), True),
|
||||
(sqlite3.OperationalError("database is locked"), True),
|
||||
(sqlite3.OperationalError("no such table: missing"), False),
|
||||
(OSError(errno.EMFILE, "too many open files"), True),
|
||||
],
|
||||
)
|
||||
def test_transient_errors_without_sqlite_constants(old_sqlite_errors, error, expected):
|
||||
assert _is_transient(error) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("os_error", [False, True])
|
||||
def test_open_recovers_without_sqlite_constants(old_sqlite_errors, tmp_path, os_error):
|
||||
path = tmp_path / "db.sqlite"
|
||||
path.touch()
|
||||
pool = Mock()
|
||||
db = SimpleNamespace(path=str(path), ds=SimpleNamespace(_read_pool_or_none=pool))
|
||||
error = (
|
||||
OSError(errno.EMFILE, "too many open files")
|
||||
if os_error
|
||||
else sqlite3.OperationalError("unable to open database file")
|
||||
)
|
||||
connection = object()
|
||||
connect = Mock(side_effect=[error, connection])
|
||||
assert connect_with_retry(db, connect) is connection
|
||||
assert connect.call_count == 2
|
||||
pool.evict_idle.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("os_error", [False, True])
|
||||
def test_open_failure_preserved_without_sqlite_constants(
|
||||
old_sqlite_errors, tmp_path, os_error
|
||||
):
|
||||
path = tmp_path / "db.sqlite"
|
||||
path.touch()
|
||||
db = SimpleNamespace(path=str(path), ds=SimpleNamespace(_read_pool_or_none=None))
|
||||
error = (
|
||||
OSError(errno.EMFILE, "too many open files")
|
||||
if os_error
|
||||
else sqlite3.OperationalError("unable to open database file")
|
||||
)
|
||||
connect = Mock(side_effect=error)
|
||||
with pytest.raises(
|
||||
DatabaseResourceError if os_error else sqlite3.OperationalError
|
||||
) as exc:
|
||||
connect_with_retry(db, connect)
|
||||
assert (exc.value.__cause__ if os_error else exc.value) is error
|
||||
assert connect.call_count == 2
|
||||
389
tests/test_write_budget.py
Normal file
389
tests/test_write_budget.py
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
import asyncio
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from datasette.app import Datasette
|
||||
from datasette.write_budget import (
|
||||
DatabaseAdmissionTimeout,
|
||||
DatabaseQueueFull,
|
||||
DatabaseReentrancyError,
|
||||
DatabaseResourceError,
|
||||
)
|
||||
|
||||
|
||||
def make_databases(tmp_path, count=3, **settings):
|
||||
paths = []
|
||||
for i in range(count):
|
||||
path = tmp_path / f"budget{i}.db"
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("create table t(id)")
|
||||
conn.close()
|
||||
paths.append(str(path))
|
||||
ds = Datasette(paths, settings={"schema_watch_interval_ms": 0, **settings})
|
||||
return ds, [ds.get_database(f"budget{i}") for i in range(count)]
|
||||
|
||||
|
||||
async def wait_for(predicate):
|
||||
deadline = time.monotonic() + 5
|
||||
while not predicate():
|
||||
assert time.monotonic() < deadline
|
||||
await asyncio.sleep(0.005)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_many_writers_bounded_fifo_and_fairness(tmp_path):
|
||||
ds, dbs = make_databases(
|
||||
tmp_path, 30, max_write_connections=2, max_pending_writes=200
|
||||
)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
order = []
|
||||
|
||||
def slow(conn):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
|
||||
first = asyncio.create_task(dbs[0].execute_write_fn(slow))
|
||||
tasks = []
|
||||
try:
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
for seq in range(4):
|
||||
for db in dbs:
|
||||
|
||||
def write(conn, name=db.name, seq=seq):
|
||||
order.append((name, seq))
|
||||
conn.execute("insert into t values (?)", (seq,))
|
||||
|
||||
tasks.append(asyncio.create_task(db.execute_write_fn(write)))
|
||||
await asyncio.sleep(0.03)
|
||||
release.set()
|
||||
await asyncio.gather(first, *tasks)
|
||||
assert ds._write_budget.snapshot()["stats"]["peak_writers"] == 2
|
||||
for db in dbs:
|
||||
assert [seq for name, seq in order if name == db.name] == list(range(4))
|
||||
assert [
|
||||
r[0] for r in (await db.execute("select id from t order by rowid")).rows
|
||||
] == list(range(4))
|
||||
assert order.index((dbs[-1].name, 0)) < order.index((dbs[0].name, 3))
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.gather(first, *tasks, return_exceptions=True)
|
||||
ds.close()
|
||||
await wait_for(lambda: ds._write_budget.snapshot()["writers"] == 0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_full_expiration_and_internal_progress(tmp_path):
|
||||
ds, dbs = make_databases(
|
||||
tmp_path,
|
||||
max_write_connections=1,
|
||||
max_pending_writes=1,
|
||||
write_queue_timeout_ms=100,
|
||||
)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
ran = []
|
||||
|
||||
def slow(conn):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
conn.execute("insert into t values (1)")
|
||||
|
||||
first = asyncio.create_task(dbs[0].execute_write_fn(slow))
|
||||
queued = None
|
||||
try:
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
queued = asyncio.create_task(
|
||||
dbs[1].execute_write_fn(lambda conn: ran.append(True))
|
||||
)
|
||||
await wait_for(lambda: ds._write_budget.snapshot()["pending"] == 1)
|
||||
with pytest.raises(DatabaseQueueFull) as exc:
|
||||
await dbs[2].execute_write_fn(lambda conn: ran.append(True))
|
||||
assert exc.value.execution_started is False
|
||||
# Internal catalog work must not compete for the single user slot.
|
||||
assert (
|
||||
await asyncio.wait_for(
|
||||
ds._internal_database.execute_write_fn(lambda conn: 42), 2
|
||||
)
|
||||
== 42
|
||||
)
|
||||
with pytest.raises(DatabaseAdmissionTimeout):
|
||||
await queued
|
||||
assert not ran
|
||||
assert dbs[1]._write_queue.qsize() == 0
|
||||
release.set()
|
||||
await first
|
||||
await dbs[2].execute_write("insert into t values (2)")
|
||||
assert ds._write_budget.snapshot()["stats"]["peak_pending"] == 1
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.gather(
|
||||
first, *([queued] if queued else []), return_exceptions=True
|
||||
)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_waiting_database_preserves_admission_deadline(tmp_path):
|
||||
ds, dbs = make_databases(
|
||||
tmp_path, max_write_connections=1, write_queue_timeout_ms=50
|
||||
)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
|
||||
def slow(conn):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
|
||||
first = asyncio.create_task(dbs[0].execute_write_fn(slow))
|
||||
try:
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
pending = asyncio.create_task(
|
||||
dbs[1].execute_write_fn(lambda conn: pytest.fail("closed callback ran"))
|
||||
)
|
||||
await wait_for(lambda: ds._write_budget.snapshot()["pending"] == 1)
|
||||
dbs[1].close()
|
||||
with pytest.raises(DatabaseAdmissionTimeout):
|
||||
await pending
|
||||
assert ds._write_budget.snapshot()["pending"] == 0
|
||||
finally:
|
||||
release.set()
|
||||
await first
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("threads", [0, 3])
|
||||
async def test_memory_pins_capacity_and_recovers_after_close(tmp_path, threads):
|
||||
ds, dbs = make_databases(
|
||||
tmp_path,
|
||||
max_write_connections=1,
|
||||
write_queue_timeout_ms=50,
|
||||
num_sql_threads=threads,
|
||||
)
|
||||
memory = ds.add_memory_database("budget_memory")
|
||||
try:
|
||||
await memory.execute_write("create table t(id)")
|
||||
with pytest.raises(DatabaseResourceError):
|
||||
await dbs[0].execute_write("insert into t values (1)")
|
||||
assert (
|
||||
await memory.execute_write_fn(
|
||||
lambda conn: conn.execute("select count(*) from t").fetchone()[0]
|
||||
)
|
||||
== 0
|
||||
)
|
||||
memory.close()
|
||||
await dbs[0].execute_write("insert into t values (1)")
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("threads", [0, 3])
|
||||
async def test_isolated_writer_does_not_hold_extra_file_connection(tmp_path, threads):
|
||||
ds, dbs = make_databases(tmp_path, max_write_connections=1, num_sql_threads=threads)
|
||||
try:
|
||||
for db in dbs:
|
||||
await db.execute_write("insert into t values (1)")
|
||||
assert (
|
||||
await db.execute_isolated_fn(
|
||||
lambda conn, db=db: len(db._all_connections)
|
||||
)
|
||||
== 1
|
||||
)
|
||||
if threads == 0:
|
||||
assert sum(db._write_connection is not None for db in dbs) <= 1
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", ["read", "write"])
|
||||
async def test_reentrant_database_call_fails_without_deadlock(tmp_path, kind):
|
||||
ds, dbs = make_databases(tmp_path, max_write_connections=1, max_open_connections=1)
|
||||
try:
|
||||
|
||||
def callback(conn):
|
||||
return asyncio.run(dbs[1].execute_write_fn(lambda conn: 1))
|
||||
|
||||
method = dbs[0].execute_fn if kind == "read" else dbs[0].execute_write_fn
|
||||
with pytest.raises(DatabaseReentrancyError):
|
||||
await asyncio.wait_for(method(callback), 2)
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admission_errors_are_http_503(tmp_path, monkeypatch):
|
||||
ds, dbs = make_databases(tmp_path)
|
||||
await ds.invoke_startup()
|
||||
|
||||
async def fail(*args, **kwargs):
|
||||
raise DatabaseQueueFull("Too many queued database writes")
|
||||
|
||||
monkeypatch.setattr(dbs[0], "execute", fail)
|
||||
try:
|
||||
response = await ds.client.get("/budget0/-/query.json?sql=select+1")
|
||||
assert response.status_code == 503, response.text
|
||||
assert response.headers["retry-after"] == "1"
|
||||
assert response.json()["code"] == "database_queue_full"
|
||||
assert response.json()["execution_started"] is False
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("threads", [0, 3])
|
||||
async def test_unfinished_explicit_transaction_is_rolled_back(tmp_path, threads):
|
||||
from datasette.connection_pool import ConnectionLeaseError
|
||||
|
||||
ds, dbs = make_databases(tmp_path, num_sql_threads=threads)
|
||||
try:
|
||||
with pytest.raises(ConnectionLeaseError, match="rolled back"):
|
||||
await dbs[0].execute_write_fn(
|
||||
lambda conn: conn.execute("insert into t values (1)"), transaction=False
|
||||
)
|
||||
await dbs[0].execute_write("insert into t values (2)")
|
||||
assert [row[0] for row in (await dbs[0].execute("select id from t")).rows] == [
|
||||
2
|
||||
]
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_nonblocking_expiration_notifies_plugin(tmp_path):
|
||||
from datasette import hookimpl
|
||||
from datasette.plugins import pm
|
||||
|
||||
notifications = []
|
||||
|
||||
class Plugin:
|
||||
@hookimpl
|
||||
async def write_task_completed(self, database, task_id, exception):
|
||||
notifications.append((database, task_id, exception))
|
||||
|
||||
plugin = Plugin()
|
||||
pm.register(plugin)
|
||||
ds, dbs = make_databases(
|
||||
tmp_path, max_write_connections=1, write_queue_timeout_ms=50
|
||||
)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
|
||||
def slow(conn):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
|
||||
first = asyncio.create_task(dbs[0].execute_write_fn(slow))
|
||||
try:
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
task_id = await dbs[1].execute_write_fn(
|
||||
lambda conn: pytest.fail("expired callback ran"), block=False
|
||||
)
|
||||
await wait_for(lambda: notifications)
|
||||
assert notifications[0][:2] == (dbs[1].name, task_id)
|
||||
assert isinstance(notifications[0][2], DatabaseAdmissionTimeout)
|
||||
release.set()
|
||||
await first
|
||||
successful_id = await dbs[1].execute_write_fn(lambda conn: 42, block=False)
|
||||
await wait_for(lambda: len(notifications) == 2)
|
||||
assert notifications[1] == (dbs[1].name, successful_id, None)
|
||||
finally:
|
||||
release.set()
|
||||
await first
|
||||
ds.close()
|
||||
pm.unregister(plugin)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connection_open_pressure_retries_before_callback(tmp_path, monkeypatch):
|
||||
import errno
|
||||
|
||||
from datasette.utils import sqlite3 as sqlite_module
|
||||
|
||||
ds, dbs = make_databases(tmp_path)
|
||||
await ds.invoke_startup()
|
||||
await dbs[1].execute("select 1")
|
||||
original = sqlite_module.connect
|
||||
calls = []
|
||||
|
||||
def connect(*args, **kwargs):
|
||||
if "budget0.db" in str(args[0]):
|
||||
calls.append(1)
|
||||
if len(calls) == 1:
|
||||
raise OSError(errno.EMFILE, "too many files")
|
||||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(sqlite_module, "connect", connect)
|
||||
ran = []
|
||||
try:
|
||||
await dbs[0].execute_write_fn(lambda conn: ran.append(1))
|
||||
assert calls == [1, 1]
|
||||
assert ran == [1]
|
||||
assert not dbs[1]._read_pool_state.idle
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("threads", [0, 3])
|
||||
async def test_resource_error_after_execution_is_not_an_admission_error(
|
||||
tmp_path, threads, monkeypatch
|
||||
):
|
||||
ds, dbs = make_databases(tmp_path, num_sql_threads=threads)
|
||||
db = dbs[0]
|
||||
await ds.invoke_startup()
|
||||
try:
|
||||
|
||||
def committed_then_failed(conn):
|
||||
conn.execute("insert into t values (1)")
|
||||
conn.commit()
|
||||
raise DatabaseQueueFull("A different operation could not be admitted")
|
||||
|
||||
with pytest.raises(RuntimeError, match="callback started") as exc:
|
||||
await db.execute_write_fn(committed_then_failed, transaction=False)
|
||||
assert not isinstance(exc.value, DatabaseResourceError)
|
||||
assert isinstance(exc.value.__cause__, DatabaseQueueFull)
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 1
|
||||
|
||||
async def refresh_failed(*args):
|
||||
raise DatabaseResourceError("Catalog connection unavailable")
|
||||
|
||||
monkeypatch.setattr(ds._schema_watcher, "after_write", refresh_failed)
|
||||
db._watch_state.needs_scan = True
|
||||
with pytest.raises(RuntimeError, match="Write completed") as exc:
|
||||
await db.execute_write("insert into t values (2)")
|
||||
assert not isinstance(exc.value, DatabaseResourceError)
|
||||
assert (await db.execute("select count(*) from t")).single_value() == 2
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retiring_writer_keeps_its_slot_until_connection_closes(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
ds, dbs = make_databases(tmp_path, max_write_connections=1)
|
||||
closing, release = threading.Event(), threading.Event()
|
||||
original = dbs[0]._forget_connection
|
||||
|
||||
def slow_close(conn):
|
||||
closing.set()
|
||||
assert release.wait(5)
|
||||
original(conn)
|
||||
|
||||
await dbs[0].execute_write("insert into t values (1)")
|
||||
monkeypatch.setattr(dbs[0], "_forget_connection", slow_close)
|
||||
pending = asyncio.create_task(dbs[1].execute_write("insert into t values (2)"))
|
||||
try:
|
||||
assert await asyncio.to_thread(closing.wait, 5)
|
||||
assert not pending.done()
|
||||
assert dbs[1]._write_thread is None
|
||||
assert ds._write_budget.snapshot()["writers"] == 1
|
||||
release.set()
|
||||
await pending
|
||||
assert ds._write_budget.snapshot()["stats"]["peak_writers"] == 1
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.gather(pending, return_exceptions=True)
|
||||
ds.close()
|
||||
250
tests/test_write_budget_shutdown.py
Normal file
250
tests/test_write_budget_shutdown.py
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
import asyncio
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from datasette.app import Datasette
|
||||
from datasette.database import Database, DatasetteClosedError
|
||||
from datasette.write_budget import RETIRE_WRITER, DatabaseAdmissionTimeout
|
||||
|
||||
|
||||
def make_databases(tmp_path, count=2, **settings):
|
||||
paths = []
|
||||
for i in range(count):
|
||||
path = tmp_path / f"shutdown{i}.db"
|
||||
with sqlite3.connect(path) as conn:
|
||||
conn.execute("create table t(id)")
|
||||
conn.close()
|
||||
paths.append(str(path))
|
||||
ds = Datasette(paths, settings={"schema_watch_interval_ms": 0, **settings})
|
||||
return ds, [ds.get_database(f"shutdown{i}") for i in range(count)]
|
||||
|
||||
|
||||
async def wait_for(predicate):
|
||||
deadline = time.monotonic() + 5
|
||||
while not predicate():
|
||||
assert time.monotonic() < deadline
|
||||
await asyncio.sleep(0.005)
|
||||
|
||||
|
||||
def rows(db):
|
||||
with sqlite3.connect(db.path) as conn:
|
||||
result = conn.execute("select id from t order by rowid").fetchall()
|
||||
conn.close()
|
||||
return [row[0] for row in result]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("retirement_requested", [False, True])
|
||||
async def test_close_drains_writes_before_worker_admission(
|
||||
tmp_path, monkeypatch, retirement_requested
|
||||
):
|
||||
ds, (db, _) = make_databases(tmp_path, max_write_connections=1)
|
||||
admitted = threading.Event()
|
||||
release = threading.Event()
|
||||
run = ds._write_budget._run
|
||||
|
||||
def paused_scheduler():
|
||||
admitted.set()
|
||||
assert release.wait(5)
|
||||
run()
|
||||
|
||||
monkeypatch.setattr(ds._write_budget, "_run", paused_scheduler)
|
||||
replies = []
|
||||
closing = None
|
||||
try:
|
||||
for i in range(20):
|
||||
_, reply = await db._send_to_write_thread(
|
||||
lambda conn, i=i: conn.execute("insert into t values (?)", [i]),
|
||||
block=False,
|
||||
)
|
||||
replies.append(reply)
|
||||
assert admitted.wait(1)
|
||||
assert db._write_thread is None
|
||||
if retirement_requested:
|
||||
# A previous retirement request can precede tasks accepted later.
|
||||
with db._write_queue.mutex:
|
||||
db._write_queue.queue.appendleft(RETIRE_WRITER)
|
||||
closing = asyncio.create_task(asyncio.to_thread(db.close))
|
||||
await wait_for(lambda: db._closed)
|
||||
release.set()
|
||||
await asyncio.wait_for(closing, 5)
|
||||
await asyncio.gather(*replies)
|
||||
assert rows(db) == list(range(20))
|
||||
assert ds._write_budget.snapshot()["writers"] == 0
|
||||
assert ds._write_budget.snapshot()["stats"]["peak_writers"] == 1
|
||||
with pytest.raises(DatasetteClosedError):
|
||||
await db.execute_write("insert into t values (21)")
|
||||
finally:
|
||||
release.set()
|
||||
if closing is not None:
|
||||
await closing
|
||||
await asyncio.gather(*replies, return_exceptions=True)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_waits_for_budget_capacity_then_drains(tmp_path):
|
||||
ds, (first, waiting) = make_databases(tmp_path, max_write_connections=1)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
|
||||
def slow(conn):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
|
||||
_, first_reply = await first._send_to_write_thread(slow, block=False)
|
||||
closing = None
|
||||
waiting_reply = None
|
||||
try:
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
_, waiting_reply = await waiting._send_to_write_thread(
|
||||
lambda conn: conn.execute("insert into t values (1)"), block=False
|
||||
)
|
||||
closing = asyncio.create_task(asyncio.to_thread(waiting.close))
|
||||
await wait_for(lambda: waiting._closed)
|
||||
assert ds._write_budget.snapshot()["pending"] == 1
|
||||
release.set()
|
||||
await asyncio.wait_for(closing, 5)
|
||||
await asyncio.gather(first_reply, waiting_reply)
|
||||
assert rows(waiting) == [1]
|
||||
assert ds._write_budget.snapshot()["stats"]["peak_writers"] == 1
|
||||
finally:
|
||||
release.set()
|
||||
if closing is not None:
|
||||
await closing
|
||||
await asyncio.gather(
|
||||
first_reply,
|
||||
*([waiting_reply] if waiting_reply else []),
|
||||
return_exceptions=True,
|
||||
)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_preserves_admission_deadline_while_capacity_pinned(tmp_path):
|
||||
ds, (waiting, _) = make_databases(
|
||||
tmp_path, max_write_connections=1, write_queue_timeout_ms=50
|
||||
)
|
||||
memory = ds.add_memory_database("pinned")
|
||||
ran = []
|
||||
try:
|
||||
await memory.execute_write("create table t(id)")
|
||||
_, reply = await waiting._send_to_write_thread(
|
||||
lambda conn: ran.append(True), block=False
|
||||
)
|
||||
await asyncio.wait_for(asyncio.to_thread(waiting.close), 2)
|
||||
with pytest.raises(DatabaseAdmissionTimeout):
|
||||
await reply
|
||||
assert not ran
|
||||
assert ds._write_budget.snapshot()["pending"] == 0
|
||||
assert ds._write_budget.snapshot()["stats"]["peak_writers"] == 1
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_datasette_shutdown_releases_later_memory_writer_before_drain(tmp_path):
|
||||
ds, (waiting, _) = make_databases(
|
||||
tmp_path, max_write_connections=1, write_queue_timeout_ms=1000
|
||||
)
|
||||
memory = ds.add_memory_database("pinned")
|
||||
try:
|
||||
await memory.execute_write("create table t(id)")
|
||||
_, reply = await waiting._send_to_write_thread(
|
||||
lambda conn: conn.execute("insert into t values (1)"), block=False
|
||||
)
|
||||
await asyncio.wait_for(asyncio.to_thread(ds.close), 5)
|
||||
await reply
|
||||
assert rows(waiting) == [1]
|
||||
assert ds._write_budget.snapshot()["writers"] == 0
|
||||
assert ds._write_budget.snapshot()["stats"]["peak_writers"] == 1
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("database_kind", ["file", "temporary", "internal"])
|
||||
async def test_close_timeout_preserves_running_connection(
|
||||
tmp_path, monkeypatch, database_kind
|
||||
):
|
||||
import datasette.database
|
||||
|
||||
monkeypatch.setattr(datasette.database, "WRITE_SHUTDOWN_TIMEOUT", 0.05)
|
||||
ds, (db, _) = make_databases(tmp_path, max_write_connections=1)
|
||||
if database_kind == "temporary":
|
||||
db = Database(ds, is_temp_disk=True)
|
||||
ds.add_database(db, name="temporary")
|
||||
await db.execute_write("create table t(id)")
|
||||
elif database_kind == "internal":
|
||||
db = ds._internal_database
|
||||
await db.execute_write("create table t(id)")
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
|
||||
def slow(conn):
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
conn.execute("insert into t values (1)")
|
||||
return conn.execute("select id from t").fetchall()[0][0]
|
||||
|
||||
_, running = await db._send_to_write_thread(slow, block=False)
|
||||
queued = None
|
||||
try:
|
||||
assert await asyncio.to_thread(entered.wait, 5)
|
||||
if database_kind != "internal":
|
||||
_, queued = await db._send_to_write_thread(
|
||||
lambda conn: pytest.fail("cancelled callback ran"), block=False
|
||||
)
|
||||
await asyncio.wait_for(asyncio.to_thread(db.close), 2)
|
||||
if queued is not None:
|
||||
with pytest.raises(DatasetteClosedError):
|
||||
await queued
|
||||
assert Path(db.path).exists()
|
||||
assert not running.done()
|
||||
release.set()
|
||||
assert await running == 1
|
||||
if database_kind == "file":
|
||||
assert rows(db) == [1]
|
||||
else:
|
||||
await wait_for(lambda: not Path(db.path).exists())
|
||||
await wait_for(lambda: db._write_thread is None)
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.gather(
|
||||
running, *([queued] if queued else []), return_exceptions=True
|
||||
)
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_timeout_during_write_connection_open(tmp_path, monkeypatch):
|
||||
import datasette.database
|
||||
|
||||
monkeypatch.setattr(datasette.database, "WRITE_SHUTDOWN_TIMEOUT", 0.05)
|
||||
ds, (db, _) = make_databases(tmp_path)
|
||||
opened, release = threading.Event(), threading.Event()
|
||||
connect = db.connect
|
||||
|
||||
def paused_connect(**kwargs):
|
||||
conn = connect(**kwargs)
|
||||
opened.set()
|
||||
assert release.wait(5)
|
||||
return conn
|
||||
|
||||
monkeypatch.setattr(db, "connect", paused_connect)
|
||||
_, reply = await db._send_to_write_thread(
|
||||
lambda conn: conn.execute("insert into t values (1)"), block=False
|
||||
)
|
||||
try:
|
||||
assert await asyncio.to_thread(opened.wait, 5)
|
||||
await asyncio.wait_for(asyncio.to_thread(db.close), 2)
|
||||
release.set()
|
||||
await reply
|
||||
assert rows(db) == [1]
|
||||
await wait_for(lambda: db._write_thread is None)
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.gather(reply, return_exceptions=True)
|
||||
ds.close()
|
||||
363
tests/test_write_thread_idle.py
Normal file
363
tests/test_write_thread_idle.py
Normal file
|
|
@ -0,0 +1,363 @@
|
|||
"""
|
||||
Tests for the connection_idle_timeout_ms setting: a database's write thread
|
||||
closes its connection and exits after sitting idle, and the next write
|
||||
starts a new one.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import queue
|
||||
import random
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from datasette.app import Datasette
|
||||
from datasette.database import Database, DatasetteClosedError
|
||||
|
||||
|
||||
def _make_db(tmp_path, name="idle", **settings):
|
||||
path = tmp_path / f"{name}.db"
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute("create table t (id integer primary key, writer integer, seq integer)")
|
||||
conn.close()
|
||||
ds = Datasette([str(path)], settings=settings)
|
||||
return ds, ds.get_database(name)
|
||||
|
||||
|
||||
async def _wait_for(predicate, timeout=5):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if predicate():
|
||||
return True
|
||||
await asyncio.sleep(0.005)
|
||||
return predicate()
|
||||
|
||||
|
||||
def _write_threads(name):
|
||||
thread_name = f"_execute_writes for database {name}"
|
||||
return [t for t in threading.enumerate() if t.name == thread_name]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_thread_exits_when_idle_and_restarts(tmp_path):
|
||||
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=50)
|
||||
try:
|
||||
await db.execute_write("insert into t (writer, seq) values (1, 1)")
|
||||
thread = db._write_thread
|
||||
assert thread is not None and thread.is_alive()
|
||||
connections_with_writer = len(db._all_connections)
|
||||
assert await _wait_for(lambda: db._write_thread is None)
|
||||
thread.join(timeout=5)
|
||||
assert not thread.is_alive()
|
||||
# Its write connection was closed and is no longer tracked
|
||||
assert len(db._all_connections) == connections_with_writer - 1
|
||||
assert not _write_threads("idle")
|
||||
# The next write starts a new thread
|
||||
await db.execute_write("insert into t (writer, seq) values (1, 2)")
|
||||
assert db._write_thread is not None and db._write_thread is not thread
|
||||
assert db._write_threads_started == 2
|
||||
count = (await db.execute("select count(*) from t")).single_value()
|
||||
assert count == 2
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_thread_idle_timeout_zero_keeps_thread(tmp_path):
|
||||
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=0)
|
||||
try:
|
||||
await db.execute_write("insert into t (writer, seq) values (1, 1)")
|
||||
thread = db._write_thread
|
||||
await asyncio.sleep(0.2)
|
||||
assert db._write_thread is thread
|
||||
assert thread.is_alive()
|
||||
finally:
|
||||
ds.close()
|
||||
assert not thread.is_alive()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("named", (True, False))
|
||||
async def test_memory_database_write_thread_never_idles(named):
|
||||
# Closing the write connection of a memory database would lose its data
|
||||
ds = Datasette(settings={"connection_idle_timeout_ms": 10})
|
||||
if named:
|
||||
db = ds.add_memory_database("idle_mem")
|
||||
else:
|
||||
db = ds.add_database(Database(ds, is_memory=True), name="idle_unnamed")
|
||||
try:
|
||||
await db.execute_write("create table t (id integer primary key)")
|
||||
await db.execute_write("insert into t (id) values (1)")
|
||||
thread = db._write_thread
|
||||
await asyncio.sleep(0.1)
|
||||
assert db._write_thread is thread and thread.is_alive()
|
||||
count = await db.execute_write_fn(
|
||||
lambda conn: conn.execute("select count(*) from t").fetchone()[0]
|
||||
)
|
||||
assert count == 1
|
||||
finally:
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_after_idle_exit(tmp_path):
|
||||
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=10)
|
||||
await db.execute_write("insert into t (writer, seq) values (1, 1)")
|
||||
thread = db._write_thread
|
||||
assert await _wait_for(lambda: db._write_thread is None)
|
||||
ds.close()
|
||||
thread.join(timeout=5)
|
||||
assert not thread.is_alive()
|
||||
with pytest.raises(DatasetteClosedError):
|
||||
await db.execute_write("insert into t (writer, seq) values (1, 2)")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_drains_queued_writes_with_idle_timeout(tmp_path):
|
||||
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=10)
|
||||
release = threading.Event()
|
||||
|
||||
def slow(conn):
|
||||
release.wait(5)
|
||||
conn.execute("insert into t (writer, seq) values (0, 0)")
|
||||
|
||||
await db.execute_write_fn(slow, block=False)
|
||||
futures = []
|
||||
for i in range(20):
|
||||
_, future = await db._send_to_write_thread(
|
||||
lambda conn, i=i: conn.execute(
|
||||
"insert into t (writer, seq) values (1, ?)", [i]
|
||||
),
|
||||
block=False,
|
||||
)
|
||||
futures.append(future)
|
||||
release.set()
|
||||
# close() is synchronous, so the queued writes complete before it returns
|
||||
db.close()
|
||||
await asyncio.wait_for(asyncio.gather(*futures), timeout=5)
|
||||
conn = sqlite3.connect(db.path)
|
||||
try:
|
||||
assert conn.execute("select count(*) from t").fetchone()[0] == 21
|
||||
finally:
|
||||
conn.close()
|
||||
ds.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_failure_retried_after_queue_drains(tmp_path):
|
||||
# With an idle timeout, a failed connect is reported to the writes that
|
||||
# were queued, then the thread exits and the next write tries again
|
||||
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=30000)
|
||||
|
||||
class ConnectError(Exception):
|
||||
pass
|
||||
|
||||
original_connect = db.connect
|
||||
|
||||
def broken_connect(write=False):
|
||||
if write:
|
||||
raise ConnectError("no")
|
||||
return original_connect(write=write)
|
||||
|
||||
db.connect = broken_connect
|
||||
try:
|
||||
with pytest.raises(ConnectError):
|
||||
await db.execute_write("insert into t (writer, seq) values (1, 1)")
|
||||
finally:
|
||||
db.connect = original_connect
|
||||
assert await _wait_for(lambda: db._write_thread is None)
|
||||
await db.execute_write("insert into t (writer, seq) values (1, 2)")
|
||||
assert db._write_threads_started == 2
|
||||
ds.close()
|
||||
|
||||
|
||||
class _RacyQueue(queue.Queue):
|
||||
"""
|
||||
The first time the write thread asks for a task while the queue is
|
||||
empty, behave as if the idle timeout fired - but first pause, so the
|
||||
test can enqueue a task in the window between the timeout and the
|
||||
thread's exit decision.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.armed = False
|
||||
self.window_open = threading.Event()
|
||||
self.proceed = threading.Event()
|
||||
|
||||
def get(self, block=True, timeout=None):
|
||||
if self.armed and self.empty():
|
||||
self.armed = False
|
||||
self.window_open.set()
|
||||
self.proceed.wait(5)
|
||||
raise queue.Empty
|
||||
return super().get(block, timeout)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_task_enqueued_between_timeout_and_exit_is_not_stranded(tmp_path):
|
||||
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=60000)
|
||||
racy = _RacyQueue()
|
||||
db._write_queue = racy
|
||||
try:
|
||||
racy.armed = True
|
||||
await db.execute_write("insert into t (writer, seq) values (1, 1)")
|
||||
# The thread has finished that write and is now inside the window
|
||||
assert await asyncio.to_thread(racy.window_open.wait, 5)
|
||||
thread = db._write_thread
|
||||
assert thread is not None
|
||||
_, future = await db._send_to_write_thread(
|
||||
lambda conn: conn.execute("insert into t (writer, seq) values (1, 2)"),
|
||||
block=False,
|
||||
)
|
||||
# The thread is still registered, so no new thread was started
|
||||
assert db._write_threads_started == 1
|
||||
racy.proceed.set()
|
||||
await asyncio.wait_for(future, timeout=5)
|
||||
assert db._write_thread is thread
|
||||
count = (await db.execute("select count(*) from t")).single_value()
|
||||
assert count == 2
|
||||
finally:
|
||||
racy.proceed.set()
|
||||
ds.close()
|
||||
|
||||
|
||||
def _recorder(log, writer, seq):
|
||||
# A cheap task (no disk write), so tasks finish in microseconds and the
|
||||
# idle timeout keeps firing between bursts
|
||||
def record(conn):
|
||||
conn.execute("select 1").fetchone()
|
||||
log.append((writer, seq, threading.get_ident()))
|
||||
|
||||
return record
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("idle_timeout_ms", (1, 5))
|
||||
async def test_idle_exit_race_stress(tmp_path, idle_timeout_ms):
|
||||
"""
|
||||
Many concurrent writers working in rounds. Each round starts once the
|
||||
previous round's tasks are done, so the write thread's idle clock starts
|
||||
at about the same moment, and the writers then wait about 0.8x-1.3x of
|
||||
the idle timeout before queueing a burst. Tasks therefore keep landing
|
||||
on both sides of, and right on top of, the thread's exit decision.
|
||||
Every task must complete, and each writer's tasks must run in the order
|
||||
they were queued.
|
||||
"""
|
||||
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=idle_timeout_ms)
|
||||
idle_timeout = idle_timeout_ms / 1000
|
||||
rng = random.Random(42)
|
||||
writers = 12
|
||||
rounds = 150 if idle_timeout_ms == 1 else 60
|
||||
seqs = [0] * writers
|
||||
log = []
|
||||
|
||||
async def writer_round(w, delay, burst):
|
||||
await asyncio.sleep(delay)
|
||||
pending = []
|
||||
for _ in range(burst):
|
||||
seq = seqs[w]
|
||||
seqs[w] += 1
|
||||
fn = _recorder(log, w, seq)
|
||||
if seq % 3 == 0:
|
||||
# Fire-and-forget write
|
||||
_, future = await db._send_to_write_thread(fn, block=False)
|
||||
pending.append(future)
|
||||
else:
|
||||
await db.execute_write_fn(fn, transaction=bool(seq % 2))
|
||||
await asyncio.gather(*pending)
|
||||
|
||||
async def all_rounds():
|
||||
for _ in range(rounds):
|
||||
# One shared delay per round close to the timeout, plus a little
|
||||
# jitter per writer, so arrivals cluster around the exit decision
|
||||
base = idle_timeout * rng.uniform(0.8, 1.2)
|
||||
await asyncio.gather(
|
||||
*(
|
||||
writer_round(
|
||||
w, base + idle_timeout * rng.uniform(0, 0.1), rng.randint(1, 3)
|
||||
)
|
||||
for w in range(writers)
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(all_rounds(), timeout=120)
|
||||
assert len(log) == sum(seqs)
|
||||
for w in range(writers):
|
||||
assert [seq for writer_id, seq, _ in log if writer_id == w] == list(
|
||||
range(seqs[w])
|
||||
)
|
||||
print(
|
||||
f"idle_timeout_ms={idle_timeout_ms} tasks={len(log)} "
|
||||
f"threads_started={db._write_threads_started} "
|
||||
f"distinct_thread_idents={len({ident for _, _, ident in log})} "
|
||||
f"exits_averted={db._write_thread_exits_averted}"
|
||||
)
|
||||
# Only meaningful if the thread really did churn
|
||||
assert db._write_threads_started > rounds // 5
|
||||
finally:
|
||||
ds.close()
|
||||
assert not _write_threads("idle")
|
||||
|
||||
|
||||
def test_idle_exit_race_stress_many_event_loops(tmp_path):
|
||||
"""
|
||||
Producers on several OS threads, each with its own event loop, race
|
||||
with the write thread's exit decision for real (no single-loop
|
||||
ordering between producers). Rounds are synchronised with a barrier,
|
||||
then each producer waits a random 0.5x-1.5x of the idle timeout.
|
||||
"""
|
||||
ds, db = _make_db(tmp_path, connection_idle_timeout_ms=1)
|
||||
loops = 6
|
||||
rounds = 200
|
||||
barrier = threading.Barrier(loops)
|
||||
errors = []
|
||||
log = []
|
||||
|
||||
def run_loop(w):
|
||||
rng = random.Random(w)
|
||||
|
||||
async def go():
|
||||
for seq in range(rounds):
|
||||
barrier.wait(timeout=30)
|
||||
# Force some completed retirements as well as racing the
|
||||
# boundary. Scheduler latency must not decide coverage.
|
||||
if seq % 4 == 0:
|
||||
deadline = time.monotonic() + 5
|
||||
while db._write_thread is not None:
|
||||
assert time.monotonic() < deadline
|
||||
await asyncio.sleep(0.001)
|
||||
barrier.wait(timeout=30)
|
||||
await asyncio.sleep(0.001 * rng.uniform(0.5, 1.5))
|
||||
await asyncio.wait_for(
|
||||
db.execute_write_fn(_recorder(log, w, seq)), timeout=10
|
||||
)
|
||||
|
||||
try:
|
||||
asyncio.run(go())
|
||||
except BaseException as e: # noqa: BLE001
|
||||
errors.append(e)
|
||||
barrier.abort()
|
||||
|
||||
threads = [threading.Thread(target=run_loop, args=(w,)) for w in range(loops)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join(timeout=120)
|
||||
try:
|
||||
assert not errors, errors
|
||||
assert len(log) == loops * rounds
|
||||
for w in range(loops):
|
||||
assert [seq for writer_id, seq, _ in log if writer_id == w] == list(
|
||||
range(rounds)
|
||||
)
|
||||
print(
|
||||
f"tasks={len(log)} threads_started={db._write_threads_started} "
|
||||
f"exits_averted={db._write_thread_exits_averted}"
|
||||
)
|
||||
assert db._write_threads_started > rounds // 5
|
||||
finally:
|
||||
ds.close()
|
||||
Loading…
Add table
Add a link
Reference in a new issue