diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 751eedfd..2a8c0ae4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,16 +11,17 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] steps: - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: pip cache-dependency-path: pyproject.toml + check-latest: true - name: Build extension for --load-extension test run: |- (cd tests && gcc ext.c -fPIC -shared -o ext.so) diff --git a/datasette/app.py b/datasette/app.py index b3c96642..42be7425 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -45,7 +45,7 @@ from . import stored_queries, write_sql from .column_types import SQLiteType from .csrf import CrossOriginProtectionMiddleware from .database import Database, QueryInterrupted -from .events import AddDatabaseEvent, Event, RemoveDatabaseEvent +from .events import Event from .plugins import DEFAULT_PLUGINS, get_plugins, pm from .renderer import json_renderer from .resources import DatabaseResource, TableResource @@ -423,9 +423,6 @@ class Datasette: ): self._startup_invoked = False self._closed = False - # Strong references to in-flight fire-and-forget event dispatch - # tasks, so they cannot be garbage-collected before completing - self._pending_event_tasks = set() assert config_dir is None or isinstance( config_dir, Path ), "config_dir= should be a pathlib.Path" @@ -456,8 +453,10 @@ class Datasette: self.databases = collections.OrderedDict() self.actions = {} # .invoke_startup() will populate this self._column_types = {} # .invoke_startup() will populate this + self._setup_db_done = False try: self._refresh_schemas_lock = asyncio.Lock() + self._startup_lock = asyncio.Lock() except RuntimeError as rex: # Workaround for intermittent test failure, see: # https://github.com/simonw/datasette/issues/1802 @@ -465,6 +464,7 @@ class Datasette: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) self._refresh_schemas_lock = asyncio.Lock() + self._startup_lock = asyncio.Lock() else: raise self.crossdb = crossdb @@ -938,14 +938,6 @@ class Datasette: new_databases[name] = db # don't mutate! that causes race conditions with live import self.databases = new_databases - self._track_event_soon( - AddDatabaseEvent( - actor=None, - database=db.name, - path=str(Path(db.path).resolve()) if db.path else None, - is_memory=db.is_memory, - ) - ) return db def add_memory_database(self, memory_name, name=None, route=None): @@ -954,41 +946,10 @@ class Datasette: ) def remove_database(self, name): - db = self.get_database(name) - # Capture event details before close() - is_temp_disk databases - # delete their backing file during close() - path = str(Path(db.path).resolve()) if db.path else None - is_memory = db.is_memory - # Fire the event only after close() returns: close() drains any - # queued writes, so listeners doing a final read of the file see - # everything - db.close() + self.get_database(name).close() new_databases = self.databases.copy() new_databases.pop(name) self.databases = new_databases - self._track_event_soon( - RemoveDatabaseEvent( - actor=None, - database=name, - path=path, - is_memory=is_memory, - ) - ) - - def _track_event_soon(self, event): - # Best-effort fire-and-forget event dispatch from synchronous code. - # If startup has not run (event classes are not yet registered) or - # there is no running event loop, the event is intentionally - # dropped - lifecycle events are documented as runtime-only - if not self._startup_invoked: - return - try: - loop = asyncio.get_running_loop() - except RuntimeError: - return - task = loop.create_task(self.track_event(event)) - self._pending_event_tasks.add(task) - task.add_done_callback(self._pending_event_tasks.discard) def close(self): """Release all resources held by this Datasette instance. @@ -2845,24 +2806,52 @@ class Datasette: raise RowNotFound(db.name, table_name, pk_values) return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first()) + async def _startup_sequence(self): + """Idempotently run the full startup sequence: table counts for + immutable databases, then invoke_startup(). Safe to call more than + once and safe to call concurrently - callers block until whichever + call got there first has finished. + + This is the single entry point used by both AsgiLifespan (so + real deployments finish startup before accepting requests) and + AsgiRunOnFirstRequest (the fallback for hosts that never send + lifespan events, e.g. DatasetteClient's httpx.ASGITransport), and + `datasette serve` (cli.py) calls it too. The fast path below checks + both `_startup_invoked` and `_setup_db_done` - not just the former - + so that a bare `await ds.invoke_startup()` made by a caller ahead of + `_startup_sequence()` (which only sets `_startup_invoked`) can't + make this method skip the immutable-database table-count precompute. + """ + if self._startup_invoked and self._setup_db_done: + return + async with self._startup_lock: + if self._startup_invoked and self._setup_db_done: + return + if not self._setup_db_done: + # First time server starts up, calculate table counts for + # immutable databases + for database in self.databases.values(): + if not database.is_mutable: + await database.table_counts(limit=60 * 60 * 1000) + self._setup_db_done = True + await self.invoke_startup() + def app(self): """Returns an ASGI app function that serves the whole of Datasette""" routes = self._routes() - async def setup_db(): - # First time server starts up, calculate table counts for immutable databases - for database in self.databases.values(): - if not database.is_mutable: - await database.table_counts(limit=60 * 60 * 1000) - async def _close_on_shutdown(): self.close() asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self) if self.setting("trace_debug"): asgi = AsgiTracer(asgi) - asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown]) - asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) + asgi = AsgiLifespan( + asgi, + on_startup=[self._startup_sequence], + on_shutdown=[_close_on_shutdown], + ) + asgi = AsgiRunOnFirstRequest(asgi, on_startup=[self._startup_sequence]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) return asgi diff --git a/datasette/cli.py b/datasette/cli.py index 57db83b6..2694c1f6 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -663,16 +663,6 @@ def serve( # Private utility mechanism for writing unit tests return ds - # Run async soundness checks before startup hooks, since invoke_startup - # now populates internal tables which requires querying each database - run_sync(lambda: check_databases(ds)) - - # Run the "startup" plugin hooks - try: - run_sync(ds.invoke_startup) - except StartupError as e: - raise click.ClickException(e.args[0]) - if headers and not get: raise click.ClickException("--headers can only be used with --get") @@ -680,6 +670,14 @@ def serve( raise click.ClickException("--token can only be used with --get") if get: + # --get means we don't run Uvicorn at all + run_sync(lambda: check_databases(ds)) + + try: + run_sync(ds.invoke_startup) + except StartupError as e: + raise click.ClickException(e.args[0]) + client = TestClient(ds) request_headers = {} if token: @@ -704,34 +702,54 @@ def serve( sys.exit(exit_code) return - # Start the server - url = None - if root: - ds.root_enabled = True - url = "http://{}:{}{}?token={}".format( - host, port, ds.urls.path("-/auth-token"), ds._root_token - ) - click.echo(url) - if open_browser: - if url is None: - # Figure out most convenient URL - to table, database or homepage - path = run_sync(lambda: initial_path_for_datasette(ds)) - url = f"http://{host}:{port}{path}" - webbrowser.open(url) - uvicorn_kwargs = { - "host": host, - "port": port, - "log_level": "info", - "lifespan": "on", - "workers": 1, - } - if uds: - uvicorn_kwargs["uds"] = uds - if ssl_keyfile: - uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile - if ssl_certfile: - uvicorn_kwargs["ssl_certfile"] = ssl_certfile - uvicorn.run(ds.app(), **uvicorn_kwargs) + # check_databases, invoke_startup() and the uvicorn server all run on a + # single event loop, so that anything a plugin's "startup" hook schedules + # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is + # still alive when the server starts handling requests. + async def _serve_async(): + # Populate internal catalog tables before invoke_startup + await check_databases(ds) + + # Run the full startup sequence (immutable-database table-count + # precompute + the "startup" plugin hooks) via the same entry point + # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when + # uvicorn's lifespan.startup fires moments later. + try: + await ds._startup_sequence() + except StartupError as e: + raise click.ClickException(e.args[0]) + + # Start the server + url = None + if root: + ds.root_enabled = True + url = "http://{}:{}{}?token={}".format( + host, port, ds.urls.path("-/auth-token"), ds._root_token + ) + click.echo(url) + if open_browser: + if url is None: + # Figure out most convenient URL - to table, database or homepage + path = await initial_path_for_datasette(ds) + url = f"http://{host}:{port}{path}" + webbrowser.open(url) + uvicorn_kwargs = { + "host": host, + "port": port, + "log_level": "info", + "lifespan": "on", + "workers": 1, + } + if uds: + uvicorn_kwargs["uds"] = uds + if ssl_keyfile: + uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile + if ssl_certfile: + uvicorn_kwargs["ssl_certfile"] = ssl_certfile + server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs)) + await server.serve() + + asyncio.run(_serve_async()) @cli.command() diff --git a/datasette/events.py b/datasette/events.py index 3583be44..5f3fd06e 100644 --- a/datasette/events.py +++ b/datasette/events.py @@ -241,54 +241,6 @@ class DeleteRowEvent(Event): pks: list -@dataclass -class AddDatabaseEvent(Event): - """ - Event name: ``add-database`` - - A new database was attached to this Datasette instance while it was - running, using :ref:`datasette.add_database() `. - - :ivar database: The name the database was attached as. - :type database: str - :ivar path: Resolved absolute filesystem path to the database file, or ``None`` for in-memory databases. - :type path: str or None - :ivar is_memory: ``True`` if this is an in-memory database. - :type is_memory: bool - """ - - name = "add-database" - database: str - path: str | None - is_memory: bool - - -@dataclass -class RemoveDatabaseEvent(Event): - """ - Event name: ``remove-database`` - - A database was detached from this Datasette instance using - :ref:`datasette.remove_database() `. The - database file itself is not deleted by Datasette core, and any queued - writes have been flushed by the time this event is delivered - with one - exception: temporary on-disk databases remove their backing file when - they are closed, so for those the path in this event no longer exists. - - :ivar database: The name the database was attached as. - :type database: str - :ivar path: Resolved absolute filesystem path to the database file, or ``None`` for in-memory databases. - :type path: str or None - :ivar is_memory: ``True`` if this was an in-memory database. - :type is_memory: bool - """ - - name = "remove-database" - database: str - path: str | None - is_memory: bool - - @hookimpl def write_wrapper(datasette, database, request, transaction): def wrapper(conn, track_event): @@ -339,6 +291,4 @@ def register_events(): UpsertRowsEvent, UpdateRowEvent, DeleteRowEvent, - AddDatabaseEvent, - RemoveDatabaseEvent, ] diff --git a/datasette/filters.py b/datasette/filters.py index 1d4e32c2..3cfb36e5 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -209,10 +209,14 @@ class TemplatedFilter(Filter): if self.numeric and converted.isdigit(): converted = int(converted) if self.no_argument: - kwargs = {"c": column} + kwargs = {"c": _quote_sqlite_identifier(column)} converted = None else: - kwargs = {"c": column, "p": f"p{param_counter}", "t": table} + kwargs = { + "c": _quote_sqlite_identifier(column), + "p": f"p{param_counter}", + "t": _quote_sqlite_identifier(table), + } return self.sql_template.format(**kwargs), converted def human_clause(self, column, value): @@ -226,6 +230,14 @@ class TemplatedFilter(Filter): return template.format(c=column, v=value) +def _quote_sqlite_identifier(identifier): + # Preserve the historic always-quoted SQL generated by TemplatedFilter. + escaped = escape_sqlite(identifier) + if escaped == identifier: + return f'"{identifier}"' + return escaped + + class InFilter(Filter): key = "in" display = "in" @@ -267,56 +279,56 @@ class Filters: TemplatedFilter( "exact", "=", - '"{c}" = :{p}', + "{c} = :{p}", lambda c, v: "{c} = {v}" if v.isdigit() else '{c} = "{v}"', ), TemplatedFilter( "not", "!=", - '"{c}" != :{p}', + "{c} != :{p}", lambda c, v: "{c} != {v}" if v.isdigit() else '{c} != "{v}"', ), TemplatedFilter( "contains", "contains", - '"{c}" like :{p}', + "{c} like :{p}", '{c} contains "{v}"', format="%{}%", ), TemplatedFilter( "notcontains", "does not contain", - '"{c}" not like :{p}', + "{c} not like :{p}", '{c} does not contain "{v}"', format="%{}%", ), TemplatedFilter( "endswith", "ends with", - '"{c}" like :{p}', + "{c} like :{p}", '{c} ends with "{v}"', format="%{}", ), TemplatedFilter( "startswith", "starts with", - '"{c}" like :{p}', + "{c} like :{p}", '{c} starts with "{v}"', format="{}%", ), - TemplatedFilter("gt", ">", '"{c}" > :{p}', "{c} > {v}", numeric=True), + TemplatedFilter("gt", ">", "{c} > :{p}", "{c} > {v}", numeric=True), TemplatedFilter( - "gte", "\u2265", '"{c}" >= :{p}', "{c} \u2265 {v}", numeric=True + "gte", "\u2265", "{c} >= :{p}", "{c} \u2265 {v}", numeric=True ), - TemplatedFilter("lt", "<", '"{c}" < :{p}', "{c} < {v}", numeric=True), + TemplatedFilter("lt", "<", "{c} < :{p}", "{c} < {v}", numeric=True), TemplatedFilter( - "lte", "\u2264", '"{c}" <= :{p}', "{c} \u2264 {v}", numeric=True + "lte", "\u2264", "{c} <= :{p}", "{c} \u2264 {v}", numeric=True ), - TemplatedFilter("like", "like", '"{c}" like :{p}', '{c} like "{v}"'), + TemplatedFilter("like", "like", "{c} like :{p}", '{c} like "{v}"'), TemplatedFilter( - "notlike", "not like", '"{c}" not like :{p}', '{c} not like "{v}"' + "notlike", "not like", "{c} not like :{p}", '{c} not like "{v}"' ), - TemplatedFilter("glob", "glob", '"{c}" glob :{p}', '{c} glob "{v}"'), + TemplatedFilter("glob", "glob", "{c} glob :{p}", '{c} glob "{v}"'), InFilter(), NotInFilter(), ] @@ -325,13 +337,13 @@ class Filters: TemplatedFilter( "arraycontains", "array contains", - """:{p} in (select value from json_each([{t}].[{c}]))""", + """:{p} in (select value from json_each({t}.{c}))""", '{c} contains "{v}"', ), TemplatedFilter( "arraynotcontains", "array does not contain", - """:{p} not in (select value from json_each([{t}].[{c}]))""", + """:{p} not in (select value from json_each({t}.{c}))""", '{c} does not contain "{v}"', ), ] @@ -339,30 +351,28 @@ class Filters: else [] ) + [ + TemplatedFilter("date", "date", "date({c}) = :{p}", '"{c}" is on date {v}'), TemplatedFilter( - "date", "date", 'date("{c}") = :{p}', '"{c}" is on date {v}' - ), - TemplatedFilter( - "isnull", "is null", '"{c}" is null', "{c} is null", no_argument=True + "isnull", "is null", "{c} is null", "{c} is null", no_argument=True ), TemplatedFilter( "notnull", "is not null", - '"{c}" is not null', + "{c} is not null", "{c} is not null", no_argument=True, ), TemplatedFilter( "isblank", "is blank", - '("{c}" is null or "{c}" = "")', + "({c} is null or {c} = '')", "{c} is blank", no_argument=True, ), TemplatedFilter( "notblank", "is not blank", - '("{c}" is not null and "{c}" != "")', + "({c} is not null and {c} != '')", "{c} is not blank", no_argument=True, ), diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 812194fd..2614ad02 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -1,3 +1,4 @@ +import asyncio import json import re from http.cookies import Morsel, SimpleCookie @@ -300,12 +301,24 @@ class AsgiLifespan: while True: message = await receive() if message["type"] == "lifespan.startup": - for fn in self.on_startup: - await fn() + try: + for fn in self.on_startup: + await fn() + except Exception as e: # noqa: BLE001 + await send( + {"type": "lifespan.startup.failed", "message": str(e)} + ) + return await send({"type": "lifespan.startup.complete"}) elif message["type"] == "lifespan.shutdown": - for fn in self.on_shutdown: - await fn() + try: + for fn in self.on_shutdown: + await fn() + except Exception as e: # noqa: BLE001 + await send( + {"type": "lifespan.shutdown.failed", "message": str(e)} + ) + return await send({"type": "lifespan.shutdown.complete"}) return else: @@ -624,10 +637,23 @@ class AsgiRunOnFirstRequest: self.asgi = asgi self.on_startup = on_startup self._started = False + # Guards against concurrent early requests interleaving with startup: + # without this, several requests could all observe `_started is + # False` and proceed before any of them finish running the hooks. + self._lock = asyncio.Lock() async def __call__(self, scope, receive, send): - if not self._started: - self._started = True - for hook in self.on_startup: - await hook() + # Leave "lifespan" scope events alone - this shim only exists as a + # fallback for hosts that never send them. It wraps AsgiLifespan, so + # if it ran on_startup here too, a startup exception would escape + # before AsgiLifespan's own try/except got a chance to turn it into + # a lifespan.startup.failed message. + if scope["type"] != "lifespan" and not self._started: + async with self._lock: + # Re-check: another request may have finished startup while + # we were waiting for the lock. + if not self._started: + for hook in self.on_startup: + await hook() + self._started = True return await self.asgi(scope, receive, send) diff --git a/datasette/version.py b/datasette/version.py index 8e238ab5..2ec12fd2 100644 --- a/datasette/version.py +++ b/datasette/version.py @@ -1,2 +1,2 @@ -__version__ = "1.0a37" +__version__ = "1.0a38" __version_info__ = tuple(__version__.split(".")) diff --git a/datasette/views/table.py b/datasette/views/table.py index 3eb80854..7c814b27 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1391,7 +1391,9 @@ class TableDropView(BaseView): # Drop table def drop_table(conn): - sqlite_utils.Database(conn)[table_name].drop() + table = sqlite_utils.Database(conn)[table_name] + table.disable_fts() + table.drop() await db.execute_write_fn(drop_table, request=request) await self.ds.track_event( diff --git a/docs/changelog.rst b/docs/changelog.rst index 670166bb..66a7caab 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,17 @@ Changelog ========= +.. _v1_0_a38: + +1.0a38 (2026-08-06) +------------------- + +This release fixes a **SQL injection** security issue that affects Datasette instances that serve a **mixture of public and private tables** in the same database, with access configured using the :ref:`Datasette permissions system `. + +Site administrators who serve private tables in this way are advised to disable the :ref:`execute-sql permission ` on that database to prevent users from accessing private tables using raw SQL queries. The bug that has been fixed would have allowed users with access to any public table to execute SQL injection attacks despite that restriction, giving them read-only access to data in private tables in the same database. + +This fix is also available in Datasette 0.65.3. + .. _v1_0_a37: 1.0a37 (2026-07-14) diff --git a/docs/events.md b/docs/events.md index c3c71057..f63d1893 100644 --- a/docs/events.md +++ b/docs/events.md @@ -9,15 +9,6 @@ Note that these events will *not* fire for changes made to a SQLite database by Plugins can listen for events using the {ref}`plugin_hook_track_event` plugin hook, which will be called with instances of the following classes - or additional classes {ref}`registered by other plugins `. -## Delivery guarantees for database lifecycle events - -The ``add-database`` and ``remove-database`` events have some specific delivery characteristics: - -- Delivery is asynchronous. Listeners run shortly after the change, not before the triggering ``add_database()`` or ``remove_database()`` call returns. -- These events fire only for changes made at runtime - while an event loop is running, after Datasette's startup has completed. Databases attached while the instance is starting up do not produce events: plugins that need to see those should iterate over ``datasette.databases`` in their own {ref}`plugin_hook_startup` hook. -- Rapid successive changes involving the same database name may reach listeners interleaved. Listeners should tolerate events arriving out of order. -- ``event.actor`` is ``None`` for programmatic calls made by Datasette itself or by plugins. - ```{eval-rst} .. automodule:: datasette.events :members: diff --git a/docs/internals.rst b/docs/internals.rst index 34e29e2c..d2bd46ef 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1357,8 +1357,6 @@ Use ``is_mutable=False`` to add an immutable database. "CREATE TABLE foo(id integer primary key)" ) -Calling this method while the instance is running - after ``invoke_startup()`` has completed, with an event loop running - emits an ``add-database`` :ref:`event `. Databases attached during startup do not emit events: plugins that need to see those should iterate over ``datasette.databases`` in their own :ref:`plugin_hook_startup` hook. - .. _datasette_add_memory_database: .add_memory_database(memory_name, name=None, route=None) @@ -1394,8 +1392,6 @@ The ``name`` and ``route`` parameters are optional and work the same way as they This removes a database that has been previously added. ``name=`` is the unique name of that database. -The database is closed but its file is not deleted. When called while the instance is running this emits a ``remove-database`` :ref:`event ` after the database has been closed - since closing flushes any queued writes first, an event listener can safely perform a final read of the database file. The exception is temporary on-disk databases, which remove their backing file when closed. - .. _datasette_close: .close() diff --git a/pyproject.toml b/pyproject.toml index cf5db905..e658955f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "hupper>=1.9", "httpx>=0.20,<1.0", "pluggy>=1.0", - "uvicorn>=0.11", + "uvicorn>=0.29", "aiofiles>=0.4", "PyYAML>=5.3", "mergedeep>=1.1.1", diff --git a/tests/conftest.py b/tests/conftest.py index a2e6aba2..12dce417 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ import importlib.metadata import os import pathlib import re +import socket import subprocess import sys import tempfile @@ -32,17 +33,31 @@ UNDOCUMENTED_PERMISSIONS = { } -def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): +def wait_until_responds(url, timeout=5.0, client=httpx, process=None, **kwargs): start = time.time() while time.time() - start < timeout: + # If the server died there is no point waiting out the timeout - fail + # now, with its output, instead of after `timeout` seconds of silence + if process is not None and process.poll() is not None: + raise AssertionError( + "Server exited early with returncode {}\n{}".format( + process.returncode, process.stdout.read().decode("utf-8") + ) + ) try: client.get(url, **kwargs) return - except httpx.ConnectError: + except httpx.TransportError: time.sleep(0.1) raise AssertionError(f"Timed out waiting for {url} to respond") +def find_free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + @pytest.fixture def bare_ds(): """ @@ -301,6 +316,71 @@ def ds_unix_domain_socket_server(tmp_path_factory): pass +@pytest.fixture +def serve_with_plugins(tmp_path): + """Factory fixture for starting ``datasette serve`` in a subprocess with + plugins written to a temporary ``--plugins-dir``. + + For tests that need the real serve path: event-loop wiring, exit codes, + signals. The usual in-process ``pm.register`` plugin pattern can't reach + a subprocess, so plugin source is written out as importable files instead. + + Unlike ``ds_localhost_http_server`` this is function-scoped and takes a + fresh port each time, because each test needs its own plugins. Call it as:: + + proc, port = serve_with_plugins({"my_plugin": PLUGIN_SOURCE}) + + ``plugins`` maps module name to Python source. Pass + ``wait_for_startup=False`` when the server is expected to fail during + startup rather than begin serving. Extra CLI arguments are passed through. + Every process started is terminated when the test ends. + """ + processes = [] + + def start(plugins, *extra_args, wait_for_startup=True): + plugins_dir = tmp_path / "plugins" + plugins_dir.mkdir(exist_ok=True) + for module_name, source in plugins.items(): + (plugins_dir / f"{module_name}.py").write_text(source, "utf-8") + port = find_free_port() + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "datasette", + "--memory", + "--plugins-dir", + str(plugins_dir), + "-h", + "127.0.0.1", + "-p", + str(port), + *extra_args, + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + # Avoid FileNotFoundError: [Errno 2] No such file or directory: + cwd=tempfile.gettempdir(), + ) + processes.append(proc) + if wait_for_startup: + wait_until_responds( + f"http://127.0.0.1:{port}/-/versions.json", process=proc + ) + return proc, port + + yield start + + for proc in processes: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + # Import fixtures from fixtures.py to make them available from .fixtures import ( # noqa: F401 TEMP_PLUGIN_SECRET_FILE, diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 7801d0a3..11ef30de 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -1,6 +1,7 @@ import time import pytest +import sqlite_utils from datasette.app import Datasette from datasette.events import RenameTableEvent @@ -1725,6 +1726,42 @@ async def test_drop_table(ds_write, scenario): assert (await ds_write.client.get("/data/docs")).status_code == 404 +@pytest.mark.asyncio +async def test_drop_table_cleans_up_fts(ds_write): + db = ds_write.get_database("data") + + def enable_fts(conn): + sqlite_utils.Database(conn)["docs"].enable_fts(["title"], create_triggers=True) + + await db.execute_write_fn(enable_fts) + assert { + row[0] + for row in await db.execute( + "select name from sqlite_master where type = 'table' and name like 'docs_fts%'" + ) + } == { + "docs_fts", + "docs_fts_config", + "docs_fts_data", + "docs_fts_docsize", + "docs_fts_idx", + } + + response = await ds_write.client.post( + "/data/docs/-/drop", + json={"confirm": True}, + headers=_headers(write_token(ds_write)), + ) + + assert response.json() == {"ok": True} + assert [ + row[0] + for row in await db.execute( + "select name from sqlite_master where type = 'table' and name like 'docs_fts%'" + ) + ] == [] + + @pytest.mark.asyncio @pytest.mark.parametrize( "input,expected_status,expected_response,expected_events", diff --git a/tests/test_cli_serve_server.py b/tests/test_cli_serve_server.py index b7604bb8..b76180fd 100644 --- a/tests/test_cli_serve_server.py +++ b/tests/test_cli_serve_server.py @@ -1,4 +1,5 @@ import socket +import time import httpx import pytest @@ -28,3 +29,119 @@ def test_serve_unix_domain_socket(ds_unix_domain_socket_server): "path": "/_memory", "tables": [], }.items() <= response.json().items() + + +# Shaped after datasette-litestream's startup hook, which schedules a +# background task with asyncio.get_running_loop().create_task(...): +# https://github.com/datasette/datasette-litestream +MARKER_TASK_PLUGIN = """ +import asyncio +from datasette import hookimpl +from datasette.utils.asgi import Response + + +@hookimpl +def startup(datasette): + datasette._startup_calls = getattr(datasette, "_startup_calls", 0) + 1 + + async def _mark(): + # Must await before setting the flag: a task with no internal + # await point could finish on the throwaway loop before it + # closed, masking the regression this test guards against. + await asyncio.sleep(0.2) + datasette._marker_task_ran = True + + asyncio.get_running_loop().create_task(_mark()) + + +@hookimpl +def register_routes(): + async def marker_status(datasette): + return Response.json( + { + "marker_task_ran": getattr(datasette, "_marker_task_ran", False), + "startup_calls": getattr(datasette, "_startup_calls", 0), + } + ) + + return [(r"^/-/marker-task-ran$", marker_status)] +""" + + +STARTUP_ERROR_PLUGIN = """ +from datasette import hookimpl +from datasette.utils import StartupError + + +@hookimpl +def startup(datasette): + raise StartupError("boom from plugin") +""" + + +@pytest.mark.serial +def test_startup_hook_background_task_runs_on_serving_loop(serve_with_plugins): + """ + Litestream-shaped regression test: a startup hook that does + asyncio.get_running_loop().create_task(...) must have that task + actually execute before/while the server is handling requests. This + only holds if invoke_startup() and uvicorn.Server.serve() share one + event loop. This test fails against unmodified main, where + invoke_startup() runs on a throwaway loop that is closed before + uvicorn opens its own loop to serve. + """ + _, port = serve_with_plugins({"marker_task_plugin": MARKER_TASK_PLUGIN}) + # The fixture has already waited for the server to answer requests. The + # marker task deliberately awaits before setting its flag, so poll for a + # moment rather than assuming it landed before the first request arrived. + deadline = time.time() + 3.0 + payload = {} + while time.time() < deadline: + payload = httpx.get( + f"http://127.0.0.1:{port}/-/marker-task-ran", timeout=1.0 + ).json() + if payload["marker_task_ran"]: + break + time.sleep(0.05) + assert payload.get("marker_task_ran"), ( + "The startup hook's asyncio.create_task(...) never ran - " + "invoke_startup() and the server are not sharing an event loop" + ) + # Polling above means this test would also pass if the startup hook were + # re-run on the serving loop by the first-request fallback - which would + # hide exactly the bug being tested. invoke_startup() is idempotent today + # so that cannot happen; assert it explicitly so that if the idempotency + # guard is ever removed this test fails loudly instead of silently + # becoming a no-op. + assert payload["startup_calls"] == 1, ( + "startup hook ran {} times - the marker may have been set by a " + "re-run on the serving loop rather than by the original task".format( + payload["startup_calls"] + ) + ) + + +@pytest.mark.serial +def test_startup_error_fails_fast_before_port_binds(serve_with_plugins): + """ + A "startup" plugin hook that raises StartupError must fail fast: print + the message, exit non-zero, and never accept a connection on the port - + the failure must happen before uvicorn.Server binds the socket. + """ + proc, port = serve_with_plugins( + {"startup_error_plugin": STARTUP_ERROR_PLUGIN}, wait_for_startup=False + ) + stdout, _ = proc.communicate(timeout=15) + output = stdout.decode("utf-8") + assert proc.returncode not in (0, None), output + assert "boom from plugin" in output, output + + # Nothing is listening on the port now the process has exited. This + # confirms the socket was not left bound; on its own it cannot prove the + # failure preceded the bind, since a port nothing ever touched also + # refuses connections. + with ( + pytest.raises(OSError), + socket.create_connection(("127.0.0.1", port), timeout=0.2), + ): + pass diff --git a/tests/test_filters.py b/tests/test_filters.py index 8d0f3512..9f201fdf 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -66,12 +66,12 @@ from datasette.utils.asgi import Request # JSON arraycontains, arraynotcontains ( (("Availability+Info__arraycontains", "yes"),), - [":p0 in (select value from json_each([table].[Availability+Info]))"], + [':p0 in (select value from json_each("table"."Availability+Info"))'], ["yes"], ), ( (("Availability+Info__arraynotcontains", "yes"),), - [":p0 not in (select value from json_each([table].[Availability+Info]))"], + [':p0 not in (select value from json_each("table"."Availability+Info"))'], ["yes"], ), ], @@ -83,6 +83,35 @@ def test_build_where(args, expected_where, expected_params): assert {f"p{i}": param for i, param in enumerate(expected_params)} == actual_params +@pytest.mark.parametrize( + "key,expected_where", + ( + ( + 'has"quote__exact', + '"has""quote" = :p0', + ), + ( + 'has"quote__isnull', + '"has""quote" is null', + ), + ( + "has]bracket__arraycontains", + ':p0 in (select value from json_each("table"."has]bracket"))', + ), + ), +) +def test_build_where_escapes_column_names(key, expected_where): + filters = Filters(((key, "value"),)) + sql_bits, _ = filters.build_where_clauses("table") + assert sql_bits == [expected_where] + + +def test_build_where_escapes_table_name(): + filters = Filters((("tags__arraycontains", "value"),)) + sql_bits, _ = filters.build_where_clauses("items]bracket") + assert sql_bits == [':p0 in (select value from json_each("items]bracket"."tags"))'] + + @pytest.mark.asyncio async def test_through_filters_from_request(ds_client): request = Request.fake( diff --git a/tests/test_internals_datasette.py b/tests/test_internals_datasette.py index e3d9b823..ed2aeaf0 100644 --- a/tests/test_internals_datasette.py +++ b/tests/test_internals_datasette.py @@ -476,98 +476,3 @@ async def test_datasette_render_template_dataclass_values_not_deep_copied(): await ds.invoke_startup() rendered = await ds.render_template("error.html", context) assert "shallow-copied-value" in rendered - - -def _lifecycle_events(ds): - return [ - event - for event in getattr(ds, "_tracked_events", []) - if event.name in ("add-database", "remove-database") - ] - - -async def _drain_event_tasks(ds): - await asyncio.gather(*ds._pending_event_tasks) - - -@pytest.mark.asyncio -async def test_add_database_fires_event(tmp_path): - ds = Datasette(memory=True) - await ds.invoke_startup() - path = str(tmp_path / "data.db") - sqlite3.connect(path).execute("vacuum") - db = ds.add_database(Database(ds, path=path, is_mutable=True)) - await _drain_event_tasks(ds) - events = _lifecycle_events(ds) - assert len(events) == 1 - event = events[0] - assert event.name == "add-database" - assert event.database == db.name == "data" - assert event.path == os.path.abspath(path) - assert event.is_memory is False - assert event.actor is None - - -@pytest.mark.asyncio -async def test_add_memory_database_fires_event(): - ds = Datasette(memory=True) - await ds.invoke_startup() - ds.add_memory_database("test_add_memory_database_event") - await _drain_event_tasks(ds) - events = _lifecycle_events(ds) - assert len(events) == 1 - event = events[0] - assert event.name == "add-database" - assert event.database == "test_add_memory_database_event" - assert event.path is None - assert event.is_memory is True - - -@pytest.mark.asyncio -async def test_remove_database_fires_event(tmp_path): - ds = Datasette(memory=True) - await ds.invoke_startup() - path = str(tmp_path / "data.db") - sqlite3.connect(path).execute("vacuum") - db = ds.add_database(Database(ds, path=path, is_mutable=True)) - ds.remove_database(db.name) - await _drain_event_tasks(ds) - events = _lifecycle_events(ds) - assert [event.name for event in events] == ["add-database", "remove-database"] - event = events[1] - assert event.database == "data" - assert event.path == os.path.abspath(path) - assert event.is_memory is False - assert event.actor is None - # remove_database never deletes the file - assert os.path.exists(path) - - -@pytest.mark.asyncio -async def test_add_database_no_event_before_startup(): - ds = Datasette(memory=True) - # invoke_startup() has not run - no event, and no AssertionError from - # track_event()'s event_classes check - ds.add_database(Database(ds, memory_name="pre_startup_db")) - assert ds._pending_event_tasks == set() - assert _lifecycle_events(ds) == [] - - -def test_add_database_no_event_without_running_loop(): - ds = Datasette(memory=True) - asyncio.run(ds.invoke_startup()) - # Startup has run but there is no running event loop now - ds.add_database(Database(ds, memory_name="no_loop_db")) - assert ds._pending_event_tasks == set() - assert _lifecycle_events(ds) == [] - - -@pytest.mark.asyncio -async def test_add_database_event_uses_renamed_name(): - ds = Datasette(memory=True) - await ds.invoke_startup() - ds.add_memory_database("first_mem", name="clash") - ds.add_memory_database("second_mem", name="clash") - await _drain_event_tasks(ds) - events = _lifecycle_events(ds) - assert [event.database for event in events] == ["clash", "clash_2"] diff --git a/tests/test_lifespan.py b/tests/test_lifespan.py new file mode 100644 index 00000000..3655285e --- /dev/null +++ b/tests/test_lifespan.py @@ -0,0 +1,259 @@ +""" +Tests for wiring Datasette startup (setup_db table counts + invoke_startup) +into the ASGI lifespan protocol. + +These exercise Datasette._startup_sequence() via three different callers: +- AsgiLifespan, by hand-driving lifespan.startup messages (no HTTP request) +- AsgiRunOnFirstRequest, the fallback for hosts that never send lifespan + events (this is what DatasetteClient / plain httpx.ASGITransport uses) +- Both at once, to prove startup hooks run at most once +""" + +import asyncio +import contextlib +import sqlite3 + +import httpx +import pytest + +from datasette import hookimpl +from datasette.app import Datasette +from datasette.database import Database +from datasette.plugins import pm + + +async def _drive_lifespan_startup(app): + """Send a single lifespan.startup message into app's ASGI lifespan loop + and return the list of messages sent back - without ever sending + lifespan.shutdown. Mirrors what a real server does: after startup + completes it parks waiting for the next event. We cancel that wait + once we've observed the startup response, rather than closing the + Datasette instance down with a shutdown message. + """ + messages_sent = [] + startup_responded = asyncio.Event() + delivered = False + + async def receive(): + nonlocal delivered + if not delivered: + delivered = True + return {"type": "lifespan.startup"} + # No further messages: block until the task is cancelled below, + # same as a real server parked waiting for lifespan.shutdown. + await asyncio.Event().wait() + + async def send(message): + messages_sent.append(message) + startup_responded.set() + + task = asyncio.create_task(app({"type": "lifespan"}, receive, send)) + try: + await asyncio.wait_for(startup_responded.wait(), timeout=5) + finally: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + return messages_sent + + +@pytest.mark.asyncio +async def test_lifespan_startup_runs_before_any_request(): + ds = Datasette(memory=True) + assert ds._startup_invoked is False + app = ds.app() + + messages = await _drive_lifespan_startup(app) + + assert {"type": "lifespan.startup.complete"} in messages + assert ds._startup_invoked is True + # Internal catalog tables should be populated too, entirely without an + # HTTP request having been made. + internal_db = ds.get_internal_database() + databases = await internal_db.execute("select * from catalog_databases") + assert len(databases.rows) >= 1 + + +@pytest.mark.asyncio +async def test_lifespan_startup_failure_reports_lifespan_startup_failed(): + class RaisingStartupPlugin: + __name__ = "RaisingStartupPlugin" + + @hookimpl + def startup(self, datasette): + async def inner(): + raise RuntimeError("boom from startup hook") + + return inner + + ds = Datasette(memory=True) + pm.register(RaisingStartupPlugin(), name="raising_startup_plugin") + try: + app = ds.app() + messages = await _drive_lifespan_startup(app) + finally: + pm.unregister(name="raising_startup_plugin") + + assert messages == [ + {"type": "lifespan.startup.failed", "message": "boom from startup hook"} + ] + # The exception happened before invoke_startup() got to the end of its + # body, so startup is not considered to have completed. + assert ds._startup_invoked is False + + +@pytest.mark.asyncio +async def test_startup_runs_exactly_once_across_lifespan_and_first_request(): + call_count = {"n": 0} + + class CountingStartupPlugin: + __name__ = "CountingStartupPlugin" + + @hookimpl + def startup(self, datasette): + async def inner(): + call_count["n"] += 1 + + return inner + + ds = Datasette(memory=True) + pm.register(CountingStartupPlugin(), name="counting_startup_plugin") + try: + # Build the ASGI app once, the way a real deployment does - and + # reuse the SAME app instance for both the lifespan drive and the + # HTTP requests below, since a fresh ds.app() call would reset the + # AsgiRunOnFirstRequest fallback's state. + app = ds.app() + + messages = await _drive_lifespan_startup(app) + assert {"type": "lifespan.startup.complete"} in messages + assert call_count["n"] == 1 + + # A first HTTP request (as if the host never sent lifespan events, + # or lifespan already ran) should not run the hook again. + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient( + transport=transport, base_url="http://localhost" + ) as client: + response1 = await client.get("/-/versions.json") + assert response1.status_code == 200 + # ... nor should a second, repeat request. + response2 = await client.get("/-/versions.json") + assert response2.status_code == 200 + finally: + pm.unregister(name="counting_startup_plugin") + + assert call_count["n"] == 1 + + +@pytest.mark.asyncio +async def test_no_lifespan_first_request_still_triggers_startup(): + # Pin today's behavior: a client that never drives ASGI lifespan events + # at all (like httpx.ASGITransport, which DatasetteClient uses) still + # gets startup armed by the AsgiRunOnFirstRequest fallback. + ds = Datasette(memory=True) + assert ds._startup_invoked is False + app = ds.app() + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient( + transport=transport, base_url="http://localhost" + ) as client: + response = await client.get("/-/versions.json") + assert response.status_code == 200 + + assert ds._startup_invoked is True + internal_db = ds.get_internal_database() + databases = await internal_db.execute("select * from catalog_databases") + assert len(databases.rows) >= 1 + + +@pytest.mark.asyncio +async def test_datasette_client_first_request_triggers_startup(): + # Same as above, but through the real DatasetteClient (ds.client) that + # plugins and tests actually use, to confirm nothing regressed there. + ds = Datasette(memory=True) + assert ds._startup_invoked is False + response = await ds.client.get("/-/versions.json") + assert response.status_code == 200 + assert ds._startup_invoked is True + + +@pytest.mark.asyncio +async def test_concurrent_first_requests_all_wait_for_slow_startup(): + call_count = {"n": 0} + + class SlowStartupPlugin: + __name__ = "SlowStartupPlugin" + + @hookimpl + def startup(self, datasette): + async def inner(): + call_count["n"] += 1 + await asyncio.sleep(0.2) + + return inner + + ds = Datasette(memory=True) + pm.register(SlowStartupPlugin(), name="slow_startup_plugin") + try: + app = ds.app() + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient( + transport=transport, base_url="http://localhost" + ) as client: + responses = await asyncio.gather( + *[client.get("/-/versions.json") for _ in range(10)] + ) + finally: + pm.unregister(name="slow_startup_plugin") + + # Every one of the 10 simultaneous first requests must have blocked + # until startup actually finished, not raced ahead of it. + assert all(response.status_code == 200 for response in responses) + assert call_count["n"] == 1 + assert ds._startup_invoked is True + + +@pytest.mark.asyncio +async def test_setup_db_still_runs_when_invoke_startup_ran_first(tmp_path, monkeypatch): + # Regression test: `datasette serve` (cli.py _serve_async) calls + # ds.invoke_startup() directly, before uvicorn ever sends a + # lifespan.startup event that drives _startup_sequence(). If + # _startup_sequence()'s fast path only checked `_startup_invoked`, it + # would see startup already done and skip the immutable-database + # table-count precompute (setup_db) entirely - a silent regression + # versus main, where AsgiRunOnFirstRequest ran setup_db unconditionally + # on request #1. + db_path = tmp_path / "immutable.db" + conn = sqlite3.connect(str(db_path)) + conn.execute("create table t (id integer primary key)") + conn.commit() + conn.close() + + ds = Datasette([], immutables=[str(db_path)]) + + call_count = {"n": 0} + original_table_counts = Database.table_counts + + async def counting_table_counts(self, *args, **kwargs): + call_count["n"] += 1 + return await original_table_counts(self, *args, **kwargs) + + monkeypatch.setattr(Database, "table_counts", counting_table_counts) + + # Simulate the CLI path: invoke_startup() runs directly and completes + # BEFORE _startup_sequence() ever gets a chance to run setup_db. + await ds.invoke_startup() + assert ds._startup_invoked is True + assert call_count["n"] == 0 + + # The lifespan/first-request path (or the CLI itself, per the fix) + # calling the shared entry point afterwards must still precompute + # table counts for immutable databases. + await ds._startup_sequence() + assert call_count["n"] == 1 + assert ds._setup_db_done is True + + # Idempotency: a second call must not recompute. + await ds._startup_sequence() + assert call_count["n"] == 1 diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 1c00bc9c..734f0fc2 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1661,9 +1661,6 @@ async def test_hook_register_events(): datasette = Datasette(memory=True) await datasette.invoke_startup() assert any(k.__name__ == "OneEvent" for k in datasette.event_classes) - # Core database lifecycle events should be registered too - registered_names = {k.name for k in datasette.event_classes} - assert {"add-database", "remove-database"} <= registered_names @pytest.mark.asyncio diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 32dd37f2..6c0c021b 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -4,7 +4,7 @@ import urllib import pytest from datasette.fixtures import generate_compound_rows, generate_sortable_rows -from datasette.utils import detect_json1 +from datasette.utils import detect_json1, tilde_encode from datasette.utils.sqlite import sqlite_version from .fixtures import make_app_client @@ -689,6 +689,34 @@ async def test_table_filter_queries_multiple_of_same_type(ds_client): ] == response.json()["rows"] +@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") +def test_table_filters_quote_identifiers(): + with make_app_client( + extra_databases={"demo.db": """ + create table "items]bracket" ( + id integer primary key, + "name""quote" text, + "tags]bracket" text + ); + insert into "items]bracket" values (1, 'Alice', '["red"]'); + """}, + ) as client: + table_path = tilde_encode("items]bracket") + exact_query = urllib.parse.urlencode( + {'name"quote__exact': "Alice", "_shape": "arrays"} + ) + exact_response = client.get(f"/demo/{table_path}.json?{exact_query}") + assert exact_response.status == 200 + assert exact_response.json["rows"] == [[1, "Alice", '["red"]']] + + array_query = urllib.parse.urlencode( + {"tags]bracket__arraycontains": "red", "_shape": "arrays"} + ) + array_response = client.get(f"/demo/{table_path}.json?{array_query}") + assert array_response.status == 200 + assert array_response.json["rows"] == [[1, "Alice", '["red"]']] + + @pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module") @pytest.mark.asyncio async def test_table_filter_json_arraycontains(ds_client):