Merge 99e12b9720 into fc2f1ca1f1

datasette/app.py

@@ -2285,6 +2285,21 @@ class Datasette:
         )
         return d
 
+    def _tasks(self):
+        return {
+            "tasks": [
+                {
+                    "name": t.name,
+                    "state": t.state,
+                    "plugin": t.plugin,
+                    "started_at": t.started_at,
+                    "exception": repr(t.exception) if t.exception else None,
+                }
+                for t in self._background_tasks.tasks()
+            ],
+            "launched": self._background_tasks.launched,
+        }
+
     def _actor(self, request):
         return {"actor": request.actor}
 
@@ -2573,6 +2588,12 @@ class Datasette:
             ),
             r"/-/threads(\.(?P<format>json))?$",
         )
+        add_route(
+            JsonDataView.as_view(
+                self, "tasks.json", self._tasks, permission="permissions-debug"
+            ),
+            r"/-/tasks(\.(?P<format>json))?$",
+        )
         add_route(
             JsonDataView.as_view(
                 self,

datasette/background_tasks.py

@@ -240,6 +240,15 @@ class BackgroundTaskSupervisor:
         """
         return list(self._tasks)
 
+    @property
+    def launched(self) -> bool:
+        """Whether :meth:`launch_all` has run yet - lets ``/-/tasks``
+        distinguish "no tasks registered" from "tasks registered but
+        nothing has armed the launch yet" without reaching for the
+        private ``_launched`` attribute.
+        """
+        return self._launched
+
 
 def _on_task_done(handle: BackgroundTask, task: asyncio.Task) -> None:
     if task.cancelled():

docs/changelog.rst

@@ -4,6 +4,46 @@
 Changelog
 =========
 
+.. _v1_0_a39:
+
+1.0a39 (unreleased)
+-------------------
+
+This alpha gives plugins a real process lifecycle. Previously, ``datasette serve`` ran ``invoke_startup()`` on a temporary event loop that was closed before the server's own loop was created, so anything a ``startup`` hook scheduled with ``asyncio.create_task()`` - or any loop-bound primitive it created - could silently die before it ever ran. Every non-CLI deployment was worse off still: startup, including populating the internal database's table catalog, didn't run at all until the *first* HTTP request arrived, so plugins compensated with ``asgi_wrapper`` bootstrap shims, ``tryfirst=True`` ordering hacks and hand-rolled "has this started yet" flags. This release fixes all three problems together: one event loop for the whole process, startup wired into ASGI lifespan, and a new supervised background-task API plus a ``shutdown`` hook, so plugins no longer need to build any of that scaffolding themselves. See :ref:`datasette_lifecycle` for the full guarantee.
+
+- ``datasette serve`` now runs startup and the server on a single ``asyncio`` event loop, instead of a temporary loop that was discarded before ``uvicorn.run()`` created the loop that actually serves requests. A ``startup`` hook can now safely call ``asyncio.create_task()``, or create loop-bound primitives such as ``asyncio.Lock``, ``asyncio.Queue`` or ``asyncio.Event``, and expect them to still be alive once the server starts handling requests.
+- Startup - the internal database's table catalog, canned queries and column type configuration, and every :ref:`plugin_hook_startup` hook - is now wired into the ASGI ``lifespan.startup`` event via ``Datasette.app()``. A spec-compliant ASGI server (uvicorn, hypercorn, and others) completes ``lifespan.startup`` before delivering any request, so startup is now guaranteed to have finished before the first request in every deployment, not only ``datasette serve``; previously the table catalog in particular only populated on the first request, even when running under ``datasette serve``. A failing ``startup`` hook now surfaces as ``lifespan.startup.failed`` with the exception message, instead of leaving the ASGI host to hang or crash ambiguously.
+- The pre-existing first-request fallback is preserved as a safety net for hosts that never send ASGI lifespan events at all - some ASGI mounts, bare ``app()`` embedding, ``datasette.client``/test clients - and is idempotent alongside the lifespan path, so it's safe for both to fire.
+- New :ref:`datasette_add_background_task` API: plugins register supervised, long-lived background work - typically from a ``startup`` hook - and core owns launching it, once every ``startup`` hook has run. Core keeps a strong reference for the life of the process (no more silently garbage-collected fire-and-forget tasks), logs crashes with a full traceback to the ``datasette.background_tasks`` logger instead of a silent "Task exception was never retrieved", and cancels every task with a five-second grace period on shutdown. There is no automatic restart of a crashed task in this release. Registration returns a :ref:`BackgroundTask <BackgroundTask>` handle (``.name``, ``.state``, ``.task``, ``.exception``, ``.cancel()``). New :ref:`await datasette.start_background_tasks() <datasette_start_background_tasks>` method lets tests and headless embedders launch registered tasks explicitly, without running a server.
+- New ``/-/tasks`` JSON debug endpoint lists every supervised background task and its state, in the style of ``/-/threads``. See :ref:`JsonDataView_tasks`. It requires the ``permissions-debug`` permission, since a crashed task's recorded exception can reveal internal details such as file paths.
+- New :ref:`plugin_hook_shutdown` plugin hook, called during graceful shutdown (Ctrl-C, ``SIGTERM``) before background tasks are cancelled and before database connections are closed, so a plugin can tell its own background work to stop gracefully while a database connection is still available to write out final state. Exceptions raised by a ``shutdown`` hook are logged, not raised, so one plugin's broken teardown code cannot block another plugin's cleanup or Datasette's own database close. It is not called on a hard kill (``SIGKILL``).
+- Plugin ``asgi_wrapper`` middleware now always runs *after* startup has completed, on every deployment path including the first-request fallback - a wrapper that short-circuits and never calls the wrapped app (an auth check returning a 401, a CORS preflight response) can no longer defer startup indefinitely. ``lifespan`` scopes are unaffected by this change and continue to flow through plugin wrappers exactly as before.
+- The ``uvicorn`` dependency floor is now ``uvicorn>=0.29``, up from ``uvicorn>=0.11``.
+- ``datasette serve --headers`` and ``--token`` are only valid alongside ``--get``; that usage error is now raised immediately after the ``Datasette`` instance is constructed and before startup runs, instead of after ``invoke_startup()`` - and therefore every plugin's ``startup`` hook - had already executed.
+
+Migrating away from ``asgi_wrapper`` bootstrap hacks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your plugin uses ``asgi_wrapper`` purely to detect "is this the first request" so that it can lazily start some background work, you can delete that code:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Before
+     - After
+   * - An ``asgi_wrapper`` that checks a module-level flag and calls ``asyncio.create_task()`` (or awaits an ``async def`` closure) the first time it sees a request scope
+     - Call ``datasette.add_background_task()`` from a :ref:`plugin_hook_startup` hook
+   * - A hand-rolled ``_ensure_started`` / ``_started`` flag guarded by a lock, to avoid starting the work twice
+     - Not needed - registration and launch are both idempotent and safe to call from multiple places
+   * - An ``asgi_wrapper`` that sniffs the ``lifespan.shutdown`` message in its receive callable to run cleanup
+     - Implement the :ref:`plugin_hook_shutdown` hook instead
+   * - A fire-and-forget ``asyncio.create_task()`` with no reference kept, plus a README caveat like "no traffic, no runs" or "ping the server to keep the scheduler alive"
+     - ``datasette.add_background_task()`` - core keeps a strong reference and launches the task once, as soon as startup finishes, whether or not any request ever arrives
+   * - ``tryfirst=True`` on a ``startup`` hook, to make sure it runs before another plugin's task-starting code
+     - Not needed - ``add_background_task()`` launch happens only after *every* ``startup`` hook across every plugin has completed, so registration order between plugins doesn't matter
+
+`datasette-cron <https://datasette.io/plugins/datasette-cron>`__ and `datasette-enrichments <https://datasette.io/plugins/datasette-enrichments>`__ are being migrated to this pattern as worked examples of the mapping above.
+
 .. _v1_0_a38:
 
 1.0a38 (2026-08-06)

docs/internals.rst

@@ -1440,7 +1440,7 @@ All three paths call the same idempotent internal methods, so it is safe for mor
     A coroutine function taking one positional argument, the ``Datasette`` instance. Core calls ``await func(datasette)``.
 
 ``name`` - string, optional
-    A name for the task, used to identify it in log messages. Defaults to ``func.__qualname__``. If the resulting name collides with an already-registered task, a ``-2``, ``-3``, ... suffix is appended.
+    A name for the task, used to identify it in the ``/-/tasks`` introspection endpoint (:ref:`JsonDataView_tasks`) and in log messages. Defaults to ``func.__qualname__``. If the resulting name collides with an already-registered task, a ``-2``, ``-3``, ... suffix is appended.
 
 Registers a piece of supervised, long-lived background work — typically called from a :ref:`plugin_hook_startup` hook, though it can be called at any point after the instance exists, including from a request handler. Returns a :ref:`BackgroundTask <BackgroundTask>` handle.
 
@@ -1470,7 +1470,7 @@ Core owns the task for the rest of the process's life:
 - **A crash is logged, not swallowed.** If ``func`` raises anything other than ``asyncio.CancelledError``, the exception (with its traceback) is logged to the ``datasette.background_tasks`` logger and recorded on the handle's ``.exception``, and the task's ``.state`` becomes ``crashed``. **There is no automatic restart in v1** — a long-running loop should catch and log its own transient errors internally if it wants to keep running after one.
 - **Cancellation is coordinated.** On shutdown, every task that is still running is cancelled and given a grace period to stop — see :ref:`datasette_lifecycle`.
 
-Raw ``asyncio.create_task()`` inside a ``startup`` hook now works correctly, because ``startup`` hooks run on the serving event loop (see the admonition in :ref:`datasette_lifecycle`) — the bug that made this unsafe is fixed. But a task created that way is unsupervised: nothing keeps a reference to it, nothing logs its exceptions, and nothing cancels it on shutdown. Prefer ``add_background_task()`` for anything long-lived.
+Raw ``asyncio.create_task()`` inside a ``startup`` hook now works correctly, because ``startup`` hooks run on the serving event loop (see the admonition in :ref:`datasette_lifecycle`) — the bug that made this unsafe is fixed. But a task created that way is unsupervised: nothing keeps a reference to it, nothing logs its exceptions, nothing cancels it on shutdown, and it will not show up in ``/-/tasks``. Prefer ``add_background_task()`` for anything long-lived.
 
 Launch matrix
 ~~~~~~~~~~~~~
@@ -1516,11 +1516,13 @@ BackgroundTask objects
     ISO 8601 UTC timestamp of when the task was launched.
 
 ``.plugin`` - string or ``None``
-    Best-effort name of the plugin that registered the task, resolved from the module ``func`` was defined in. Used in log messages; ``None`` if it cannot be determined.
+    Best-effort name of the plugin that registered the task, resolved from the module ``func`` was defined in. Used by ``/-/tasks`` and log messages; ``None`` if it cannot be determined.
 
 ``.cancel()``
     Cancel the task. If it has already launched, this cancels the underlying ``asyncio.Task`` — ``.state`` becomes ``cancelled`` once the cancellation is observed. If it has not launched yet, it is removed from the queue so it never runs.
 
+This is also the shape of each entry returned by the ``/-/tasks`` JSON introspection endpoint — see :ref:`JsonDataView_tasks`.
+
 .. _datasette_start_background_tasks:
 
 await .start_background_tasks()

docs/introspection.rst

@@ -9,7 +9,7 @@ Each of these pages can be viewed in your browser. Add ``.json`` to the URL to g
 
 JSON responses that return an object include an ``"ok": true`` key, consistent with the rest of the :ref:`JSON API <json_api>`.
 
-The introspection endpoints documented on this page are covered by the :ref:`JSON API stability promise <json_api_stability>`, with the exception of the debug endpoints ``/-/threads`` and ``/-/actions``, whose shapes may change in future releases.
+The introspection endpoints documented on this page are covered by the :ref:`JSON API stability promise <json_api_stability>`, with the exception of the debug endpoints ``/-/threads``, ``/-/tasks`` and ``/-/actions``, whose shapes may change in future releases.
 
 .. _JsonDataView_metadata:
 
@@ -278,6 +278,57 @@ Shows details of threads and ``asyncio`` tasks. This endpoint requires the ``per
         ]
     }
 
+.. _JsonDataView_tasks:
+
+/-/tasks
+--------
+
+Shows the state of every supervised background task registered with
+:ref:`datasette.add_background_task() <datasette_add_background_task>`; see also
+:ref:`BackgroundTask <BackgroundTask>` for what each field below means, and
+:ref:`datasette_lifecycle` for when tasks are launched. This endpoint requires
+the ``permissions-debug`` permission, since a crashed task's ``exception``
+field can reveal internals such as file paths or query text:
+
+.. code-block:: json
+
+    {
+        "ok": true,
+        "tasks": [
+            {
+                "name": "my_plugin.poll_for_updates",
+                "state": "running",
+                "plugin": "my-plugin",
+                "started_at": "2026-07-30T12:00:00+00:00",
+                "exception": null
+            },
+            {
+                "name": "my_plugin.broken_task",
+                "state": "crashed",
+                "plugin": "my-plugin",
+                "started_at": "2026-07-30T12:00:00+00:00",
+                "exception": "ValueError('something went wrong')"
+            }
+        ],
+        "launched": true
+    }
+
+Each entry's ``state`` is one of ``registered`` (added but not yet
+launched), ``running``, ``completed``, ``crashed`` or ``cancelled``.
+``exception`` is a one-line ``repr()`` of the exception for a ``crashed``
+task, or ``null`` otherwise - the full traceback is written to the
+``datasette.background_tasks`` logger instead, to keep this payload
+skimmable.
+
+The top-level ``launched`` flag reports whether the instance has run its
+one-time background task launch (after ``startup`` hooks finish, or via
+lifespan/first-request/:ref:`start_background_tasks() <datasette_start_background_tasks>`).
+It distinguishes "no tasks have been registered" (``tasks`` is empty either
+way) from "tasks are registered but nothing has armed the launch yet"
+(``launched`` is ``false`` and every task's ``state`` is still
+``registered``) - useful when debugging a host that never triggers
+Datasette's lifespan events.
+
 .. _JsonDataView_actor:
 
 /-/actor

docs/json_api.rst

@@ -48,6 +48,7 @@ Some JSON endpoints are **exempt** from this promise:
   debug playground.
 - Debug and support endpoints are documented so you can use them, but their
   JSON shapes are not frozen: :ref:`/-/threads <JsonDataView_threads>`,
+  :ref:`/-/tasks <JsonDataView_tasks>`,
   :ref:`/-/actions <JsonDataView_actions>`,
   the :ref:`permission debug endpoints <PermissionsDebugView>`
   (``/-/allowed``, ``/-/rules``, ``/-/check``) and the

tests/test_permissions.py

@@ -520,6 +520,7 @@ def view_instance_client():
         "/-/plugins",
         "/-/settings",
         "/-/threads",
+        "/-/tasks",
         "/-/databases",
         "/-/permissions",
         "/-/messages",

tests/test_success_envelope.py

@@ -59,6 +59,7 @@ async def test_success_object_has_ok_true(ds_client, path):
         "/-/rules.json?action=view-instance",
         "/-/check.json?action=view-instance",
         "/-/threads.json",
+        "/-/tasks.json",
     ),
 )
 async def test_permission_debug_success_has_ok_true(ds_envelope, path):

tests/test_tasks_endpoint.py

@@ -0,0 +1,112 @@
+"""
+Tests for the /-/tasks introspection endpoint.
+
+/-/tasks exposes datasette._background_tasks (see tests/test_background_tasks.py
+for the supervisor machinery itself) the same way /-/threads exposes threading
+internals: gated behind the permissions-debug permission, JSON-only.
+"""
+
+import asyncio
+import contextlib
+
+import pytest
+
+from datasette.app import Datasette
+
+
+@pytest.mark.asyncio
+async def test_tasks_requires_permissions_debug():
+    ds = Datasette(memory=True)
+    ds.root_enabled = True
+
+    denied = await ds.client.get("/-/tasks.json")
+    assert denied.status_code == 403
+
+    allowed = await ds.client.get("/-/tasks.json", actor={"id": "root"})
+    assert allowed.status_code == 200
+    data = allowed.json()
+    assert data["ok"] is True
+    assert "tasks" in data
+    assert "launched" in data
+
+
+@pytest.mark.asyncio
+async def test_running_and_crashed_task_states():
+    ds = Datasette(memory=True)
+    ds.root_enabled = True
+
+    async def long_running(datasette):
+        await asyncio.Event().wait()
+
+    async def crashing_task(datasette):
+        raise RuntimeError("kaboom")
+
+    long_handle = ds.add_background_task(long_running, name="long-runner")
+    crash_handle = ds.add_background_task(crashing_task, name="crashing_task")
+
+    await ds.start_background_tasks()
+
+    # Let the crashing_task run to completion and its done-callback (which sets
+    # handle.state = "crashed") actually fire before we read state back out.
+    await asyncio.wait_for(
+        asyncio.gather(crash_handle.task, return_exceptions=True), timeout=5
+    )
+    await asyncio.sleep(0)
+
+    try:
+        response = await ds.client.get("/-/tasks.json", actor={"id": "root"})
+        assert response.status_code == 200
+        data = response.json()
+        assert data["launched"] is True
+
+        by_name = {t["name"]: t for t in data["tasks"]}
+        assert by_name["long-runner"]["state"] == "running"
+        assert by_name["long-runner"]["exception"] is None
+        assert by_name["long-runner"]["started_at"] is not None
+
+        crashed = by_name["crashing_task"]
+        assert crashed["state"] == "crashed"
+        assert crashed["exception"] is not None
+        assert isinstance(crashed["exception"], str)
+        assert "kaboom" in crashed["exception"]
+        assert "RuntimeError" in crashed["exception"]
+    finally:
+        long_handle.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await long_handle.task
+
+
+@pytest.mark.asyncio
+async def test_cold_instance_launched_false_and_task_registered():
+    ds = Datasette(memory=True)
+    ds.root_enabled = True
+
+    async def task(datasette):
+        pass
+
+    ds.add_background_task(task, name="cold-task")
+
+    # ds.client / httpx.ASGITransport routes through the full ASGI app,
+    # including AsgiRunOnFirstRequest - the first-request fallback that
+    # itself launches background tasks (_launch_background_tasks) so hosts
+    # without lifespan support still get supervised tasks running. That
+    # means a plain HTTP request here would launch "cold-task" before we
+    # ever get a response, making the "never launched" state impossible to
+    # observe over HTTP. _suppress_background_tasks is the same switch the
+    # `datasette --get` CLI path sets to stop its one-shot request from
+    # launching long-lived work (see _launch_background_tasks's docstring
+    # in datasette/app.py) - setting it here keeps this one request from
+    # arming the launch, so we can still exercise the real permission-gated
+    # HTTP endpoint while asserting on a genuinely pre-launch snapshot.
+    ds._suppress_background_tasks = True
+
+    response = await ds.client.get("/-/tasks.json", actor={"id": "root"})
+    assert response.status_code == 200
+    data = response.json()
+    assert data["launched"] is False
+    assert len(data["tasks"]) == 1
+    task_data = data["tasks"][0]
+    assert task_data["name"] == "cold-task"
+    assert task_data["state"] == "registered"
+    assert task_data["started_at"] is None
+    assert task_data["exception"] is None