From 8194cb5a1d90d734ae52375033648383ba345b7d Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 17:30:52 -0700 Subject: [PATCH 01/24] Add opentelemetry-api dependency and datasette/telemetry.py scaffolding Datasette core is gaining OpenTelemetry spans alongside the existing hand-rolled tracer. This commit only lays the groundwork - no span is emitted yet. Core takes a runtime dependency on opentelemetry-api and nothing more. It deliberately never creates a TracerProvider, configures an exporter, or touches sampling: that belongs to whoever runs Datasette, normally via an opentelemetry-instrument agent. Owning a provider in core was tried in an earlier design and produced a cross-request span leak, a process-global provider that tests could not tear down, and a sampling env var that silently blanked output. With no provider installed every span is a NonRecordingSpan and costs approximately nothing. datasette/telemetry.py exposes the module-level tracer plus sql_attribute(), which truncates SQL to 2048 characters. On a public instance the SQL is attacker-controlled and unbounded - someone can paste a 10MB query into ?sql= - so it must never reach a telemetry pipeline verbatim. opentelemetry-sdk goes in the dev dependency group only, because the test suite needs it to assert on spans while the package itself must not import it. tests/test_telemetry.py enforces that by importing datasette in a fresh interpreter and inspecting sys.modules, which catches a lazy import inside a function body that a grep would miss. conftest.py gains a session-scoped autouse fixture installing an SDK provider with an InMemorySpanExporter. It has to be session-scoped because set_tracer_provider() is effectively once-per-process - a second call logs a warning and is ignored. SimpleSpanProcessor rather than BatchSpanProcessor, so assertions made right after a request never race a background export thread. The otel_spans fixture that later tickets assert against is added here too. test_datasette_package_never_imports_the_sdk is moved to the front of the run. Late in a serial run the pytest process holds enough threads that the fork half of subprocess' fork+exec segfaults the interpreter on macOS/CPython 3.13. That reproduces with any subprocess call in that position on an unmodified tree, so it is a pre-existing hazard rather than something this commit introduces; the repo already moves its other subprocess-spawning tests to the front for related reasons. Co-Authored-By: Claude Opus 5 --- datasette/telemetry.py | 24 ++++++++++++++++ pyproject.toml | 2 ++ tests/conftest.py | 62 +++++++++++++++++++++++++++++++++++++++++ tests/test_telemetry.py | 25 +++++++++++++++++ 4 files changed, 113 insertions(+) create mode 100644 datasette/telemetry.py create mode 100644 tests/test_telemetry.py diff --git a/datasette/telemetry.py b/datasette/telemetry.py new file mode 100644 index 00000000..6f5ed093 --- /dev/null +++ b/datasette/telemetry.py @@ -0,0 +1,24 @@ +""" +OpenTelemetry integration for Datasette core. + +Core depends on `opentelemetry-api` only. It never creates a +`TracerProvider`, never configures an exporter, and never touches +sampling - that is the responsibility of whoever is running Datasette +(an `opentelemetry-instrument` agent, a future plugin, or a test +harness). With no provider installed every span produced here is a +`NonRecordingSpan` and costs approximately nothing. +""" + +from opentelemetry import trace as otel_trace + +tracer = otel_trace.get_tracer("datasette") + +MAX_SQL_LENGTH = 2048 + + +def sql_attribute(sql: str) -> str: + "Truncate SQL text so it is safe to attach to a span as an attribute." + sql = sql.strip() + if len(sql) <= MAX_SQL_LENGTH: + return sql + return sql[:MAX_SQL_LENGTH] + "…[truncated]" diff --git a/pyproject.toml b/pyproject.toml index e658955f..9dd231d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "setuptools", "pip", "pydantic>=2", + "opentelemetry-api>=1.37", ] [project.urls] @@ -70,6 +71,7 @@ dev = [ "cogapp>=3.3.0", "multipart-form-data-conformance==0.1a0", "ruff>=0.16.0", + "opentelemetry-sdk>=1.37", # docs "Sphinx==7.4.7", "furo==2025.9.25", diff --git a/tests/conftest.py b/tests/conftest.py index 12dce417..b01f111d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -58,6 +58,62 @@ def find_free_port(): return sock.getsockname()[1] +_otel_span_exporter = None + + +@pytest.fixture(scope="session", autouse=True) +def _otel_provider(): + """ + Install a real OTel SDK TracerProvider + InMemorySpanExporter exactly + once, before any span is ever created in this process. + + This has to be session-scoped and autouse because + `opentelemetry.trace.set_tracer_provider()` is effectively + once-per-process: a second call logs a warning and is ignored. So the + install must happen exactly once, before anything asserts on spans. + + `datasette.telemetry.tracer` is a module-level `ProxyTracer`. Once a + provider exists, the first span it starts resolves a concrete tracer + and caches it permanently. It does *not* cache the no-op tracer, so + any span started before this fixture runs is merely lost rather than + poisoning the tracer for the rest of the process. If the SDK isn't + installed, do nothing: core spans stay no-op `NonRecordingSpan`s and + the rest of the suite is unaffected. + """ + global _otel_span_exporter + try: + from opentelemetry import trace as otel_trace + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, + ) + except ImportError: + return + exporter = InMemorySpanExporter() + provider = TracerProvider() + # SimpleSpanProcessor exports synchronously on span end - no background + # batching thread, so assertions immediately after a request never race. + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel_trace.set_tracer_provider(provider) + _otel_span_exporter = exporter + + +@pytest.fixture +def otel_spans(): + """ + Function-scoped access to the finished-spans exporter: clears any spans + left over from previous tests, then yields the exporter so a test can + call `.get_finished_spans()` after making requests. Skips (rather than + fails) if the OTel SDK is not installed. + """ + pytest.importorskip("opentelemetry.sdk") + if _otel_span_exporter is None: + pytest.skip("OpenTelemetry SDK provider was not installed") + _otel_span_exporter.clear() + yield _otel_span_exporter + + @pytest.fixture def bare_ds(): """ @@ -168,6 +224,12 @@ def pytest_collection_modifyitems(config, items): move_to_front(items, "test_spatialite_error_if_attempt_to_open_spatialite") move_to_front(items, "test_package") move_to_front(items, "test_package_with_port") + # Same reason: this one shells out to a fresh interpreter. Late in a serial + # run the pytest process holds enough threads that the fork half of + # subprocess' fork+exec crashes the interpreter on macOS/CPython 3.13 + # (SIGSEGV/SIGBUS inside _execute_child). Reproduces with any subprocess + # call placed there, on an unmodified tree - running it first avoids it. + move_to_front(items, "test_datasette_package_never_imports_the_sdk") def move_to_front(items, test_name): diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py new file mode 100644 index 00000000..b71be504 --- /dev/null +++ b/tests/test_telemetry.py @@ -0,0 +1,25 @@ +import subprocess +import sys + + +def test_datasette_package_never_imports_the_sdk(): + """ + Core depends on opentelemetry-api only. The SDK is a test dependency. + + Checked by importing datasette in a fresh process and inspecting + sys.modules, rather than by grepping, so a lazy `import + opentelemetry.sdk` inside a function body cannot slip past. + + conftest.py's pytest_collection_modifyitems() moves this test to the + front of the run by name - if you rename it, rename it there too. + """ + code = ( + "import datasette.app, datasette.database, datasette.telemetry, sys; " + "print([m for m in sys.modules if m.startswith('opentelemetry.sdk')])" + ) + result = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ) + assert ( + result.stdout.strip() == "[]" + ), f"datasette imported the OpenTelemetry SDK: {result.stdout.strip()}" From b40b06f1cbd87115336b2aa3f3eda7c083b6bbf2 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 17:45:32 -0700 Subject: [PATCH 02/24] Emit a db.query span around Database.execute() Datasette's existing tracer times a "sql" block that wraps a good deal more than the query itself - queueing onto the thread pool, the pool wait, and result marshalling all disappear into one number. That is simonw/datasette#1730, "SQL tracing should much more closely track the SQL query execution", open since 2022. A db.query span here is the outer half of the answer; a later change adds the inner span drawn around the sqlite3 call itself, and the gap between the two is exactly the thread pool wait the current tracer folds away. The span carries OTel semantic-convention attributes (db.system, db.namespace, db.query.text) plus a few datasette.* ones. db.query.text goes through sql_attribute(), which caps it at 2048 characters, because on a public instance the SQL is attacker-supplied and unbounded. Only len(params) is recorded, never a parameter value. The existing `with trace(...)` wrapper stays exactly where it is and the new span nests inside it. This change removes nothing: ?_trace=1 and the trace_debug setting keep working unchanged. The two systems are independent code paths. Exception handling on the span is explicit rather than inherited from start_as_current_span's defaults, which would record the exception and set StatusCode.ERROR on anything passing through. That is wrong here because some SQL failures are the expected answer. ArrayFacet.suggest() runs json_type() against every column precisely to discover which ones raise "malformed JSON", and passes log_sql_errors=False to say so. Left to the defaults, a table with N text columns marks N queries per page as failed - burying genuine failures and tripping any alerting keyed on span status. Measured on a plain table page before this: 4 error spans out of 225, all expected. Suppressed errors now leave the status UNSET and set datasette.sql_error_suppressed instead, so they stay discoverable without reading as failures. QueryInterrupted still sets ERROR unconditionally. That is not quite right either - facet suggestion is designed to time out - but the fix needs its own reasoning and lands separately. Behaviour change worth calling out: time_limit_ms is hoisted out of sql_operation_in_thread so the span can record it on the event loop. It is therefore read at call time rather than at thread-execution time. Benign in practice, since ds.sql_time_limit_ms is set at startup, but it is a real change. Co-Authored-By: Claude Opus 5 --- datasette/database.py | 54 +++++++++++-- tests/test_telemetry.py | 169 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 217 insertions(+), 6 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index e162d34e..f78eb925 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -11,8 +11,10 @@ from collections import namedtuple from pathlib import Path import sqlite_utils +from opentelemetry.trace import Status, StatusCode from .inspect import inspect_hash +from .telemetry import sql_attribute, tracer from .tracer import trace from .utils import ( call_with_supported_arguments, @@ -529,12 +531,11 @@ class Database: """Executes sql against db_name in a thread""" self._check_not_closed() page_size = page_size or self.ds.page_size + time_limit_ms = self.ds.sql_time_limit_ms + if custom_time_limit and custom_time_limit < time_limit_ms: + time_limit_ms = custom_time_limit def sql_operation_in_thread(conn): - time_limit_ms = self.ds.sql_time_limit_ms - if custom_time_limit and custom_time_limit < time_limit_ms: - time_limit_ms = custom_time_limit - with sqlite_timelimit(conn, time_limit_ms): try: cursor = conn.cursor() @@ -565,8 +566,49 @@ class Database: else: return Results(rows, False, cursor.description) - with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_fn(sql_operation_in_thread) + # SIM117 wants these two context managers merged. They are kept nested + # deliberately: the hand-rolled tracer's wrapper is on its way out, and + # nesting makes removing it a single-line deletion. + with trace( # noqa: SIM117 + "sql", database=self.name, sql=sql.strip(), params=params + ): + # Exception handling is explicit rather than left to the context + # manager's defaults, so that callers passing log_sql_errors=False + # can be honoured - see the comment on the generic handler below. + with tracer.start_as_current_span( + "db.query", + record_exception=False, + set_status_on_exception=False, + ) as span: + span.set_attribute("db.system", "sqlite") + span.set_attribute("db.namespace", self.name) + span.set_attribute("db.query.text", sql_attribute(sql)) + span.set_attribute("datasette.time_limit_ms", time_limit_ms) + if params: + span.set_attribute("datasette.param_count", len(params)) + try: + results = await self.execute_fn(sql_operation_in_thread) + except QueryInterrupted as e: + span.set_status(Status(StatusCode.ERROR, str(e))) + span.set_attribute("datasette.interrupted", True) + span.record_exception(e) + raise + except Exception as e: + # log_sql_errors=False means the caller is probing and + # treats failure as an expected answer, not an error. + # Facet suggestion is the big one: it runs json_type() + # against every column precisely to find out which ones + # raise, so a table with N text columns would otherwise + # mark N queries per page as failed - burying real errors + # and setting off any alerting based on span status. + if log_sql_errors: + span.record_exception(e) + span.set_status(Status(StatusCode.ERROR, str(e))) + else: + span.set_attribute("datasette.sql_error_suppressed", True) + raise + span.set_attribute("datasette.truncated", results.truncated) + span.set_attribute("datasette.rows_returned", len(results.rows)) return results @property diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index b71be504..a13898ff 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,6 +1,31 @@ +import json +import sqlite3 import subprocess import sys +import pytest +from opentelemetry.trace import StatusCode + +from datasette.telemetry import MAX_SQL_LENGTH, sql_attribute + +SECRET_PARAM_VALUE = "SUPER_SECRET_PARAM_VALUE_XYZ_123" + +INVALID_SQL = "select this_is_not_valid_sql from nowhere" + + +def _db_query_spans(otel_spans): + return [span for span in otel_spans.get_finished_spans() if span.name == "db.query"] + + +def _all_attribute_values(otel_spans): + "Every attribute value across every finished span, for the 'no leaked param values' test." + values = [] + for span in otel_spans.get_finished_spans(): + values.extend((span.attributes or {}).values()) + for event in span.events: + values.extend((event.attributes or {}).values()) + return values + def test_datasette_package_never_imports_the_sdk(): """ @@ -23,3 +48,147 @@ def test_datasette_package_never_imports_the_sdk(): assert ( result.stdout.strip() == "[]" ), f"datasette imported the OpenTelemetry SDK: {result.stdout.strip()}" + + +@pytest.mark.asyncio +async def test_db_query_span_basic_attributes(ds_client, otel_spans): + response = await ds_client.get("/fixtures/-/query.json?sql=select+1") + assert response.status_code == 200 + + spans = _db_query_spans(otel_spans) + assert spans, "expected at least one db.query span" + span = spans[-1] + + assert span.attributes["db.system"] == "sqlite" + assert span.attributes["db.namespace"] == "fixtures" + assert span.attributes["db.query.text"] == "select 1" + assert span.attributes["datasette.rows_returned"] == 1 + assert span.attributes["datasette.truncated"] is False + assert isinstance(span.attributes["datasette.time_limit_ms"], int) + assert span.status.status_code == StatusCode.UNSET + + +@pytest.mark.asyncio +async def test_facetable_request_produces_db_query_spans(ds_client, otel_spans): + response = await ds_client.get("/fixtures/facetable.json") + assert response.status_code == 200 + + spans = _db_query_spans(otel_spans) + assert spans, "expected at least one db.query span" + assert all(span.attributes["db.system"] == "sqlite" for span in spans) + assert all(span.attributes["db.query.text"] for span in spans) + # Rendering the page also queries the internal database, so only some of + # these spans belong to "fixtures". + assert any(span.attributes["db.namespace"] == "fixtures" for span in spans) + + +def test_sql_attribute_truncates_at_2048(): + short_sql = "select 1" + assert sql_attribute(short_sql) == "select 1" + # Whitespace is stripped, so the same query logged twice with different + # surrounding whitespace produces one attribute value, not two. + assert sql_attribute(" select 1\n") == "select 1" + + long_sql = "select 1 -- " + ("x" * 3000) + truncated = sql_attribute(long_sql) + assert len(truncated) == MAX_SQL_LENGTH + len("…[truncated]") + assert truncated.startswith("select 1 -- ") + assert truncated.endswith("…[truncated]") + + +@pytest.mark.asyncio +async def test_db_query_text_is_truncated_in_real_span(ds_client, otel_spans): + # A long trailing SQL comment keeps the query valid and executable while + # pushing db.query.text well past the 2048 char cap. + long_sql = "select 1 -- " + ("x" * 3000) + response = await ds_client.get("/fixtures/-/query.json", params={"sql": long_sql}) + assert response.status_code == 200 + + spans = _db_query_spans(otel_spans) + assert spans + assert any(len(span.attributes["db.query.text"]) > 100 for span in spans), ( + "expected the long query to reach a span - otherwise this test would " + "pass even if truncation were never applied" + ) + for span in spans: + recorded = span.attributes["db.query.text"] + assert len(recorded) <= MAX_SQL_LENGTH + len("…[truncated]") + + +@pytest.mark.asyncio +async def test_no_span_attribute_ever_contains_a_parameter_value(ds_client, otel_spans): + response = await ds_client.get( + "/fixtures/-/query.json", + params={"sql": "select :secret", "secret": SECRET_PARAM_VALUE}, + ) + assert response.status_code == 200 + # Sanity check the value really did flow through as a bound parameter, + # not inlined into the SQL text, otherwise this test would be vacuous. + assert SECRET_PARAM_VALUE in json.dumps(response.json()) + + for value in _all_attribute_values(otel_spans): + if isinstance(value, str): + assert SECRET_PARAM_VALUE not in value + elif isinstance(value, (list, tuple)): + for item in value: + if isinstance(item, str): + assert SECRET_PARAM_VALUE not in item + + spans = _db_query_spans(otel_spans) + assert spans + span = spans[-1] + assert "select :secret" in span.attributes["db.query.text"] + assert span.attributes.get("datasette.param_count") == 1 + + +@pytest.mark.asyncio +async def test_query_interrupted_sets_error_status(ds_client, otel_spans): + response = await ds_client.get( + "/fixtures/-/query.json", + params={"sql": "select sleep(0.05)", "_timelimit": 5}, + ) + assert response.status_code == 400 + + spans = _db_query_spans(otel_spans) + assert spans + span = spans[-1] + assert span.status.status_code == StatusCode.ERROR + assert span.attributes["datasette.interrupted"] is True + assert span.events + assert all(event.name == "exception" for event in span.events) + + +@pytest.mark.asyncio +async def test_unsuppressed_sql_error_is_a_span_error(ds_client, otel_spans): + db = ds_client.ds.get_database("fixtures") + with pytest.raises(sqlite3.OperationalError): + await db.execute(INVALID_SQL) + + spans = _db_query_spans(otel_spans) + assert spans + span = spans[-1] + assert span.status.status_code == StatusCode.ERROR + assert any(event.name == "exception" for event in span.events) + assert "datasette.sql_error_suppressed" not in span.attributes + + +@pytest.mark.asyncio +async def test_suppressed_sql_error_is_not_a_span_error(ds_client, otel_spans): + """ + log_sql_errors=False means the caller is probing and expects failures. + + Facet suggestion runs `json_type(column)` against every column precisely + to discover which ones raise, so marking those spans as errors would put + two red spans per text column on every table page - burying real failures + and tripping any alerting keyed on span status. + """ + db = ds_client.ds.get_database("fixtures") + with pytest.raises(sqlite3.OperationalError): + await db.execute(INVALID_SQL, log_sql_errors=False) + + spans = _db_query_spans(otel_spans) + assert spans + span = spans[-1] + assert span.status.status_code == StatusCode.UNSET + assert span.attributes["datasette.sql_error_suppressed"] is True + assert not [event for event in span.events if event.name == "exception"] From 59bfa495cc9b31f1ba5920aa50f8916c6ef6d53a Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 17:52:55 -0700 Subject: [PATCH 03/24] Emit db.query spans from the three write entry points execute_write(), execute_write_script() and execute_write_many() were the only Database methods that ran SQL without producing an OpenTelemetry span, so any instance doing writes - which is every instance, since Datasette builds its internal catalog through these methods at startup - showed reads in a trace and nothing else. The same db.system, db.namespace and db.query.text attributes the read path already sets now appear here, with db.query.text going through sql_attribute() so attacker-supplied SQL cannot put an unbounded string on a span. execute_write_many() records the parameter-set count as datasette.param_sets, not datasette.rows_returned. executemany() consumes parameter sets and returns no rows at all, so a rows_returned name would be describing something that does not exist - and a consumer building a "rows written" dashboard on top of it would be charting the wrong number. These spans only cover the event-loop side of a write. The time actually spent waiting on the write queue and executing on the write thread is not attributed yet; that needs context propagation across the thread boundary and lands separately. Writes with block=False are worse still - execute_write_fn returns before the write happens, so the span closes early. Span links fix that later. As with the read path, the existing `with trace(...)` wrappers stay put and the new spans nest inside them, so ?_trace=1 keeps working unchanged - including execute_write_many's `count`, which the old tracer stashes through the context manager's return value. Co-Authored-By: Claude Opus 5 --- datasette/database.py | 50 +++++++++++++++++++++------ tests/test_telemetry.py | 75 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 11 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index f78eb925..cc35c038 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -259,10 +259,21 @@ class Database: cursor, return_all=return_all, returning_limit=returning_limit ) - with trace("sql", database=self.name, sql=sql.strip(), params=params): - results = await self.execute_write_fn( - _inner, block=block, request=request, transaction=transaction - ) + # SIM117 wants these two context managers merged. They are kept nested + # deliberately: the hand-rolled tracer's wrapper is on its way out, and + # nesting makes removing it a single-line deletion. + with trace( # noqa: SIM117 + "sql", database=self.name, sql=sql.strip(), params=params + ): + with tracer.start_as_current_span("db.query") as span: + span.set_attribute("db.system", "sqlite") + span.set_attribute("db.namespace", self.name) + span.set_attribute("db.query.text", sql_attribute(sql)) + if params: + span.set_attribute("datasette.param_count", len(params)) + results = await self.execute_write_fn( + _inner, block=block, request=request, transaction=transaction + ) return results async def execute_write_script(self, sql, block=True, request=None): @@ -271,10 +282,18 @@ class Database: def _inner(conn): return conn.executescript(sql) - with trace("sql", database=self.name, sql=sql.strip(), executescript=True): - results = await self.execute_write_fn( - _inner, block=block, transaction=False, request=request - ) + # Nested on purpose - see the note in execute_write(). + with trace( # noqa: SIM117 + "sql", database=self.name, sql=sql.strip(), executescript=True + ): + with tracer.start_as_current_span("db.query") as span: + span.set_attribute("db.system", "sqlite") + span.set_attribute("db.namespace", self.name) + span.set_attribute("db.query.text", sql_attribute(sql)) + span.set_attribute("datasette.executescript", True) + results = await self.execute_write_fn( + _inner, block=block, transaction=False, request=request + ) return results async def execute_write_many(self, sql, params_seq, block=True, request=None): @@ -291,12 +310,21 @@ class Database: return conn.executemany(sql, count_params(params_seq)), count + # Nested on purpose - see the note in execute_write(). with trace( "sql", database=self.name, sql=sql.strip(), executemany=True ) as kwargs: - results, count = await self.execute_write_fn( - _inner, block=block, request=request - ) + with tracer.start_as_current_span("db.query") as span: + span.set_attribute("db.system", "sqlite") + span.set_attribute("db.namespace", self.name) + span.set_attribute("db.query.text", sql_attribute(sql)) + span.set_attribute("datasette.executemany", True) + results, count = await self.execute_write_fn( + _inner, block=block, request=request + ) + # count is the number of parameter *sets* consumed by + # executemany(), not a row count - executemany returns no rows. + span.set_attribute("datasette.param_sets", count) kwargs["count"] = count return results diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index a13898ff..d8e89446 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -6,6 +6,7 @@ import sys import pytest from opentelemetry.trace import StatusCode +from datasette.app import Datasette from datasette.telemetry import MAX_SQL_LENGTH, sql_attribute SECRET_PARAM_VALUE = "SUPER_SECRET_PARAM_VALUE_XYZ_123" @@ -17,6 +18,21 @@ def _db_query_spans(otel_spans): return [span for span in otel_spans.get_finished_spans() if span.name == "db.query"] +def _spans_for_namespace(otel_spans, namespace): + """ + db.query spans belonging to one database. + + Datasette queries its internal catalog constantly - including while a + Datasette instance is being constructed - so a test that just grabbed + every db.query span would be reading someone else's traffic. + """ + return [ + span + for span in _db_query_spans(otel_spans) + if span.attributes["db.namespace"] == namespace + ] + + def _all_attribute_values(otel_spans): "Every attribute value across every finished span, for the 'no leaked param values' test." values = [] @@ -192,3 +208,62 @@ async def test_suppressed_sql_error_is_not_a_span_error(ds_client, otel_spans): assert span.status.status_code == StatusCode.UNSET assert span.attributes["datasette.sql_error_suppressed"] is True assert not [event for event in span.events if event.name == "exception"] + + +@pytest.mark.asyncio +async def test_execute_write_produces_db_query_span(otel_spans): + # Named in-memory databases are shared-cache, so every test in this file + # needs its own name or the second `create table` hits an existing table. + db = Datasette(memory=True).add_memory_database("t03_write_span") + await db.execute_write("create table docs (id integer primary key, name text)") + await db.execute_write("insert into docs (id, name) values (?, ?)", [1, "one"]) + + spans = _spans_for_namespace(otel_spans, "t03_write_span") + assert spans, "expected db.query spans from execute_write()" + span = spans[-1] + + assert span.attributes["db.system"] == "sqlite" + assert span.attributes["db.namespace"] == "t03_write_span" + assert span.attributes["db.query.text"] == ( + "insert into docs (id, name) values (?, ?)" + ) + assert span.attributes["datasette.param_count"] == 2 + + +@pytest.mark.asyncio +async def test_execute_write_script_sets_executescript_attribute(otel_spans): + db = Datasette(memory=True).add_memory_database("t03_write_script_span") + await db.execute_write_script( + "create table docs (id integer primary key);\n" + "insert into docs (id) values (1);" + ) + + spans = _spans_for_namespace(otel_spans, "t03_write_script_span") + assert spans, "expected a db.query span from execute_write_script()" + span = spans[-1] + + assert span.attributes["db.system"] == "sqlite" + assert span.attributes["datasette.executescript"] is True + assert "insert into docs" in span.attributes["db.query.text"] + + +@pytest.mark.asyncio +async def test_execute_write_many_records_param_sets_not_rows_returned(otel_spans): + db = Datasette(memory=True).add_memory_database("t03_write_many_span") + await db.execute_write("create table docs (id integer primary key)") + await db.execute_write_many( + "insert into docs (id) values (?)", [[i] for i in range(1, 6)] + ) + + spans = _spans_for_namespace(otel_spans, "t03_write_many_span") + many_spans = [ + span for span in spans if span.attributes.get("datasette.executemany") is True + ] + assert len(many_spans) == 1 + span = many_spans[0] + + assert span.attributes["datasette.param_sets"] == 5 + # executemany() consumes parameter sets and returns no rows at all, so + # calling this a row count would be a lie. Asserted explicitly because the + # attribute really was named datasette.rows_returned at one point. + assert "datasette.rows_returned" not in span.attributes From 582d79a148f65e4b6e1295719dbbe3886536612d Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 18:05:34 -0700 Subject: [PATCH 04/24] Propagate otel context across the thread boundaries Spans created on a worker thread resolve their parent from that thread's ambient context, so without this every span produced below Database came back as an unparented root, disconnected from the request that caused it. Carrying the caller's context across each boundary is also what makes the thread-pool wait visible: db.query covers the full round trip, the new db.query.execute covers only the work inside the worker, and the gap between them is the queueing the old tracer folds invisibly into one number. - execute_fn()'s executor.submit() and execute_isolated_fn()'s run_in_executor() (immutable databases) now run the callable inside a contextvars.copy_context(). A *fresh* copy per submit is required: concurrently entering one shared Context raises "RuntimeError: cannot enter context ... already entered". - WriteTask carries the otel Context captured on the event loop at enqueue time plus an enqueued_at_ns timestamp (both need __slots__ entries, or they fail with AttributeError at runtime). _execute_writes attaches that context right after the _SHUTDOWN check and detaches it in a finally spanning all three execution branches - the write thread is persistent and shared, so a leaked token would grow its context stack for every write processed afterwards, and a wrong-token detach only logs rather than raising. - New spans: db.query.execute (read worker thread), db.write.queue_wait (explicit start/end timestamps, so its duration is the real enqueue -> dequeue wait rather than the microseconds spent building the span) and db.write.execute (skipped in the conn_exception branch, where fn never runs). db.query.execute honours log_sql_errors for the same reason db.query does: facet suggestion probes with log_sql_errors=False and would otherwise paint two red spans per text column on every table page. - The write-thread warm-up prepare_connection is left as a documented orphan root - no caller context exists that early. Tests assert actual parent/child span-id relationships in a shared trace, not just that spans exist, since an unparented root looks identical to a correct span if you only check presence. Note that copy_context() copies every ContextVar, not just OTel's, so Datasette's own context vars (_skip_permission_checks, _permission_check_cache, _in_datasette_client) now flow into worker threads where they previously did not. Co-Authored-By: Claude Opus 5 --- datasette/database.py | 228 ++++++++++++++++++++++--------- tests/test_internals_database.py | 109 +++++++++++++++ tests/test_telemetry.py | 192 +++++++++++++++++++++++++- 3 files changed, 463 insertions(+), 66 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index cc35c038..d7cbc99b 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -1,16 +1,19 @@ import asyncio import atexit +import contextvars import inspect import os import queue import sys import tempfile import threading +import time import uuid from collections import namedtuple from pathlib import Path import sqlite_utils +from opentelemetry import context as otel_context_api from opentelemetry.trace import Status, StatusCode from .inspect import inspect_hash @@ -351,9 +354,15 @@ class Database: return _run() if not write: # Immutable database - no writes can ever occur, so there is no - # write queue to block; run against a fresh read-only connection + # write queue to block; run against a fresh read-only connection. + # A fresh copy_context() is required per submit (not one shared + # copy reused across calls): concurrent execution of the same + # Context raises "RuntimeError: cannot enter context ... already + # entered". This propagates the caller's otel context (e.g. the + # enclosing db.query span) onto the worker thread. + ctx = contextvars.copy_context() return await asyncio.get_running_loop().run_in_executor( - self.ds.executor, _run + self.ds.executor, ctx.run, _run ) # Threaded mode - send to write thread return await self._send_to_write_thread(fn, isolated_connection=True) @@ -458,8 +467,21 @@ class Database: task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") loop = asyncio.get_running_loop() reply_future = loop.create_future() + # Captured here, on the event loop, at enqueue time: the otel + # Context (carrying the enclosing db.query span, if any) and the + # timestamp used to build the db.write.queue_wait span once this + # task is dequeued on the write thread. self._write_queue.put( - WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction) + WriteTask( + fn, + task_id, + loop, + reply_future, + isolated_connection, + transaction, + otel_context_api.get_current(), + time.time_ns(), + ) ) if block: return await reply_future @@ -473,6 +495,9 @@ class Database: conn = None try: conn = self.connect(write=True) + # This warm-up runs before any write has ever been queued, so + # there is no caller otel context yet to attach - any spans + # created by plugin hooks here are orphans (roots). self.ds._prepare_connection(conn, self.name) except Exception as e: # noqa: BLE001 # Stored and re-raised to whoever queues the next write @@ -487,40 +512,79 @@ class Database: # Best-effort close as the write thread exits pass return - exception = None - result = None - if conn_exception is not None: - exception = conn_exception - elif task.isolated_connection: - try: - isolated_connection = self.connect(write=True) + # Restore the caller's otel context (captured on the event loop + # at enqueue time) so spans created while processing this task + # parent correctly to the request that queued it. Must be + # detached below in `finally` - a leaked token silently poisons + # this thread's ambient context for every write processed after + # it, and a *wrong*-token detach only logs a warning rather than + # raising, so this pairing is load-bearing and easy to get wrong + # silently. + token = otel_context_api.attach(task.otel_context) + try: + exception = None + result = None + # Explicit start_time/end_time rather than a `with` block: + # this span's duration is the time the task actually spent + # waiting in the queue (enqueue -> dequeue), not the near- + # zero time spent constructing/ending the span object here. + tracer.start_span( + "db.write.queue_wait", start_time=task.enqueued_at_ns + ).end(end_time=time.time_ns()) + if conn_exception is not None: + # fn never runs in this branch, so there is nothing to + # wrap in a db.write.execute span. + exception = conn_exception + elif task.isolated_connection: try: - result = task.fn(isolated_connection) - finally: - isolated_connection.close() - try: - self._all_file_connections.remove(isolated_connection) - except ValueError: - # Was probably a memory connection - pass - except Exception as e: # noqa: BLE001 - # Write thread must survive any task failure or the database wedges - sys.stderr.write(f"{e}\n") - sys.stderr.flush() - exception = e - else: - try: - if task.transaction: - with conn: - conn.execute("BEGIN IMMEDIATE") - result = task.fn(conn) - else: - result = task.fn(conn) - except Exception as e: # noqa: BLE001 - sys.stderr.write(f"{e}\n") - sys.stderr.flush() - exception = e - _deliver_write_result(task, result, exception) + with tracer.start_as_current_span("db.write.execute") as span: + span.set_attribute( + "datasette.isolated_connection", + task.isolated_connection, + ) + span.set_attribute( + "datasette.transaction", task.transaction + ) + isolated_connection = self.connect(write=True) + try: + result = task.fn(isolated_connection) + finally: + isolated_connection.close() + try: + self._all_file_connections.remove( + isolated_connection + ) + except ValueError: + # Was probably a memory connection + pass + except Exception as e: # noqa: BLE001 + # Write thread must survive any task failure or the database wedges + sys.stderr.write(f"{e}\n") + sys.stderr.flush() + exception = e + else: + try: + with tracer.start_as_current_span("db.write.execute") as span: + span.set_attribute( + "datasette.isolated_connection", + task.isolated_connection, + ) + span.set_attribute( + "datasette.transaction", task.transaction + ) + if task.transaction: + with conn: + conn.execute("BEGIN IMMEDIATE") + result = task.fn(conn) + else: + result = task.fn(conn) + except Exception as e: # noqa: BLE001 + sys.stderr.write(f"{e}\n") + sys.stderr.flush() + exception = e + _deliver_write_result(task, result, exception) + finally: + otel_context_api.detach(token) async def execute_fn(self, fn): self._check_not_closed() @@ -542,7 +606,13 @@ class Database: with self._pending_execute_futures_lock: self._check_not_closed() - future = self.ds.executor.submit(in_thread) + # A fresh copy_context() is required per submit (not one shared + # copy reused across calls): concurrent execution of the same + # Context raises "RuntimeError: cannot enter context ... + # already entered". This propagates the caller's otel context + # (e.g. the enclosing db.query span) onto the worker thread. + ctx = contextvars.copy_context() + future = self.ds.executor.submit(ctx.run, in_thread) self._pending_execute_futures.add(future) future.add_done_callback(self._remove_pending_execute_future) return await asyncio.wrap_future(future) @@ -564,35 +634,51 @@ class Database: time_limit_ms = custom_time_limit def sql_operation_in_thread(conn): - with sqlite_timelimit(conn, time_limit_ms): - try: - cursor = conn.cursor() - cursor.execute(sql, params if params is not None else {}) - max_returned_rows = self.ds.max_returned_rows - if max_returned_rows == page_size: - max_returned_rows += 1 - if max_returned_rows and truncate: - rows = cursor.fetchmany(max_returned_rows + 1) - truncated = len(rows) > max_returned_rows - rows = rows[:max_returned_rows] - else: - rows = cursor.fetchall() - truncated = False - except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: - if e.args == ("interrupted",): - raise QueryInterrupted(e, sql, params) - if log_sql_errors: - sys.stderr.write( - f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" - ) - sys.stderr.flush() - raise + # This span is created inside the worker thread. Its parent is + # resolved from the ambient otel context, which was propagated + # onto this thread via copy_context() at the executor.submit() + # boundary in execute_fn() (or run_in_executor() for immutable + # databases) - so it parents correctly to the enclosing + # db.query span despite running on a different thread. + # + # Callers passing log_sql_errors=False are probing and treat a + # failure as an expected answer - see the matching handling on the + # db.query span in execute(). Without this, facet suggestion marks + # two spans per text column as failed on every table page. + with tracer.start_as_current_span( + "db.query.execute", + record_exception=log_sql_errors, + set_status_on_exception=log_sql_errors, + ): + with sqlite_timelimit(conn, time_limit_ms): + try: + cursor = conn.cursor() + cursor.execute(sql, params if params is not None else {}) + max_returned_rows = self.ds.max_returned_rows + if max_returned_rows == page_size: + max_returned_rows += 1 + if max_returned_rows and truncate: + rows = cursor.fetchmany(max_returned_rows + 1) + truncated = len(rows) > max_returned_rows + rows = rows[:max_returned_rows] + else: + rows = cursor.fetchall() + truncated = False + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + if e.args == ("interrupted",): + raise QueryInterrupted(e, sql, params) + if log_sql_errors: + sys.stderr.write( + f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" + ) + sys.stderr.flush() + raise - if truncate: - return Results(rows, truncated, cursor.description) + if truncate: + return Results(rows, truncated, cursor.description) - else: - return Results(rows, False, cursor.description) + else: + return Results(rows, False, cursor.description) # SIM117 wants these two context managers merged. They are kept nested # deliberately: the hand-rolled tracer's wrapper is on its way out, and @@ -924,16 +1010,26 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: __slots__ = ( + "enqueued_at_ns", "fn", "isolated_connection", "loop", + "otel_context", "reply_future", "task_id", "transaction", ) def __init__( - self, fn, task_id, loop, reply_future, isolated_connection, transaction + self, + fn, + task_id, + loop, + reply_future, + isolated_connection, + transaction, + otel_context, + enqueued_at_ns, ): self.fn = fn self.task_id = task_id @@ -941,6 +1037,8 @@ class WriteTask: self.reply_future = reply_future self.isolated_connection = isolated_connection self.transaction = transaction + self.otel_context = otel_context + self.enqueued_at_ns = enqueued_at_ns def _deliver_write_result(task, result, exception): diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py index b1093b1c..ce0d7a53 100644 --- a/tests/test_internals_database.py +++ b/tests/test_internals_database.py @@ -3,11 +3,13 @@ Tests for the datasette.database.Database class """ import asyncio +import threading import uuid from types import SimpleNamespace import pytest import sqlite_utils +from opentelemetry import context as otel_context_api from datasette.app import Datasette from datasette.database import ( @@ -1223,3 +1225,110 @@ async def test_database_close_is_idempotent(tmpdir): # Second call should be a no-op, not raise db.close() ds._internal_database.close() + + +_CONTEXT_LEAK_MARKER_KEY = "otel-context-leak-marker" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("num_sql_threads", (0, 1)) +async def test_write_thread_context_is_detached_between_tasks( + tmp_path, monkeypatch, num_sql_threads +): + """ + The write thread attaches each task's otel Context and must detach it + again before picking up the next task. The thread is persistent and + shared, so a leaked token would grow that thread's context stack for the + rest of the process - and a *wrong*-token detach only logs a warning + rather than raising, so "does it throw" cannot catch either mistake. + + Two things are asserted, because neither alone is sufficient: + + 1. Each task observes the context value that was current on the event + loop when it was queued. This is what fails if the Context is not + carried on WriteTask, or is never attached. It does *not* catch a + missing detach: attach() replaces the current Context wholesale, so a + leftover one from a previous task is simply overwritten. + 2. The write thread's attach depth is identical at the same point in + every task. This is what fails if detach is missing - the stack grows + by one per task - and it holds across a task that raises, because the + detach lives in a `finally`. + + An otel context value is used rather than a plain contextvars.ContextVar: + a plain var set on the event loop never crosses into the write thread, so + the probe would read None every time and the test could not fail. + """ + name = f"context_leak_test_{num_sql_threads}" + db_path = tmp_path / f"{name}.db" + sqlite3.connect(db_path).close() + ds = Datasette([str(db_path)], settings={"num_sql_threads": num_sql_threads}) + db = ds.get_database(name) + await db.execute_write("create table t (id integer primary key)") + + write_thread_name = f"_execute_writes for database {name}" + depth = {"value": 0} + real_attach = otel_context_api.attach + real_detach = otel_context_api.detach + + def counting_attach(context): + token = real_attach(context) + if threading.current_thread().name == write_thread_name: + depth["value"] += 1 + return token + + def counting_detach(token): + real_detach(token) + if threading.current_thread().name == write_thread_name: + depth["value"] -= 1 + + # Patched on the opentelemetry.context module itself, which is what both + # database.py and opentelemetry.trace.use_span() look the functions up on. + monkeypatch.setattr(otel_context_api, "attach", counting_attach) + monkeypatch.setattr(otel_context_api, "detach", counting_detach) + + seen_markers = [] + seen_depths = [] + + def probe(conn): + seen_markers.append(otel_context_api.get_value(_CONTEXT_LEAK_MARKER_KEY)) + seen_depths.append(depth["value"]) + + def failing_probe(conn): + probe(conn) + # Exercises the write thread's exception path: the detach still has + # to happen, which is why it lives in a `finally`. + raise ValueError("deliberate failure inside a write task") + + try: + for i in range(5): + ctx = otel_context_api.set_value(_CONTEXT_LEAK_MARKER_KEY, f"marker-{i}") + token = real_attach(ctx) + try: + if i == 2: + with pytest.raises(ValueError): + await db.execute_write_fn(failing_probe) + else: + await db.execute_write_fn(probe) + finally: + real_detach(token) + + # Sanity check: no marker is active in *this* (event loop) context + # right now, so the final probe is a fair test of the write thread's + # own state rather than something this test forgot to clean up. + assert otel_context_api.get_value(_CONTEXT_LEAK_MARKER_KEY) is None + await db.execute_write_fn(probe) + finally: + db.close() + + assert seen_markers == [ + "marker-0", + "marker-1", + "marker-2", + "marker-3", + "marker-4", + None, + ] + assert len(set(seen_depths)) == 1, ( + f"write thread context stack grew across tasks: {seen_depths} - " + "a token was attached without being detached" + ) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index d8e89446..f232a39d 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -2,12 +2,15 @@ import json import sqlite3 import subprocess import sys +import time import pytest +import sqlite_utils from opentelemetry.trace import StatusCode from datasette.app import Datasette -from datasette.telemetry import MAX_SQL_LENGTH, sql_attribute +from datasette.database import Database +from datasette.telemetry import MAX_SQL_LENGTH, sql_attribute, tracer SECRET_PARAM_VALUE = "SUPER_SECRET_PARAM_VALUE_XYZ_123" @@ -33,6 +36,26 @@ def _spans_for_namespace(otel_spans, namespace): ] +def _children_named(otel_spans, name, parent_span_context): + """ + Finished spans called `name` whose parent really is `parent_span_context`. + + Parentage is matched on span id, not on "a span with this name exists" - + a span can exist and still be an unparented root if a thread boundary + dropped the otel context, which is the exact failure these tests exist + to catch. + """ + return [ + span + for span in otel_spans.get_finished_spans() + if span.name == name + and span.parent is not None + and span.parent.span_id == parent_span_context.span_id + and span.parent.trace_id == parent_span_context.trace_id + and span.context.trace_id == parent_span_context.trace_id + ] + + def _all_attribute_values(otel_spans): "Every attribute value across every finished span, for the 'no leaked param values' test." values = [] @@ -267,3 +290,170 @@ async def test_execute_write_many_records_param_sets_not_rows_returned(otel_span # calling this a row count would be a lie. Asserted explicitly because the # attribute really was named datasette.rows_returned at one point. assert "datasette.rows_returned" not in span.attributes + + +# --- Context propagation across thread boundaries -------------------------- +# +# Every assertion below checks parentage (child.parent.span_id == +# expected_parent.span_id, in the same trace), not merely that spans exist. +# Spans can exist and still be wrongly parented - or be unparented roots - if +# a thread boundary drops the otel context, which is exactly the failure mode +# these tests exist to prevent. + + +@pytest.mark.asyncio +async def test_db_query_execute_parents_to_db_query(ds_client, otel_spans): + # execute_fn()'s executor.submit() is thread boundary #1. The + # db.query.execute span is created inside the worker thread; without the + # copy_context() propagation it comes back as an unparented root span + # rather than a child of db.query. + response = await ds_client.get("/fixtures/-/query.json?sql=select+1") + assert response.status_code == 200 + + query_spans = [ + span + for span in _spans_for_namespace(otel_spans, "fixtures") + if span.attributes["db.query.text"] == "select 1" + ] + assert query_spans, "expected a db.query span for 'select 1'" + query_span = query_spans[-1] + + assert [ + span + for span in otel_spans.get_finished_spans() + if span.name == "db.query.execute" + ], "expected at least one db.query.execute span" + children = _children_named(otel_spans, "db.query.execute", query_span.context) + assert len(children) == 1, "expected exactly one db.query.execute child of db.query" + # The execute span is strictly contained by the round-trip span, and the + # gap between the two is the thread-pool wait. + assert query_span.start_time <= children[0].start_time + assert children[0].end_time <= query_span.end_time + + +@pytest.mark.asyncio +async def test_immutable_database_propagates_context(tmp_path, otel_spans): + # Thread boundary #3, the easy one to miss: immutable databases route + # execute_isolated_fn() through loop.run_in_executor() directly rather + # than through the write thread. A span created inside that worker must + # still parent to whatever was current when execute_isolated_fn() was + # awaited, or every immutable-database operation emits orphan roots. + db_path = tmp_path / "t04_immutable.db" + sqlite_utils.Database(str(db_path))["t"].insert({"id": 1}, pk="id") + + ds = Datasette() + db = Database(ds, path=str(db_path), is_mutable=False) + ds.add_database(db, name="t04_immutable") + + def fn(conn): + with tracer.start_as_current_span("t04-child-in-isolated-worker"): + pass + + try: + with tracer.start_as_current_span("t04-parent-on-event-loop") as parent: + parent_context = parent.get_span_context() + await db.execute_isolated_fn(fn) + finally: + ds.remove_database("t04_immutable") + + assert [ + span + for span in otel_spans.get_finished_spans() + if span.name == "t04-child-in-isolated-worker" + ], "expected a span created inside execute_isolated_fn's worker thread" + children = _children_named( + otel_spans, "t04-child-in-isolated-worker", parent_context + ) + assert len(children) == 1 + + +@pytest.mark.asyncio +async def test_write_spans_parent_to_db_query(otel_spans): + # Thread boundary #2: WriteTask -> queue.Queue -> the write thread. + # db.write.queue_wait and db.write.execute are both direct children of + # the db.query span that was current on the event loop at enqueue time, + # so they are siblings rather than nested inside one another. + db = Datasette(memory=True).add_memory_database("t04_write_spans") + await db.execute_write("create table docs (id integer primary key)") + + query_spans = _spans_for_namespace(otel_spans, "t04_write_spans") + assert query_spans, "expected a db.query span from execute_write()" + query_span = query_spans[-1] + + queue_wait_children = _children_named( + otel_spans, "db.write.queue_wait", query_span.context + ) + execute_children = _children_named( + otel_spans, "db.write.execute", query_span.context + ) + assert len(queue_wait_children) == 1 + assert len(execute_children) == 1 + + execute_span = execute_children[0] + assert execute_span.attributes["datasette.isolated_connection"] is False + assert execute_span.attributes["datasette.transaction"] is True + # Siblings, not parent/child: the queue wait is over by the time the + # write begins. + assert queue_wait_children[0].end_time <= execute_span.start_time + + +@pytest.mark.asyncio +async def test_write_queue_wait_duration_reflects_real_wait(otel_spans): + # db.write.queue_wait is built from explicit start/end timestamps - + # task.enqueued_at_ns, captured on the event loop, through to the moment + # the write thread dequeued it. If it were a plain `with` block on the + # write thread it would instead measure the microseconds spent building + # the span object, and this assertion would fail. + ds = Datasette(memory=True) + db = ds.add_memory_database("t04_queue_wait") + await db.execute_write("create table docs (id integer primary key)") + + def slow_write(conn): + time.sleep(0.1) + + # Queue a deliberately slow write without waiting for it, then queue a + # second write immediately behind it: the second task sits in the queue + # for roughly the duration of the first. + _, slow_future = await db._send_to_write_thread(slow_write, block=False) + await db.execute_write("insert into docs (id) values (1)") + await slow_future + + query_spans = [ + span + for span in _spans_for_namespace(otel_spans, "t04_queue_wait") + if span.attributes["db.query.text"] == "insert into docs (id) values (1)" + ] + assert query_spans, "expected a db.query span for the queued-behind insert" + queue_wait_children = _children_named( + otel_spans, "db.write.queue_wait", query_spans[-1].context + ) + assert len(queue_wait_children) == 1 + duration_ns = queue_wait_children[0].end_time - queue_wait_children[0].start_time + # The slow write sleeps 100ms; anything above 10ms is far beyond the + # microseconds a mis-timestamped span would report. + assert duration_ns > 10_000_000, f"queue wait was only {duration_ns}ns" + + +@pytest.mark.asyncio +async def test_suppressed_error_does_not_mark_execute_span(ds_client, otel_spans): + """ + The inner db.query.execute span must honour log_sql_errors too. + + It is created inside the worker thread, so without record_exception / + set_status_on_exception being passed through it would mark every facet + suggestion probe as failed even though the outer db.query span correctly + reports the failure as suppressed. + """ + db = ds_client.ds.get_database("fixtures") + with pytest.raises(sqlite3.OperationalError): + await db.execute(INVALID_SQL, log_sql_errors=False) + + execute_spans = [ + span + for span in otel_spans.get_finished_spans() + if span.name == "db.query.execute" + ] + assert execute_spans + span = execute_spans[-1] + assert span.status.status_code == StatusCode.UNSET + assert not [event for event in span.events if event.name == "exception"] From 4ebec0b1ea4a9e49cbf062a8882d774cd2fa9a36 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 18:16:46 -0700 Subject: [PATCH 05/24] Give startup's ~20 orphan spans somewhere to belong invoke_startup() runs before any request exists, so nothing it does has an ambient span to nest under. Measured on a fresh instance: 19 distinct traces, 19 of them single- or few-span roots - the register_* hook dispatches, the internal catalog's db.query reads and its db.write.* catalog writes. In a trace UI that is nineteen pieces of noise sitting next to every real trace, which for an operator opening Jaeger for the first time is the difference between "this works" and "this is unusable". Bracketing the whole method body in one datasette.startup span takes that to 1. This is not a propagation fix - ticket 04's context propagation was already correct, it simply had nothing to propagate. The bulk of the app.py diff is re-indentation; `git diff -w` shows the real change (plus one line-length rewrap black applied to the StartupError raise). register_output_renderer and asgi_wrapper stay orphans deliberately: both are dispatched from Datasette.__init__ / .app(), before invoke_startup() exists to be called, and wrapping them would mean holding a span open across object construction in library code that may never serve a request. Suppressing instrumentation during warm-up was rejected as an alternative: a slow prepare_connection runs on every connection, not just at startup, and is exactly what tracing should reveal. Also corrects the stale write-thread warm-up comment in database.py. It is still a root, but for a reason worth stating precisely: a raw threading.Thread does not inherit the starting thread's context, so the datasette.startup span current on the event loop does not reach it. Read connections do warm up under copy_context() and nest correctly. Co-Authored-By: Claude Opus 5 --- datasette/app.py | 109 ++++++++++++++++++++++------------------ datasette/database.py | 11 +++- tests/test_telemetry.py | 86 +++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 50 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 42be7425..d3626ab2 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -49,6 +49,7 @@ from .events import Event from .plugins import DEFAULT_PLUGINS, get_plugins, pm from .renderer import json_renderer from .resources import DatabaseResource, TableResource +from .telemetry import tracer from .tokens import TokenInvalid from .tracer import AsgiTracer from .url_builder import Urls @@ -778,57 +779,69 @@ class Datasette: # This must be called for Datasette to be in a usable state if self._startup_invoked: return - # Register event classes - event_classes = [] - for hook in pm.hook.register_events(datasette=self): - extra_classes = await await_me_maybe(hook) - if extra_classes: - event_classes.extend(extra_classes) - self.event_classes = tuple(event_classes) + # invoke_startup() runs before any request exists, so every span its + # children create - the register_* hook dispatches, the internal + # catalog's db.query/db.write spans, and the prepare_connection + # warm-up of the read connections those touch - would otherwise be + # its own orphan root trace: around twenty of them on a fresh + # instance. Bracketing the whole thing gives them somewhere to belong. + # A connection warmed lazily later, by a request touching a new + # database for the first time, nests under that request instead: + # this span has already ended by then. + with tracer.start_as_current_span("datasette.startup"): + # Register event classes + event_classes = [] + for hook in pm.hook.register_events(datasette=self): + extra_classes = await await_me_maybe(hook) + if extra_classes: + event_classes.extend(extra_classes) + self.event_classes = tuple(event_classes) - # Register actions, but watch out for duplicate name/abbr - action_names = {} - action_abbrs = {} - for hook in pm.hook.register_actions(datasette=self): - if hook: - for action in hook: - if ( - action.name in action_names - and action != action_names[action.name] - ): - raise StartupError(f"Duplicate action name: {action.name}") - if ( - action.abbr - and action.abbr in action_abbrs - and action != action_abbrs[action.abbr] - ): - raise StartupError(f"Duplicate action abbr: {action.abbr}") - action_names[action.name] = action - if action.abbr: - action_abbrs[action.abbr] = action - self.actions[action.name] = action + # Register actions, but watch out for duplicate name/abbr + action_names = {} + action_abbrs = {} + for hook in pm.hook.register_actions(datasette=self): + if hook: + for action in hook: + if ( + action.name in action_names + and action != action_names[action.name] + ): + raise StartupError(f"Duplicate action name: {action.name}") + if ( + action.abbr + and action.abbr in action_abbrs + and action != action_abbrs[action.abbr] + ): + raise StartupError(f"Duplicate action abbr: {action.abbr}") + action_names[action.name] = action + if action.abbr: + action_abbrs[action.abbr] = action + self.actions[action.name] = action - # Register column types (classes, not instances) - self._column_types = {} - for hook in pm.hook.register_column_types(datasette=self): - if hook: - for ct_cls in hook: - if ct_cls.name in self._column_types: - raise StartupError(f"Duplicate column type name: {ct_cls.name}") - self._column_types[ct_cls.name] = ct_cls + # Register column types (classes, not instances) + self._column_types = {} + for hook in pm.hook.register_column_types(datasette=self): + if hook: + for ct_cls in hook: + if ct_cls.name in self._column_types: + raise StartupError( + f"Duplicate column type name: {ct_cls.name}" + ) + self._column_types[ct_cls.name] = ct_cls - for hook in pm.hook.prepare_jinja2_environment( - env=self._jinja_env, datasette=self - ): - await await_me_maybe(hook) - # Ensure internal tables and metadata are populated before startup hooks - await self._refresh_schemas() - await self._save_queries_from_config() - # Load column_types from config into internal DB - await self._apply_column_types_config() - for hook in pm.hook.startup(datasette=self): - await await_me_maybe(hook) - self._startup_invoked = True + for hook in pm.hook.prepare_jinja2_environment( + env=self._jinja_env, datasette=self + ): + await await_me_maybe(hook) + # Ensure internal tables and metadata are populated before startup hooks + await self._refresh_schemas() + await self._save_queries_from_config() + # Load column_types from config into internal DB + await self._apply_column_types_config() + for hook in pm.hook.startup(datasette=self): + await await_me_maybe(hook) + self._startup_invoked = True def sign(self, value, namespace="default"): return URLSafeSerializer(self._secret, namespace).dumps(value) diff --git a/datasette/database.py b/datasette/database.py index d7cbc99b..38b38f48 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -496,8 +496,15 @@ class Database: try: conn = self.connect(write=True) # This warm-up runs before any write has ever been queued, so - # there is no caller otel context yet to attach - any spans - # created by plugin hooks here are orphans (roots). + # there is no captured caller context to attach - and a raw + # threading.Thread does not inherit the context of whoever started + # it. Spans created by plugin hooks here are therefore roots even + # when the write thread is started from inside invoke_startup(): + # its datasette.startup span is current on the event loop but does + # not cross this thread boundary. Read connections differ - they + # warm up inside executor tasks submitted with copy_context(), so + # their prepare_connection spans do nest under whoever triggered + # them. self.ds._prepare_connection(conn, self.name) except Exception as e: # noqa: BLE001 # Stored and re-raised to whoever queues the next write diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index f232a39d..7894670c 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -6,6 +6,7 @@ import time import pytest import sqlite_utils +from opentelemetry import trace as otel_trace from opentelemetry.trace import StatusCode from datasette.app import Datasette @@ -56,6 +57,27 @@ def _children_named(otel_spans, name, parent_span_context): ] +def _descends_from(span, ancestor_span_context, by_span_id): + """ + True if `span` reaches `ancestor_span_context` by walking parent links. + + Walks real span ids rather than trusting a shared trace id: a span can + carry the right trace id and still hang off the wrong parent. + """ + seen = set() + current = span + while current.parent is not None: + if current.parent.span_id == ancestor_span_context.span_id: + return current.parent.trace_id == ancestor_span_context.trace_id + if current.parent.span_id in seen: + return False + seen.add(current.parent.span_id) + current = by_span_id.get(current.parent.span_id) + if current is None: + return False + return False + + def _all_attribute_values(otel_spans): "Every attribute value across every finished span, for the 'no leaked param values' test." values = [] @@ -457,3 +479,67 @@ async def test_suppressed_error_does_not_mark_execute_span(ds_client, otel_spans span = execute_spans[-1] assert span.status.status_code == StatusCode.UNSET assert not [event for event in span.events if event.name == "exception"] + + +@pytest.mark.asyncio +async def test_invoke_startup_produces_one_trace_not_dozens_of_orphans(otel_spans): + """ + invoke_startup() runs with no request, so nothing it does has an ambient + span to nest under. Without datasette.startup every register_* hook, every + internal-catalog read and every catalog write becomes its own single-span + root trace - around twenty of them per fresh instance. + """ + ds = Datasette(memory=True) + # Named in-memory databases are shared-cache, so this needs its own name. + ds.add_memory_database("t05_startup_db") + # Constructing a Datasette already touches the internal catalog, and that + # work is genuinely outside startup. Clear so the assertions below describe + # invoke_startup() alone. + otel_spans.clear() + + # Deliberately no ambient span: this mirrors the ASGI lifespan path, where + # startup runs before any request exists. If something did wrap this call + # the "one root" assertion below would pass for the wrong reason. + assert ( + not otel_trace.get_current_span().get_span_context().is_valid + ), "this test must run with no ambient span" + + await ds.invoke_startup() + + spans = otel_spans.get_finished_spans() + assert len(spans) > 10, f"expected startup to emit many spans, got {len(spans)}" + + startup_spans = [span for span in spans if span.name == "datasette.startup"] + assert len(startup_spans) == 1 + startup = startup_spans[0] + assert startup.parent is None, "datasette.startup should be a root span" + + trace_ids = {span.context.trace_id for span in spans} + assert trace_ids == {startup.context.trace_id}, ( + f"startup produced {len(trace_ids)} distinct traces; every span it " + "causes should share the datasette.startup trace" + ) + + roots = [span for span in spans if span.parent is None] + assert [span.name for span in roots] == ["datasette.startup"] + + by_span_id = {span.context.span_id: span for span in spans} + + # The internal catalog reads are what made up the bulk of the orphans. + internal_queries = [ + span + for span in spans + if span.name == "db.query" and span.attributes["db.namespace"] == "__INTERNAL__" + ] + assert internal_queries, "expected internal-catalog db.query spans during startup" + assert all( + _descends_from(span, startup.context, by_span_id) for span in internal_queries + ) + + # ...and the catalog writes, which reach the span through the write thread, + # so they also prove the ticket-04 context capture survives startup. + write_spans = [span for span in spans if span.name.startswith("db.write.")] + assert write_spans, "expected db.write.* spans during startup" + assert all( + _descends_from(span, startup.context, by_span_id) for span in write_spans + ) From 77b025be281b1215c51c2f62c75fb3d6b1c7092d Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 18:29:16 -0700 Subject: [PATCH 06/24] Make db.query spans match OpenTelemetry semantic conventions Three corrections to the emitted data, bundled because changing what is on the wire after operators have built dashboards on it is a breaking change - so they belong in the first release that ships spans at all, not a later one. db.query is now SpanKind.CLIENT. Trace UIs key their database rendering off the span kind rather than off db.system, so the spans rendered as ordinary internal work despite carrying db.system and db.query.text. The three child spans stay INTERNAL on purpose: db.query.execute, db.write.execute and db.write.queue_wait are Datasette's decomposition of one logical query, not three database calls, and queue_wait touches no database at all - marking them CLIENT would make one query look like several to anything counting spans by kind. The instrumentation scope now carries the Datasette version and a schema URL, so a backend can tell which Datasette produced a span. The URL is 1.29.0 rather than the latest semconv release because that is the highest version at which every name emitted here is the current spelling: db.system was renamed to db.system.name in 1.30.0 and this code still emits the older form. Claiming a later schema would be false, and would stop a consumer translating that name forward, since the claim asserts the rename already happened. db.operation.name is the statement's leading keyword matched against a fixed allowlist, not a parse. On a public instance the SQL is attacker-controlled and this attribute is a candidate metric dimension in a later phase, so echoing back an arbitrary first token would let a visitor's typo mint a permanent series. Anything unrecognised gets no attribute rather than a wrong one. execute_write_script() does not set it at all, since semantic conventions say not to extract an operation name from query text that can hold several statements. db.collection.name comes only from a new table= argument on Database.execute(), and is never derived from the SQL: deriving it would be a parse, and on an instance where anyone can create a table the value set has no ceiling. It is passed from every query in the table and row views that targets exactly one user table. Internal-catalog reads and the row view's cross-table foreign key counts are deliberately left without it. Co-Authored-By: Claude Opus 5 --- datasette/database.py | 38 +++++-- datasette/telemetry.py | 89 ++++++++++++++++- datasette/views/row.py | 9 +- datasette/views/table.py | 16 ++- tests/test_telemetry.py | 209 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 346 insertions(+), 15 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 38b38f48..1b9a5076 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -14,10 +14,10 @@ from pathlib import Path import sqlite_utils from opentelemetry import context as otel_context_api -from opentelemetry.trace import Status, StatusCode +from opentelemetry.trace import SpanKind, Status, StatusCode from .inspect import inspect_hash -from .telemetry import sql_attribute, tracer +from .telemetry import sql_attribute, sql_operation_name, tracer from .tracer import trace from .utils import ( call_with_supported_arguments, @@ -268,10 +268,13 @@ class Database: with trace( # noqa: SIM117 "sql", database=self.name, sql=sql.strip(), params=params ): - with tracer.start_as_current_span("db.query") as span: + with tracer.start_as_current_span("db.query", kind=SpanKind.CLIENT) as span: span.set_attribute("db.system", "sqlite") span.set_attribute("db.namespace", self.name) span.set_attribute("db.query.text", sql_attribute(sql)) + operation_name = sql_operation_name(sql) + if operation_name: + span.set_attribute("db.operation.name", operation_name) if params: span.set_attribute("datasette.param_count", len(params)) results = await self.execute_write_fn( @@ -289,7 +292,11 @@ class Database: with trace( # noqa: SIM117 "sql", database=self.name, sql=sql.strip(), executescript=True ): - with tracer.start_as_current_span("db.query") as span: + # No db.operation.name here, deliberately: executescript() runs + # several semicolon-separated statements, and semantic conventions + # say the attribute should not be extracted from query text that + # can hold more than one operation - see sql_operation_name(). + with tracer.start_as_current_span("db.query", kind=SpanKind.CLIENT) as span: span.set_attribute("db.system", "sqlite") span.set_attribute("db.namespace", self.name) span.set_attribute("db.query.text", sql_attribute(sql)) @@ -317,11 +324,16 @@ class Database: with trace( "sql", database=self.name, sql=sql.strip(), executemany=True ) as kwargs: - with tracer.start_as_current_span("db.query") as span: + with tracer.start_as_current_span("db.query", kind=SpanKind.CLIENT) as span: span.set_attribute("db.system", "sqlite") span.set_attribute("db.namespace", self.name) span.set_attribute("db.query.text", sql_attribute(sql)) span.set_attribute("datasette.executemany", True) + # A single statement run with many parameter sets, so unlike + # execute_write_script() there is exactly one operation to name. + operation_name = sql_operation_name(sql) + if operation_name: + span.set_attribute("db.operation.name", operation_name) results, count = await self.execute_write_fn( _inner, block=block, request=request ) @@ -632,8 +644,16 @@ class Database: custom_time_limit=None, page_size=None, log_sql_errors=True, + table=None, ): - """Executes sql against db_name in a thread""" + """Executes sql against db_name in a thread + + `table`, if passed, is recorded as the `db.collection.name` span + attribute. It exists for callers that already know which table the + query targets - the table and row views - and is never derived from + `sql` itself: deriving it would be a parse, and on an instance where + anyone can create a table the resulting value set has no ceiling. + """ self._check_not_closed() page_size = page_size or self.ds.page_size time_limit_ms = self.ds.sql_time_limit_ms @@ -698,6 +718,7 @@ class Database: # can be honoured - see the comment on the generic handler below. with tracer.start_as_current_span( "db.query", + kind=SpanKind.CLIENT, record_exception=False, set_status_on_exception=False, ) as span: @@ -705,6 +726,11 @@ class Database: span.set_attribute("db.namespace", self.name) span.set_attribute("db.query.text", sql_attribute(sql)) span.set_attribute("datasette.time_limit_ms", time_limit_ms) + operation_name = sql_operation_name(sql) + if operation_name: + span.set_attribute("db.operation.name", operation_name) + if table: + span.set_attribute("db.collection.name", table) if params: span.set_attribute("datasette.param_count", len(params)) try: diff --git a/datasette/telemetry.py b/datasette/telemetry.py index 6f5ed093..37195584 100644 --- a/datasette/telemetry.py +++ b/datasette/telemetry.py @@ -9,9 +9,34 @@ harness). With no provider installed every span produced here is a `NonRecordingSpan` and costs approximately nothing. """ +import re + from opentelemetry import trace as otel_trace -tracer = otel_trace.get_tracer("datasette") +from .version import __version__ + +# The semantic-convention version whose spellings this instrumentation +# actually emits. Deliberately NOT the latest release. +# +# A schema URL is a machine-readable claim: a consumer doing schema +# translation replays the renames between the declared version and the one +# it wants, so the claim has to name the version whose spellings are on the +# wire. A wrong one makes translation wrong rather than merely uninformative. +# +# Datasette emits `db.system`, which was renamed to `db.system.name` in +# semconv 1.30.0. Everything else it emits (`db.namespace`, `db.query.text`, +# `db.operation.name`, `db.collection.name`) has been current since 1.26.0. +# So 1.29.0 is the highest version at which every name emitted here is the +# current spelling. Everything under `datasette.*` is Datasette's own and +# outside semconv, so it is unaffected either way. +# +# Declaring 1.43.0 would be false about `db.system`, and would actively STOP +# a consumer translating it forward, because it asserts the rename already +# happened. Bump this deliberately, in the same commit as the attribute +# renames it implies - it is a claim about the names, not decoration. +SCHEMA_URL = "https://opentelemetry.io/schemas/1.29.0" + +tracer = otel_trace.get_tracer("datasette", __version__, schema_url=SCHEMA_URL) MAX_SQL_LENGTH = 2048 @@ -22,3 +47,65 @@ def sql_attribute(sql: str) -> str: if len(sql) <= MAX_SQL_LENGTH: return sql return sql[:MAX_SQL_LENGTH] + "…[truncated]" + + +# db.operation.name is the leading keyword of a statement matched against a +# fixed allowlist - deliberately not a parse. +# +# This runs against arbitrary user-supplied SQL (the `?sql=` query string, +# canned queries, anything typed into the query editor), and the attribute is +# a candidate dimension on a query-duration metric in a later phase. A metric +# series is keyed by its attribute values, so echoing back an arbitrary first +# token would let one visitor's typo mint a new, permanent series. The +# allowlist bounds that at a fixed, small set regardless of what anyone sends. +DB_OPERATION_ALLOWLIST = frozenset( + { + "SELECT", + "INSERT", + "UPDATE", + "DELETE", + "CREATE", + "DROP", + "ALTER", + "PRAGMA", + "EXPLAIN", + "REPLACE", + "VACUUM", + "ANALYZE", + "WITH", + } +) + +_LEADING_KEYWORD = re.compile(r"^\s*([A-Za-z]+)") + + +def sql_operation_name(sql: str) -> str | None: + """ + The statement's leading keyword, if it is one we recognise. + + Returns None - never a guess - for anything not on the allowlist, + including a statement that opens with a comment or with punctuation such + as the "(" of a parenthesised SELECT. + + Known limitation: a statement beginning with a CTE reports `WITH` rather + than the operation inside it, and a substantial share of Datasette's own + reads take that form. Extracting more than the leading keyword means + handling comment stripping, parenthesised `(SELECT ...) UNION` and + compound names like `CREATE TABLE` - each a special case a hand-rolled + matcher would accrete and eventually get wrong. Omitting a name beats + guessing at one. + + Only safe to call with a single statement: `execute_write_script()` runs + several separated by semicolons, and semantic conventions say + `db.operation.name` "SHOULD NOT be extracted from db.query.text, when the + database system supports query text with multiple operations in non-batch + operations" - so that call site does not use this at all rather than + reporting only the first statement's operation. + """ + match = _LEADING_KEYWORD.match(sql) + if not match: + return None + keyword = match.group(1).upper() + if keyword in DB_OPERATION_ALLOWLIST: + return keyword + return None diff --git a/datasette/views/row.py b/datasette/views/row.py index b1388299..b5196a83 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -407,7 +407,7 @@ class RowView(BaseView): raise Forbidden("You do not have permission to view this table") results = await resolved.db.execute( - resolved.sql, resolved.params, truncate=True + resolved.sql, resolved.params, truncate=True, table=table ) columns = [r[0] for r in results.description] rows = list(results.rows) @@ -652,6 +652,9 @@ class RowView(BaseView): ] ) try: + # No table= here: this counts incoming references across every + # foreign key pointing at this row, so it spans many tables and + # there is no single value db.collection.name could take. rows = list(await db.execute(sql, {"id": pk_values[0]})) except QueryInterrupted: # Almost certainly hit the timeout @@ -840,7 +843,7 @@ class RowUpdateView(BaseView): returned_row = None if data.get("return"): results = await resolved.db.execute( - resolved.sql, resolved.params, truncate=True + resolved.sql, resolved.params, truncate=True, table=resolved.table ) returned_row = results.dicts()[0] result["rows"] = [returned_row] @@ -858,7 +861,7 @@ class RowUpdateView(BaseView): message_row = returned_row if message_row is None: results = await resolved.db.execute( - resolved.sql, resolved.params, truncate=True + resolved.sql, resolved.params, truncate=True, table=resolved.table ) message_row = results.first() self.ds.add_message( diff --git a/datasette/views/table.py b/datasette/views/table.py index 7c814b27..6d920fd8 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -1170,6 +1170,7 @@ class TableInsertView(BaseView): "rowid, " if pks == ["rowid"] else "", table_name, where_clause ), args, + table=table_name, ) result["rows"] = fetched_rows.dicts() else: @@ -1382,7 +1383,9 @@ class TableDropView(BaseView): "database": database_name, "table": table_name, "row_count": ( - await db.execute(f"select count(*) from [{table_name}]") + await db.execute( + f"select count(*) from [{table_name}]", table=table_name + ) ).single_value(), "message": 'Pass "confirm": true to confirm', }, @@ -1576,7 +1579,10 @@ class TableAutocompleteView(BaseView): try: results = await db.execute( - sql, params, custom_time_limit=AUTOCOMPLETE_TIME_LIMIT_MS + sql, + params, + custom_time_limit=AUTOCOMPLETE_TIME_LIMIT_MS, + table=table_name, ) except QueryInterrupted: fallback_where = _autocomplete_prefix_like(pks[0]) @@ -1597,6 +1603,7 @@ class TableAutocompleteView(BaseView): fallback_sql, params, custom_time_limit=AUTOCOMPLETE_TIME_LIMIT_MS, + table=table_name, ) except QueryInterrupted: return Response.json({"ok": True, "rows": []}) @@ -2163,7 +2170,9 @@ async def table_view_data( # Execute the main query! try: - results = await db.execute(sql, params, truncate=True, **extra_args) + results = await db.execute( + sql, params, truncate=True, table=table_name, **extra_args + ) except (sqlite3.OperationalError, InvalidSql) as e: raise DatasetteError(str(e), title="Invalid SQL", status=400) @@ -2439,6 +2448,7 @@ async def _next_value_and_url( await db.execute( prefix_lookup_sql, {**{f"pk{i}": rows[-2][pk] for i, pk in enumerate(pks)}}, + table=table_name, ) ).single_value() if isinstance(prefix, dict) and "value" in prefix: diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 7894670c..c49a11cf 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -7,11 +7,18 @@ import time import pytest import sqlite_utils from opentelemetry import trace as otel_trace -from opentelemetry.trace import StatusCode +from opentelemetry.trace import SpanKind, StatusCode from datasette.app import Datasette from datasette.database import Database -from datasette.telemetry import MAX_SQL_LENGTH, sql_attribute, tracer +from datasette.telemetry import ( + MAX_SQL_LENGTH, + SCHEMA_URL, + sql_attribute, + sql_operation_name, + tracer, +) +from datasette.version import __version__ SECRET_PARAM_VALUE = "SUPER_SECRET_PARAM_VALUE_XYZ_123" @@ -543,3 +550,201 @@ async def test_invoke_startup_produces_one_trace_not_dozens_of_orphans(otel_span assert all( _descends_from(span, startup.context, by_span_id) for span in write_spans ) + + +# --- Semantic conventions: span kind, scope, db.operation/collection ------- + + +@pytest.mark.asyncio +async def test_db_query_is_client_kind_and_children_are_internal(otel_spans): + """ + db.query is a database client span; Datasette's decomposition of it is not. + + Trace UIs key their database rendering off the span kind rather than off + db.system, so db.query has to be CLIENT. db.query.execute, + db.write.execute and db.write.queue_wait deliberately stay INTERNAL: they + are parts of one logical query rather than three separate database calls, + and queue_wait touches no database at all - marking them CLIENT would + make one query look like several to anything counting spans by kind. + """ + # Named in-memory databases are shared-cache, so this needs its own name. + db = Datasette(memory=True).add_memory_database("t06_span_kind") + # All four db.query entry points, so a missed `kind=` on any one of them + # fails here - plus the write path (db.write.queue_wait, + # db.write.execute) and the read path (db.query.execute) children. + await db.execute_write("create table docs (id integer primary key)") + await db.execute_write_many( + "insert into docs (id) values (?)", [[i] for i in range(1, 4)] + ) + await db.execute_write_script("insert into docs (id) values (99);") + await db.execute("select id from docs") + + query_spans = _spans_for_namespace(otel_spans, "t06_span_kind") + assert len(query_spans) == 4, "expected a db.query span per entry point" + for span in query_spans: + text = span.attributes["db.query.text"] + assert span.kind == SpanKind.CLIENT, f"db.query for {text!r} should be CLIENT" + + for name in ("db.query.execute", "db.write.execute", "db.write.queue_wait"): + children = [ + span for span in otel_spans.get_finished_spans() if span.name == name + ] + assert children, f"expected at least one {name} span" + for span in children: + assert span.kind == SpanKind.INTERNAL, f"{name} should be INTERNAL" + + +@pytest.mark.asyncio +async def test_instrumentation_scope_declares_version_and_schema_url( + ds_client, otel_spans +): + """ + Spans say which Datasette produced them and which semconv version their + attribute names follow. + + Before get_tracer() was given a version and a schema URL every exported + scope was name='datasette' version='' schema_url='', so nothing + downstream could tell which Datasette a span came from, or whether + `db.system` meant `db.system` or the post-1.30.0 `db.system.name`. + """ + response = await ds_client.get("/fixtures/-/query.json?sql=select+1") + assert response.status_code == 200 + + spans = _db_query_spans(otel_spans) + assert spans, "expected at least one db.query span" + scope = spans[-1].instrumentation_scope + + assert scope.name == "datasette" + assert scope.version == __version__ + # The literal URL, not the SCHEMA_URL constant: comparing the span + # against the same constant the instrumentation is built from would only + # catch a dropped argument, never a wrong value. Bumping this is a claim + # about the attribute names on the wire - see SCHEMA_URL in telemetry.py. + assert scope.schema_url == "https://opentelemetry.io/schemas/1.29.0" + assert SCHEMA_URL == "https://opentelemetry.io/schemas/1.29.0" + assert __version__, "the scope version must not be empty" + + +def test_db_operation_name_from_leading_keyword(): + assert sql_operation_name("select 1") == "SELECT" + assert sql_operation_name(" insert into x (a) values (1)") == "INSERT" + # A leading CTE reports WITH rather than the operation inside it. That is + # the documented limitation, not an accident - see sql_operation_name(). + assert sql_operation_name("with foo as (select 1) select * from foo") == "WITH" + # Unrecognised leading keyword: no attribute rather than a wrong one, and + # no unbounded value set derived from attacker-supplied SQL. + assert sql_operation_name("gibberish 1") is None + # Not a parser: a parenthesised SELECT and a leading comment both yield + # nothing rather than a guess. + assert sql_operation_name("(select 1) union select 2") is None + assert sql_operation_name("-- a comment\nselect 1") is None + assert sql_operation_name("") is None + + +@pytest.mark.asyncio +async def test_db_operation_name_on_real_span(ds_client, otel_spans): + response = await ds_client.get("/fixtures/-/query.json?sql=select+1") + assert response.status_code == 200 + + spans = [ + span + for span in _spans_for_namespace(otel_spans, "fixtures") + if span.attributes["db.query.text"] == "select 1" + ] + assert spans, "expected a db.query span for 'select 1'" + assert spans[-1].attributes["db.operation.name"] == "SELECT" + + +@pytest.mark.asyncio +async def test_execute_write_sets_db_operation_name(otel_spans): + db = Datasette(memory=True).add_memory_database("t06_write_operation") + await db.execute_write("create table docs (id integer primary key)") + await db.execute_write_many( + "insert into docs (id) values (?)", [[i] for i in range(1, 4)] + ) + + spans = _spans_for_namespace(otel_spans, "t06_write_operation") + by_operation = { + span.attributes["db.query.text"]: span.attributes.get("db.operation.name") + for span in spans + } + assert by_operation["create table docs (id integer primary key)"] == "CREATE" + assert by_operation["insert into docs (id) values (?)"] == "INSERT" + + +@pytest.mark.asyncio +async def test_execute_write_script_has_no_operation_name(otel_spans): + """ + executescript() runs several statements, so naming the operation after + the first one would be a lie. Semantic conventions say db.operation.name + should not be extracted from query text that can hold more than one + operation, so the attribute is absent entirely. + + The script deliberately starts with `create`, which *is* on the + allowlist - so this fails if the call site ever starts calling + sql_operation_name(). + """ + db = Datasette(memory=True).add_memory_database("t06_script_operation") + await db.execute_write_script( + "create table docs (id integer primary key);\n" + "insert into docs (id) values (1);" + ) + + spans = _spans_for_namespace(otel_spans, "t06_script_operation") + script_spans = [ + span for span in spans if span.attributes.get("datasette.executescript") is True + ] + assert len(script_spans) == 1 + assert "db.operation.name" not in script_spans[0].attributes + + +@pytest.mark.asyncio +async def test_db_collection_name_set_from_table_argument(ds_client, otel_spans): + db = ds_client.ds.get_database("fixtures") + await db.execute("select pk from facetable limit 1", table="facetable") + + spans = _spans_for_namespace(otel_spans, "fixtures") + assert spans + assert spans[-1].attributes["db.collection.name"] == "facetable" + + +@pytest.mark.asyncio +async def test_db_collection_name_absent_without_table_argument(ds_client, otel_spans): + """ + db.collection.name comes only from an explicit table= argument and is + never derived from the SQL. + + Deriving it would be a parse, and on an instance where anybody can create + a table the value set has no ceiling. Without this test the one above + would still pass if the table name were being read out of the query text. + """ + db = ds_client.ds.get_database("fixtures") + await db.execute("select pk from facetable limit 1") + + spans = _spans_for_namespace(otel_spans, "fixtures") + assert spans + span = spans[-1] + assert span.attributes["db.query.text"] == "select pk from facetable limit 1" + assert "db.collection.name" not in span.attributes + + +@pytest.mark.parametrize( + "path,table", + ( + ("/fixtures/facetable.json", "facetable"), + ("/fixtures/simple_primary_key/1.json", "simple_primary_key"), + ), +) +@pytest.mark.asyncio +async def test_table_and_row_pages_set_db_collection_name( + ds_client, otel_spans, path, table +): + "The table and row views know their table, so their queries carry it." + response = await ds_client.get(path) + assert response.status_code == 200 + + spans = _spans_for_namespace(otel_spans, "fixtures") + assert spans + assert any( + span.attributes.get("db.collection.name") == table for span in spans + ), f"expected a db.query span from {path} carrying db.collection.name" From e24a2c122fb646d72f3db2f2b6cd50e78130726c Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 18:36:49 -0700 Subject: [PATCH 07/24] Link block=False write spans to their enqueuer instead of parenting them A block=False write returns without awaiting the reply future, so the enclosing db.query span finishes - and exports - before db.write.queue_wait and db.write.execute even exist. They were still parented to it, which produced a child bar ending ~50ms after its already-closed parent: legal OpenTelemetry, but it renders as nonsense in a trace UI. Parenting asserts containment; a link asserts causation without containment. The enqueueing request causes the write without containing it, which is exactly what a span link is for. So for block=False both write spans are now roots - started with an explicit empty Context, so the write thread's ambient context cannot supply a parent either - each carrying one link back to the enqueueing span. block=True is untouched, since there the caller really does await the reply and containment is accurate. The link carries no attributes. There is only one kind of link here, so naming the relationship would be a constant conveying nothing the link's existence does not already say. Accepted trade-off: a linked span will not appear inside the request's waterfall in most trace UIs. It shows up as its own trace with a "linked from" reference rather than a bar under the request. For a fire-and-forget write whose latency the request never pays, that is the right trade - correctness over at-a-glance nesting for a case the request-latency view was never accurate for anyway. This does add root traces, which looks like it cuts against the startup span work that spent its whole diff removing them. The difference is reachability: those roots were orphans, whereas these are reachable from the request that caused them via the link. Nothing in core issues block=False writes today - it is a plugin-facing path - so this changes no trace Datasette produces on its own. Co-Authored-By: Claude Opus 5 --- datasette/database.py | 85 +++++++++++++++++---- tests/test_telemetry.py | 161 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 231 insertions(+), 15 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 1b9a5076..3c14c1c0 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -14,7 +14,7 @@ from pathlib import Path import sqlite_utils from opentelemetry import context as otel_context_api -from opentelemetry.trace import SpanKind, Status, StatusCode +from opentelemetry.trace import Link, SpanKind, Status, StatusCode, get_current_span from .inspect import inspect_hash from .telemetry import sql_attribute, sql_operation_name, tracer @@ -482,7 +482,9 @@ class Database: # Captured here, on the event loop, at enqueue time: the otel # Context (carrying the enclosing db.query span, if any) and the # timestamp used to build the db.write.queue_wait span once this - # task is dequeued on the write thread. + # task is dequeued on the write thread. `block` travels with the + # task too, because it decides whether that context is this task's + # parent or only a link target - see `_execute_writes`. self._write_queue.put( WriteTask( fn, @@ -493,6 +495,7 @@ class Database: transaction, otel_context_api.get_current(), time.time_ns(), + block, ) ) if block: @@ -531,15 +534,52 @@ class Database: # Best-effort close as the write thread exits pass return - # Restore the caller's otel context (captured on the event loop - # at enqueue time) so spans created while processing this task - # parent correctly to the request that queued it. Must be - # detached below in `finally` - a leaked token silently poisons - # this thread's ambient context for every write processed after - # it, and a *wrong*-token detach only logs a warning rather than - # raising, so this pairing is load-bearing and easy to get wrong - # silently. - token = otel_context_api.attach(task.otel_context) + # `task.block` decides how this task's spans relate to the + # context captured at enqueue time: + # + # - block=True: the caller genuinely awaits the reply, so + # containment is accurate. Restore that context as current + # (attach below) so db.write.queue_wait/db.write.execute parent + # normally to the request that queued them. The token must be + # detached below in `finally` - a leaked token silently + # poisons this thread's ambient context for every write + # processed after it, and a *wrong*-token detach only logs a + # warning rather than raising, so this pairing is load-bearing + # and easy to get wrong silently. + # - block=False: the caller returned already without awaiting, + # so the enqueueing span may already have closed (and + # exported) before this task's spans even start - parenting to + # it would make a child appear to outlive its already-closed + # parent, which OTel allows but which renders badly in most + # trace UIs. The enqueueing request *caused* this write + # without *containing* it, so nothing is attached here - + # instead each write span is started as its own root (explicit + # empty `context=`, so the write thread's ambient context + # cannot supply a parent either) carrying one `Link` back to + # the enqueueing span's context, built once into + # `write_span_kwargs` and spread into every start_span call + # below. + token = None + write_span_kwargs = {} + if task.block: + token = otel_context_api.attach(task.otel_context) + else: + enqueueing_span_context = get_current_span( + task.otel_context + ).get_span_context() + # No attributes on the link: there is only one kind of link + # here, so naming the relationship would be a constant that + # carries no information a consumer does not already have + # from the link's existence. + links = ( + [Link(enqueueing_span_context)] + if enqueueing_span_context.is_valid + else [] + ) + write_span_kwargs = { + "context": otel_context_api.Context(), + "links": links, + } try: exception = None result = None @@ -548,7 +588,9 @@ class Database: # waiting in the queue (enqueue -> dequeue), not the near- # zero time spent constructing/ending the span object here. tracer.start_span( - "db.write.queue_wait", start_time=task.enqueued_at_ns + "db.write.queue_wait", + start_time=task.enqueued_at_ns, + **write_span_kwargs, ).end(end_time=time.time_ns()) if conn_exception is not None: # fn never runs in this branch, so there is nothing to @@ -556,7 +598,9 @@ class Database: exception = conn_exception elif task.isolated_connection: try: - with tracer.start_as_current_span("db.write.execute") as span: + with tracer.start_as_current_span( + "db.write.execute", **write_span_kwargs + ) as span: span.set_attribute( "datasette.isolated_connection", task.isolated_connection, @@ -583,7 +627,9 @@ class Database: exception = e else: try: - with tracer.start_as_current_span("db.write.execute") as span: + with tracer.start_as_current_span( + "db.write.execute", **write_span_kwargs + ) as span: span.set_attribute( "datasette.isolated_connection", task.isolated_connection, @@ -603,7 +649,8 @@ class Database: exception = e _deliver_write_result(task, result, exception) finally: - otel_context_api.detach(token) + if token is not None: + otel_context_api.detach(token) async def execute_fn(self, fn): self._check_not_closed() @@ -1043,6 +1090,7 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event): class WriteTask: __slots__ = ( + "block", "enqueued_at_ns", "fn", "isolated_connection", @@ -1063,6 +1111,7 @@ class WriteTask: transaction, otel_context, enqueued_at_ns, + block, ): self.fn = fn self.task_id = task_id @@ -1072,6 +1121,12 @@ class WriteTask: self.transaction = transaction self.otel_context = otel_context self.enqueued_at_ns = enqueued_at_ns + # Whether the enqueueing caller awaits the reply future. Decides how + # `_execute_writes` relates this task's spans to `otel_context`: + # parent (block=True) or span-link target (block=False). See the + # comment at the WriteTask construction site in + # `_send_to_write_thread`. + self.block = block def _deliver_write_result(task, result, exception): diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index c49a11cf..0a6d0192 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -2,10 +2,12 @@ import json import sqlite3 import subprocess import sys +import threading import time import pytest import sqlite_utils +from opentelemetry import context as otel_context_api from opentelemetry import trace as otel_trace from opentelemetry.trace import SpanKind, StatusCode @@ -463,6 +465,165 @@ async def test_write_queue_wait_duration_reflects_real_wait(otel_spans): assert duration_ns > 10_000_000, f"queue wait was only {duration_ns}ns" +async def _write_spans_from_one_enqueue(otel_spans, name, block): + """ + Run exactly one write through the write thread from inside a span of our + own, and return (enqueueing span context, {span name: span}). + + `_send_to_write_thread` is called directly rather than `execute_write()` + because `execute_write()` opens its own db.query span, which would then + be the span current at enqueue time - so the parent/link would point at + that span rather than at the one this test controls. + + The exporter is cleared immediately before the enqueue so the write spans + collected here can only have come from this one write. + """ + db = Datasette(memory=True).add_memory_database(name) + await db.execute_write("create table docs (id integer primary key)") + + def insert(conn): + conn.execute("insert into docs (id) values (1)") + + otel_spans.clear() + with tracer.start_as_current_span("enqueueing-span") as enqueuer: + enqueuer_context = enqueuer.get_span_context() + queued = await db._send_to_write_thread(insert, block=block) + if not block: + # The point of block=False is that the write happens after the + # caller has returned and the enqueueing span above has closed. + # Awaiting the reply future outside that `with` waits for the write + # thread deterministically - it is resolved only after both write + # spans have ended and been exported. + _, reply_future = queued + await reply_future + + spans = {} + for span in otel_spans.get_finished_spans(): + if span.name in ("db.write.queue_wait", "db.write.execute"): + assert span.name not in spans, f"more than one {span.name} span" + spans[span.name] = span + assert set(spans) == {"db.write.queue_wait", "db.write.execute"} + return enqueuer_context, spans + + +@pytest.mark.asyncio +async def test_blocking_write_spans_still_parent_normally(otel_spans): + # Regression guard for ticket 07: block=True genuinely has containment - + # the caller awaits the reply future - so those spans must keep parenting + # to the enqueueing span, and must not grow links. + enqueuer_context, spans = await _write_spans_from_one_enqueue( + otel_spans, "t07_blocking_write", block=True + ) + for name, span in spans.items(): + assert span.parent is not None, f"{name} lost its parent" + assert span.parent.span_id == enqueuer_context.span_id, name + assert span.parent.trace_id == enqueuer_context.trace_id, name + assert span.context.trace_id == enqueuer_context.trace_id, name + assert span.links == (), f"{name} should be parented, not linked" + + +@pytest.mark.asyncio +async def test_nonblocking_write_spans_are_roots_with_a_link(otel_spans): + # block=False returns before the write runs, so the enqueueing span has + # already ended (and exported) by the time these spans start. Parenting + # them to it would draw a child outliving its closed parent, so they are + # roots in their own traces, linked back to the span that caused them. + enqueuer_context, spans = await _write_spans_from_one_enqueue( + otel_spans, "t07_nonblocking_write", block=False + ) + assert enqueuer_context.is_valid, "test's own enqueueing span was not recorded" + for name, span in spans.items(): + assert span.parent is None, f"{name} is still parented" + # A link does not join the linked trace: each of these is its own + # root trace, which is the correct shape and not a workaround. + assert span.context.trace_id != enqueuer_context.trace_id, name + assert len(span.links) == 1, f"{name} has links {span.links}" + link_context = span.links[0].context + assert link_context.trace_id == enqueuer_context.trace_id, name + assert link_context.span_id == enqueuer_context.span_id, name + # The two write spans are independent roots, not nested in one another. + assert ( + spans["db.write.queue_wait"].context.trace_id + != spans["db.write.execute"].context.trace_id + ) + + +@pytest.mark.asyncio +async def test_nonblocking_write_link_has_no_attributes(otel_spans): + # There is only one kind of link here, so a relationship-name attribute + # would be a constant conveying nothing the link's existence does not. + _, spans = await _write_spans_from_one_enqueue( + otel_spans, "t07_nonblocking_link_attrs", block=False + ) + for name, span in spans.items(): + assert len(span.links) == 1, name + assert dict(span.links[0].attributes or {}) == {}, name + + +@pytest.mark.asyncio +async def test_nonblocking_write_spans_ignore_the_write_threads_ambient_context( + otel_spans, +): + """ + block=False spans pass an explicit empty Context, not merely "no attach". + + Nothing is attached for a block=False task, but "nothing attached" is not + the same as "no ambient context": the write thread is persistent, and + anything running on it - a prepare_connection plugin hook, say - can + attach a context and never detach it. Without the explicit `context=` + these spans would silently parent to that leftover span instead of being + roots, and no other test here would notice, because in every other test + the write thread's ambient context happens to be empty. + + So this test leaks exactly such a context on the write thread, the way a + careless plugin would, and then checks the write spans are still roots. + """ + ds = Datasette(memory=True) + db = ds.add_memory_database("t07_ambient_write_thread") + write_thread_name = "_execute_writes for database t07_ambient_write_thread" + real_prepare_connection = ds._prepare_connection + leaked = {} + + def prepare_connection(conn, database): + if threading.current_thread().name == write_thread_name: + # Runs once, on the write thread, before any task is dequeued - + # and never detaches, which is the whole point. + span = tracer.start_span("leaked-write-thread-ambient-span") + leaked["span_id"] = span.get_span_context().span_id + otel_context_api.attach(otel_trace.set_span_in_context(span)) + return real_prepare_connection(conn, database) + + ds._prepare_connection = prepare_connection + try: + await db.execute_write("create table docs (id integer primary key)") + + def insert(conn): + conn.execute("insert into docs (id) values (1)") + + otel_spans.clear() + with tracer.start_as_current_span("enqueueing-span") as enqueuer: + enqueuer_context = enqueuer.get_span_context() + _, reply_future = await db._send_to_write_thread(insert, block=False) + await reply_future + finally: + ds._prepare_connection = real_prepare_connection + db.close() + + assert "span_id" in leaked, "the ambient context was never leaked - test is vacuous" + write_spans = [ + span + for span in otel_spans.get_finished_spans() + if span.name in ("db.write.queue_wait", "db.write.execute") + ] + assert len(write_spans) == 2 + for span in write_spans: + assert span.parent is None, ( + f"{span.name} parented to the write thread's leftover ambient " + "context instead of being a root" + ) + assert span.links[0].context.span_id == enqueuer_context.span_id + + @pytest.mark.asyncio async def test_suppressed_error_does_not_mark_execute_span(ds_client, otel_spans): """ From 8d32eac89508453eb519ae6d4377950aa03a19a0 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 18:46:34 -0700 Subject: [PATCH 08/24] Name every span and attribute once, in a registry the docs are built from The span and attribute names were string literals spread across four call sites in database.py and one in app.py, with a hand-written reference page that would have been true only on the day it was written. That drift is not hypothetical: an earlier iteration of this work carried a README asserting parameter values were never recorded for two branches after that had stopped being true. datasette/telemetry_registry.py now holds each name once, with its documentation. Attribute and SpanName subclass str, so a registry entry *is* the string OpenTelemetry wants - no wrapper API over the OTel calls, no parallel structure to keep in step, and a typo becomes an ImportError rather than a silently misnamed attribute. docs/internals.rst renders the span reference from it via cog, and `cog --check docs/*.rst` already runs in CI, so the reference cannot drift from the definitions. Nothing changes on the wire: the emitted span names and attribute keys are byte-identical before and after, verified by diffing a dump of both. tests/test_telemetry_registry.py exercises a real workload and compares it against the registry in both directions - emitted-but-unregistered catches instrumentation added without documentation, registered-but-never-emitted catches documentation that has outlived its code. Because the call sites now take their names from the registry, neither direction can catch a rename: move DB_NAMESPACE to "db.namespace2" and code and registry still agree while every dashboard breaks. So the literal names are also written out in the test and asserted against the registry and against the wire separately. That pair is the only comparison in the file not derived from the registry itself. Co-Authored-By: Claude Opus 5 --- datasette/app.py | 3 +- datasette/database.py | 106 ++++++----- datasette/telemetry_registry.py | 269 +++++++++++++++++++++++++++ docs/internals.rst | 72 +++++++ docs/telemetry_doc.py | 37 ++++ tests/test_telemetry_registry.py | 309 +++++++++++++++++++++++++++++++ 6 files changed, 751 insertions(+), 45 deletions(-) create mode 100644 datasette/telemetry_registry.py create mode 100644 docs/telemetry_doc.py create mode 100644 tests/test_telemetry_registry.py diff --git a/datasette/app.py b/datasette/app.py index d3626ab2..7499c6f9 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -50,6 +50,7 @@ from .plugins import DEFAULT_PLUGINS, get_plugins, pm from .renderer import json_renderer from .resources import DatabaseResource, TableResource from .telemetry import tracer +from .telemetry_registry import STARTUP from .tokens import TokenInvalid from .tracer import AsgiTracer from .url_builder import Urls @@ -788,7 +789,7 @@ class Datasette: # A connection warmed lazily later, by a request touching a new # database for the first time, nests under that request instead: # this span has already ended by then. - with tracer.start_as_current_span("datasette.startup"): + with tracer.start_as_current_span(STARTUP): # Register event classes event_classes = [] for hook in pm.hook.register_events(datasette=self): diff --git a/datasette/database.py b/datasette/database.py index 3c14c1c0..22984d6f 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -14,10 +14,32 @@ from pathlib import Path import sqlite_utils from opentelemetry import context as otel_context_api -from opentelemetry.trace import Link, SpanKind, Status, StatusCode, get_current_span +from opentelemetry.trace import Link, Status, StatusCode, get_current_span from .inspect import inspect_hash from .telemetry import sql_attribute, sql_operation_name, tracer +from .telemetry_registry import ( + DB_COLLECTION_NAME, + DB_NAMESPACE, + DB_OPERATION_NAME, + DB_QUERY, + DB_QUERY_EXECUTE, + DB_QUERY_TEXT, + DB_SYSTEM, + DB_WRITE_EXECUTE, + DB_WRITE_QUEUE_WAIT, + EXECUTEMANY, + EXECUTESCRIPT, + INTERRUPTED, + ISOLATED_CONNECTION, + PARAM_COUNT, + PARAM_SETS, + ROWS_RETURNED, + SQL_ERROR_SUPPRESSED, + TIME_LIMIT_MS, + TRANSACTION, + TRUNCATED, +) from .tracer import trace from .utils import ( call_with_supported_arguments, @@ -268,15 +290,15 @@ class Database: with trace( # noqa: SIM117 "sql", database=self.name, sql=sql.strip(), params=params ): - with tracer.start_as_current_span("db.query", kind=SpanKind.CLIENT) as span: - span.set_attribute("db.system", "sqlite") - span.set_attribute("db.namespace", self.name) - span.set_attribute("db.query.text", sql_attribute(sql)) + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) operation_name = sql_operation_name(sql) if operation_name: - span.set_attribute("db.operation.name", operation_name) + span.set_attribute(DB_OPERATION_NAME, operation_name) if params: - span.set_attribute("datasette.param_count", len(params)) + span.set_attribute(PARAM_COUNT, len(params)) results = await self.execute_write_fn( _inner, block=block, request=request, transaction=transaction ) @@ -296,11 +318,11 @@ class Database: # several semicolon-separated statements, and semantic conventions # say the attribute should not be extracted from query text that # can hold more than one operation - see sql_operation_name(). - with tracer.start_as_current_span("db.query", kind=SpanKind.CLIENT) as span: - span.set_attribute("db.system", "sqlite") - span.set_attribute("db.namespace", self.name) - span.set_attribute("db.query.text", sql_attribute(sql)) - span.set_attribute("datasette.executescript", True) + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) + span.set_attribute(EXECUTESCRIPT, True) results = await self.execute_write_fn( _inner, block=block, transaction=False, request=request ) @@ -324,22 +346,22 @@ class Database: with trace( "sql", database=self.name, sql=sql.strip(), executemany=True ) as kwargs: - with tracer.start_as_current_span("db.query", kind=SpanKind.CLIENT) as span: - span.set_attribute("db.system", "sqlite") - span.set_attribute("db.namespace", self.name) - span.set_attribute("db.query.text", sql_attribute(sql)) - span.set_attribute("datasette.executemany", True) + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) + span.set_attribute(EXECUTEMANY, True) # A single statement run with many parameter sets, so unlike # execute_write_script() there is exactly one operation to name. operation_name = sql_operation_name(sql) if operation_name: - span.set_attribute("db.operation.name", operation_name) + span.set_attribute(DB_OPERATION_NAME, operation_name) results, count = await self.execute_write_fn( _inner, block=block, request=request ) # count is the number of parameter *sets* consumed by # executemany(), not a row count - executemany returns no rows. - span.set_attribute("datasette.param_sets", count) + span.set_attribute(PARAM_SETS, count) kwargs["count"] = count return results @@ -588,7 +610,7 @@ class Database: # waiting in the queue (enqueue -> dequeue), not the near- # zero time spent constructing/ending the span object here. tracer.start_span( - "db.write.queue_wait", + DB_WRITE_QUEUE_WAIT, start_time=task.enqueued_at_ns, **write_span_kwargs, ).end(end_time=time.time_ns()) @@ -599,15 +621,13 @@ class Database: elif task.isolated_connection: try: with tracer.start_as_current_span( - "db.write.execute", **write_span_kwargs + DB_WRITE_EXECUTE, **write_span_kwargs ) as span: span.set_attribute( - "datasette.isolated_connection", + ISOLATED_CONNECTION, task.isolated_connection, ) - span.set_attribute( - "datasette.transaction", task.transaction - ) + span.set_attribute(TRANSACTION, task.transaction) isolated_connection = self.connect(write=True) try: result = task.fn(isolated_connection) @@ -628,15 +648,13 @@ class Database: else: try: with tracer.start_as_current_span( - "db.write.execute", **write_span_kwargs + DB_WRITE_EXECUTE, **write_span_kwargs ) as span: span.set_attribute( - "datasette.isolated_connection", + ISOLATED_CONNECTION, task.isolated_connection, ) - span.set_attribute( - "datasette.transaction", task.transaction - ) + span.set_attribute(TRANSACTION, task.transaction) if task.transaction: with conn: conn.execute("BEGIN IMMEDIATE") @@ -720,7 +738,7 @@ class Database: # db.query span in execute(). Without this, facet suggestion marks # two spans per text column as failed on every table page. with tracer.start_as_current_span( - "db.query.execute", + DB_QUERY_EXECUTE, record_exception=log_sql_errors, set_status_on_exception=log_sql_errors, ): @@ -764,27 +782,27 @@ class Database: # manager's defaults, so that callers passing log_sql_errors=False # can be honoured - see the comment on the generic handler below. with tracer.start_as_current_span( - "db.query", - kind=SpanKind.CLIENT, + DB_QUERY, + kind=DB_QUERY.kind, record_exception=False, set_status_on_exception=False, ) as span: - span.set_attribute("db.system", "sqlite") - span.set_attribute("db.namespace", self.name) - span.set_attribute("db.query.text", sql_attribute(sql)) - span.set_attribute("datasette.time_limit_ms", time_limit_ms) + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) + span.set_attribute(TIME_LIMIT_MS, time_limit_ms) operation_name = sql_operation_name(sql) if operation_name: - span.set_attribute("db.operation.name", operation_name) + span.set_attribute(DB_OPERATION_NAME, operation_name) if table: - span.set_attribute("db.collection.name", table) + span.set_attribute(DB_COLLECTION_NAME, table) if params: - span.set_attribute("datasette.param_count", len(params)) + span.set_attribute(PARAM_COUNT, len(params)) try: results = await self.execute_fn(sql_operation_in_thread) except QueryInterrupted as e: span.set_status(Status(StatusCode.ERROR, str(e))) - span.set_attribute("datasette.interrupted", True) + span.set_attribute(INTERRUPTED, True) span.record_exception(e) raise except Exception as e: @@ -799,10 +817,10 @@ class Database: span.record_exception(e) span.set_status(Status(StatusCode.ERROR, str(e))) else: - span.set_attribute("datasette.sql_error_suppressed", True) + span.set_attribute(SQL_ERROR_SUPPRESSED, True) raise - span.set_attribute("datasette.truncated", results.truncated) - span.set_attribute("datasette.rows_returned", len(results.rows)) + span.set_attribute(TRUNCATED, results.truncated) + span.set_attribute(ROWS_RETURNED, len(results.rows)) return results @property diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py new file mode 100644 index 00000000..40b32cc5 --- /dev/null +++ b/datasette/telemetry_registry.py @@ -0,0 +1,269 @@ +""" +The single source of truth for every span and span attribute that Datasette +core emits. + +Three things read this module, which is the point of it existing: + +1. **The instrumentation itself.** `Attribute` and `SpanName` subclass `str`, + so a registry entry *is* the string OpenTelemetry wants. Call sites pass + `DB_NAMESPACE` where they used to pass `"db.namespace"` - no wrapper API + over the OTel calls, no parallel structure to keep in step, and a typo is + now an `ImportError` instead of a silently misnamed attribute. + +2. **The documentation.** `docs/telemetry_doc.py` renders the span reference + in `docs/internals.rst` from these definitions using cog, and + `cog --check` runs in CI - so the docs cannot drift from the code. + +3. **A conformance test.** `tests/test_telemetry_registry.py` makes real + requests, collects every span and attribute actually emitted, and compares + both directions: emitted-but-unregistered catches instrumentation added + without documentation, registered-but-never-emitted catches documentation + describing something that no longer exists. Neither the type system nor + the generated docs can catch that second case. +""" + +from opentelemetry.trace import SpanKind + + +class Attribute(str): + """ + A span attribute key, carrying its own documentation. + + Subclasses `str` so it can be handed straight to `set_attribute()`. + """ + + __slots__ = ("description", "optional") + + def __new__(cls, name, description, optional=False): + self = super().__new__(cls, name) + self.description = description + self.optional = optional + return self + + def __repr__(self): + return f"Attribute({str(self)!r})" + + +class SpanName(str): + "A span name, carrying its documentation and the attributes it may set." + + __slots__ = ("attributes", "description", "kind", "prefix") + + def __new__( + cls, name, description, attributes=(), prefix=False, kind=SpanKind.INTERNAL + ): + self = super().__new__(cls, name) + self.description = description + self.attributes = tuple(attributes) + # True for a span family whose emitted names carry a variable suffix, + # so the conformance test matches by prefix rather than equality. + # Nothing sets it yet. + self.prefix = prefix + # SpanKind.INTERNAL by default - every span Datasette emits describes + # its own internal work. db.query is the one exception: it is a real + # database call, so semantic conventions (and trace UIs, which key + # their database styling off this) expect SpanKind.CLIENT. + self.kind = kind + return self + + def __repr__(self): + return f"SpanName({str(self)!r})" + + +# --- Attributes ----------------------------------------------------------- +# +# Shared attributes are defined once and referenced by every span that sets +# them, so "which spans carry db.namespace?" is answerable by grep. + +DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.") +DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.") +DB_QUERY_TEXT = Attribute( + "db.query.text", + "The SQL, truncated to 2048 characters. Never the parameter values.", +) +DB_OPERATION_NAME = Attribute( + "db.operation.name", + "The statement's leading keyword - ``SELECT``, ``INSERT``, ``CREATE``, and " + "so on - matched against a small fixed allowlist. Omitted rather than set " + "to an arbitrary value: the allowlist exists because this attribute is a " + "candidate dimension for a query-duration metric in a later phase, and " + "echoing an unrecognised first token from user-supplied SQL would be an " + "unbounded-cardinality hazard. Also omitted for " + "``execute_write_script()``, which runs multiple statements - per " + "semantic conventions, the operation name should not be extracted from " + "query text that can contain more than one operation. Note that a " + "statement beginning with a CTE reports ``WITH``, not the operation " + "inside it - a substantial share of Datasette's own reads take that " + "form. Resolving it further would mean parsing.", + optional=True, +) +DB_COLLECTION_NAME = Attribute( + "db.collection.name", + "The primary table, set only where the view already knows it - the table " + "and row pages. Omitted for arbitrary ``?sql=`` queries, where determining " + "the table would mean parsing the query.", + optional=True, +) + +PARAM_COUNT = Attribute( + "datasette.param_count", + "Number of bound parameters. Recorded instead of the values themselves.", + optional=True, +) +PARAM_SETS = Attribute( + "datasette.param_sets", + "Number of parameter sets consumed by ``execute_write_many()``. Not a row " + "count - ``executemany()`` returns no rows. The parameter values " + "themselves are never recorded: that sequence can hold thousands of rows.", + optional=True, +) +TIME_LIMIT_MS = Attribute( + "datasette.time_limit_ms", + "The :ref:`setting_sql_time_limit_ms` value this query ran under. Set on " + "reads, which are the queries that time limit applies to.", + optional=True, +) +ROWS_RETURNED = Attribute( + "datasette.rows_returned", + "Number of rows a read returned. Set on the read path only, and only when " + "the read succeeded.", + optional=True, +) +TRUNCATED = Attribute( + "datasette.truncated", + "True if the result was cut short by :ref:`setting_max_returned_rows`.", + optional=True, +) +INTERRUPTED = Attribute( + "datasette.interrupted", + "True if the query was cancelled for exceeding the time limit. The span " + "status is also set to ``ERROR``.", + optional=True, +) +SQL_ERROR_SUPPRESSED = Attribute( + "datasette.sql_error_suppressed", + "True when the query failed but the caller passed ``log_sql_errors=False``, " + "meaning it was probing and treats failure as an expected answer. Facet " + "suggestion does this against every column.", + optional=True, +) +EXECUTESCRIPT = Attribute( + "datasette.executescript", + "True for ``execute_write_script()``, which runs multiple statements.", + optional=True, +) +EXECUTEMANY = Attribute( + "datasette.executemany", + "True for ``execute_write_many()``, which runs one statement against many " + "parameter sets.", + optional=True, +) +ISOLATED_CONNECTION = Attribute( + "datasette.isolated_connection", + "True if the write ran on its own connection rather than the shared write " + "connection.", +) +TRANSACTION = Attribute( + "datasette.transaction", + "False for statements such as ``VACUUM`` that cannot run inside a transaction.", +) + + +# --- Spans ---------------------------------------------------------------- + +DB_QUERY = SpanName( + "db.query", + "A SQL operation issued by Datasette, covering the full round trip " + "including any time spent queued for a thread.", + ( + DB_SYSTEM, + DB_NAMESPACE, + DB_QUERY_TEXT, + DB_OPERATION_NAME, + DB_COLLECTION_NAME, + PARAM_COUNT, + PARAM_SETS, + TIME_LIMIT_MS, + ROWS_RETURNED, + TRUNCATED, + INTERRUPTED, + SQL_ERROR_SUPPRESSED, + EXECUTESCRIPT, + EXECUTEMANY, + ), + kind=SpanKind.CLIENT, +) + +DB_QUERY_EXECUTE = SpanName( + "db.query.execute", + "The read executing inside a SQL worker thread. Child of ``db.query``; the " + "gap between the two is time spent waiting for a thread.", +) + +DB_WRITE_QUEUE_WAIT = SpanName( + "db.write.queue_wait", + "Time a write spent waiting in its database's write queue before the write " + "thread picked it up. Child of ``db.query`` for a ``block=True`` write, " + "where the caller awaits the write and containment is accurate. For a " + "``block=False`` write the caller does not await it - the enqueueing " + "request *caused* the write without *containing* it, and the write's " + "spans can outlive the request's own - so this is a root span instead, " + "carrying an OpenTelemetry link back to the enqueueing span rather than " + "a parent. A link records causation without asserting containment, which " + "is exactly the distinction here.", +) + +DB_WRITE_EXECUTE = SpanName( + "db.write.execute", + "The write executing on the write thread. Child of ``db.query`` for a " + "``block=True`` write; for ``block=False`` a root span with a link back " + "to the enqueueing span instead - see ``db.write.queue_wait`` above.", + (ISOLATED_CONNECTION, TRANSACTION), +) + +STARTUP = SpanName( + "datasette.startup", + "``invoke_startup()`` running: ``register_events``, ``register_actions``, " + "``register_column_types``, ``prepare_jinja2_environment``, internal-database " + "schema catalog refresh (including the ``prepare_connection`` warm-up this " + "triggers for each database touched for the first time), saved queries, " + "column type config and the ``startup`` hook. Runs once per process, before " + "any request exists, so without this span every child it creates would be " + "its own orphan root trace. A connection warmed later - lazily, the first " + "time a *request* touches a new database or thread - nests under that " + "request's own span instead, not under this one, since this span has " + "already ended by then.", +) + +SPANS = ( + DB_QUERY, + DB_QUERY_EXECUTE, + DB_WRITE_QUEUE_WAIT, + DB_WRITE_EXECUTE, + STARTUP, +) + + +def span_for(emitted_name): + """ + Resolve an emitted span name to its registry entry, or None. + + Handles span families whose emitted names carry a suffix that is not + knowable in advance - `prefix=True` entries. Phase 1 has none, but the + lookup is what the conformance test calls, so it lives here rather than + in the test. + """ + for span in SPANS: + if span.prefix: + if emitted_name.startswith(span): + return span + elif emitted_name == span: + return span + return None + + +def attribute_allowed(span, emitted_key): + "Whether `emitted_key` is a registered attribute of `span`." + if span is None: + return False + return emitted_key in span.attributes diff --git a/docs/internals.rst b/docs/internals.rst index d2bd46ef..b2215d9d 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2313,6 +2313,78 @@ The ``Database`` class also provides properties and methods for introspecting th } } +.. _internals_telemetry: + +OpenTelemetry +============= + +Datasette core depends on `opentelemetry-api `__ only. It never creates a ``TracerProvider``, never configures an exporter and never sets a sampler. With no OpenTelemetry SDK provider installed, every span described below is a no-op ``NonRecordingSpan`` - the overhead is close to zero and nothing is recorded or exported anywhere. + +Turning tracing on is entirely an operational decision made outside of Datasette itself: run Datasette under the standard ``opentelemetry-instrument`` agent, or embed Datasette inside a host application that installs its own provider. + +Everything Datasette emits carries the instrumentation scope ``datasette``, versioned with the running Datasette version and declaring the `semantic conventions schema `__ its attribute names follow. + +Span reference +-------------- + +Attribute names use the ``datasette.*`` prefix for Datasette-specific data, alongside standard OpenTelemetry attributes such as ``db.system``. + +This reference is generated from ``datasette/telemetry_registry.py``, the single source of truth for every span and attribute Datasette emits. A conformance test makes real requests and compares what is actually emitted against that registry in both directions, so nothing here is hand-maintained and nothing can silently drift out of date. + +Spans are ``SpanKind.INTERNAL`` unless a kind is listed below. Only ``db.query`` is ``CLIENT``: it is the one span that represents a call to a database rather than Datasette's own work, and trace UIs use the kind to decide whether to render a span as a database call. Its children stay ``INTERNAL`` because they are Datasette's decomposition of that one query - marking them ``CLIENT`` too would make a single query look like several database calls to anything counting by kind. + +.. [[[cog + from telemetry_doc import spans + spans(cog) +.. ]]] + +``db.query`` + A SQL operation issued by Datasette, covering the full round trip including any time spent queued for a thread. + + Kind: ``CLIENT``. + + Attributes: + + - ``db.system`` - Always ``sqlite``. + - ``db.namespace`` - Name of the database being queried. + - ``db.query.text`` - The SQL, truncated to 2048 characters. Never the parameter values. + - ``db.operation.name`` *(optional)* - The statement's leading keyword - ``SELECT``, ``INSERT``, ``CREATE``, and so on - matched against a small fixed allowlist. Omitted rather than set to an arbitrary value: the allowlist exists because this attribute is a candidate dimension for a query-duration metric in a later phase, and echoing an unrecognised first token from user-supplied SQL would be an unbounded-cardinality hazard. Also omitted for ``execute_write_script()``, which runs multiple statements - per semantic conventions, the operation name should not be extracted from query text that can contain more than one operation. Note that a statement beginning with a CTE reports ``WITH``, not the operation inside it - a substantial share of Datasette's own reads take that form. Resolving it further would mean parsing. + - ``db.collection.name`` *(optional)* - The primary table, set only where the view already knows it - the table and row pages. Omitted for arbitrary ``?sql=`` queries, where determining the table would mean parsing the query. + - ``datasette.param_count`` *(optional)* - Number of bound parameters. Recorded instead of the values themselves. + - ``datasette.param_sets`` *(optional)* - Number of parameter sets consumed by ``execute_write_many()``. Not a row count - ``executemany()`` returns no rows. The parameter values themselves are never recorded: that sequence can hold thousands of rows. + - ``datasette.time_limit_ms`` *(optional)* - The :ref:`setting_sql_time_limit_ms` value this query ran under. Set on reads, which are the queries that time limit applies to. + - ``datasette.rows_returned`` *(optional)* - Number of rows a read returned. Set on the read path only, and only when the read succeeded. + - ``datasette.truncated`` *(optional)* - True if the result was cut short by :ref:`setting_max_returned_rows`. + - ``datasette.interrupted`` *(optional)* - True if the query was cancelled for exceeding the time limit. The span status is also set to ``ERROR``. + - ``datasette.sql_error_suppressed`` *(optional)* - True when the query failed but the caller passed ``log_sql_errors=False``, meaning it was probing and treats failure as an expected answer. Facet suggestion does this against every column. + - ``datasette.executescript`` *(optional)* - True for ``execute_write_script()``, which runs multiple statements. + - ``datasette.executemany`` *(optional)* - True for ``execute_write_many()``, which runs one statement against many parameter sets. + +``db.query.execute`` + The read executing inside a SQL worker thread. Child of ``db.query``; the gap between the two is time spent waiting for a thread. + + No attributes. + +``db.write.queue_wait`` + Time a write spent waiting in its database's write queue before the write thread picked it up. Child of ``db.query`` for a ``block=True`` write, where the caller awaits the write and containment is accurate. For a ``block=False`` write the caller does not await it - the enqueueing request *caused* the write without *containing* it, and the write's spans can outlive the request's own - so this is a root span instead, carrying an OpenTelemetry link back to the enqueueing span rather than a parent. A link records causation without asserting containment, which is exactly the distinction here. + + No attributes. + +``db.write.execute`` + The write executing on the write thread. Child of ``db.query`` for a ``block=True`` write; for ``block=False`` a root span with a link back to the enqueueing span instead - see ``db.write.queue_wait`` above. + + Attributes: + + - ``datasette.isolated_connection`` - True if the write ran on its own connection rather than the shared write connection. + - ``datasette.transaction`` - False for statements such as ``VACUUM`` that cannot run inside a transaction. + +``datasette.startup`` + ``invoke_startup()`` running: ``register_events``, ``register_actions``, ``register_column_types``, ``prepare_jinja2_environment``, internal-database schema catalog refresh (including the ``prepare_connection`` warm-up this triggers for each database touched for the first time), saved queries, column type config and the ``startup`` hook. Runs once per process, before any request exists, so without this span every child it creates would be its own orphan root trace. A connection warmed later - lazily, the first time a *request* touches a new database or thread - nests under that request's own span instead, not under this one, since this span has already ended by then. + + No attributes. + +.. [[[end]]] + .. _internals_csrf: CSRF protection diff --git a/docs/telemetry_doc.py b/docs/telemetry_doc.py new file mode 100644 index 00000000..3551ef0e --- /dev/null +++ b/docs/telemetry_doc.py @@ -0,0 +1,37 @@ +""" +Render the span reference in ``internals.rst`` from +``datasette/telemetry_registry.py``. + +Driven by cog, and ``cog --check docs/*.rst`` runs in CI - so adding a span +without documenting it, or documenting one that no longer exists, is a build +failure rather than something a reader discovers later. +""" + + +def _attribute_lines(cog, attributes): + if not attributes: + cog.out(" No attributes.\n\n") + return + cog.out(" Attributes:\n\n") + for attribute in attributes: + suffix = " *(optional)*" if attribute.optional else "" + cog.out(f" - ``{attribute}``{suffix} - {attribute.description}\n") + cog.out("\n") + + +def spans(cog): + from opentelemetry.trace import SpanKind + + from datasette.telemetry_registry import SPANS + + cog.out("\n") + for span in SPANS: + title = f"{span}*" if span.prefix else str(span) + cog.out(f"``{title}``\n") + cog.out(f" {span.description}\n\n") + # INTERNAL is the default and the overwhelming majority of spans - + # printing it on every one would be noise. Only the exceptional case, + # a real database call, is worth calling out. + if span.kind != SpanKind.INTERNAL: + cog.out(f" Kind: ``{span.kind.name}``.\n\n") + _attribute_lines(cog, span.attributes) diff --git a/tests/test_telemetry_registry.py b/tests/test_telemetry_registry.py new file mode 100644 index 00000000..335eb946 --- /dev/null +++ b/tests/test_telemetry_registry.py @@ -0,0 +1,309 @@ +""" +Two-way conformance between `datasette/telemetry_registry.py` and what +Datasette actually emits. + +This is the test that makes the generated documentation trustworthy. cog +guarantees the docs match the registry; this guarantees the registry matches +the code. Without it, both could agree with each other and be wrong. + +It checks both directions, and the second one is the one nothing else catches: + +- **emitted but not registered** - instrumentation was added without + documenting it, so the reference page silently omits it. +- **registered but never emitted** - the reference page describes a span or + attribute that no longer exists, which is worse than omitting it, because a + reader will build a dashboard on it. + +Both of those directions compare the code against the registry. Neither can +catch a *rename*, because the call sites now take their names from the +registry - move `DB_NAMESPACE` to `"db.namespace2"` and code and registry +still agree with each other, while every existing dashboard breaks. So the +literal names live here too, spelled out, and are asserted against both the +registry and the wire. That is the one comparison in this file that is not +made against a value derived from the registry itself. +""" + +import itertools + +import pytest +import pytest_asyncio + +pytest.importorskip("opentelemetry.sdk") + +from datasette import telemetry_registry as reg +from datasette.app import Datasette +from datasette.database import QueryInterrupted +from datasette.utils.sqlite import sqlite3 + +# The names as they appear on the wire, written out rather than read from the +# registry. If a change to the registry makes one of these fail, that change +# is renaming something a user's dashboards and saved queries depend on - +# which is a decision to take deliberately, here, not a line to re-derive. +EXPECTED_ATTRIBUTES = { + "db.query": { + "db.system", + "db.namespace", + "db.query.text", + "db.operation.name", + "db.collection.name", + "datasette.param_count", + "datasette.param_sets", + "datasette.time_limit_ms", + "datasette.rows_returned", + "datasette.truncated", + "datasette.interrupted", + "datasette.sql_error_suppressed", + "datasette.executescript", + "datasette.executemany", + }, + "db.query.execute": set(), + "db.write.queue_wait": set(), + "db.write.execute": { + "datasette.isolated_connection", + "datasette.transaction", + }, + "datasette.startup": set(), +} +EXPECTED_SPANS = set(EXPECTED_ATTRIBUTES) + +# Named in-memory databases are shared-cache, so two Datasette instances using +# the same name share one SQLite database - and the second `create table` +# fails. Every workload below therefore gets its own name. +_names = itertools.count() + + +def _unique(prefix): + return f"{prefix}{next(_names)}" + + +async def exercise(): + """ + Drive enough of Datasette to emit every span and attribute the registry + claims exists. + + Each call is here because it is the only thing that produces some span or + attribute - see the comments. If you add instrumentation on a path this + does not reach, add the path rather than loosening the assertions. + + Returns the instance so the caller can close it; startup happens inside + so that the `datasette.startup` span lands in the collected set. + """ + name = _unique("registry") + ds = Datasette(memory=True) + ds.add_memory_database(name) + # datasette.startup - and the internal catalog work nested under it + await ds.invoke_startup() + db = ds.get_database(name) + + # Writes: db.write.queue_wait, db.write.execute, db.query + await db.execute_write("create table t (id integer primary key, v text)") + # datasette.executemany, datasette.param_sets + await db.execute_write_many( + "insert into t (id, v) values (?, ?)", [[i, f"v{i}"] for i in range(30)] + ) + # datasette.executescript + await db.execute_write_script("create table t2 (id integer); drop table t2;") + # datasette.transaction=False - VACUUM cannot run inside a transaction + await db.execute_write("vacuum", transaction=False) + # datasette.isolated_connection=True + await db.execute_isolated_fn(lambda conn: conn.execute("select 1").fetchone()) + + # Reads: db.query.execute, datasette.rows_returned, datasette.truncated, + # datasette.param_count, datasette.time_limit_ms + await db.execute("select * from t where id > :n", {"n": 5}) + await db.execute("select * from t", truncate=True) + + # datasette.sql_error_suppressed - the caller is probing and treats + # failure as an expected answer + with pytest.raises(sqlite3.OperationalError): + await db.execute("select nope from t", log_sql_errors=False) + + # datasette.interrupted - only ever set when a query exceeds its time + # limit, so the workload has to force one rather than exempt it. An + # unbounded recursive CTE cannot finish, so 1ms is always exceeded. + with pytest.raises(QueryInterrupted): + await db.execute( + "with recursive c(x) as (select 0 union all select x+1 from c) " + "select * from c", + custom_time_limit=1, + ) + + # db.collection.name - set only by views that already know their table + assert (await ds.client.get(f"/{name}/t?_facet=v")).status_code == 200 + assert (await ds.client.get(f"/{name}/t/1.json")).status_code == 200 + return ds + + +@pytest_asyncio.fixture +async def emitted(otel_spans): + "Every span name and (span name, attribute key) pair a broad workload emits." + # otel_spans has already cleared the exporter, and nothing is cleared + # after this point: the workload's own startup emits datasette.startup. + ds = await exercise() + spans = otel_spans.get_finished_spans() + assert spans, "no spans captured - the fixture is not exercising anything" + names = set() + pairs = set() + for span in spans: + # str() because span.name is the registry's SpanName instance, and a + # set of those would compare equal to literals but read confusingly + # in a failure message. + names.add(str(span.name)) + for key in span.attributes or {}: + pairs.add((str(span.name), str(key))) + ds.close() + return {"names": names, "pairs": pairs} + + +def _keys_by_span(pairs): + by_span = {} + for span_name, key in pairs: + by_span.setdefault(span_name, set()).add(key) + return by_span + + +@pytest.mark.asyncio +async def test_workload_emits_exactly_the_expected_names(emitted): + """ + The wire format, pinned to literals. + + Not derived from the registry, so this is what catches a rename that the + registry and the call sites make together. + """ + assert emitted["names"] == EXPECTED_SPANS + by_span = _keys_by_span(emitted["pairs"]) + assert {name: by_span.get(name, set()) for name in emitted["names"]} == ( + EXPECTED_ATTRIBUTES + ) + + +def test_registry_matches_the_expected_names(): + "The other half of the rename check: the registry against the same literals." + assert {str(span) for span in reg.SPANS} == EXPECTED_SPANS + for span in reg.SPANS: + assert {str(attribute) for attribute in span.attributes} == EXPECTED_ATTRIBUTES[ + str(span) + ], f"{span} attributes have drifted" + + +@pytest.mark.asyncio +async def test_every_emitted_span_is_registered(emitted): + "A span added without a registry entry would be missing from the docs." + unregistered = sorted( + name for name in emitted["names"] if reg.span_for(name) is None + ) + assert ( + not unregistered + ), f"these spans are emitted but not in telemetry_registry.SPANS: {unregistered}" + + +@pytest.mark.asyncio +async def test_every_emitted_attribute_is_registered(emitted): + "An attribute added without a registry entry would be missing from the docs." + unregistered = sorted( + f"{span_name} -> {key}" + for span_name, key in emitted["pairs"] + if not reg.attribute_allowed(reg.span_for(span_name), key) + ) + assert ( + not unregistered + ), "these span attributes are emitted but not registered: " + ", ".join( + unregistered + ) + + +@pytest.mark.asyncio +async def test_every_registered_span_is_emitted(emitted): + """ + The direction nothing else catches: the docs must not describe a span that + no longer exists. + """ + missing = sorted( + str(span) + for span in reg.SPANS + if not any(reg.span_for(name) is span for name in emitted["names"]) + ) + assert not missing, ( + f"these spans are documented but never emitted by the workload: {missing}. " + "Either the instrumentation was removed, or exercise() no longer reaches it." + ) + + +@pytest.mark.asyncio +async def test_every_registered_attribute_is_emitted(emitted): + """ + Every registered attribute, optional or not, must actually be set at least + once by the workload. + + `optional` describes whether a reader should expect it on every span, not + whether the code still sets it - so an attribute deleted from the code but + left in the docs has to fail here even when it is marked optional. If a + new attribute only appears in some rare case, extend exercise() to reach + that case. + """ + by_span = _keys_by_span(emitted["pairs"]) + missing = [] + for span in reg.SPANS: + emitted_keys = by_span.get(str(span), set()) + for attribute in span.attributes: + if attribute not in emitted_keys: + missing.append(f"{span} -> {attribute}") + assert not missing, ( + "these attributes are documented but never emitted by the workload: " + + ", ".join(sorted(missing)) + ) + + +def test_registry_has_no_duplicate_names(): + assert len(set(reg.SPANS)) == len(reg.SPANS) + for span in reg.SPANS: + assert len(set(span.attributes)) == len( + span.attributes + ), f"{span} lists an attribute twice" + + +def test_registry_entries_are_documented(): + "Every entry carries a description - the docs are generated from these." + for span in reg.SPANS: + assert span.description.strip(), f"{span} has no description" + for attribute in span.attributes: + assert attribute.description.strip(), f"{span} -> {attribute} has none" + + +def test_registry_entries_are_usable_as_plain_strings(): + "The str subclassing is the whole reason call sites need no wrapper API." + assert isinstance(reg.DB_QUERY, str) + assert isinstance(reg.DB_NAMESPACE, str) + assert reg.DB_QUERY == "db.query" + assert reg.DB_NAMESPACE == "db.namespace" + assert f"{reg.DB_QUERY}.execute" == "db.query.execute" + + +def test_span_and_attribute_lookup(): + assert reg.span_for("db.query") is reg.DB_QUERY + assert reg.span_for("datasette.startup") is reg.STARTUP + assert reg.span_for("not.a.datasette.span") is None + assert reg.attribute_allowed(reg.DB_QUERY, "db.namespace") + assert not reg.attribute_allowed(reg.DB_QUERY, "db.namespace.extra") + assert not reg.attribute_allowed(reg.DB_QUERY, "datasette.isolated_connection") + assert not reg.attribute_allowed(None, "db.namespace") + + +def test_prefix_span_lookup(): + """ + `prefix=True` matching, exercised directly. + + Phase 1 registers no prefix spans, so without this the branch in + `span_for()` would be untested code that the conformance tests silently + never reach. + """ + hook = reg.SpanName("datasette.hook.", "A hypothetical span family", prefix=True) + original = reg.SPANS + reg.SPANS = original + (hook,) + try: + assert reg.span_for("datasette.hook.render_cell") is hook + assert reg.span_for("datasette.hook.anything") is hook + assert reg.span_for("datasette.hookish") is None + assert reg.span_for("db.query") is reg.DB_QUERY + finally: + reg.SPANS = original From b74231c190ce4ca71be5ce7f69513c63d30625fe Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 18:58:53 -0700 Subject: [PATCH 09/24] Stop marking a deliberately-short query budget as a span error Datasette has a family of callers that run a query under a tiny time limit and treat "did not finish" as a usable answer. table_counts() is the loudest: the homepage counts every table with a 10ms budget and stores None for the ones that blow it. The QueryInterrupted handler on the db.query span was unconditional, so on a two-table database that produced four ERROR spans - two db.query and two db.query.execute - on every homepage hit. Measured on a 30MB two-table database: 4 red spans before, 0 after. Honouring log_sql_errors here would have silenced none of it. Only the three ArrayFacet json_type() probes pass log_sql_errors=False, and they are not the queries that time out; table_counts() and ColumnFacet.suggest both leave it at its True default. The signal that does separate the two cases is the budget itself: a caller asking for less time than sql_time_limit_ms is saying the query may not finish. Keying off that needs no new API and no changes outside database.py. A query that runs out the instance-wide limit is still an error. datasette.interrupted is still set in every case - it is the signal worth having, and only the ERROR status becomes conditional. Its registry description said the status is "also set to ERROR" full stop, which is now wrong, and that string is published in docs/internals.rst. The inner db.query.execute span carried the same bug through set_status_on_exception=log_sql_errors, so its exception handling is now explicit, matching the db.query span above it. The context manager's flags apply to every exception type alike and this span has to tell two apart. test_query_interrupted_sets_error_status forced its timeout with ?_timelimit=5, which is exactly the signal now reclassified as expected. It now forces one via sql_time_limit_ms so it still tests what it was written to test. Also documents, at the copy_context() sites, that context propagation carries Datasette's non-OTel ContextVars into worker threads too. Verified harmless: nothing reads _skip_permission_checks, _permission_check_cache or _in_datasette_client off the event loop, and Context.run() restores the thread's previous context on return, so no value can reach the next task on the shared pool. Co-Authored-By: Claude Opus 5 --- datasette/database.py | 116 +++++++++++++++++++++--------- datasette/telemetry_registry.py | 6 +- docs/internals.rst | 2 +- tests/test_telemetry.py | 120 +++++++++++++++++++++++++++++--- 4 files changed, 201 insertions(+), 43 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 22984d6f..7114a691 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -394,6 +394,9 @@ class Database: # Context raises "RuntimeError: cannot enter context ... already # entered". This propagates the caller's otel context (e.g. the # enclosing db.query span) onto the worker thread. + # + # It also propagates every *other* ContextVar - see the note in + # execute_fn() for why that is safe. ctx = contextvars.copy_context() return await asyncio.get_running_loop().run_in_executor( self.ds.executor, ctx.run, _run @@ -695,6 +698,22 @@ class Database: # Context raises "RuntimeError: cannot enter context ... # already entered". This propagates the caller's otel context # (e.g. the enclosing db.query span) onto the worker thread. + # + # copy_context() is not selective: it also carries Datasette's own + # ContextVars - _skip_permission_checks and _permission_check_cache + # (datasette/permissions.py), _in_datasette_client (app.py) and, + # until the hand-rolled tracer goes, trace_task_id (tracer.py) - + # into worker threads, where they previously took their defaults. + # That is safe, for two reasons. Nothing reads them on a worker + # thread: the permission code that reads the first two is async and + # only ever runs on the event loop. And Context.run() restores the + # thread's previous context when the callable returns, so a value + # cannot outlive the submit that carried it and reach the next task + # on this shared pool - "skip permission checks" in particular can + # never bleed from one request into another's query. Where a value + # would be read - a plugin calling datasette.in_client() or trace() + # from inside an execute_fn callable - seeing the submitting + # request's value is the more accurate answer, not a leak. ctx = contextvars.copy_context() future = self.ds.executor.submit(ctx.run, in_thread) self._pending_execute_futures.add(future) @@ -722,7 +741,19 @@ class Database: self._check_not_closed() page_size = page_size or self.ds.page_size time_limit_ms = self.ds.sql_time_limit_ms - if custom_time_limit and custom_time_limit < time_limit_ms: + # A caller that hands in a budget shorter than the instance-wide + # sql_time_limit_ms is saying "this may not finish, and that is an + # answer I can use" - and every such caller in core does treat the + # timeout as normal: table_counts() stores None per table, facet + # suggestion moves on to the next column, autocomplete falls back to a + # prefix query. Those timeouts are therefore not span errors. Without + # this, the homepage alone emits one red span per table (it counts + # every table under a 10ms budget) on every single hit. + # + # A query that runs out the instance-wide limit is a different event - + # nobody asked for a short budget, so it stays an error. + timeout_expected = bool(custom_time_limit) and custom_time_limit < time_limit_ms + if timeout_expected: time_limit_ms = custom_time_limit def sql_operation_in_thread(conn): @@ -733,38 +764,53 @@ class Database: # databases) - so it parents correctly to the enclosing # db.query span despite running on a different thread. # - # Callers passing log_sql_errors=False are probing and treat a - # failure as an expected answer - see the matching handling on the - # db.query span in execute(). Without this, facet suggestion marks - # two spans per text column as failed on every table page. + # Exception handling is explicit rather than left to the context + # manager's flags, which apply to every exception type alike. This + # span needs to tell two apart: an expected timeout is never an + # error, while a genuine SQL failure is one unless the caller + # passed log_sql_errors=False, meaning it was probing and treats + # failure as an expected answer. Without the latter, facet + # suggestion marks two spans per text column as failed on every + # table page; without the former, so does every homepage hit. with tracer.start_as_current_span( DB_QUERY_EXECUTE, - record_exception=log_sql_errors, - set_status_on_exception=log_sql_errors, - ): - with sqlite_timelimit(conn, time_limit_ms): - try: - cursor = conn.cursor() - cursor.execute(sql, params if params is not None else {}) - max_returned_rows = self.ds.max_returned_rows - if max_returned_rows == page_size: - max_returned_rows += 1 - if max_returned_rows and truncate: - rows = cursor.fetchmany(max_returned_rows + 1) - truncated = len(rows) > max_returned_rows - rows = rows[:max_returned_rows] - else: - rows = cursor.fetchall() - truncated = False - except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: - if e.args == ("interrupted",): - raise QueryInterrupted(e, sql, params) - if log_sql_errors: - sys.stderr.write( - f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" - ) - sys.stderr.flush() - raise + record_exception=False, + set_status_on_exception=False, + ) as execute_span: + try: + with sqlite_timelimit(conn, time_limit_ms): + try: + cursor = conn.cursor() + cursor.execute(sql, params if params is not None else {}) + max_returned_rows = self.ds.max_returned_rows + if max_returned_rows == page_size: + max_returned_rows += 1 + if max_returned_rows and truncate: + rows = cursor.fetchmany(max_returned_rows + 1) + truncated = len(rows) > max_returned_rows + rows = rows[:max_returned_rows] + else: + rows = cursor.fetchall() + truncated = False + except (sqlite3.OperationalError, sqlite3.DatabaseError) as e: + if e.args == ("interrupted",): + raise QueryInterrupted(e, sql, params) + if log_sql_errors: + sys.stderr.write( + f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n" + ) + sys.stderr.flush() + raise + except QueryInterrupted as e: + if not timeout_expected: + execute_span.record_exception(e) + execute_span.set_status(Status(StatusCode.ERROR, str(e))) + raise + except Exception as e: + if log_sql_errors: + execute_span.record_exception(e) + execute_span.set_status(Status(StatusCode.ERROR, str(e))) + raise if truncate: return Results(rows, truncated, cursor.description) @@ -801,9 +847,13 @@ class Database: try: results = await self.execute_fn(sql_operation_in_thread) except QueryInterrupted as e: - span.set_status(Status(StatusCode.ERROR, str(e))) + # datasette.interrupted is set either way - it is the + # signal worth having. Only the ERROR status is + # conditional; see the timeout_expected comment above. span.set_attribute(INTERRUPTED, True) - span.record_exception(e) + if not timeout_expected: + span.set_status(Status(StatusCode.ERROR, str(e))) + span.record_exception(e) raise except Exception as e: # log_sql_errors=False means the caller is probing and diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py index 40b32cc5..2d0692a8 100644 --- a/datasette/telemetry_registry.py +++ b/datasette/telemetry_registry.py @@ -137,7 +137,11 @@ TRUNCATED = Attribute( INTERRUPTED = Attribute( "datasette.interrupted", "True if the query was cancelled for exceeding the time limit. The span " - "status is also set to ``ERROR``.", + "status is also set to ``ERROR``, unless the caller asked for a budget " + "shorter than :ref:`setting_sql_time_limit_ms` - as table counts, facet " + "suggestion and autocomplete all do - in which case running out of time " + "is an expected answer rather than a failure and the status is left " + "unset.", optional=True, ) SQL_ERROR_SUPPRESSED = Attribute( diff --git a/docs/internals.rst b/docs/internals.rst index b2215d9d..e7028637 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2355,7 +2355,7 @@ Spans are ``SpanKind.INTERNAL`` unless a kind is listed below. Only ``db.query`` - ``datasette.time_limit_ms`` *(optional)* - The :ref:`setting_sql_time_limit_ms` value this query ran under. Set on reads, which are the queries that time limit applies to. - ``datasette.rows_returned`` *(optional)* - Number of rows a read returned. Set on the read path only, and only when the read succeeded. - ``datasette.truncated`` *(optional)* - True if the result was cut short by :ref:`setting_max_returned_rows`. - - ``datasette.interrupted`` *(optional)* - True if the query was cancelled for exceeding the time limit. The span status is also set to ``ERROR``. + - ``datasette.interrupted`` *(optional)* - True if the query was cancelled for exceeding the time limit. The span status is also set to ``ERROR``, unless the caller asked for a budget shorter than :ref:`setting_sql_time_limit_ms` - as table counts, facet suggestion and autocomplete all do - in which case running out of time is an expected answer rather than a failure and the status is left unset. - ``datasette.sql_error_suppressed`` *(optional)* - True when the query failed but the caller passed ``log_sql_errors=False``, meaning it was probing and treats failure as an expected answer. Facet suggestion does this against every column. - ``datasette.executescript`` *(optional)* - True for ``execute_write_script()``, which runs multiple statements. - ``datasette.executemany`` *(optional)* - True for ``execute_write_many()``, which runs one statement against many parameter sets. diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 0a6d0192..0022883a 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -12,7 +12,7 @@ from opentelemetry import trace as otel_trace from opentelemetry.trace import SpanKind, StatusCode from datasette.app import Datasette -from datasette.database import Database +from datasette.database import Database, QueryInterrupted from datasette.telemetry import ( MAX_SQL_LENGTH, SCHEMA_URL, @@ -26,6 +26,15 @@ SECRET_PARAM_VALUE = "SUPER_SECRET_PARAM_VALUE_XYZ_123" INVALID_SQL = "select this_is_not_valid_sql from nowhere" +# Bounded so a broken time limit fails the test instead of hanging it, but far +# too long to finish inside any of the millisecond budgets used below. +SLOW_SQL = """ +with recursive counter(x) as ( + select 1 union all select x + 1 from counter where x < 50000000 +) +select max(x) from counter +""" + def _db_query_spans(otel_spans): return [span for span in otel_spans.get_finished_spans() if span.name == "db.query"] @@ -212,14 +221,22 @@ async def test_no_span_attribute_ever_contains_a_parameter_value(ds_client, otel @pytest.mark.asyncio -async def test_query_interrupted_sets_error_status(ds_client, otel_spans): - response = await ds_client.get( - "/fixtures/-/query.json", - params={"sql": "select sleep(0.05)", "_timelimit": 5}, - ) - assert response.status_code == 400 +async def test_query_interrupted_sets_error_status(otel_spans): + """ + A query that runs out the instance-wide sql_time_limit_ms is an error. - spans = _db_query_spans(otel_spans) + This used to force the timeout with `?_timelimit=5`, but a caller-supplied + budget shorter than the instance limit is now the signal that the timeout + was expected - see test_expected_timeout_is_not_a_span_error - so the + timeout has to come from the setting for this to still test what it was + written to test. + """ + ds = Datasette(memory=True, settings={"sql_time_limit_ms": 20}) + db = ds.add_memory_database("t09_instance_limit_timeout") + with pytest.raises(QueryInterrupted): + await db.execute(SLOW_SQL) + + spans = _spans_for_namespace(otel_spans, "t09_instance_limit_timeout") assert spans span = spans[-1] assert span.status.status_code == StatusCode.ERROR @@ -228,6 +245,93 @@ async def test_query_interrupted_sets_error_status(ds_client, otel_spans): assert all(event.name == "exception" for event in span.events) +async def _expected_timeout_count_span(otel_spans, database_name): + """ + Drive the real table_counts() path into a timeout; return its db.query span. + + table_counts() is where the headline instance of this lives: the homepage + counts every table under a 10ms budget and stores None for any table that + does not finish in time. Before this was fixed, a two-table database + produced four ERROR spans - two db.query and two db.query.execute - on + every single homepage hit. + """ + db = Datasette(memory=True).add_memory_database(database_name) + await db.execute_write("create table big (id integer primary key, t text)") + await db.execute_write_many( + "insert into big (t) values (?)", [["x" * 50] for _ in range(11000)] + ) + # count_limit caps the scan at 10001 rows, and below 20ms sqlite_timelimit() + # runs its progress handler on every VM instruction, so 1ms is not a close + # call - a scan of that size takes single-digit milliseconds at best. + counts = await db.table_counts(1) + assert counts == { + "big": None + }, "the count did not actually time out, so the rest of this test is vacuous" + + spans = [ + span + for span in _spans_for_namespace(otel_spans, database_name) + if "count(*)" in span.attributes["db.query.text"] + ] + assert len(spans) == 1 + return spans[0] + + +@pytest.mark.asyncio +async def test_expected_timeout_is_not_a_span_error(otel_spans): + span = await _expected_timeout_count_span(otel_spans, "t09_expected_timeout") + # The useful signal survives; only the red status goes away. + assert span.attributes["datasette.interrupted"] is True + assert span.status.status_code != StatusCode.ERROR + assert not [event for event in span.events if event.name == "exception"] + + +@pytest.mark.asyncio +async def test_expected_timeout_does_not_error_the_inner_execute_span(otel_spans): + """ + The same fix has to reach db.query.execute, which sets its own status. + + Half of the original bug lived here: the inner span passed + set_status_on_exception=log_sql_errors, and table_counts() leaves + log_sql_errors at its True default, so it went ERROR too. + """ + span = await _expected_timeout_count_span(otel_spans, "t09_expected_timeout_inner") + children = _children_named(otel_spans, "db.query.execute", span.context) + assert len(children) == 1 + child = children[0] + assert child.status.status_code != StatusCode.ERROR + assert not [event for event in child.events if event.name == "exception"] + + +@pytest.mark.asyncio +async def test_unexpected_timeout_is_still_a_span_error(otel_spans): + """ + A custom_time_limit *above* sql_time_limit_ms is not a short budget. + + This is the half of the rule that stops the fix collapsing into "never + report timeouts": the caller asked for 5 seconds, the instance overruled it + at 20ms, and nobody expected that. + """ + ds = Datasette(memory=True, settings={"sql_time_limit_ms": 20}) + db = ds.add_memory_database("t09_custom_limit_ignored") + with pytest.raises(QueryInterrupted): + await db.execute(SLOW_SQL, custom_time_limit=5000) + + spans = _spans_for_namespace(otel_spans, "t09_custom_limit_ignored") + assert spans + span = spans[-1] + # Proves the caller's larger budget really was discarded - otherwise this + # would be asserting on a query that ran under a 5s limit. + assert span.attributes["datasette.time_limit_ms"] == 20 + assert span.attributes["datasette.interrupted"] is True + assert span.status.status_code == StatusCode.ERROR + assert any(event.name == "exception" for event in span.events) + + children = _children_named(otel_spans, "db.query.execute", span.context) + assert len(children) == 1 + assert children[0].status.status_code == StatusCode.ERROR + + @pytest.mark.asyncio async def test_unsuppressed_sql_error_is_a_span_error(ds_client, otel_spans): db = ds_client.ds.get_database("fixtures") From 1052dc5c7b676193d175810f8f97580aae6b4493 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 19:10:04 -0700 Subject: [PATCH 10/24] Document what the database-layer spans emit, and how to turn them on The span reference itself is generated from the registry, so this adds the prose the generated list cannot supply: how to actually see a span, what is deliberately never recorded, and where the instrumentation stops short. The "how to turn it on" part is the part people get wrong. Core installs no provider, so OTEL_TRACES_EXPORTER=console against a plain `datasette` process emits nothing at all - that variable is read by the SDK auto-configuration which only runs under `opentelemetry-instrument`. Documented as a warning because it reads like a bug when you hit it. Two more measured facts get the same treatment: the SDK's BatchSpanProcessor default schedule delay is 5000ms (checked, not assumed - `BatchSpanProcessor._default_schedule_delay_millis()` on opentelemetry-sdk 1.44), so nothing appears for five seconds; and without OTEL_SERVICE_NAME the default resource reports service.name=unknown_service. Privacy properties are stated positively rather than left implicit: SQL truncated at 2048 characters, parameter values never recorded, no actor identifiers, table names only from an explicit `table=` argument. The last of those is now documented on db.execute() itself, since it is public API. The limitations section claims only what was measured. An earlier draft said two traces per process are orphaned by the register_output_renderer and asgi_wrapper hooks; measuring it showed a default install emits zero spans from either, because Datasette queries no database there - it is a plugin that would produce the orphan. Corrected to say that. It also deliberately does NOT say an embedder must install its provider before Datasette's first span or get nothing. That claim is false: ProxyTracer._tracer returns the no-op tracer without caching it when no provider is set, so early spans are dropped and nothing is poisoned. The telemetry.py docstring said no-op spans "cost approximately nothing". The benchmark for this diff does not support a claim that strong - a table page emits ~58 spans - so it now states the measurement instead: median 9.80ms to 9.98ms across 15 runs, inside a 1.4ms run-to-run spread. Co-Authored-By: Claude Opus 5 --- datasette/telemetry.py | 10 +++++-- docs/changelog.rst | 10 +++++++ docs/internals.rst | 62 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/datasette/telemetry.py b/datasette/telemetry.py index 37195584..3a205cee 100644 --- a/datasette/telemetry.py +++ b/datasette/telemetry.py @@ -5,8 +5,14 @@ Core depends on `opentelemetry-api` only. It never creates a `TracerProvider`, never configures an exporter, and never touches sampling - that is the responsibility of whoever is running Datasette (an `opentelemetry-instrument` agent, a future plugin, or a test -harness). With no provider installed every span produced here is a -`NonRecordingSpan` and costs approximately nothing. +harness). + +With no provider installed every span produced here is a +`NonRecordingSpan`. That is not free - a table page emits ~58 spans - +but it is below what an end-to-end page benchmark can resolve: measured +across 15 runs of a 5,000-row table page, the median moved 9.80ms to +9.98ms while run-to-run spread was 1.4ms. Installing an SDK provider is +what costs something measurable. """ import re diff --git a/docs/changelog.rst b/docs/changelog.rst index 66a7caab..cec8e92d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,16 @@ Changelog ========= +.. _v_unreleased: + +Unreleased +---------- + +- Datasette's database layer now emits `OpenTelemetry `__ spans: one per query, covering the full round trip including time spent waiting for a SQL worker thread, plus separate child spans for the execution itself and for time spent in the write queue. Datasette core depends on ``opentelemetry-api`` only and never installs an SDK provider, an exporter or a sampler, so there is no effect and no measurable overhead unless tracing is switched on externally - normally with the standard ``opentelemetry-instrument`` agent. See :ref:`internals_telemetry`. (:issue:`1730`) +- :ref:`db.execute(sql, ..., table=None) ` has a new optional ``table=`` parameter, naming the table a query is about so it can be recorded on that query's OpenTelemetry span. It has no effect on query execution, and Datasette never derives it from the SQL. (:issue:`1730`) + +Nothing is removed by this change: the ``?_trace=1`` query string parameter, the ``trace_debug`` setting and the :ref:`internals_tracer` module all continue to work as before. + .. _v1_0_a38: 1.0a38 (2026-08-06) diff --git a/docs/internals.rst b/docs/internals.rst index e7028637..31666e0e 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1963,6 +1963,9 @@ Executes a SQL query against the database and returns the resulting rows (see :r ``log_sql_errors`` - boolean Should any SQL errors be logged to the console in addition to being raised as an error? Defaults to ``True``. +``table`` - string + The name of the table this query is about, if the caller already knows it. This has no effect on how the query executes - it is recorded as the ``db.collection.name`` attribute on the :ref:`OpenTelemetry span ` for the query. Datasette never derives this from the SQL, so leave it unset for queries that do not have one obvious table. + .. _database_results: Results @@ -2318,16 +2321,47 @@ The ``Database`` class also provides properties and methods for introspecting th OpenTelemetry ============= -Datasette core depends on `opentelemetry-api `__ only. It never creates a ``TracerProvider``, never configures an exporter and never sets a sampler. With no OpenTelemetry SDK provider installed, every span described below is a no-op ``NonRecordingSpan`` - the overhead is close to zero and nothing is recorded or exported anywhere. +Datasette core depends on `opentelemetry-api `__ only. It never creates a ``TracerProvider``, never configures an exporter and never sets a sampler. With no OpenTelemetry SDK provider installed, every span described below is a no-op ``NonRecordingSpan``: nothing is recorded, nothing is exported, and the cost does not show up in page latency. Benchmarking a table page with and without this instrumentation, the median moved by less than the run-to-run variation of the benchmark itself. Turning tracing on is entirely an operational decision made outside of Datasette itself: run Datasette under the standard ``opentelemetry-instrument`` agent, or embed Datasette inside a host application that installs its own provider. Everything Datasette emits carries the instrumentation scope ``datasette``, versioned with the running Datasette version and declaring the `semantic conventions schema `__ its attribute names follow. +This is separate from, and does not replace, the built-in :ref:`internals_tracer` mechanism behind ``?_trace=1`` and the :ref:`setting_trace_debug` setting. Both continue to work exactly as before. + +.. _internals_telemetry_turning_on: + +Turning tracing on +------------------ + +Install an OpenTelemetry SDK, an exporter and the instrumentation agent, then launch Datasette through ``opentelemetry-instrument``: + +.. code-block:: bash + + pip install opentelemetry-distro opentelemetry-exporter-otlp + OTEL_SERVICE_NAME=datasette \ + OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \ + OTEL_METRICS_EXPORTER=none \ + OTEL_LOGS_EXPORTER=none \ + opentelemetry-instrument datasette mydb.db + +Point ``OTEL_EXPORTER_OTLP_ENDPOINT`` at whichever tracing backend you use. To print spans straight to the terminal instead, with no backend at all, drop that variable and set ``OTEL_TRACES_EXPORTER=console`` in its place. + +A few things catch people out the first time: + +.. warning:: + ``OTEL_TRACES_EXPORTER=console datasette mydb.db`` produces **nothing**. That environment variable is read by the OpenTelemetry SDK's auto-configuration, which only runs when the ``opentelemetry-instrument`` agent wraps the process. Datasette core installs no provider, so a plain ``datasette`` process emits nothing at all, whatever ``OTEL_`` variables are set. + +Spans do not appear immediately. The SDK's default ``BatchSpanProcessor`` flushes on a timer, every 5 seconds. Either wait, or stop the process - shutdown triggers a final flush - or set ``OTEL_BSP_SCHEDULE_DELAY=1000`` while you are experimenting. That last one is for demos, not for production. + +Always set ``OTEL_SERVICE_NAME``. Without it the SDK's default resource reports a ``service.name`` of ``unknown_service``, and your traces will be filed under that instead of under a name you can search for. + +Setting ``OTEL_METRICS_EXPORTER=none`` and ``OTEL_LOGS_EXPORTER=none`` is worth doing unless your backend accepts those signals too - ``opentelemetry-distro`` defaults every signal to OTLP, and a traces-only backend will reject the other two noisily. Datasette itself emits no metrics and no logs through OpenTelemetry. + Span reference -------------- -Attribute names use the ``datasette.*`` prefix for Datasette-specific data, alongside standard OpenTelemetry attributes such as ``db.system``. +Datasette emits five spans. Four of them describe the database layer - one per query, one for the work that query does inside a SQL worker thread, and two more for the write queue - and the fifth covers startup. Attribute names use the ``datasette.*`` prefix for Datasette-specific data, alongside standard OpenTelemetry attributes such as ``db.system``. This reference is generated from ``datasette/telemetry_registry.py``, the single source of truth for every span and attribute Datasette emits. A conformance test makes real requests and compares what is actually emitted against that registry in both directions, so nothing here is hand-maintained and nothing can silently drift out of date. @@ -2385,6 +2419,30 @@ Spans are ``SpanKind.INTERNAL`` unless a kind is listed below. Only ``db.query`` .. [[[end]]] +.. _internals_telemetry_privacy: + +Privacy and safety +------------------ + +Spans leave your infrastructure whenever you configure an exporter, so what goes into them is a security decision. Datasette's rules are: + +- **SQL text is truncated to 2048 characters.** On a public instance the SQL is supplied by visitors and is unbounded in length, so ``db.query.text`` is cut off - with a ``…[truncated]`` marker - rather than allowed to set the size of a span. +- **SQL parameter values are never recorded.** Only ``datasette.param_count``, a count. Parameter values are the part of a query most likely to hold something sensitive, and separating them from the SQL is the reason bound parameters exist. +- **No actor identifiers are recorded.** No actor ID, no actor JSON, no client IP address. Nothing on a span identifies who made the request. +- **Table names come only from an explicit** ``table=`` **argument.** ``db.collection.name`` is set by callers that already know which table they are working with, and is never derived from the SQL. Deriving it would mean parsing, and on an instance where visitors can create tables the set of possible values has no ceiling. + +The SQL itself, though, *is* recorded, and on a public instance that means anything a visitor types into the query editor or passes as ``?sql=`` will be exported along with the span. That is the trade-off tracing a query engine makes. + +.. _internals_telemetry_limitations: + +Known limitations +----------------- + +- **Datasette does not create a span for the HTTP request itself.** Every span listed above is therefore a root span unless something above Datasette - an ASGI instrumentation layer, or the web framework embedding it - has already started one for the request, in which case Datasette's spans nest underneath it correctly. +- **Two plugin hooks run outside the** ``datasette.startup`` **span.** ``register_output_renderer`` is dispatched from ``Datasette.__init__()`` and ``asgi_wrapper`` from ``Datasette.app()``, both of which happen before ``invoke_startup()``. Datasette itself queries no database in either, so a default install emits nothing there - but a plugin that does will produce a root trace. Covering these would mean holding a span open across object construction, which is worse than the orphan. +- ``db.operation.name`` **reports** ``WITH`` **for a statement that opens with a common table expression**, rather than the operation inside it, and a substantial share of Datasette's own reads take that form. The attribute is a leading-keyword match against a fixed allowlist, deliberately not a parse. +- **Spans emitted before a provider is installed are not recorded.** If you are embedding Datasette in a host application, install your ``TracerProvider`` before serving traffic. This is ordinary OpenTelemetry behaviour rather than anything Datasette controls; nothing is permanently affected, those particular spans are simply dropped. + .. _internals_csrf: CSRF protection From 69162875038bad1f7186860264d494153c4a7721 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Wed, 2 Sep 2026 11:25:10 -0700 Subject: [PATCH 11/24] Review polish: drop unused prefix machinery, tighten comments and docs - Remove the registry's unused prefix=True slot, its span_for() branch, its doc-rendering case and its test - nothing in the stack sets it. - Stop promising a "later phase" query-duration metric dimension in the db.operation.name description; the cardinality rationale stands alone. - Replace baked-in benchmark numbers in the telemetry module docstring with the docs' own phrasing (below run-to-run variation). - Compact the duplicated copy_context() and enqueue-site comments in database.py to pointers at their canonical tellings. - Make the "catch people out" gotchas skimmable as a bullet list and give the changelog's "nothing is removed" line a clear antecedent. - Add a test that a result cut short by max_returned_rows records datasette.truncated=True - previously only ever asserted False. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012U7coQfVu8nK2R4q2mCULA --- datasette/database.py | 20 ++++++-------------- datasette/telemetry.py | 7 +++---- datasette/telemetry_registry.py | 28 ++++++++-------------------- docs/changelog.rst | 2 +- docs/internals.rst | 8 ++++---- docs/telemetry_doc.py | 3 +-- tests/test_telemetry.py | 23 +++++++++++++++++++++++ tests/test_telemetry_registry.py | 20 -------------------- 8 files changed, 46 insertions(+), 65 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 7114a691..300a71f6 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -389,14 +389,9 @@ class Database: if not write: # Immutable database - no writes can ever occur, so there is no # write queue to block; run against a fresh read-only connection. - # A fresh copy_context() is required per submit (not one shared - # copy reused across calls): concurrent execution of the same - # Context raises "RuntimeError: cannot enter context ... already - # entered". This propagates the caller's otel context (e.g. the - # enclosing db.query span) onto the worker thread. - # - # It also propagates every *other* ContextVar - see the note in - # execute_fn() for why that is safe. + # copy_context() carries the caller's otel context onto the worker + # thread - see the notes in execute_fn() for why it must be a + # fresh copy per submit and why carrying every ContextVar is safe. ctx = contextvars.copy_context() return await asyncio.get_running_loop().run_in_executor( self.ds.executor, ctx.run, _run @@ -504,12 +499,9 @@ class Database: task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io") loop = asyncio.get_running_loop() reply_future = loop.create_future() - # Captured here, on the event loop, at enqueue time: the otel - # Context (carrying the enclosing db.query span, if any) and the - # timestamp used to build the db.write.queue_wait span once this - # task is dequeued on the write thread. `block` travels with the - # task too, because it decides whether that context is this task's - # parent or only a link target - see `_execute_writes`. + # The otel Context and enqueue timestamp are captured here, on the + # event loop, for the db.write.queue_wait span built at dequeue time. + # `block` travels too - it decides parent vs. link; see `_execute_writes`. self._write_queue.put( WriteTask( fn, diff --git a/datasette/telemetry.py b/datasette/telemetry.py index 3a205cee..3705d89f 100644 --- a/datasette/telemetry.py +++ b/datasette/telemetry.py @@ -9,10 +9,9 @@ harness). With no provider installed every span produced here is a `NonRecordingSpan`. That is not free - a table page emits ~58 spans - -but it is below what an end-to-end page benchmark can resolve: measured -across 15 runs of a 5,000-row table page, the median moved 9.80ms to -9.98ms while run-to-run spread was 1.4ms. Installing an SDK provider is -what costs something measurable. +but end-to-end page benchmarks put the overhead below their own +run-to-run variation. Installing an SDK provider is what costs +something measurable. """ import re diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py index 2d0692a8..c2b9db32 100644 --- a/datasette/telemetry_registry.py +++ b/datasette/telemetry_registry.py @@ -47,18 +47,12 @@ class Attribute(str): class SpanName(str): "A span name, carrying its documentation and the attributes it may set." - __slots__ = ("attributes", "description", "kind", "prefix") + __slots__ = ("attributes", "description", "kind") - def __new__( - cls, name, description, attributes=(), prefix=False, kind=SpanKind.INTERNAL - ): + def __new__(cls, name, description, attributes=(), kind=SpanKind.INTERNAL): self = super().__new__(cls, name) self.description = description self.attributes = tuple(attributes) - # True for a span family whose emitted names carry a variable suffix, - # so the conformance test matches by prefix rather than equality. - # Nothing sets it yet. - self.prefix = prefix # SpanKind.INTERNAL by default - every span Datasette emits describes # its own internal work. db.query is the one exception: it is a real # database call, so semantic conventions (and trace UIs, which key @@ -85,10 +79,9 @@ DB_OPERATION_NAME = Attribute( "db.operation.name", "The statement's leading keyword - ``SELECT``, ``INSERT``, ``CREATE``, and " "so on - matched against a small fixed allowlist. Omitted rather than set " - "to an arbitrary value: the allowlist exists because this attribute is a " - "candidate dimension for a query-duration metric in a later phase, and " - "echoing an unrecognised first token from user-supplied SQL would be an " - "unbounded-cardinality hazard. Also omitted for " + "to an arbitrary value: the attribute must stay safe to use as a metric " + "dimension, and echoing an unrecognised first token from user-supplied " + "SQL would be an unbounded-cardinality hazard. Also omitted for " "``execute_write_script()``, which runs multiple statements - per " "semantic conventions, the operation name should not be extracted from " "query text that can contain more than one operation. Note that a " @@ -252,16 +245,11 @@ def span_for(emitted_name): """ Resolve an emitted span name to its registry entry, or None. - Handles span families whose emitted names carry a suffix that is not - knowable in advance - `prefix=True` entries. Phase 1 has none, but the - lookup is what the conformance test calls, so it lives here rather than - in the test. + The lookup is what the conformance test calls, so it lives here rather + than in the test. """ for span in SPANS: - if span.prefix: - if emitted_name.startswith(span): - return span - elif emitted_name == span: + if emitted_name == span: return span return None diff --git a/docs/changelog.rst b/docs/changelog.rst index cec8e92d..03d913b1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,7 +12,7 @@ Unreleased - Datasette's database layer now emits `OpenTelemetry `__ spans: one per query, covering the full round trip including time spent waiting for a SQL worker thread, plus separate child spans for the execution itself and for time spent in the write queue. Datasette core depends on ``opentelemetry-api`` only and never installs an SDK provider, an exporter or a sampler, so there is no effect and no measurable overhead unless tracing is switched on externally - normally with the standard ``opentelemetry-instrument`` agent. See :ref:`internals_telemetry`. (:issue:`1730`) - :ref:`db.execute(sql, ..., table=None) ` has a new optional ``table=`` parameter, naming the table a query is about so it can be recorded on that query's OpenTelemetry span. It has no effect on query execution, and Datasette never derives it from the SQL. (:issue:`1730`) -Nothing is removed by this change: the ``?_trace=1`` query string parameter, the ``trace_debug`` setting and the :ref:`internals_tracer` module all continue to work as before. +Nothing is removed by the OpenTelemetry work: the ``?_trace=1`` query string parameter, the ``trace_debug`` setting and the :ref:`internals_tracer` module all continue to work as before. .. _v1_0_a38: diff --git a/docs/internals.rst b/docs/internals.rst index 31666e0e..aee51969 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2352,11 +2352,11 @@ A few things catch people out the first time: .. warning:: ``OTEL_TRACES_EXPORTER=console datasette mydb.db`` produces **nothing**. That environment variable is read by the OpenTelemetry SDK's auto-configuration, which only runs when the ``opentelemetry-instrument`` agent wraps the process. Datasette core installs no provider, so a plain ``datasette`` process emits nothing at all, whatever ``OTEL_`` variables are set. -Spans do not appear immediately. The SDK's default ``BatchSpanProcessor`` flushes on a timer, every 5 seconds. Either wait, or stop the process - shutdown triggers a final flush - or set ``OTEL_BSP_SCHEDULE_DELAY=1000`` while you are experimenting. That last one is for demos, not for production. +- **Spans do not appear immediately.** The SDK's default ``BatchSpanProcessor`` flushes on a timer, every 5 seconds. Either wait, or stop the process - shutdown triggers a final flush - or set ``OTEL_BSP_SCHEDULE_DELAY=1000`` while you are experimenting. That last one is for demos, not for production. -Always set ``OTEL_SERVICE_NAME``. Without it the SDK's default resource reports a ``service.name`` of ``unknown_service``, and your traces will be filed under that instead of under a name you can search for. +- **Always set** ``OTEL_SERVICE_NAME``. Without it the SDK's default resource reports a ``service.name`` of ``unknown_service``, and your traces will be filed under that instead of under a name you can search for. -Setting ``OTEL_METRICS_EXPORTER=none`` and ``OTEL_LOGS_EXPORTER=none`` is worth doing unless your backend accepts those signals too - ``opentelemetry-distro`` defaults every signal to OTLP, and a traces-only backend will reject the other two noisily. Datasette itself emits no metrics and no logs through OpenTelemetry. +- **Setting** ``OTEL_METRICS_EXPORTER=none`` **and** ``OTEL_LOGS_EXPORTER=none`` is worth doing unless your backend accepts those signals too - ``opentelemetry-distro`` defaults every signal to OTLP, and a traces-only backend will reject the other two noisily. Datasette itself emits no metrics and no logs through OpenTelemetry. Span reference -------------- @@ -2382,7 +2382,7 @@ Spans are ``SpanKind.INTERNAL`` unless a kind is listed below. Only ``db.query`` - ``db.system`` - Always ``sqlite``. - ``db.namespace`` - Name of the database being queried. - ``db.query.text`` - The SQL, truncated to 2048 characters. Never the parameter values. - - ``db.operation.name`` *(optional)* - The statement's leading keyword - ``SELECT``, ``INSERT``, ``CREATE``, and so on - matched against a small fixed allowlist. Omitted rather than set to an arbitrary value: the allowlist exists because this attribute is a candidate dimension for a query-duration metric in a later phase, and echoing an unrecognised first token from user-supplied SQL would be an unbounded-cardinality hazard. Also omitted for ``execute_write_script()``, which runs multiple statements - per semantic conventions, the operation name should not be extracted from query text that can contain more than one operation. Note that a statement beginning with a CTE reports ``WITH``, not the operation inside it - a substantial share of Datasette's own reads take that form. Resolving it further would mean parsing. + - ``db.operation.name`` *(optional)* - The statement's leading keyword - ``SELECT``, ``INSERT``, ``CREATE``, and so on - matched against a small fixed allowlist. Omitted rather than set to an arbitrary value: the attribute must stay safe to use as a metric dimension, and echoing an unrecognised first token from user-supplied SQL would be an unbounded-cardinality hazard. Also omitted for ``execute_write_script()``, which runs multiple statements - per semantic conventions, the operation name should not be extracted from query text that can contain more than one operation. Note that a statement beginning with a CTE reports ``WITH``, not the operation inside it - a substantial share of Datasette's own reads take that form. Resolving it further would mean parsing. - ``db.collection.name`` *(optional)* - The primary table, set only where the view already knows it - the table and row pages. Omitted for arbitrary ``?sql=`` queries, where determining the table would mean parsing the query. - ``datasette.param_count`` *(optional)* - Number of bound parameters. Recorded instead of the values themselves. - ``datasette.param_sets`` *(optional)* - Number of parameter sets consumed by ``execute_write_many()``. Not a row count - ``executemany()`` returns no rows. The parameter values themselves are never recorded: that sequence can hold thousands of rows. diff --git a/docs/telemetry_doc.py b/docs/telemetry_doc.py index 3551ef0e..d0af5dbd 100644 --- a/docs/telemetry_doc.py +++ b/docs/telemetry_doc.py @@ -26,8 +26,7 @@ def spans(cog): cog.out("\n") for span in SPANS: - title = f"{span}*" if span.prefix else str(span) - cog.out(f"``{title}``\n") + cog.out(f"``{span}``\n") cog.out(f" {span.description}\n\n") # INTERNAL is the default and the overwhelming majority of spans - # printing it on every one would be noise. Only the exceptional case, diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 0022883a..83e13b6e 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -147,6 +147,29 @@ async def test_db_query_span_basic_attributes(ds_client, otel_spans): assert span.status.status_code == StatusCode.UNSET +@pytest.mark.asyncio +async def test_truncated_result_sets_truncated_attribute(otel_spans): + """ + A result actually cut short by max_returned_rows records truncated=True. + + Every other test asserts the attribute is False, so a regression that + recorded the flag before the slice (or inverted it) would pass the rest + of the suite. + """ + ds = Datasette(memory=True, settings={"max_returned_rows": 5}) + db = ds.add_memory_database("t04_truncated") + results = await db.execute( + "select value from json_each('[1,2,3,4,5,6,7,8,9,10]')", truncate=True + ) + assert results.truncated + + spans = _spans_for_namespace(otel_spans, "t04_truncated") + assert spans + span = spans[-1] + assert span.attributes["datasette.truncated"] is True + assert span.attributes["datasette.rows_returned"] == 5 + + @pytest.mark.asyncio async def test_facetable_request_produces_db_query_spans(ds_client, otel_spans): response = await ds_client.get("/fixtures/facetable.json") diff --git a/tests/test_telemetry_registry.py b/tests/test_telemetry_registry.py index 335eb946..75a183b4 100644 --- a/tests/test_telemetry_registry.py +++ b/tests/test_telemetry_registry.py @@ -287,23 +287,3 @@ def test_span_and_attribute_lookup(): assert not reg.attribute_allowed(reg.DB_QUERY, "db.namespace.extra") assert not reg.attribute_allowed(reg.DB_QUERY, "datasette.isolated_connection") assert not reg.attribute_allowed(None, "db.namespace") - - -def test_prefix_span_lookup(): - """ - `prefix=True` matching, exercised directly. - - Phase 1 registers no prefix spans, so without this the branch in - `span_for()` would be untested code that the conformance tests silently - never reach. - """ - hook = reg.SpanName("datasette.hook.", "A hypothetical span family", prefix=True) - original = reg.SPANS - reg.SPANS = original + (hook,) - try: - assert reg.span_for("datasette.hook.render_cell") is hook - assert reg.span_for("datasette.hook.anything") is hook - assert reg.span_for("datasette.hookish") is None - assert reg.span_for("db.query") is reg.DB_QUERY - finally: - reg.SPANS = original From f73128dea6f7e944fa654552d133e230168e44d6 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Wed, 2 Sep 2026 11:48:56 -0700 Subject: [PATCH 12/24] Trace callback-style calls: execute_fn, execute_write_fn, execute_isolated_fn The database instrumentation covered the four SQL-string entry points but not the callback entry points, which are the documented way for plugins to run arbitrary SQL - so the JSON write API's inserts and deletes, the startup catalog scan, and every plugin built on execute_fn/execute_write_fn were invisible to a trace, or worse, showed orphan-looking db.write.* spans with no db.query above them. Each callback method now opens the same db.query CLIENT span as its SQL-string sibling, carrying a new optional datasette.callback attribute (the callable's qualified name, captured before _wrap_fn_with_hooks() can rename it) in place of db.query.text, which is now marked optional. A bare execute_fn() also wraps the callback in a db.query.execute child, so the "gap between the spans is thread-wait" story holds for plugin callbacks too. No db.operation.name: there is no statement to take a keyword from, and the registry says that attribute is omitted rather than guessed. The previous bodies move to private _execute_fn()/_execute_write_fn() and the SQL-string methods call those, so an execute() emits exactly the spans it did before - pinned by test_execute_does_not_double_wrap. Database's own introspection helpers stay on the public method deliberately: they are real SQLite round trips, which lifts a table page from ~58 to ~100 (no-op) spans. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012U7coQfVu8nK2R4q2mCULA --- datasette/database.py | 106 ++++++++++++---- datasette/telemetry.py | 12 +- datasette/telemetry_registry.py | 24 +++- docs/changelog.rst | 2 +- docs/internals.rst | 9 +- tests/test_telemetry.py | 206 ++++++++++++++++++++++++++++++- tests/test_telemetry_registry.py | 12 ++ 7 files changed, 342 insertions(+), 29 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 300a71f6..f8adc5fa 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -17,8 +17,9 @@ from opentelemetry import context as otel_context_api from opentelemetry.trace import Link, Status, StatusCode, get_current_span from .inspect import inspect_hash -from .telemetry import sql_attribute, sql_operation_name, tracer +from .telemetry import callback_name, sql_attribute, sql_operation_name, tracer from .telemetry_registry import ( + CALLBACK, DB_COLLECTION_NAME, DB_NAMESPACE, DB_OPERATION_NAME, @@ -299,7 +300,7 @@ class Database: span.set_attribute(DB_OPERATION_NAME, operation_name) if params: span.set_attribute(PARAM_COUNT, len(params)) - results = await self.execute_write_fn( + results = await self._execute_write_fn( _inner, block=block, request=request, transaction=transaction ) return results @@ -323,7 +324,7 @@ class Database: span.set_attribute(DB_NAMESPACE, self.name) span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) span.set_attribute(EXECUTESCRIPT, True) - results = await self.execute_write_fn( + results = await self._execute_write_fn( _inner, block=block, transaction=False, request=request ) return results @@ -356,7 +357,7 @@ class Database: operation_name = sql_operation_name(sql) if operation_name: span.set_attribute(DB_OPERATION_NAME, operation_name) - results, count = await self.execute_write_fn( + results, count = await self._execute_write_fn( _inner, block=block, request=request ) # count is the number of parameter *sets* consumed by @@ -383,21 +384,31 @@ class Database: # Was probably a memory connection pass - if self.ds.executor is None: - # non-threaded mode - return _run() - if not write: - # Immutable database - no writes can ever occur, so there is no - # write queue to block; run against a fresh read-only connection. - # copy_context() carries the caller's otel context onto the worker - # thread - see the notes in execute_fn() for why it must be a - # fresh copy per submit and why carrying every ContextVar is safe. - ctx = contextvars.copy_context() - return await asyncio.get_running_loop().run_in_executor( - self.ds.executor, ctx.run, _run - ) - # Threaded mode - send to write thread - return await self._send_to_write_thread(fn, isolated_connection=True) + # One db.query span here, like execute_fn() / execute_write_fn(). + # The wrap must NOT move into _send_to_write_thread(): that is the + # shared tail for every write, and for block=False it is where the + # link back to this span is captured - a span opened there would be + # the link target for its own children. + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(CALLBACK, callback_name(fn)) + if self.ds.executor is None: + # non-threaded mode + return _run() + if not write: + # Immutable database - no writes can ever occur, so there is + # no write queue to block; run against a fresh read-only + # connection. copy_context() carries the caller's otel context + # onto the worker thread - see the notes in _execute_fn() for + # why it must be a fresh copy per submit and why carrying + # every ContextVar is safe. + ctx = contextvars.copy_context() + return await asyncio.get_running_loop().run_in_executor( + self.ds.executor, ctx.run, _run + ) + # Threaded mode - send to write thread + return await self._send_to_write_thread(fn, isolated_connection=True) async def analyze_sql(self, sql, params=None) -> SQLAnalysis: self._check_not_closed() @@ -407,6 +418,30 @@ class Database: ) async def execute_write_fn(self, fn, block=True, transaction=True, request=None): + """Run `fn(conn)` on the write connection, traced as one database call. + + The public entry point for callback-style writes. Instrumented like + `execute_write()`: one `db.query` span (with `datasette.callback` in + place of `db.query.text`) above the `db.write.queue_wait` and + `db.write.execute` spans the write thread emits. The SQL-string write + methods call `_execute_write_fn()` directly, so they never get a + second span. For `block=False` this span ends at enqueue and the + write-thread spans become roots carrying a link back to it, exactly + as for `execute_write(block=False)`. + """ + self._check_not_closed() + # The raw fn's name, before _wrap_fn_with_hooks() replaces it with a + # wrapper - otherwise every write would report the wrapper's name. + name = callback_name(fn) + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(CALLBACK, name) + return await self._execute_write_fn( + fn, block=block, transaction=transaction, request=request + ) + + async def _execute_write_fn(self, fn, block=True, transaction=True, request=None): self._check_not_closed() pending_events = [] @@ -666,6 +701,35 @@ class Database: otel_context_api.detach(token) async def execute_fn(self, fn): + """Run `fn(conn)` on a read connection, traced as one database call. + + The public entry point for callback-style reads - plugins and core + both use it to run arbitrary Python against a connection. It is + instrumented exactly like `execute()`: one `db.query` span (with + `datasette.callback` in place of `db.query.text`, since there is no + SQL string to record) and a `db.query.execute` child covering the + time actually spent on the worker thread. `execute()` itself calls + `_execute_fn()` directly, so a SQL read never gets a second span. + """ + self._check_not_closed() + + def fn_in_execute_span(conn): + # Created on the worker thread; parents to the db.query span via + # the copy_context() propagation in _execute_fn(). The gap + # between the two spans is time spent waiting for a free thread. + with tracer.start_as_current_span(DB_QUERY_EXECUTE): + return fn(conn) + + with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span: + span.set_attribute(DB_SYSTEM, "sqlite") + span.set_attribute(DB_NAMESPACE, self.name) + span.set_attribute(CALLBACK, callback_name(fn)) + # Default exception handling applies, unlike execute(): there is + # no log_sql_errors=False probing caller and no expected-timeout + # budget on this path, so a raised exception is an error. + return await self._execute_fn(fn_in_execute_span) + + async def _execute_fn(self, fn): self._check_not_closed() if self.ds.executor is None: # non-threaded mode @@ -752,7 +816,7 @@ class Database: # This span is created inside the worker thread. Its parent is # resolved from the ambient otel context, which was propagated # onto this thread via copy_context() at the executor.submit() - # boundary in execute_fn() (or run_in_executor() for immutable + # boundary in _execute_fn() (or run_in_executor() for immutable # databases) - so it parents correctly to the enclosing # db.query span despite running on a different thread. # @@ -837,7 +901,7 @@ class Database: if params: span.set_attribute(PARAM_COUNT, len(params)) try: - results = await self.execute_fn(sql_operation_in_thread) + results = await self._execute_fn(sql_operation_in_thread) except QueryInterrupted as e: # datasette.interrupted is set either way - it is the # signal worth having. Only the ERROR status is diff --git a/datasette/telemetry.py b/datasette/telemetry.py index 3705d89f..641dfeae 100644 --- a/datasette/telemetry.py +++ b/datasette/telemetry.py @@ -8,7 +8,7 @@ sampling - that is the responsibility of whoever is running Datasette harness). With no provider installed every span produced here is a -`NonRecordingSpan`. That is not free - a table page emits ~58 spans - +`NonRecordingSpan`. That is not free - a table page emits ~100 spans - but end-to-end page benchmarks put the overhead below their own run-to-run variation. Installing an SDK provider is what costs something measurable. @@ -54,6 +54,16 @@ def sql_attribute(sql: str) -> str: return sql[:MAX_SQL_LENGTH] + "…[truncated]" +def callback_name(fn) -> str: + """ + The name recorded as `datasette.callback` for a callback-style call. + + `functools.partial` objects (and other callables) have no `__qualname__`, + so fall back to the type's name rather than fail the query over telemetry. + """ + return getattr(fn, "__qualname__", type(fn).__name__) + + # db.operation.name is the leading keyword of a statement matched against a # fixed allowlist - deliberately not a parse. # diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py index c2b9db32..183d514a 100644 --- a/datasette/telemetry_registry.py +++ b/datasette/telemetry_registry.py @@ -73,7 +73,23 @@ DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.") DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.") DB_QUERY_TEXT = Attribute( "db.query.text", - "The SQL, truncated to 2048 characters. Never the parameter values.", + "The SQL, truncated to 2048 characters. Never the parameter values. " + "Absent for a callback-style call (``execute_fn()`` and friends), where " + "there is no SQL string to record - ``datasette.callback`` is set " + "instead.", + optional=True, +) +CALLBACK = Attribute( + "datasette.callback", + "The qualified name of the Python callable passed to ``execute_fn()``, " + "``execute_write_fn()`` or ``execute_isolated_fn()`` - for example " + "``TableInsertView.post..insert_or_upsert_rows``. Set instead of " + "``db.query.text``, which does not exist for a callback: the SQL is " + "whatever the function chooses to run. A lambda reports ````, " + "which is why callers wanting a recognisable span should pass a named " + "function. Bounded cardinality: the set of callables is fixed by the " + "installed code, not by request input.", + optional=True, ) DB_OPERATION_NAME = Attribute( "db.operation.name", @@ -171,11 +187,15 @@ TRANSACTION = Attribute( DB_QUERY = SpanName( "db.query", "A SQL operation issued by Datasette, covering the full round trip " - "including any time spent queued for a thread.", + "including any time spent queued for a thread. Callback-style calls - " + "``execute_fn()``, ``execute_write_fn()`` and ``execute_isolated_fn()`` - " + "appear here too, distinguished by ``datasette.callback`` in place of " + "``db.query.text``.", ( DB_SYSTEM, DB_NAMESPACE, DB_QUERY_TEXT, + CALLBACK, DB_OPERATION_NAME, DB_COLLECTION_NAME, PARAM_COUNT, diff --git a/docs/changelog.rst b/docs/changelog.rst index 03d913b1..1488f3a7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ Changelog Unreleased ---------- -- Datasette's database layer now emits `OpenTelemetry `__ spans: one per query, covering the full round trip including time spent waiting for a SQL worker thread, plus separate child spans for the execution itself and for time spent in the write queue. Datasette core depends on ``opentelemetry-api`` only and never installs an SDK provider, an exporter or a sampler, so there is no effect and no measurable overhead unless tracing is switched on externally - normally with the standard ``opentelemetry-instrument`` agent. See :ref:`internals_telemetry`. (:issue:`1730`) +- Datasette's database layer now emits `OpenTelemetry `__ spans: one per query, covering the full round trip including time spent waiting for a SQL worker thread, plus separate child spans for the execution itself and for time spent in the write queue. Callback-style calls - :ref:`db.execute_fn() `, :ref:`db.execute_write_fn() ` and ``db.execute_isolated_fn()``, the documented way for plugins to run arbitrary SQL - are covered too, carrying ``datasette.callback`` in place of the SQL text. Datasette core depends on ``opentelemetry-api`` only and never installs an SDK provider, an exporter or a sampler, so there is no effect and no measurable overhead unless tracing is switched on externally - normally with the standard ``opentelemetry-instrument`` agent. See :ref:`internals_telemetry`. (:issue:`1730`) - :ref:`db.execute(sql, ..., table=None) ` has a new optional ``table=`` parameter, naming the table a query is about so it can be recorded on that query's OpenTelemetry span. It has no effect on query execution, and Datasette never derives it from the SQL. (:issue:`1730`) Nothing is removed by the OpenTelemetry work: the ``?_trace=1`` query string parameter, the ``trace_debug`` setting and the :ref:`internals_tracer` module all continue to work as before. diff --git a/docs/internals.rst b/docs/internals.rst index aee51969..bc3d2fb1 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2024,6 +2024,8 @@ Example usage: version = await db.execute_fn(get_version) +The call is traced as a ``db.query`` OpenTelemetry span carrying ``datasette.callback`` (the function's qualified name) rather than ``db.query.text``, since the SQL is whatever the function chooses to run - see :ref:`internals_telemetry`. Passing a named function gives the span a readable identity; a lambda reports ````. + .. _database_execute_write: await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True) @@ -2100,6 +2102,8 @@ This method works like ``.execute_write()``, but instead of a SQL statement you The function can then perform multiple actions, safe in the knowledge that it has exclusive access to the single writable connection for as long as it is executing. +Like ``execute_fn()``, the call is traced as a ``db.query`` OpenTelemetry span carrying ``datasette.callback`` rather than ``db.query.text``, above the write-queue spans - see :ref:`internals_telemetry`. A named function gives the span a readable identity; a lambda reports ````. + .. warning:: ``fn`` needs to be a regular function, not an ``async def`` function. @@ -2373,7 +2377,7 @@ Spans are ``SpanKind.INTERNAL`` unless a kind is listed below. Only ``db.query`` .. ]]] ``db.query`` - A SQL operation issued by Datasette, covering the full round trip including any time spent queued for a thread. + A SQL operation issued by Datasette, covering the full round trip including any time spent queued for a thread. Callback-style calls - ``execute_fn()``, ``execute_write_fn()`` and ``execute_isolated_fn()`` - appear here too, distinguished by ``datasette.callback`` in place of ``db.query.text``. Kind: ``CLIENT``. @@ -2381,7 +2385,8 @@ Spans are ``SpanKind.INTERNAL`` unless a kind is listed below. Only ``db.query`` - ``db.system`` - Always ``sqlite``. - ``db.namespace`` - Name of the database being queried. - - ``db.query.text`` - The SQL, truncated to 2048 characters. Never the parameter values. + - ``db.query.text`` *(optional)* - The SQL, truncated to 2048 characters. Never the parameter values. Absent for a callback-style call (``execute_fn()`` and friends), where there is no SQL string to record - ``datasette.callback`` is set instead. + - ``datasette.callback`` *(optional)* - The qualified name of the Python callable passed to ``execute_fn()``, ``execute_write_fn()`` or ``execute_isolated_fn()`` - for example ``TableInsertView.post..insert_or_upsert_rows``. Set instead of ``db.query.text``, which does not exist for a callback: the SQL is whatever the function chooses to run. A lambda reports ````, which is why callers wanting a recognisable span should pass a named function. Bounded cardinality: the set of callables is fixed by the installed code, not by request input. - ``db.operation.name`` *(optional)* - The statement's leading keyword - ``SELECT``, ``INSERT``, ``CREATE``, and so on - matched against a small fixed allowlist. Omitted rather than set to an arbitrary value: the attribute must stay safe to use as a metric dimension, and echoing an unrecognised first token from user-supplied SQL would be an unbounded-cardinality hazard. Also omitted for ``execute_write_script()``, which runs multiple statements - per semantic conventions, the operation name should not be extracted from query text that can contain more than one operation. Note that a statement beginning with a CTE reports ``WITH``, not the operation inside it - a substantial share of Datasette's own reads take that form. Resolving it further would mean parsing. - ``db.collection.name`` *(optional)* - The primary table, set only where the view already knows it - the table and row pages. Omitted for arbitrary ``?sql=`` queries, where determining the table would mean parsing the query. - ``datasette.param_count`` *(optional)* - Number of bound parameters. Recorded instead of the values themselves. diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 83e13b6e..a5ccc3fb 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -178,7 +178,14 @@ async def test_facetable_request_produces_db_query_spans(ds_client, otel_spans): spans = _db_query_spans(otel_spans) assert spans, "expected at least one db.query span" assert all(span.attributes["db.system"] == "sqlite" for span in spans) - assert all(span.attributes["db.query.text"] for span in spans) + # Every db.query names what ran: SQL text for the string methods, + # datasette.callback for callback-style calls (schema introspection here). + assert all( + span.attributes.get("db.query.text") + or span.attributes.get("datasette.callback") + for span in spans + ) + assert any(span.attributes.get("db.query.text") for span in spans) # Rendering the page also queries the internal database, so only some of # these spans belong to "fixtures". assert any(span.attributes["db.namespace"] == "fixtures" for span in spans) @@ -519,8 +526,14 @@ async def test_immutable_database_propagates_context(tmp_path, otel_spans): for span in otel_spans.get_finished_spans() if span.name == "t04-child-in-isolated-worker" ], "expected a span created inside execute_isolated_fn's worker thread" + # execute_isolated_fn() now opens its own db.query span, so the chain is + # event-loop parent -> db.query -> worker child. The worker child + # parenting to that db.query span, across the thread, is the propagation + # this test exists to prove. + query_spans = _children_named(otel_spans, "db.query", parent_context) + assert len(query_spans) == 1 children = _children_named( - otel_spans, "t04-child-in-isolated-worker", parent_context + otel_spans, "t04-child-in-isolated-worker", query_spans[0].context ) assert len(children) == 1 @@ -1036,3 +1049,192 @@ async def test_table_and_row_pages_set_db_collection_name( assert any( span.attributes.get("db.collection.name") == table for span in spans ), f"expected a db.query span from {path} carrying db.collection.name" + + +# --- Callback-style calls: execute_fn / execute_write_fn / execute_isolated_fn + + +@pytest.mark.asyncio +async def test_execute_fn_produces_db_query_span(otel_spans): + db = Datasette(memory=True).add_memory_database("t16_execute_fn") + await db.execute_write("create table t (id integer primary key)") + + def count_rows(conn): + return conn.execute("select count(*) from t").fetchone()[0] + + otel_spans.clear() + assert await db.execute_fn(count_rows) == 0 + + spans = _spans_for_namespace(otel_spans, "t16_execute_fn") + assert len(spans) == 1 + span = spans[0] + assert span.kind == SpanKind.CLIENT + assert span.attributes["db.system"] == "sqlite" + assert ( + span.attributes["datasette.callback"] + == "test_execute_fn_produces_db_query_span..count_rows" + ) + # There is no SQL string for a callback, and no statement to take a + # leading keyword from - absent beats guessed. + assert "db.query.text" not in span.attributes + assert "db.operation.name" not in span.attributes + children = _children_named(otel_spans, "db.query.execute", span.context) + assert len(children) == 1 + + +@pytest.mark.asyncio +async def test_execute_fn_lambda_reports_lambda(otel_spans): + # Pins the documented behaviour rather than pretending lambdas have names. + db = Datasette(memory=True).add_memory_database("t16_lambda") + otel_spans.clear() + await db.execute_fn(lambda conn: conn.execute("select 1").fetchone()) + spans = _spans_for_namespace(otel_spans, "t16_lambda") + assert len(spans) == 1 + assert spans[0].attributes["datasette.callback"].endswith("") + + +@pytest.mark.asyncio +async def test_execute_write_fn_produces_db_query_span(otel_spans): + db = Datasette(memory=True).add_memory_database("t16_write_fn") + + def create_table(conn): + conn.execute("create table t (id integer primary key)") + + otel_spans.clear() + await db.execute_write_fn(create_table) + + spans = _spans_for_namespace(otel_spans, "t16_write_fn") + assert len(spans) == 1 + span = spans[0] + assert span.kind == SpanKind.CLIENT + assert ( + span.attributes["datasette.callback"] + == "test_execute_write_fn_produces_db_query_span..create_table" + ) + assert "db.query.text" not in span.attributes + # The write-thread spans are this span's children, same as execute_write() + for name in ("db.write.queue_wait", "db.write.execute"): + assert len(_children_named(otel_spans, name, span.context)) == 1, name + + +@pytest.mark.asyncio +async def test_execute_write_fn_callback_name_is_not_the_hook_wrapper(otel_spans): + # A callback that declares track_event is the case where + # _wrap_fn_with_hooks() actually replaces fn with a wrapper - the span + # must still report the caller's function, not the wrapper's name. + db = Datasette(memory=True).add_memory_database("t16_wrapper_name") + + def create_with_events(conn, track_event): + conn.execute("create table t (id integer primary key)") + + otel_spans.clear() + await db.execute_write_fn(create_with_events) + spans = _spans_for_namespace(otel_spans, "t16_wrapper_name") + assert len(spans) == 1 + assert spans[0].attributes["datasette.callback"] == ( + "test_execute_write_fn_callback_name_is_not_the_hook_wrapper" + "..create_with_events" + ) + + +@pytest.mark.asyncio +async def test_execute_write_fn_nonblocking_spans_link_to_the_new_span(otel_spans): + # For block=False the public db.query span ends at enqueue and the + # write-thread spans become roots. Their link must target that new span, + # not whatever was current around the execute_write_fn() call. + db = Datasette(memory=True).add_memory_database("t16_nonblocking") + await db.execute_write("create table docs (id integer primary key)") + + def insert(conn): + conn.execute("insert into docs (id) values (1)") + + otel_spans.clear() + with tracer.start_as_current_span("t16-enqueueing-span") as enqueuer: + enqueuer_context = enqueuer.get_span_context() + await db.execute_write_fn(insert, block=False) + # Writes are serialized on the write thread, so a blocking write behind + # the non-blocking one waits for it deterministically. + await db.execute_write("insert into docs (id) values (2)") + + query_spans = [ + span + for span in _spans_for_namespace(otel_spans, "t16_nonblocking") + if span.attributes.get("datasette.callback") + ] + assert len(query_spans) == 1 + fn_span_context = query_spans[0].context + linked = [ + span + for span in otel_spans.get_finished_spans() + if span.name in ("db.write.queue_wait", "db.write.execute") and span.links + ] + assert len(linked) == 2 + for span in linked: + assert span.parent is None, f"{span.name} is still parented" + assert span.links[0].context.span_id == fn_span_context.span_id, span.name + assert span.links[0].context.span_id != enqueuer_context.span_id, span.name + + +@pytest.mark.asyncio +async def test_execute_does_not_double_wrap(otel_spans): + # The regression guard for the refactor: execute() and the SQL-string + # write methods call the private _execute_fn/_execute_write_fn, so they + # must not gain a second db.query span from the public wrappers. + db = Datasette(memory=True).add_memory_database("t16_no_double_wrap") + otel_spans.clear() + await db.execute_write("create table t (id integer primary key)") + assert len(_spans_for_namespace(otel_spans, "t16_no_double_wrap")) == 1 + otel_spans.clear() + await db.execute("select * from t") + spans = _spans_for_namespace(otel_spans, "t16_no_double_wrap") + assert len(spans) == 1 + assert len(_children_named(otel_spans, "db.query.execute", spans[0].context)) == 1 + + +@pytest.mark.asyncio +async def test_execute_isolated_fn_span_on_mutable_and_immutable(tmp_path, otel_spans): + def read_one(conn): + return conn.execute("select 1").fetchone()[0] + + mutable = Datasette(memory=True).add_memory_database("t16_isolated_mutable") + otel_spans.clear() + assert await mutable.execute_isolated_fn(read_one) == 1 + spans = _spans_for_namespace(otel_spans, "t16_isolated_mutable") + assert len(spans) == 1 + assert spans[0].attributes["datasette.callback"].endswith("read_one") + # Mutable databases route through the write thread, so the write spans + # appear as children; immutable ones run on the pool and get none. + assert _children_named(otel_spans, "db.write.execute", spans[0].context) + + db_path = tmp_path / "t16_isolated_immutable.db" + sqlite_utils.Database(str(db_path))["t"].insert({"id": 1}) + ds = Datasette() + immutable = Database(ds, path=str(db_path), is_mutable=False) + ds.add_database(immutable, name="t16_isolated_immutable") + try: + otel_spans.clear() + assert await immutable.execute_isolated_fn(read_one) == 1 + finally: + ds.remove_database("t16_isolated_immutable") + spans = _spans_for_namespace(otel_spans, "t16_isolated_immutable") + assert len(spans) == 1 + assert spans[0].attributes["datasette.callback"].endswith("read_one") + assert not _children_named(otel_spans, "db.write.execute", spans[0].context) + + +@pytest.mark.asyncio +async def test_execute_fn_exception_marks_span_error(otel_spans): + # Unlike execute(), there is no probing caller on this path - a callback + # that raises is an error, with the default record_exception behaviour. + db = Datasette(memory=True).add_memory_database("t16_fn_error") + + def boom(conn): + raise ValueError("callback failed") + + otel_spans.clear() + with pytest.raises(ValueError): + await db.execute_fn(boom) + spans = _spans_for_namespace(otel_spans, "t16_fn_error") + assert len(spans) == 1 + assert spans[0].status.status_code == StatusCode.ERROR + assert any(event.name == "exception" for event in spans[0].events) diff --git a/tests/test_telemetry_registry.py b/tests/test_telemetry_registry.py index 75a183b4..ef7c93a3 100644 --- a/tests/test_telemetry_registry.py +++ b/tests/test_telemetry_registry.py @@ -44,6 +44,7 @@ EXPECTED_ATTRIBUTES = { "db.system", "db.namespace", "db.query.text", + "datasette.callback", "db.operation.name", "db.collection.name", "datasette.param_count", @@ -108,6 +109,17 @@ async def exercise(): # datasette.isolated_connection=True await db.execute_isolated_fn(lambda conn: conn.execute("select 1").fetchone()) + # datasette.callback, with named functions so the conformance run sees the + # attribute's documented value shape (a qualname, not just "") + def registry_read_callback(conn): + return conn.execute("select count(*) from t").fetchone() + + def registry_write_callback(conn): + conn.execute("insert into t (id, v) values (100, 'callback')") + + await db.execute_fn(registry_read_callback) + await db.execute_write_fn(registry_write_callback) + # Reads: db.query.execute, datasette.rows_returned, datasette.truncated, # datasette.param_count, datasette.time_limit_ms await db.execute("select * from t where id > :n", {"n": 5}) From c1719811117d8f123b185ecc6536c7f0c5f7de63 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Wed, 2 Sep 2026 13:33:03 -0700 Subject: [PATCH 13/24] Name core's own callback functions so their spans are greppable The introspection wrappers (table_columns, primary_keys, fts_table, table_column_details), analyze_sql and the inspect CLI passed lambdas to execute_fn/execute_isolated_fn, so their db.query spans reported datasette.callback values like "Database.primary_keys..". Named inner functions give each span a greppable identity - the exact guidance the plugin telemetry docs give, applied to core's own highest-frequency callback sites. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012U7coQfVu8nK2R4q2mCULA --- datasette/cli.py | 6 +++++- datasette/database.py | 32 +++++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/datasette/cli.py b/datasette/cli.py index 2694c1f6..c41b3919 100644 --- a/datasette/cli.py +++ b/datasette/cli.py @@ -157,7 +157,11 @@ async def inspect_(files, sqlite_extensions): app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) data = {} for name, database in app.databases.items(): - tables = await database.execute_fn(lambda conn: inspect_tables(conn, {})) + + def _inspect_tables(conn): + return inspect_tables(conn, {}) + + tables = await database.execute_fn(_inspect_tables) data[name] = { "hash": database.hash, "size": database.size, diff --git a/datasette/database.py b/datasette/database.py index f8adc5fa..6cad4889 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -413,9 +413,10 @@ class Database: async def analyze_sql(self, sql, params=None) -> SQLAnalysis: self._check_not_closed() - return await self.execute_isolated_fn( - lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name) - ) + def _analyze_sql(conn): + return analyze_sql_tables(conn, sql, params, database_name=self.name) + + return await self.execute_isolated_fn(_analyze_sql) async def execute_write_fn(self, fn, block=True, transaction=True, request=None): """Run `fn(conn)` on the write connection, traced as one database call. @@ -1017,17 +1018,34 @@ class Database: ) return [r[0] for r in results.rows] + # These callbacks are named functions rather than lambdas so that their + # db.query spans carry a greppable datasette.callback - exactly the + # guidance the plugin telemetry docs give, applied to core's own + # highest-frequency introspection calls. + async def table_columns(self, table): - return await self.execute_fn(lambda conn: table_columns(conn, table)) + def _table_columns(conn): + return table_columns(conn, table) + + return await self.execute_fn(_table_columns) async def table_column_details(self, table): - return await self.execute_fn(lambda conn: table_column_details(conn, table)) + def _table_column_details(conn): + return table_column_details(conn, table) + + return await self.execute_fn(_table_column_details) async def primary_keys(self, table): - return await self.execute_fn(lambda conn: detect_primary_keys(conn, table)) + def _primary_keys(conn): + return detect_primary_keys(conn, table) + + return await self.execute_fn(_primary_keys) async def fts_table(self, table): - return await self.execute_fn(lambda conn: detect_fts(conn, table)) + def _fts_table(conn): + return detect_fts(conn, table) + + return await self.execute_fn(_fts_table) async def label_column_for_table(self, table): explicit_label_column = (await self.ds.table_config(self.name, table)).get( From ad9da9959f3b7063349728293fb7e84251471d45 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 19:33:04 -0700 Subject: [PATCH 14/24] Give every span a request to belong to Nothing in Datasette created a span for the HTTP request itself, so every span the database layer emits was a root span. Measured on this branch: one faceted table page produces 70 spans in 36 separate traces, none of which carries a URL. A trace UI shows that as dozens of unrelated single-span traces per page, interleaved across concurrent requests - worse than ?_trace=1 at the exact job people reach for tracing to do. With the request span it is 71 spans in 1 trace. `opentelemetry-instrument` does not fix this on its own: auto-instrumentation only picks up frameworks that ship an instrumentor entry point, and Datasette's raw ASGI app is not one. TelemetryMiddleware is mounted outermost in Datasette.app(), after the asgi_wrapper() plugin loop, so plugin middleware and the CSRF layer run *inside* the span. Putting it in DatasetteRouter instead would leave a span created by an instrumented plugin as an orphan root - reintroducing the problem for exactly the code most likely to be instrumented. It stays at ~90 lines, against roughly 700 for opentelemetry-instrumentation-asgi, because Datasette's app does not return before its body is sent: route_path awaits response.asgi_send(send), and a streaming CSV export runs its generator inline inside AsgiStream.asgi_send. So a plain `finally` covers the response body and no deferred-end machinery is needed. Two decisions worth flagging for review: - Inbound W3C traceparent and baggage are extracted, using the *global* propagator. That is the ecosystem norm (Flask, Django, FastAPI, the ASGI instrumentation), and going through the global propagator leaves the operator in control with no Datasette setting to invent: OTEL_PROPAGATORS=none disables it entirely. A public instance that does not want client-influenced traces should strip those headers at the proxy. - url.query is not recorded, anywhere. Datasette query strings carry user-supplied SQL in ?sql= and canned query parameters. client.address is not recorded either. The status code is sniffed from the ASGI http.response.start message rather than read off a Response, because asgi_static, the favicon route, AsgiStream and AsgiFileDownload all send that message themselves and never build one. Only a >= 500 sets an error status - per semantic conventions a 4xx is the client's mistake, and Datasette 404s are routine enough that treating them as errors would bury a real 500. The registry gains a `dynamic` flag, because this span's name is composed at runtime and so can never equal a fixed registry string. Dynamic entries resolve by span kind instead, and only after exact and prefix matching has failed, so they cannot shadow a span that does have a registered name. Co-Authored-By: Claude Opus 5 --- datasette/app.py | 24 ++- datasette/telemetry.py | 194 +++++++++++++++++ datasette/telemetry_registry.py | 105 +++++++++- docs/internals.rst | 21 +- tests/test_http_span.py | 344 +++++++++++++++++++++++++++++++ tests/test_telemetry_registry.py | 172 ++++++++++++---- 6 files changed, 811 insertions(+), 49 deletions(-) create mode 100644 tests/test_http_span.py diff --git a/datasette/app.py b/datasette/app.py index 7499c6f9..939c8cf8 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -49,7 +49,7 @@ from .events import Event from .plugins import DEFAULT_PLUGINS, get_plugins, pm from .renderer import json_renderer from .resources import DatabaseResource, TableResource -from .telemetry import tracer +from .telemetry import TelemetryMiddleware, tracer from .telemetry_registry import STARTUP from .tokens import TokenInvalid from .tracer import AsgiTracer @@ -780,12 +780,16 @@ class Datasette: # This must be called for Datasette to be in a usable state if self._startup_invoked: return - # invoke_startup() runs before any request exists, so every span its - # children create - the register_* hook dispatches, the internal - # catalog's db.query/db.write spans, and the prepare_connection - # warm-up of the read connections those touch - would otherwise be - # its own orphan root trace: around twenty of them on a fresh - # instance. Bracketing the whole thing gives them somewhere to belong. + # `datasette serve` calls invoke_startup() before uvicorn starts, so + # on the CLI path every span its children create - the register_* + # hook dispatches, the internal catalog's db.query/db.write spans, + # and the prepare_connection warm-up of the read connections those + # touch - would otherwise be its own orphan root trace: around twenty + # of them on a fresh instance. Bracketing the whole thing gives them + # somewhere to belong. An ASGI-hosted or programmatic deployment + # reaches here instead through AsgiRunOnFirstRequest, in which case + # this span nests under the first request's own span - honest enough, + # since it genuinely is that request's latency. # A connection warmed lazily later, by a request touching a new # database for the first time, nests under that request instead: # this span has already ended by then. @@ -2868,6 +2872,12 @@ class Datasette: asgi = AsgiRunOnFirstRequest(asgi, on_startup=[self._startup_sequence]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) + # Outermost, deliberately: plugin asgi_wrapper() middleware, the + # CSRF layer and the first-request startup fallback all run *inside* + # this span, so a span created by an instrumented plugin - or by + # startup work triggered by the first request - parents to the + # request instead of becoming its own orphan root trace. + asgi = TelemetryMiddleware(asgi) return asgi diff --git a/datasette/telemetry.py b/datasette/telemetry.py index 641dfeae..11391d3a 100644 --- a/datasette/telemetry.py +++ b/datasette/telemetry.py @@ -17,7 +17,19 @@ something measurable. import re from opentelemetry import trace as otel_trace +from opentelemetry.propagate import extract +from opentelemetry.propagators.textmap import Getter +from opentelemetry.trace import SpanKind, Status, StatusCode +from .telemetry_registry import ( + ERROR_TYPE, + HTTP_REQUEST_METHOD, + HTTP_RESPONSE_STATUS_CODE, + SERVER_ADDRESS, + URL_PATH, + URL_SCHEME, + USER_AGENT_ORIGINAL, +) from .version import __version__ # The semantic-convention version whose spellings this instrumentation @@ -124,3 +136,185 @@ def sql_operation_name(sql: str) -> str | None: if keyword in DB_OPERATION_ALLOWLIST: return keyword return None + + +# --- The HTTP request span ------------------------------------------------ + + +class _ScopeHeadersGetter(Getter): + """ + Read W3C trace context out of an ASGI scope's headers. + + `scope["headers"]` is a list of `(bytes, bytes)` pairs, lowercased by the + server per the ASGI spec - but `.lower()` is applied again here because + that is a spec promise about servers, not something this process + controls. Header bytes are latin-1 by RFC 9110. + """ + + def get(self, carrier, key): + wanted = key.lower().encode("latin-1") + values = [v.decode("latin-1") for k, v in carrier if k.lower() == wanted] + return values or None + + def keys(self, carrier): + return [k.decode("latin-1") for k, _ in carrier] + + +_HEADERS_GETTER = _ScopeHeadersGetter() + + +# An unclamped method is an unbounded dimension a client controls: anyone can +# send `FOO / HTTP/1.1`. Semantic conventions say map anything unrecognised to +# `_OTHER`. These nine are the methods of RFC 9110 plus PATCH (RFC 5789). +_KNOWN_METHODS = frozenset( + {"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"} +) + + +def clamp_http_method(method): + "The request method if it is one we recognise, else ``_OTHER``." + method = (method or "").upper() + return method if method in _KNOWN_METHODS else "_OTHER" + + +def _first_header(headers, name): + "The first value of a header, decoded, or None." + for key, value in headers: + if key.lower() == name: + return value.decode("latin-1") + return None + + +def _url_path(scope): + """ + The request path, with any query string removed. + + `raw_path` is preferred because it is the bytes the client sent, before + percent-decoding - Datasette routes on database and table names that can + contain encoded slashes, which `scope["path"]` has already collapsed. + + The split on "?" is not decoration. The ASGI spec's `raw_path` excludes + the query string, and uvicorn honours that, but the name is used the + other way round elsewhere in this same dependency tree: httpx's + `URL.raw_path` is documented as "raw bytes of both the path and query". + A server that followed that reading would hand us `?sql=...` here, and + Datasette's query strings carry user-supplied SQL, which core never + records. A literal "?" cannot appear unencoded in a path, so the split + costs nothing when the server is well behaved. + """ + raw_path = scope.get("raw_path") + if raw_path: + if isinstance(raw_path, bytes): + raw_path = raw_path.decode("latin-1") + return raw_path.split("?", 1)[0] + return scope.get("path", "") + + +class TelemetryMiddleware: + """ + One `SpanKind.SERVER` span per HTTP request. + + Mounted outermost in `Datasette.app()`, so every other span raised while + serving a request - database queries, plugin middleware, startup work on + a cold ASGI-hosted deployment - has somewhere to belong instead of + becoming its own root trace. + + Deliberately much smaller than `opentelemetry-instrumentation-asgi`, + which needs several hundred lines of deferred-end machinery for + applications that return before their body is sent. Datasette does not: + `DatasetteRouter.route_path` awaits `response.asgi_send(send)`, and for a + streaming CSV export `AsgiStream.asgi_send` runs the generator inline. + All of it happens inside the single `await self.app(...)` below, so + ending the span in a `finally` covers the response body too. + """ + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + # First, before anything else: `AsgiLifespan` is *inside* this + # middleware, so lifespan startup and shutdown have to pass through + # untouched or the server never starts. Same for websockets. + if scope["type"] != "http": + await self.app(scope, receive, send) + return + headers = scope.get("headers") or [] + # The *global* propagator, deliberately: it leaves the operator in + # control with no Datasette-specific setting - OTEL_PROPAGATORS=none + # disables extraction entirely, OTEL_PROPAGATORS=tracecontext drops + # baggage - and core configuring propagation itself would be the same + # mistake as core configuring sampling. + context = extract(headers, getter=_HEADERS_GETTER) + method = clamp_http_method(scope.get("method", "")) + # The method, not the URL: a span name has to be low cardinality, and + # the method is what is known out here at the edge, before any routing + # has happened. + with tracer.start_as_current_span( + method, context=context, kind=SpanKind.SERVER + ) as span: + if not span.is_recording(): + # No provider installed, or a sampler dropped this trace. + # Everything below would be discarded, so skip building the + # `send` wrapper and let a default install pay almost + # nothing. Note this cannot be `get_span_context().is_valid`: + # with no provider but an inbound `traceparent`, the API's + # NoOpTracer returns a NonRecordingSpan carrying the *remote* + # context, which is perfectly valid and still records nothing. + await self.app(scope, receive, send) + return + span.set_attribute(HTTP_REQUEST_METHOD, method) + span.set_attribute(URL_PATH, _url_path(scope)) + scheme = scope.get("scheme") + if scheme: + span.set_attribute(URL_SCHEME, scheme) + host = _first_header(headers, b"host") + if host: + span.set_attribute(SERVER_ADDRESS, host) + user_agent = _first_header(headers, b"user-agent") + if user_agent: + span.set_attribute(USER_AGENT_ORIGINAL, user_agent) + + # The status cannot be read off a Response object: `asgi_static`, + # the favicon route, `AsgiStream` and `AsgiFileDownload` all call + # `send` directly and never build one. Wrapping `send` is the only + # thing that sees every response, including the 404 and 500 + # handlers. + status_holder = {} + + async def wrapped_send(message): + if ( + message["type"] == "http.response.start" + and "status" not in status_holder + ): + status_holder["status"] = message["status"] + await send(message) + + escaped = False + try: + # Positional (scope, receive, send) throughout this codebase - + # `wrapped_send` is the third argument. `receive` is passed + # through unwrapped. + await self.app(scope, receive, wrapped_send) + except BaseException as exception: + # BaseException, not Exception: `route_path` turns almost + # everything into a 500 itself, but `asyncio.CancelledError` + # on client disconnect is a BaseException its `except + # Exception` deliberately does not catch. + escaped = True + span.set_attribute(ERROR_TYPE, type(exception).__name__) + span.set_status(Status(StatusCode.ERROR, str(exception))) + raise + finally: + status = status_holder.get("status") + if status is not None: + span.set_attribute(HTTP_RESPONSE_STATUS_CODE, status) + # 4xx is NOT an error for a SERVER span per semantic + # conventions - the client made the mistake, not us. + # + # `not escaped` because this block still runs when an + # exception is on its way out, and a response can have + # started before it: the exception's class name is more + # use than the string "500", so it wins. + if status >= 500 and not escaped: + span.set_status(Status(StatusCode.ERROR)) + span.set_attribute(ERROR_TYPE, str(status)) diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py index 183d514a..0cc9c156 100644 --- a/datasette/telemetry_registry.py +++ b/datasette/telemetry_registry.py @@ -47,12 +47,26 @@ class Attribute(str): class SpanName(str): "A span name, carrying its documentation and the attributes it may set." - __slots__ = ("attributes", "description", "kind") + __slots__ = ("attributes", "description", "dynamic", "kind") - def __new__(cls, name, description, attributes=(), kind=SpanKind.INTERNAL): + def __new__( + cls, + name, + description, + attributes=(), + dynamic=False, + kind=SpanKind.INTERNAL, + ): self = super().__new__(cls, name) self.description = description self.attributes = tuple(attributes) + # True when the emitted name is composed at runtime and shares no + # fixed prefix with the registry entry - the HTTP request span, whose + # name is the request method. There is no substring of the entry that + # could be matched against the wire, so `span_for()` resolves these by + # span kind instead, and the entry's own string is a template written + # for a human reading the generated reference. + self.dynamic = dynamic # SpanKind.INTERNAL by default - every span Datasette emits describes # its own internal work. db.query is the one exception: it is a real # database call, so semantic conventions (and trace UIs, which key @@ -69,6 +83,53 @@ class SpanName(str): # Shared attributes are defined once and referenced by every span that sets # them, so "which spans carry db.namespace?" is answerable by grep. +HTTP_REQUEST_METHOD = Attribute( + "http.request.method", + "The HTTP method, clamped to the nine methods RFC 9110 and RFC 5789 " + "define. Anything else is reported as ``_OTHER``: the method is a " + "client-controlled string, so echoing it back unbounded would be a " + "cardinality hazard.", +) +HTTP_RESPONSE_STATUS_CODE = Attribute( + "http.response.status_code", + "The status of the response, read from the ASGI ``http.response.start`` " + "message rather than from a :ref:`internals_response` object - several " + "views, including static files, file downloads and streaming CSV, send " + "that message themselves and never build one. Omitted if the connection " + "closed before anything was sent.", + optional=True, +) +URL_PATH = Attribute( + "url.path", + "The path portion of the URL. The query string is deliberately **not** " + "recorded, on this or any other span: Datasette puts user-supplied SQL in " + "``?sql=`` and canned query parameters in the query string, so exporting " + "it by default would export exactly the data the rest of this " + "instrumentation is careful with.", +) +URL_SCHEME = Attribute("url.scheme", "``http`` or ``https``.") +SERVER_ADDRESS = Attribute( + "server.address", + "The ``Host`` header. Client-controlled, so treat it as untrusted input " + "rather than as the identity of the server.", + optional=True, +) +USER_AGENT_ORIGINAL = Attribute( + "user_agent.original", + "The ``User-Agent`` header, verbatim. Omitted if the client sent none. " + "The client's IP address is deliberately not recorded: core records no " + "identifier that would tie a span to a person.", + optional=True, +) +ERROR_TYPE = Attribute( + "error.type", + "Set when the request failed: the exception class name if one escaped the " + "application, otherwise the status code as a string for a 5xx response. " + "A 4xx does **not** set this and does not set an error status - per " + "semantic conventions a client error is not a server span's failure.", + optional=True, +) + DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.") DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.") DB_QUERY_TEXT = Attribute( @@ -184,6 +245,30 @@ TRANSACTION = Attribute( # --- Spans ---------------------------------------------------------------- +HTTP_REQUEST = SpanName( + "{http.request.method}", + "One span per HTTP request, created by the outermost layer of the ASGI " + "stack - so plugin ``asgi_wrapper()`` middleware, CSRF protection and " + "every database span raised while serving the request all nest inside " + "it. Without it each of those would be its own root trace. The span name " + "is not a fixed string: it is the value of ``http.request.method``. " + "W3C ``traceparent`` and ``baggage`` headers are extracted using the " + "global propagator, so a request arriving from an already-traced caller " + "continues that trace; set ``OTEL_PROPAGATORS=none`` to turn that off, " + "and strip those headers at your proxy if your instance is public.", + ( + HTTP_REQUEST_METHOD, + URL_PATH, + URL_SCHEME, + SERVER_ADDRESS, + USER_AGENT_ORIGINAL, + HTTP_RESPONSE_STATUS_CODE, + ERROR_TYPE, + ), + dynamic=True, + kind=SpanKind.SERVER, +) + DB_QUERY = SpanName( "db.query", "A SQL operation issued by Datasette, covering the full round trip " @@ -253,6 +338,7 @@ STARTUP = SpanName( ) SPANS = ( + HTTP_REQUEST, DB_QUERY, DB_QUERY_EXECUTE, DB_WRITE_QUEUE_WAIT, @@ -261,16 +347,25 @@ SPANS = ( ) -def span_for(emitted_name): +def span_for(emitted_name, kind=None): """ Resolve an emitted span name to its registry entry, or None. - The lookup is what the conformance test calls, so it lives here rather - than in the test. + Handles `dynamic=True` entries, whose emitted names are not knowable in + advance: the name has no fixed part at all, so it is matched on `kind` + instead and the caller has to supply one. Exact entries are tried first, + so a dynamic entry can never shadow a span that does have a registered + name. """ for span in SPANS: + if span.dynamic: + continue if emitted_name == span: return span + if kind is not None: + for span in SPANS: + if span.dynamic and span.kind == kind: + return span return None diff --git a/docs/internals.rst b/docs/internals.rst index bc3d2fb1..ba749f95 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2365,17 +2365,34 @@ A few things catch people out the first time: Span reference -------------- -Datasette emits five spans. Four of them describe the database layer - one per query, one for the work that query does inside a SQL worker thread, and two more for the write queue - and the fifth covers startup. Attribute names use the ``datasette.*`` prefix for Datasette-specific data, alongside standard OpenTelemetry attributes such as ``db.system``. +Datasette emits six spans. One covers the HTTP request, and is the root everything else raised while serving that request hangs from. Four describe the database layer - one per query, one for the work that query does inside a SQL worker thread, and two more for the write queue. The sixth covers startup. Attribute names use the ``datasette.*`` prefix for Datasette-specific data, alongside standard OpenTelemetry attributes such as ``db.system``. This reference is generated from ``datasette/telemetry_registry.py``, the single source of truth for every span and attribute Datasette emits. A conformance test makes real requests and compares what is actually emitted against that registry in both directions, so nothing here is hand-maintained and nothing can silently drift out of date. -Spans are ``SpanKind.INTERNAL`` unless a kind is listed below. Only ``db.query`` is ``CLIENT``: it is the one span that represents a call to a database rather than Datasette's own work, and trace UIs use the kind to decide whether to render a span as a database call. Its children stay ``INTERNAL`` because they are Datasette's decomposition of that one query - marking them ``CLIENT`` too would make a single query look like several database calls to anything counting by kind. +Spans are ``SpanKind.INTERNAL`` unless a kind is listed below. Two are not: the request span is ``SERVER``, and ``db.query`` is ``CLIENT`` because it is the one span that represents a call to a database rather than Datasette's own work. Trace UIs use the kind to decide whether to render a span as an inbound request or as a database call. ``db.query``'s children stay ``INTERNAL`` because they are Datasette's decomposition of that one query - marking them ``CLIENT`` too would make a single query look like several database calls to anything counting by kind. + +The request span's name is the only one that is not a fixed string - it is composed from the request, so the heading below shows the template rather than a literal you will see in a trace. .. [[[cog from telemetry_doc import spans spans(cog) .. ]]] +``{http.request.method}`` + One span per HTTP request, created by the outermost layer of the ASGI stack - so plugin ``asgi_wrapper()`` middleware, CSRF protection and every database span raised while serving the request all nest inside it. Without it each of those would be its own root trace. The span name is not a fixed string: it is the value of ``http.request.method``. W3C ``traceparent`` and ``baggage`` headers are extracted using the global propagator, so a request arriving from an already-traced caller continues that trace; set ``OTEL_PROPAGATORS=none`` to turn that off, and strip those headers at your proxy if your instance is public. + + Kind: ``SERVER``. + + Attributes: + + - ``http.request.method`` - The HTTP method, clamped to the nine methods RFC 9110 and RFC 5789 define. Anything else is reported as ``_OTHER``: the method is a client-controlled string, so echoing it back unbounded would be a cardinality hazard. + - ``url.path`` - The path portion of the URL. The query string is deliberately **not** recorded, on this or any other span: Datasette puts user-supplied SQL in ``?sql=`` and canned query parameters in the query string, so exporting it by default would export exactly the data the rest of this instrumentation is careful with. + - ``url.scheme`` - ``http`` or ``https``. + - ``server.address`` *(optional)* - The ``Host`` header. Client-controlled, so treat it as untrusted input rather than as the identity of the server. + - ``user_agent.original`` *(optional)* - The ``User-Agent`` header, verbatim. Omitted if the client sent none. The client's IP address is deliberately not recorded: core records no identifier that would tie a span to a person. + - ``http.response.status_code`` *(optional)* - The status of the response, read from the ASGI ``http.response.start`` message rather than from a :ref:`internals_response` object - several views, including static files, file downloads and streaming CSV, send that message themselves and never build one. Omitted if the connection closed before anything was sent. + - ``error.type`` *(optional)* - Set when the request failed: the exception class name if one escaped the application, otherwise the status code as a string for a 5xx response. A 4xx does **not** set this and does not set an error status - per semantic conventions a client error is not a server span's failure. + ``db.query`` A SQL operation issued by Datasette, covering the full round trip including any time spent queued for a thread. Callback-style calls - ``execute_fn()``, ``execute_write_fn()`` and ``execute_isolated_fn()`` - appear here too, distinguished by ``datasette.callback`` in place of ``db.query.text``. diff --git a/tests/test_http_span.py b/tests/test_http_span.py new file mode 100644 index 00000000..740d437d --- /dev/null +++ b/tests/test_http_span.py @@ -0,0 +1,344 @@ +""" +The HTTP request span. + +`tests/test_telemetry_registry.py` already pins the span's name shape, kind +and attribute keys against literals, so this file deliberately does not +repeat that. What it covers is the three properties of the middleware that +the registry conformance test structurally cannot see: + +- **where the middleware sits.** Outermost is the entire point - moving it + inside the plugin `asgi_wrapper()` loop leaves plugin middleware creating + orphan root traces, which is the problem this span exists to fix, and every + attribute assertion still passes. +- **method clamping**, which a workload of ordinary GETs can never exercise. +- **the query string never being recorded**, which only fails if a request + actually carries one. +""" + +import asyncio +import itertools + +import pytest +import pytest_asyncio + +pytest.importorskip("opentelemetry.sdk") + +from opentelemetry.trace import SpanKind, StatusCode + +from datasette import hookimpl +from datasette.app import Datasette +from datasette.telemetry import TelemetryMiddleware, tracer + +# Named in-memory databases are shared-cache: two Datasette instances given +# the same name share one SQLite database and the second `create table` +# fails. +_names = itertools.count() + + +PLUGIN_MIDDLEWARE_SPAN = "test.plugin.middleware" + + +class _MiddlewarePlugin: + "A plugin asgi_wrapper() that creates a span, standing in for a real one." + + __name__ = "HttpSpanMiddlewarePlugin" + + @hookimpl + def asgi_wrapper(self, datasette): + def wrap(app): + async def wrapped(scope, receive, send): + with tracer.start_as_current_span(PLUGIN_MIDDLEWARE_SPAN): + await app(scope, receive, send) + + return wrapped + + return wrap + + +class _RaisingMiddlewarePlugin: + """ + A plugin asgi_wrapper() that raises. + + `route_path` converts almost every exception into a 500 itself, so an + exception escaping into the request span is only reachable from *outside* + the router - a plugin wrapper, or a failure inside the 500 handler. + """ + + __name__ = "HttpSpanRaisingMiddlewarePlugin" + + def __init__(self, call_app_first): + self.call_app_first = call_app_first + + @hookimpl + def asgi_wrapper(self, datasette): + call_app_first = self.call_app_first + + def wrap(app): + async def wrapped(scope, receive, send): + if call_app_first: + await app(scope, receive, send) + raise RuntimeError("wrapper exploded") + + return wrapped + + return wrap + + +class _BoomPlugin: + "A route that raises, which route_path turns into a 500." + + __name__ = "HttpSpanBoomPlugin" + + @hookimpl + def register_routes(self): + return [(r"^/-/http-span-boom$", lambda: 1 / 0)] + + +@pytest_asyncio.fixture +async def ds(): + name = f"httpspan{next(_names)}" + instance = Datasette(memory=True) + instance.add_memory_database(name) + await instance.invoke_startup() + await instance.get_database(name).execute_write( + "create table t (id integer primary key, v text)" + ) + instance.db_name = name + try: + yield instance + finally: + instance.close() + + +def _server_spans(otel_spans): + return [ + span for span in otel_spans.get_finished_spans() if span.kind is SpanKind.SERVER + ] + + +@pytest.mark.asyncio +async def test_plugin_asgi_wrapper_middleware_runs_inside_the_request_span( + ds, otel_spans +): + """ + The placement check. + + A span created by a plugin `asgi_wrapper()` must be a *child* of the + request span. If the middleware is mounted anywhere inside the plugin + loop the two swap places - the plugin's span becomes the root and the + request span its child - which is exactly the orphaning this is meant to + prevent, and which no attribute assertion notices. + """ + ds.pm.register(_MiddlewarePlugin(), name="httpspan-middleware") + try: + otel_spans.clear() + response = await ds.client.get(f"/{ds.db_name}/t") + assert response.status_code == 200 + finally: + ds.pm.unregister(name="httpspan-middleware") + + spans = otel_spans.get_finished_spans() + server = [span for span in spans if span.kind is SpanKind.SERVER] + assert len(server) == 1, "expected exactly one SERVER span per request" + request_span = server[0] + assert request_span.parent is None, "the request span should be the trace root" + + plugin_spans = [span for span in spans if span.name == PLUGIN_MIDDLEWARE_SPAN] + assert len(plugin_spans) == 1 + assert plugin_spans[0].parent is not None + assert plugin_spans[0].parent.span_id == request_span.context.span_id + assert plugin_spans[0].context.trace_id == request_span.context.trace_id + + # And the database work is in the same trace, not off on its own. + queries = [span for span in spans if span.name == "db.query"] + assert queries, "a table page should have issued at least one query" + for query in queries: + assert query.context.trace_id == request_span.context.trace_id + + +@pytest.mark.asyncio +async def test_unrecognised_method_is_clamped(ds, otel_spans): + """ + Anyone can send `FROB / HTTP/1.1`. An unclamped method is an unbounded + dimension a client controls, so semantic conventions map anything off the + known list to `_OTHER` - and the span name is the method, so an unclamped + one would put attacker-supplied text in the span name too. + """ + otel_spans.clear() + await ds.client.request("FROB", f"/{ds.db_name}/t") + server = _server_spans(otel_spans) + assert len(server) == 1 + assert server[0].name == "_OTHER" + assert server[0].attributes["http.request.method"] == "_OTHER" + + +@pytest.mark.asyncio +async def test_known_method_is_not_clamped(ds, otel_spans): + "The other half of clamping: a real method must survive it verbatim." + otel_spans.clear() + await ds.client.get(f"/{ds.db_name}/t") + server = _server_spans(otel_spans) + assert len(server) == 1 + assert server[0].name == "GET" + assert server[0].attributes["http.request.method"] == "GET" + + +@pytest.mark.asyncio +async def test_the_query_string_is_never_recorded(ds, otel_spans): + """ + Datasette puts user-supplied SQL in `?sql=` and canned query parameters in + the query string, so no span may carry it. Asserting on the absence of a + `url.query` key alone would not catch it arriving under some other name, + so this searches every attribute value of every span for the marker. + """ + marker = "canary-9f2b1c" + otel_spans.clear() + await ds.client.get(f"/{ds.db_name}/t?_facet=v&_nosuch={marker}") + spans = otel_spans.get_finished_spans() + assert _server_spans(otel_spans), "no request span was emitted" + leaked = [ + f"{span.name} -> {key}={value!r}" + for span in spans + for key, value in (span.attributes or {}).items() + if marker in str(value) or key == "url.query" + ] + assert not leaked, "the query string reached a span attribute: " + ", ".join(leaked) + + +@pytest.mark.asyncio +async def test_url_path_is_recorded_without_the_query_string(ds, otel_spans): + otel_spans.clear() + await ds.client.get(f"/{ds.db_name}/t?_facet=v") + server = _server_spans(otel_spans) + assert len(server) == 1 + assert server[0].attributes["url.path"] == f"/{ds.db_name}/t" + + +@pytest.mark.asyncio +async def test_escaping_exception_sets_error_type_and_reraises(ds, otel_spans): + """ + An exception that gets past `route_path` must be recorded, not swallowed. + + No response ever started, so there is no status code to record either. + """ + ds.pm.register( + _RaisingMiddlewarePlugin(call_app_first=False), name="httpspan-raiser" + ) + try: + otel_spans.clear() + with pytest.raises(RuntimeError): + await ds.client.get(f"/{ds.db_name}/t") + finally: + ds.pm.unregister(name="httpspan-raiser") + server = _server_spans(otel_spans) + assert len(server) == 1 + assert server[0].attributes["error.type"] == "RuntimeError" + assert "http.response.status_code" not in server[0].attributes + assert server[0].status.status_code is StatusCode.ERROR + + +@pytest.mark.asyncio +async def test_an_escaping_exception_beats_the_status_code_for_error_type( + ds, otel_spans +): + """ + Both paths can fire on one request: a 500 response is sent and *then* + something raises on the way out. The `finally` block runs while the + exception is propagating, so without the guard it would overwrite the + exception's class name with the string "500" - strictly less information + about what actually went wrong. + """ + ds.pm.register(_BoomPlugin(), name="httpspan-boom") + ds.pm.register( + _RaisingMiddlewarePlugin(call_app_first=True), name="httpspan-raiser" + ) + try: + otel_spans.clear() + with pytest.raises(RuntimeError): + await ds.client.get("/-/http-span-boom") + finally: + ds.pm.unregister(name="httpspan-raiser") + ds.pm.unregister(name="httpspan-boom") + server = _server_spans(otel_spans) + assert len(server) == 1 + # The 500 really was sent, so the status is still recorded ... + assert server[0].attributes["http.response.status_code"] == 500 + # ... but error.type names the exception, not the status. + assert server[0].attributes["error.type"] == "RuntimeError" + + +@pytest.mark.asyncio +async def test_a_404_is_not_an_error(ds, otel_spans): + """ + Per semantic conventions a 4xx is the client's mistake, not the server's, + so a SERVER span must record the status and leave both its own status and + `error.type` alone. Datasette 404s are routine - every missing table, and + every bot probing for /wp-login.php - so treating them as errors would + drown a real 500 in noise. + """ + otel_spans.clear() + response = await ds.client.get("/no-such-database-at-all") + assert response.status_code == 404 + server = _server_spans(otel_spans) + assert len(server) == 1 + assert server[0].attributes["http.response.status_code"] == 404 + assert "error.type" not in server[0].attributes + assert server[0].status.status_code is StatusCode.UNSET + + +@pytest.mark.asyncio +async def test_only_the_first_http_response_start_is_recorded(otel_spans): + """ + The `send` wrapper keeps the first status it sees. + + Nothing in Datasette sends two `http.response.start` messages, so this + drives the middleware directly rather than pretending a request could + reach it. Without the guard a misbehaving plugin's second start message + would silently replace the status the client actually received. + """ + + async def two_starts(scope, receive, send): + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.start", "status": 503, "headers": []}) + await send({"type": "http.response.body", "body": b""}) + + middleware = TelemetryMiddleware(two_starts) + scope = { + "type": "http", + "method": "GET", + "path": "/twice", + "raw_path": b"/twice", + "scheme": "http", + "headers": [], + } + otel_spans.clear() + await middleware(scope, None, lambda message: asyncio.sleep(0)) + server = _server_spans(otel_spans) + assert len(server) == 1 + assert server[0].attributes["http.response.status_code"] == 200 + assert "error.type" not in server[0].attributes + + +@pytest.mark.asyncio +async def test_lifespan_scope_passes_through_unspanned(otel_spans): + """ + `AsgiLifespan` sits *inside* this middleware, so the scope-type check has + to come first or startup and shutdown events never reach it. A SERVER + span for a lifespan scope is the symptom of that check being missing or + late. + """ + instance = Datasette(memory=True) + app = instance.app() + events = iter([{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}]) + sent = [] + + async def receive(): + return next(events) + + async def send(message): + sent.append(message["type"]) + + otel_spans.clear() + await app({"type": "lifespan"}, receive, send) + assert sent == ["lifespan.startup.complete", "lifespan.shutdown.complete"] + assert not _server_spans(otel_spans) diff --git a/tests/test_telemetry_registry.py b/tests/test_telemetry_registry.py index ef7c93a3..a6037345 100644 --- a/tests/test_telemetry_registry.py +++ b/tests/test_telemetry_registry.py @@ -30,6 +30,9 @@ import pytest_asyncio pytest.importorskip("opentelemetry.sdk") +from opentelemetry.trace import SpanKind + +from datasette import hookimpl from datasette import telemetry_registry as reg from datasette.app import Datasette from datasette.database import QueryInterrupted @@ -67,6 +70,31 @@ EXPECTED_ATTRIBUTES = { } EXPECTED_SPANS = set(EXPECTED_ATTRIBUTES) +# The HTTP request span is handled separately because its name is composed at +# runtime - it is the request method - so there is no fixed string to pin it +# to. What can still be pinned, and is what a dashboard depends on, is the +# shape of the name and the attribute keys. The workload below only issues +# GETs, so a change that stopped clamping the method, or that started naming +# the span after the path, fails here. +EXPECTED_HTTP_SPAN_NAME = "{http.request.method}" +EXPECTED_HTTP_SPAN_NAMES = {"GET"} +EXPECTED_HTTP_ATTRIBUTES = { + "http.request.method", + "url.path", + "url.scheme", + "server.address", + "user_agent.original", + "http.response.status_code", + "error.type", +} + +# The registry's own name for the request span is that template, not anything +# that appears on the wire. +EXPECTED_REGISTRY_ATTRIBUTES = dict( + EXPECTED_ATTRIBUTES, **{EXPECTED_HTTP_SPAN_NAME: EXPECTED_HTTP_ATTRIBUTES} +) +EXPECTED_REGISTRY_NAMES = set(EXPECTED_REGISTRY_ATTRIBUTES) + # Named in-memory databases are shared-cache, so two Datasette instances using # the same name share one SQLite database - and the second `create table` # fails. Every workload below therefore gets its own name. @@ -77,6 +105,23 @@ def _unique(prefix): return f"{prefix}{next(_names)}" +class _BoomPlugin: + """ + A route that raises. + + `error.type` on the request span is only ever set by a 5xx, and nothing + in Datasette returns one on a healthy instance - `route_path` converts + exceptions into a 500 itself, so the workload has to supply the + exception. + """ + + __name__ = "TelemetryRegistryBoomPlugin" + + @hookimpl + def register_routes(self): + return [(r"^/-/telemetry-registry-boom$", lambda: 1 / 0)] + + async def exercise(): """ Drive enough of Datasette to emit every span and attribute the registry @@ -140,37 +185,62 @@ async def exercise(): custom_time_limit=1, ) - # db.collection.name - set only by views that already know their table + # db.collection.name - set only by views that already know their table. + # These requests are also what produces the HTTP request span and its + # http.request.method / url.path / url.scheme / server.address / + # user_agent.original / http.response.status_code attributes. assert (await ds.client.get(f"/{name}/t?_facet=v")).status_code == 200 assert (await ds.client.get(f"/{name}/t/1.json")).status_code == 200 + + # error.type on the request span, which only a 5xx sets + ds.pm.register(_BoomPlugin(), name="telemetry-registry-boom") + try: + response = await ds.client.get("/-/telemetry-registry-boom") + assert response.status_code == 500 + finally: + ds.pm.unregister(name="telemetry-registry-boom") return ds @pytest_asyncio.fixture async def emitted(otel_spans): - "Every span name and (span name, attribute key) pair a broad workload emits." + """ + Every (span name, span kind, attribute keys) triple a broad workload emits. + + The kind is carried because the request span's name is composed at + runtime, so `span_for()` resolves it by kind instead. + """ # otel_spans has already cleared the exporter, and nothing is cleared # after this point: the workload's own startup emits datasette.startup. ds = await exercise() spans = otel_spans.get_finished_spans() assert spans, "no spans captured - the fixture is not exercising anything" - names = set() - pairs = set() - for span in spans: - # str() because span.name is the registry's SpanName instance, and a - # set of those would compare equal to literals but read confusingly - # in a failure message. - names.add(str(span.name)) - for key in span.attributes or {}: - pairs.add((str(span.name), str(key))) + # str() because span.name is the registry's SpanName instance, and a set + # of those would compare equal to literals but read confusingly in a + # failure message. + collected = tuple( + ( + str(span.name), + span.kind, + frozenset(str(key) for key in span.attributes or {}), + ) + for span in spans + ) ds.close() - return {"names": names, "pairs": pairs} + return collected -def _keys_by_span(pairs): +def _partition(emitted): + "The statically named spans, and the dynamically named request spans." + static = [record for record in emitted if record[1] is not SpanKind.SERVER] + server = [record for record in emitted if record[1] is SpanKind.SERVER] + return static, server + + +def _keys_by_span(records): by_span = {} - for span_name, key in pairs: - by_span.setdefault(span_name, set()).add(key) + for name, _kind, keys in records: + by_span.setdefault(name, set()).update(keys) return by_span @@ -182,27 +252,34 @@ async def test_workload_emits_exactly_the_expected_names(emitted): Not derived from the registry, so this is what catches a rename that the registry and the call sites make together. """ - assert emitted["names"] == EXPECTED_SPANS - by_span = _keys_by_span(emitted["pairs"]) - assert {name: by_span.get(name, set()) for name in emitted["names"]} == ( - EXPECTED_ATTRIBUTES - ) + static, server = _partition(emitted) + by_span = _keys_by_span(static) + assert set(by_span) == EXPECTED_SPANS + assert by_span == EXPECTED_ATTRIBUTES + + assert server, "the workload made HTTP requests but no SERVER span was emitted" + server_keys = _keys_by_span(server) + assert set(server_keys) == EXPECTED_HTTP_SPAN_NAMES + union = set() + for keys in server_keys.values(): + union |= keys + assert union == EXPECTED_HTTP_ATTRIBUTES def test_registry_matches_the_expected_names(): "The other half of the rename check: the registry against the same literals." - assert {str(span) for span in reg.SPANS} == EXPECTED_SPANS + assert {str(span) for span in reg.SPANS} == EXPECTED_REGISTRY_NAMES for span in reg.SPANS: - assert {str(attribute) for attribute in span.attributes} == EXPECTED_ATTRIBUTES[ - str(span) - ], f"{span} attributes have drifted" + assert { + str(attribute) for attribute in span.attributes + } == EXPECTED_REGISTRY_ATTRIBUTES[str(span)], f"{span} attributes have drifted" @pytest.mark.asyncio async def test_every_emitted_span_is_registered(emitted): "A span added without a registry entry would be missing from the docs." unregistered = sorted( - name for name in emitted["names"] if reg.span_for(name) is None + {name for name, kind, _ in emitted if reg.span_for(name, kind) is None} ) assert ( not unregistered @@ -213,9 +290,12 @@ async def test_every_emitted_span_is_registered(emitted): async def test_every_emitted_attribute_is_registered(emitted): "An attribute added without a registry entry would be missing from the docs." unregistered = sorted( - f"{span_name} -> {key}" - for span_name, key in emitted["pairs"] - if not reg.attribute_allowed(reg.span_for(span_name), key) + { + f"{name} -> {key}" + for name, kind, keys in emitted + for key in keys + if not reg.attribute_allowed(reg.span_for(name, kind), key) + } ) assert ( not unregistered @@ -230,11 +310,10 @@ async def test_every_registered_span_is_emitted(emitted): The direction nothing else catches: the docs must not describe a span that no longer exists. """ - missing = sorted( - str(span) - for span in reg.SPANS - if not any(reg.span_for(name) is span for name in emitted["names"]) - ) + # By identity, not by name: a dynamic entry's own string never appears on + # the wire, so comparing strings would be comparing the wrong things. + resolved = {id(reg.span_for(name, kind)) for name, kind, _ in emitted} + missing = sorted(str(span) for span in reg.SPANS if id(span) not in resolved) assert not missing, ( f"these spans are documented but never emitted by the workload: {missing}. " "Either the instrumentation was removed, or exercise() no longer reaches it." @@ -253,10 +332,14 @@ async def test_every_registered_attribute_is_emitted(emitted): new attribute only appears in some rare case, extend exercise() to reach that case. """ - by_span = _keys_by_span(emitted["pairs"]) + by_entry = {} + for name, kind, keys in emitted: + entry = reg.span_for(name, kind) + if entry is not None: + by_entry.setdefault(id(entry), set()).update(keys) missing = [] for span in reg.SPANS: - emitted_keys = by_span.get(str(span), set()) + emitted_keys = by_entry.get(id(span), set()) for attribute in span.attributes: if attribute not in emitted_keys: missing.append(f"{span} -> {attribute}") @@ -291,6 +374,25 @@ def test_registry_entries_are_usable_as_plain_strings(): assert f"{reg.DB_QUERY}.execute" == "db.query.execute" +def test_dynamic_span_lookup(): + """ + `dynamic=True` matching, which is how the request span resolves. + + The last two assertions are the ones worth having: a dynamic entry must + not swallow a span that does have a registered name, and must not match at + all when the caller supplies no kind - otherwise every unregistered span + in the suite would silently resolve to the request span and the + emitted-but-not-registered direction would stop catching anything. + """ + assert reg.span_for("GET", SpanKind.SERVER) is reg.HTTP_REQUEST + assert reg.span_for("POST /^/(?P[^/]+)$", SpanKind.SERVER) is ( + reg.HTTP_REQUEST + ) + assert reg.span_for("GET") is None + assert reg.span_for("anything at all", SpanKind.INTERNAL) is None + assert reg.span_for("db.query", SpanKind.SERVER) is reg.DB_QUERY + + def test_span_and_attribute_lookup(): assert reg.span_for("db.query") is reg.DB_QUERY assert reg.span_for("datasette.startup") is reg.STARTUP From 8b5956a77bda9d2c83e076d1a776dd52a22b7ee2 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 19:52:12 -0700 Subject: [PATCH 15/24] Name the request span after the route it matched The request span was created at the ASGI edge, before anything knew which route would match, so it carried nothing but the method: every request in a trace UI showed up as "GET", and the only URL on it was url.path, which is unbounded on a public instance and useless as a grouping key. Routing resolves in DatasetteRouter, so that is where the span gets http.route and its semconv `{method} {route}` name. http.route is the compiled route pattern, not a prettified /{database}/{table} template. Datasette routes with compiled regexes and the route table is fixed when the app is built, so the pattern is exact, bounded and needs no parsing; the transform into something prettier accretes edge cases, and Django's instrumentation ships regex-flavoured routes for the same reason. A request that matches no route gets no http.route and keeps its bare method name, which is what semantic conventions ask for. Two things the obvious implementation gets wrong, both found by testing it: - The router must not read `get_current_span()`. A plugin asgi_wrapper() runs *inside* the request middleware, so an instrumented plugin makes its own span current for the whole request - and the route then lands on that plugin's INTERNAL span, renaming it, while the actual request span never gets the one attribute a trace UI groups by. It reproduces with a five-line plugin. The span is passed through the ASGI scope instead, falling back to the current span so an externally-created SERVER span is still enriched. - The method has to be clamped again here. The middleware clamps it for the attribute, but the name is rebuilt from request.method, which is the raw client string - so an unclamped rename put `FROB /(?P...` back into the span name that the middleware had just kept it out of. Both guards are `is_recording()`, not `get_span_context().is_valid`: with no provider but an inbound traceparent the API returns a NonRecordingSpan carrying the remote context, which is valid and records nothing, so an is_valid guard would do the work on every request from a traced caller. Tests cover the route and name, the unrouted 404 fallback, the full attribute set, db.query spans reaching the request span by parent walk, a 500, an inbound traceparent becoming a remote parent, ?sql= never reaching a span attribute, and - in a subprocess, because the suite's provider fixture is session-scoped and unavoidable - the no-provider fast path handing the app the original `send`. The streaming test uses a table larger than one page so the export genuinely issues queries during the body send; without that it passes however early the span ends. Measured on this branch against fixtures.db: a faceted table page went from 112 spans in 56 traces to 113 spans in 1. Co-Authored-By: Claude Opus 5 --- datasette/app.py | 27 +- datasette/telemetry.py | 36 +++ datasette/telemetry_registry.py | 31 +- docs/internals.rst | 38 ++- tests/conftest.py | 1 + tests/test_http_span.py | 537 ++++++++++++++++++++++++++++++- tests/test_telemetry_registry.py | 52 ++- 7 files changed, 681 insertions(+), 41 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 939c8cf8..37c4e882 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -49,8 +49,13 @@ from .events import Event from .plugins import DEFAULT_PLUGINS, get_plugins, pm from .renderer import json_renderer from .resources import DatabaseResource, TableResource -from .telemetry import TelemetryMiddleware, tracer -from .telemetry_registry import STARTUP +from .telemetry import ( + TelemetryMiddleware, + clamp_http_method, + request_span, + tracer, +) +from .telemetry_registry import HTTP_ROUTE, STARTUP from .tokens import TokenInvalid from .tracer import AsgiTracer from .url_builder import Urls @@ -2958,8 +2963,26 @@ class DatasetteRouter: match, view = resolve_routes(self.routes, path) if match is None: + # No route matched, so the span keeps the bare method name it was + # given at the edge and gets no http.route. That is what semantic + # conventions ask for when the route is unknown. return await self.handle_404(request, send) + # The request span was started at the ASGI edge, before routing, so it + # carries only the method as a name. Now that the route is known, give + # it the `{method} {route}` shape semantic conventions want, and the + # http.route attribute - the low-cardinality counterpart to url.path, + # and so the one to group by. + span = request_span(scope) + if span is not None: + route = match.re.pattern + span.set_attribute(HTTP_ROUTE, route) + # Clamped, for the same reason the middleware clamps it: the method + # is a client-controlled string, and an unclamped one here would + # put attacker-supplied text back into the span name that the + # middleware just kept out of it. + span.update_name(f"{clamp_http_method(request.method)} {route}") + new_scope = dict(scope, url_route={"kwargs": match.groupdict()}) request.scope = new_scope try: diff --git a/datasette/telemetry.py b/datasette/telemetry.py index 11391d3a..53193eb0 100644 --- a/datasette/telemetry.py +++ b/datasette/telemetry.py @@ -210,6 +210,38 @@ def _url_path(scope): return scope.get("path", "") +# The request span is handed to `DatasetteRouter.route_path` through the ASGI +# scope rather than through `get_current_span()`, because by the time routing +# happens the current span may well be something else: a plugin +# `asgi_wrapper()` runs *inside* this middleware, and an instrumented one makes +# its own span current for the whole request. Reading the current span there +# would set `http.route` on that plugin's span - and rename it - while leaving +# the actual request span without the one attribute a trace UI groups by. Not +# hypothetical: an ordinary tracing plugin triggers it. +# +# Namespaced per the ASGI spec's rules for extension keys. Absent when the span +# is not recording, which is exactly when the router should skip the work too. +REQUEST_SPAN_SCOPE_KEY = "datasette.telemetry.request_span" + + +def request_span(scope): + """ + The recording request span for an ASGI scope, or None. + + Falls back to the current span so that a `DatasetteRouter` running under + some other instrumentation - one that started a SERVER span but of course + knows nothing about this scope key - still gets enriched. + """ + span = scope.get(REQUEST_SPAN_SCOPE_KEY) + if span is None: + span = otel_trace.get_current_span() + # is_recording(), not `get_span_context().is_valid`: with no provider but + # an inbound `traceparent`, the API's NoOpTracer hands back a + # NonRecordingSpan carrying the *remote* context, which is perfectly valid + # and still records nothing. + return span if span.is_recording() else None + + class TelemetryMiddleware: """ One `SpanKind.SERVER` span per HTTP request. @@ -274,6 +306,10 @@ class TelemetryMiddleware: if user_agent: span.set_attribute(USER_AGENT_ORIGINAL, user_agent) + # A copy, not a mutation: the scope belongs to the server, and + # every other layer in Datasette extends it the same way. + scope = dict(scope, **{REQUEST_SPAN_SCOPE_KEY: span}) + # The status cannot be read off a Response object: `asgi_static`, # the favicon route, `AsgiStream` and `AsgiFileDownload` all call # `send` directly and never build one. Wrapping `send` is the only diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py index 0cc9c156..469a0191 100644 --- a/datasette/telemetry_registry.py +++ b/datasette/telemetry_registry.py @@ -62,10 +62,11 @@ class SpanName(str): self.attributes = tuple(attributes) # True when the emitted name is composed at runtime and shares no # fixed prefix with the registry entry - the HTTP request span, whose - # name is the request method. There is no substring of the entry that - # could be matched against the wire, so `span_for()` resolves these by - # span kind instead, and the entry's own string is a template written - # for a human reading the generated reference. + # name is the request method followed by the matched route. There is + # no substring of the entry that could be matched against the wire, so + # `span_for()` resolves these by span kind instead, and the entry's own + # string is a template written for a human reading the generated + # reference. self.dynamic = dynamic # SpanKind.INTERNAL by default - every span Datasette emits describes # its own internal work. db.query is the one exception: it is a real @@ -99,6 +100,20 @@ HTTP_RESPONSE_STATUS_CODE = Attribute( "closed before anything was sent.", optional=True, ) +HTTP_ROUTE = Attribute( + "http.route", + "The route the request matched, as the compiled regular expression " + "pattern Datasette routes with - for example " + "``/(?P[^\\/\\.]+)/(?P[^\\/\\.]+)(\\.(?P\\w+))?$`` " + "for a table page. It is deliberately the pattern rather than a prettified " + "``/{database}/{table}`` template: the route table is fixed when the app " + "is built, so the pattern is exact, bounded and needs no parsing, whereas " + "the transform into something prettier accretes edge cases. Unlike " + "``url.path`` this is low cardinality, so it is the attribute to group by. " + "Omitted when no route matched - a 404 - which is also when the span name " + "falls back to the bare method.", + optional=True, +) URL_PATH = Attribute( "url.path", "The path portion of the URL. The query string is deliberately **not** " @@ -246,18 +261,22 @@ TRANSACTION = Attribute( # --- Spans ---------------------------------------------------------------- HTTP_REQUEST = SpanName( - "{http.request.method}", + "{http.request.method} {http.route}", "One span per HTTP request, created by the outermost layer of the ASGI " "stack - so plugin ``asgi_wrapper()`` middleware, CSRF protection and " "every database span raised while serving the request all nest inside " "it. Without it each of those would be its own root trace. The span name " - "is not a fixed string: it is the value of ``http.request.method``. " + "is not a fixed string: it is the method followed by the matched route, " + "and just the method for a request that matched no route. The span starts " + "at the ASGI edge, before routing has happened, so it is named for the " + "method there and renamed once the route is known. " "W3C ``traceparent`` and ``baggage`` headers are extracted using the " "global propagator, so a request arriving from an already-traced caller " "continues that trace; set ``OTEL_PROPAGATORS=none`` to turn that off, " "and strip those headers at your proxy if your instance is public.", ( HTTP_REQUEST_METHOD, + HTTP_ROUTE, URL_PATH, URL_SCHEME, SERVER_ADDRESS, diff --git a/docs/internals.rst b/docs/internals.rst index ba749f95..1283b899 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2371,21 +2371,26 @@ This reference is generated from ``datasette/telemetry_registry.py``, the single Spans are ``SpanKind.INTERNAL`` unless a kind is listed below. Two are not: the request span is ``SERVER``, and ``db.query`` is ``CLIENT`` because it is the one span that represents a call to a database rather than Datasette's own work. Trace UIs use the kind to decide whether to render a span as an inbound request or as a database call. ``db.query``'s children stay ``INTERNAL`` because they are Datasette's decomposition of that one query - marking them ``CLIENT`` too would make a single query look like several database calls to anything counting by kind. -The request span's name is the only one that is not a fixed string - it is composed from the request, so the heading below shows the template rather than a literal you will see in a trace. +The request span's name is the only one that is not a fixed string - it is composed from the request, so the heading below shows the template rather than a literal you will see in a trace. A request to a table page produces a span named, in full:: + + GET /(?P[^\/\.]+)/(?P
[^\/\.]+)(\.(?P\w+))?$ + +That is the route's compiled regular expression, not a prettified ``/{database}/{table}`` template. It is deliberate: Datasette routes with compiled patterns and the route table is fixed when the app is built, so the pattern is exact, bounded and needs no parsing, while transforming it into something prettier accretes edge cases. Django's own instrumentation ships regex-flavoured routes for the same reason. .. [[[cog from telemetry_doc import spans spans(cog) .. ]]] -``{http.request.method}`` - One span per HTTP request, created by the outermost layer of the ASGI stack - so plugin ``asgi_wrapper()`` middleware, CSRF protection and every database span raised while serving the request all nest inside it. Without it each of those would be its own root trace. The span name is not a fixed string: it is the value of ``http.request.method``. W3C ``traceparent`` and ``baggage`` headers are extracted using the global propagator, so a request arriving from an already-traced caller continues that trace; set ``OTEL_PROPAGATORS=none`` to turn that off, and strip those headers at your proxy if your instance is public. +``{http.request.method} {http.route}`` + One span per HTTP request, created by the outermost layer of the ASGI stack - so plugin ``asgi_wrapper()`` middleware, CSRF protection and every database span raised while serving the request all nest inside it. Without it each of those would be its own root trace. The span name is not a fixed string: it is the method followed by the matched route, and just the method for a request that matched no route. The span starts at the ASGI edge, before routing has happened, so it is named for the method there and renamed once the route is known. W3C ``traceparent`` and ``baggage`` headers are extracted using the global propagator, so a request arriving from an already-traced caller continues that trace; set ``OTEL_PROPAGATORS=none`` to turn that off, and strip those headers at your proxy if your instance is public. Kind: ``SERVER``. Attributes: - ``http.request.method`` - The HTTP method, clamped to the nine methods RFC 9110 and RFC 5789 define. Anything else is reported as ``_OTHER``: the method is a client-controlled string, so echoing it back unbounded would be a cardinality hazard. + - ``http.route`` *(optional)* - The route the request matched, as the compiled regular expression pattern Datasette routes with - for example ``/(?P[^\/\.]+)/(?P
[^\/\.]+)(\.(?P\w+))?$`` for a table page. It is deliberately the pattern rather than a prettified ``/{database}/{table}`` template: the route table is fixed when the app is built, so the pattern is exact, bounded and needs no parsing, whereas the transform into something prettier accretes edge cases. Unlike ``url.path`` this is low cardinality, so it is the attribute to group by. Omitted when no route matched - a 404 - which is also when the span name falls back to the bare method. - ``url.path`` - The path portion of the URL. The query string is deliberately **not** recorded, on this or any other span: Datasette puts user-supplied SQL in ``?sql=`` and canned query parameters in the query string, so exporting it by default would export exactly the data the rest of this instrumentation is careful with. - ``url.scheme`` - ``http`` or ``https``. - ``server.address`` *(optional)* - The ``Host`` header. Client-controlled, so treat it as untrusted input rather than as the identity of the server. @@ -2441,6 +2446,29 @@ The request span's name is the only one that is not a fixed string - it is compo .. [[[end]]] +.. _internals_telemetry_requests: + +Requests and inbound trace context +---------------------------------- + +Datasette creates the request span itself, at the outermost layer of the ASGI stack, so a trace is complete out of the box with no plugin and no extra instrumentation package. Everything raised while serving the request - plugin ``asgi_wrapper()`` middleware, CSRF protection, every database query - nests inside it. + +**Inbound trace context is trusted by default.** W3C ``traceparent`` and ``baggage`` headers are extracted from every request using the global propagator, so a request arriving from an already-traced caller continues that trace instead of starting a new one. That is what every other framework instrumentation does - Flask, Django, FastAPI and ``opentelemetry-instrumentation-asgi`` all extract unconditionally - but on an instance open to the internet it means an arbitrary client can influence your traces: + +- **Trace-ID pollution.** The client chooses the trace ID its request is filed under. +- **Sampling control.** The SDK's default sampler is ``parentbased_always_on``, so under any parent-based sampler a client's sampled flag can force recording - a telemetry-cost denial of service - or suppress it. +- **Baggage injection**, through the default composite propagator. + +Because extraction goes through the *global* propagator there is no Datasette setting to configure, and the remedies are the standard OpenTelemetry ones: + +- Strip ``traceparent``, ``tracestate`` and ``baggage`` at your reverse proxy, which is the right answer for a public instance fronted by one. +- Set ``OTEL_PROPAGATORS=none`` to disable extraction entirely, or ``OTEL_PROPAGATORS=tracecontext`` to keep trace continuation and drop baggage. +- Use a sampler that is not parent-based, which neutralises the sampling concern on its own. + +**Installing an ASGI instrumentation as well is harmless.** If you wire up ``opentelemetry-instrumentation-asgi`` through an ``asgi_wrapper()`` plugin, its middleware lands *inside* Datasette's own, so its span becomes a redundant child ``SERVER`` span in the same trace. Nothing is re-orphaned. There is no setting to turn Datasette's request span off, because "turn it off" is already covered by installing no provider, or by ``OTEL_SDK_DISABLED=true``. + +**Where** ``datasette.startup`` **lands depends on how you run Datasette.** ``datasette serve`` calls ``invoke_startup()`` before the server starts accepting connections, so the startup span is its own trace. An ASGI-hosted or programmatic deployment reaches startup lazily, on the first request, so there the startup span nests under that first request - which is honest, since it genuinely is that request's latency. + .. _internals_telemetry_privacy: Privacy and safety @@ -2452,6 +2480,7 @@ Spans leave your infrastructure whenever you configure an exporter, so what goes - **SQL parameter values are never recorded.** Only ``datasette.param_count``, a count. Parameter values are the part of a query most likely to hold something sensitive, and separating them from the SQL is the reason bound parameters exist. - **No actor identifiers are recorded.** No actor ID, no actor JSON, no client IP address. Nothing on a span identifies who made the request. - **Table names come only from an explicit** ``table=`` **argument.** ``db.collection.name`` is set by callers that already know which table they are working with, and is never derived from the SQL. Deriving it would mean parsing, and on an instance where visitors can create tables the set of possible values has no ceiling. +- **The query string is never recorded.** There is no ``url.query`` attribute on the request span or on any other span. Datasette puts user-supplied SQL in ``?sql=`` and canned query parameters in the query string, so recording it by default would export exactly the class of data the rules above are careful with. Only ``url.path`` and ``http.route`` are recorded. The SQL itself, though, *is* recorded, and on a public instance that means anything a visitor types into the query editor or passes as ``?sql=`` will be exported along with the span. That is the trade-off tracing a query engine makes. @@ -2460,7 +2489,8 @@ The SQL itself, though, *is* recorded, and on a public instance that means anyth Known limitations ----------------- -- **Datasette does not create a span for the HTTP request itself.** Every span listed above is therefore a root span unless something above Datasette - an ASGI instrumentation layer, or the web framework embedding it - has already started one for the request, in which case Datasette's spans nest underneath it correctly. +- ``http.route`` **is a compiled regular expression, not a pretty route template.** See :ref:`internals_telemetry_requests` above for why. +- **Inbound trace context is trusted by default**, which on a public instance means a client can influence your trace IDs, your sampling and your baggage. :ref:`internals_telemetry_requests` lists the remedies. - **Two plugin hooks run outside the** ``datasette.startup`` **span.** ``register_output_renderer`` is dispatched from ``Datasette.__init__()`` and ``asgi_wrapper`` from ``Datasette.app()``, both of which happen before ``invoke_startup()``. Datasette itself queries no database in either, so a default install emits nothing there - but a plugin that does will produce a root trace. Covering these would mean holding a span open across object construction, which is worse than the orphan. - ``db.operation.name`` **reports** ``WITH`` **for a statement that opens with a common table expression**, rather than the operation inside it, and a substantial share of Datasette's own reads take that form. The attribute is a leading-keyword match against a fixed allowlist, deliberately not a parse. - **Spans emitted before a provider is installed are not recorded.** If you are embedding Datasette in a host application, install your ``TracerProvider`` before serving traffic. This is ordinary OpenTelemetry behaviour rather than anything Datasette controls; nothing is permanently affected, those particular spans are simply dropped. diff --git a/tests/conftest.py b/tests/conftest.py index b01f111d..b2453133 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -230,6 +230,7 @@ def pytest_collection_modifyitems(config, items): # (SIGSEGV/SIGBUS inside _execute_child). Reproduces with any subprocess # call placed there, on an unmodified tree - running it first avoids it. move_to_front(items, "test_datasette_package_never_imports_the_sdk") + move_to_front(items, "test_no_provider_takes_the_fast_path") def move_to_front(items, test_name): diff --git a/tests/test_http_span.py b/tests/test_http_span.py index 740d437d..ff792489 100644 --- a/tests/test_http_span.py +++ b/tests/test_http_span.py @@ -3,31 +3,53 @@ The HTTP request span. `tests/test_telemetry_registry.py` already pins the span's name shape, kind and attribute keys against literals, so this file deliberately does not -repeat that. What it covers is the three properties of the middleware that -the registry conformance test structurally cannot see: +repeat that. What it covers is the properties of the middleware and of the +router's `http.route` enrichment that the registry conformance test +structurally cannot see: - **where the middleware sits.** Outermost is the entire point - moving it inside the plugin `asgi_wrapper()` loop leaves plugin middleware creating orphan root traces, which is the problem this span exists to fix, and every attribute assertion still passes. +- **which span the route lands on**, which only diverges once something else + has made a span current. - **method clamping**, which a workload of ordinary GETs can never exercise. - **the query string never being recorded**, which only fails if a request actually carries one. +- **the span outliving a streamed response body**, which only a paging export + can distinguish from ending far too early. """ import asyncio import itertools +import json +import subprocess +import sys +import textwrap +import time import pytest import pytest_asyncio pytest.importorskip("opentelemetry.sdk") -from opentelemetry.trace import SpanKind, StatusCode +from opentelemetry.trace import ( + NonRecordingSpan, + SpanContext, + SpanKind, + StatusCode, + TraceFlags, +) from datasette import hookimpl from datasette.app import Datasette -from datasette.telemetry import TelemetryMiddleware, tracer +from datasette.telemetry import ( + REQUEST_SPAN_SCOPE_KEY, + TelemetryMiddleware, + request_span, + tracer, +) +from datasette.utils import resolve_routes # Named in-memory databases are shared-cache: two Datasette instances given # the same name share one SQLite database and the second `create table` @@ -110,12 +132,51 @@ async def ds(): instance.close() +@pytest_asyncio.fixture +async def ds_paging(): + """ + An instance whose table is bigger than `max_returned_rows`. + + That is what makes `?_stream=1` genuinely page: `stream_csv` loops calling + `fetch_data` for each page *inside* the response body send, so the trace + contains `db.query` spans that start after the response has begun. On a + table that fits in one page every query finishes before the body starts + and the span-covers-the-body assertion cannot fail. + """ + name = f"httpspanpaging{next(_names)}" + # Both settings matter. `?_stream=1` forces `_size=max`, which is + # `max_returned_rows` - so lowering only that gives one page of five rows + # and no `next` token, and the export never loops. + instance = Datasette( + memory=True, settings={"max_returned_rows": 5, "default_page_size": 3} + ) + instance.add_memory_database(name) + await instance.invoke_startup() + db = instance.get_database(name) + await db.execute_write("create table t (id integer primary key, v text)") + await db.execute_write_many( + "insert into t (id, v) values (?, ?)", [[i, f"v{i}"] for i in range(40)] + ) + instance.db_name = name + try: + yield instance + finally: + instance.close() + + def _server_spans(otel_spans): return [ span for span in otel_spans.get_finished_spans() if span.kind is SpanKind.SERVER ] +def _route_for(ds, path): + "The compiled pattern Datasette's own router resolves `path` to." + match, _view = resolve_routes(ds._routes(), path) + assert match is not None, f"{path} matches no route" + return match.re.pattern + + @pytest.mark.asyncio async def test_plugin_asgi_wrapper_middleware_runs_inside_the_request_span( ds, otel_spans @@ -140,20 +201,20 @@ async def test_plugin_asgi_wrapper_middleware_runs_inside_the_request_span( spans = otel_spans.get_finished_spans() server = [span for span in spans if span.kind is SpanKind.SERVER] assert len(server) == 1, "expected exactly one SERVER span per request" - request_span = server[0] - assert request_span.parent is None, "the request span should be the trace root" + server_span = server[0] + assert server_span.parent is None, "the request span should be the trace root" plugin_spans = [span for span in spans if span.name == PLUGIN_MIDDLEWARE_SPAN] assert len(plugin_spans) == 1 assert plugin_spans[0].parent is not None - assert plugin_spans[0].parent.span_id == request_span.context.span_id - assert plugin_spans[0].context.trace_id == request_span.context.trace_id + assert plugin_spans[0].parent.span_id == server_span.context.span_id + assert plugin_spans[0].context.trace_id == server_span.context.trace_id # And the database work is in the same trace, not off on its own. queries = [span for span in spans if span.name == "db.query"] assert queries, "a table page should have issued at least one query" for query in queries: - assert query.context.trace_id == request_span.context.trace_id + assert query.context.trace_id == server_span.context.trace_id @pytest.mark.asyncio @@ -161,15 +222,20 @@ async def test_unrecognised_method_is_clamped(ds, otel_spans): """ Anyone can send `FROB / HTTP/1.1`. An unclamped method is an unbounded dimension a client controls, so semantic conventions map anything off the - known list to `_OTHER` - and the span name is the method, so an unclamped - one would put attacker-supplied text in the span name too. + known list to `_OTHER`. + + The span name is checked too, and it is the reason the router clamps the + method a second time when it renames the span: the middleware's clamping + protects the attribute, but the name is rebuilt from `request.method` in + `route_path`, which is the raw client string. An unclamped rename would + put attacker-supplied text straight back into the span name. """ otel_spans.clear() await ds.client.request("FROB", f"/{ds.db_name}/t") server = _server_spans(otel_spans) assert len(server) == 1 - assert server[0].name == "_OTHER" assert server[0].attributes["http.request.method"] == "_OTHER" + assert server[0].name == f"_OTHER {server[0].attributes['http.route']}" @pytest.mark.asyncio @@ -179,8 +245,8 @@ async def test_known_method_is_not_clamped(ds, otel_spans): await ds.client.get(f"/{ds.db_name}/t") server = _server_spans(otel_spans) assert len(server) == 1 - assert server[0].name == "GET" assert server[0].attributes["http.request.method"] == "GET" + assert server[0].name == f"GET {server[0].attributes['http.route']}" @pytest.mark.asyncio @@ -275,6 +341,10 @@ async def test_a_404_is_not_an_error(ds, otel_spans): `error.type` alone. Datasette 404s are routine - every missing table, and every bot probing for /wp-login.php - so treating them as errors would drown a real 500 in noise. + + Note this 404 *does* match a route: `/no-such-database-at-all` matches the + database pattern and the view then raises `NotFound`. Most Datasette 404s + are that shape rather than the unrouted one below. """ otel_spans.clear() response = await ds.client.get("/no-such-database-at-all") @@ -286,6 +356,29 @@ async def test_a_404_is_not_an_error(ds, otel_spans): assert server[0].status.status_code is StatusCode.UNSET +@pytest.mark.asyncio +async def test_an_unrouted_404_has_no_route_and_a_bare_method_name(ds, otel_spans): + """ + When no route matches there is nothing to set `http.route` to, so the span + keeps the bare method name it was given at the edge - which is exactly the + fallback semantic conventions specify for an unknown route. + + `/a/b/c/d/e` is used rather than a plausible-looking missing name because + Datasette's route table is greedy: `/no-such-database-at-all` matches the + database pattern, and `/-/nope/deeper` matches the row pattern. Only a + path deeper than any route matches nothing at all. + """ + otel_spans.clear() + response = await ds.client.get("/a/b/c/d/e") + assert response.status_code == 404 + server = _server_spans(otel_spans) + assert len(server) == 1 + assert server[0].name == "GET" + assert "http.route" not in server[0].attributes + assert server[0].attributes["http.response.status_code"] == 404 + assert server[0].status.status_code is StatusCode.UNSET + + @pytest.mark.asyncio async def test_only_the_first_http_response_start_is_recorded(otel_spans): """ @@ -342,3 +435,421 @@ async def test_lifespan_scope_passes_through_unspanned(otel_spans): await app({"type": "lifespan"}, receive, send) assert sent == ["lifespan.startup.complete", "lifespan.shutdown.complete"] assert not _server_spans(otel_spans) + + +@pytest.mark.asyncio +async def test_http_route_is_the_compiled_pattern(ds, otel_spans): + """ + `http.route` is the route's compiled regex, not a prettified template. + + Asserted against what Datasette's own router resolves rather than against + a copied literal, so this pins the *relationship* - the attribute is the + matched route - and does not break when a core pattern is edited. + """ + path = f"/{ds.db_name}/t" + expected = _route_for(ds, path) + otel_spans.clear() + assert (await ds.client.get(path)).status_code == 200 + server = _server_spans(otel_spans) + assert len(server) == 1 + assert server[0].attributes["http.route"] == expected + assert server[0].name == f"GET {expected}" + # The pattern really is the ugly one, and that is deliberate - if someone + # adds a prettifier this is the assertion that should make them argue for + # it rather than slip it in. + assert "(?P" in expected + + +@pytest.mark.asyncio +async def test_the_route_lands_on_the_request_span_not_a_plugins_current_span( + ds, otel_spans +): + """ + The route is set on the span the middleware started, found through the + ASGI scope - not on whatever span happens to be current when routing + resolves. + + Those are the same span only until a plugin `asgi_wrapper()` starts one of + its own. A plugin wrapper runs *inside* this middleware, so an instrumented + plugin makes its span current for the whole request: reading the current + span in `route_path` renames that plugin's INTERNAL span to + `GET ` and hangs `http.route` off it, while the actual request span + keeps a bare method name and never gets the one attribute a trace UI + groups requests by. Verified by reproducing it, not by reasoning about it. + """ + ds.pm.register(_MiddlewarePlugin(), name="httpspan-middleware") + try: + otel_spans.clear() + path = f"/{ds.db_name}/t" + expected = _route_for(ds, path) + assert (await ds.client.get(path)).status_code == 200 + finally: + ds.pm.unregister(name="httpspan-middleware") + + spans = otel_spans.get_finished_spans() + server = _server_spans(otel_spans) + assert len(server) == 1 + assert server[0].attributes["http.route"] == expected + assert server[0].name == f"GET {expected}" + # And the plugin's span is untouched: same name, no route attribute. + plugin_spans = [span for span in spans if span.name == PLUGIN_MIDDLEWARE_SPAN] + assert len(plugin_spans) == 1 + assert "http.route" not in (plugin_spans[0].attributes or {}) + + +@pytest.mark.asyncio +async def test_request_span_attributes(ds, otel_spans): + "The whole attribute set on one ordinary request." + path = f"/{ds.db_name}/t" + otel_spans.clear() + assert (await ds.client.get(path)).status_code == 200 + server = _server_spans(otel_spans) + assert len(server) == 1 + attributes = server[0].attributes + assert attributes["http.request.method"] == "GET" + assert attributes["url.path"] == path + assert attributes["url.scheme"] == "http" + assert attributes["http.response.status_code"] == 200 + assert attributes["http.route"] == _route_for(ds, path) + assert server[0].status.status_code is StatusCode.UNSET + # Never, on any span: an IP is borderline PII and the query string carries + # user-supplied SQL. + assert "client.address" not in attributes + assert "url.query" not in attributes + + +@pytest.mark.asyncio +async def test_db_query_spans_are_children_of_the_request_span(ds, otel_spans): + """ + The point of the whole PR. + + Not just "same trace ID" - every `db.query` span must reach the request + span by walking parents, and the request span must be the only root. A + stray root would show up in a trace UI as its own single-span trace, which + is the state this replaces. + """ + otel_spans.clear() + assert (await ds.client.get(f"/{ds.db_name}/t?_facet=v")).status_code == 200 + spans = otel_spans.get_finished_spans() + server = _server_spans(otel_spans) + assert len(server) == 1 + server_span = server[0] + assert server_span.parent is None + + by_span_id = {span.context.span_id: span for span in spans} + roots = [span for span in spans if span.parent is None] + assert [span.name for span in roots] == [server_span.name], ( + "every span from a request should hang off the request span, but these " + f"are roots: {sorted(span.name for span in roots)}" + ) + + queries = [span for span in spans if span.name == "db.query"] + assert queries, "a faceted table page should have issued queries" + for query in queries: + assert query.context.trace_id == server_span.context.trace_id + # Walk up to the root, which must be the request span. + current = query + seen = 0 + while current.parent is not None: + current = by_span_id[current.parent.span_id] + seen += 1 + assert seen < 20, "parent chain did not terminate" + assert current is server_span + + +@pytest.mark.asyncio +async def test_500_sets_error_status_and_error_type(ds, otel_spans): + """ + A plain 500 - no exception escaping the app, because `route_path` converts + it into a response itself. The status is the only signal the middleware + gets, so `error.type` is the status as a string. + """ + ds.pm.register(_BoomPlugin(), name="httpspan-boom") + try: + otel_spans.clear() + response = await ds.client.get("/-/http-span-boom") + assert response.status_code == 500 + finally: + ds.pm.unregister(name="httpspan-boom") + server = _server_spans(otel_spans) + assert len(server) == 1 + assert server[0].attributes["http.response.status_code"] == 500 + assert server[0].attributes["error.type"] == "500" + assert server[0].status.status_code is StatusCode.ERROR + + +@pytest.mark.asyncio +async def test_csv_stream_span_covers_the_body_send(ds_paging, otel_spans): + """ + The span must not end when the handler returns - it has to cover the + response body. + + `stream_csv` runs its generator inline inside `AsgiStream.asgi_send`, and + that call happens inside the single `await self.app(...)` the middleware + makes, so a plain `finally` is enough and no deferred-end machinery is + needed. This is the assertion that holds that claim up: a `db.query` that + starts during the body send must still finish before the request span + does. + + Only meaningful on an export that actually pages, hence `ds_paging` - on a + single-page table every query is over before the body begins and this + passes however early the span ends. The middle assertion below, that some + query *started* after `http.response.start` went out, is what keeps the + test honest about that; it is why the app is driven as raw ASGI rather + than through `ds.client`, which cannot timestamp the response start. + + `time.time_ns()` is the same clock the SDK stamps spans with, so the two + are directly comparable. + """ + app = ds_paging.app() + body = [] + response_started_at = None + + async def receive(): + return {"type": "http.request", "body": b"", "more_body": False} + + async def send(message): + nonlocal response_started_at + if message["type"] == "http.response.start": + assert message["status"] == 200 + response_started_at = time.time_ns() + else: + body.append(message.get("body") or b"") + + otel_spans.clear() + await app( + { + "type": "http", + "http_version": "1.1", + "method": "GET", + "path": f"/{ds_paging.db_name}/t.csv", + "raw_path": f"/{ds_paging.db_name}/t.csv".encode("latin-1"), + "query_string": b"_stream=1", + "scheme": "http", + "headers": [(b"host", b"localhost")], + }, + receive, + send, + ) + # 40 rows plus a header - the export really did read past one page + assert len(b"".join(body).decode("utf-8").strip().splitlines()) == 41 + assert response_started_at is not None + + spans = otel_spans.get_finished_spans() + server = _server_spans(otel_spans) + assert len(server) == 1 + server_span = server[0] + queries = [span for span in spans if span.name == "db.query"] + assert len(queries) > 1 + during_body = [span for span in queries if span.start_time > response_started_at] + assert during_body, ( + "no query ran after the response started, so this workload cannot " + "distinguish a span that covers the body send from one that ends when " + "the handler returns - the export is not paging" + ) + last_query_end = max(span.end_time for span in queries) + assert server_span.end_time > last_query_end, ( + "the request span ended before the last query of a streaming export - " + "it is not covering the response body" + ) + for query in queries: + assert query.context.trace_id == server_span.context.trace_id + + +@pytest.mark.asyncio +async def test_inbound_traceparent_becomes_the_parent(ds, otel_spans): + """ + W3C trace context is extracted with the global propagator, so a request + from an already-traced caller continues that trace. + + The sampled flag has to be set: the SDK's default sampler is + parentbased_always_on, so a `-00` flag would drop the span and the test + would fail for a reason that has nothing to do with propagation. + """ + trace_id = "4bf92f3577b34da6a3ce929d0e0e4736" + parent_span_id = "00f067aa0ba902b7" + otel_spans.clear() + response = await ds.client.get( + f"/{ds.db_name}/t", + headers={"traceparent": f"00-{trace_id}-{parent_span_id}-01"}, + ) + assert response.status_code == 200 + server = _server_spans(otel_spans) + assert len(server) == 1 + server_span = server[0] + assert f"{server_span.context.trace_id:032x}" == trace_id + assert server_span.parent is not None + assert f"{server_span.parent.span_id:016x}" == parent_span_id + assert server_span.parent.is_remote + # And the database spans joined the caller's trace too, not a new one. + queries = [ + span for span in otel_spans.get_finished_spans() if span.name == "db.query" + ] + assert queries + for query in queries: + assert f"{query.context.trace_id:032x}" == trace_id + + +@pytest.mark.asyncio +async def test_user_supplied_sql_in_the_query_string_is_never_recorded(ds, otel_spans): + """ + The `?sql=` case specifically, which is the one that matters: this is the + request where the query string *is* user-supplied SQL, and it reaches a + view that runs it. The marker is searched for across every attribute of + every span in the trace, not just for a `url.query` key, so recording it + under some other name fails too. + + `db.query.text` legitimately contains the SQL - that is documented and + deliberate - so the marker is checked against the request span's own + attributes, and against `url.*` and `http.*` keys everywhere. + """ + marker = "secret_marker_5b1f" + otel_spans.clear() + # `/{db}?sql=` 302s to the query view, so go straight there - a redirect + # would leave the SQL only on a span for a request that never ran it. + response = await ds.client.get(f"/{ds.db_name}/-/query?sql=select+'{marker}'") + assert response.status_code == 200 + spans = otel_spans.get_finished_spans() + server = _server_spans(otel_spans) + assert len(server) == 1 + leaked = [ + f"{span.name} -> {key}={value!r}" + for span in spans + for key, value in (span.attributes or {}).items() + if (span is server[0] or str(key).startswith(("url.", "http."))) + and (marker in str(value) or str(key) == "url.query") + ] + assert not leaked, "the query string reached a span attribute: " + ", ".join(leaked) + # The request really did carry the marker, so the search above had + # something to find. + assert marker in response.text + + +def test_request_span_skips_a_valid_but_non_recording_span(): + """ + `request_span()` is guarded on `is_recording()`, not on + `get_span_context().is_valid`, and this is the case that separates them. + + With no provider installed but an inbound `traceparent`, the API's + NoOpTracer hands back a `NonRecordingSpan` carrying the *remote* span + context - valid, sampled, and recording nothing. An `is_valid` guard would + wave that through and the router would build the name string and call + `set_attribute`/`update_name` on a span that discards both. + + Tested at this level deliberately: through a real request the two guards + are indistinguishable, because every call the router makes on a + NonRecordingSpan is already a no-op. The only difference is the work done + to get there, so the guard itself is what has to be asserted on. + """ + remote = SpanContext( + trace_id=0x4BF92F3577B34DA6A3CE929D0E0E4736, + span_id=0x00F067AA0BA902B7, + is_remote=True, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + ) + assert remote.is_valid + non_recording = NonRecordingSpan(remote) + assert non_recording.is_recording() is False + assert request_span({REQUEST_SPAN_SCOPE_KEY: non_recording}) is None + # Nothing current, nothing in the scope: the INVALID_SPAN fallback. + assert request_span({}) is None + # And the case it must not skip. + with tracer.start_as_current_span("test.request_span.recording") as span: + assert request_span({REQUEST_SPAN_SCOPE_KEY: span}) is span + # Falling back to the current span is how an externally installed + # SERVER span still gets enriched. + assert request_span({}) is span + + +NO_PROVIDER_PROGRAM = textwrap.dedent(""" + import asyncio, json, sys + + from datasette.telemetry import TelemetryMiddleware + + seen = {} + + + async def inner(scope, receive, send): + seen.setdefault("sends", []).append(send) + seen.setdefault("scopes", []).append(scope) + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b""}) + + + async def real_send(message): + pass + + + async def main(): + middleware = TelemetryMiddleware(inner) + for headers in ([], [(b"traceparent", b"00-" + b"a" * 32 + b"-" + b"b" * 16 + b"-01")]): + await middleware( + { + "type": "http", + "method": "GET", + "path": "/", + "raw_path": b"/", + "scheme": "http", + "headers": headers, + }, + None, + real_send, + ) + print( + json.dumps( + { + "unwrapped": [send is real_send for send in seen["sends"]], + "scope_keys": [ + "datasette.telemetry.request_span" in scope + for scope in seen["scopes"] + ], + "sdk_imported": any( + name.startswith("opentelemetry.sdk") for name in sys.modules + ), + } + ) + ) + + + asyncio.run(main()) + """) + + +def test_no_provider_takes_the_fast_path(): + """ + With no `TracerProvider` installed the middleware must hand the + application the *original* `send`, not a wrapper - a default Datasette + install should pay essentially nothing for instrumentation it is not + using. + + This has to run in a subprocess. The suite's `_otel_provider` fixture is + session-scoped and autouse, and `set_tracer_provider()` is effectively + once-per-process, so in-process every span is recording and the fast path + is unreachable. + + The second case, with an inbound `traceparent`, is the one that pins the + check itself. With no provider the API's NoOpTracer returns a + NonRecordingSpan carrying the *remote* span context: its + `get_span_context().is_valid` is True while `is_recording()` is False. A + fast path guarded on `is_valid` would therefore silently stop working for + exactly the requests that arrive from an already-traced caller - which on + a real deployment behind an instrumented proxy is all of them. + + conftest.py's pytest_collection_modifyitems() moves this test to the front + of the run by name - if you rename it, rename it there too. + """ + result = subprocess.run( + [sys.executable, "-c", NO_PROVIDER_PROGRAM], + capture_output=True, + text=True, + check=True, + ) + report = json.loads(result.stdout) + assert report["sdk_imported"] is False, "the SDK loaded in a fresh interpreter" + assert report["unwrapped"] == [True, True], ( + "the middleware wrapped `send` with no provider installed; the second " + "entry is the inbound-traceparent case, which fails if the fast path " + "is guarded on is_valid instead of is_recording()" + ) + # Same fast path, other observable: nothing is stashed in the scope either. + assert report["scope_keys"] == [False, False] diff --git a/tests/test_telemetry_registry.py b/tests/test_telemetry_registry.py index a6037345..f37dc8f0 100644 --- a/tests/test_telemetry_registry.py +++ b/tests/test_telemetry_registry.py @@ -71,15 +71,22 @@ EXPECTED_ATTRIBUTES = { EXPECTED_SPANS = set(EXPECTED_ATTRIBUTES) # The HTTP request span is handled separately because its name is composed at -# runtime - it is the request method - so there is no fixed string to pin it -# to. What can still be pinned, and is what a dashboard depends on, is the -# shape of the name and the attribute keys. The workload below only issues -# GETs, so a change that stopped clamping the method, or that started naming -# the span after the path, fails here. -EXPECTED_HTTP_SPAN_NAME = "{http.request.method}" -EXPECTED_HTTP_SPAN_NAMES = {"GET"} +# runtime - the request method, then the route it matched - so there is no +# fixed string to pin it to. What can still be pinned, and is what a dashboard +# depends on, is the shape of that name and the attribute keys. +# +# The route half is deliberately not spelled out as a literal: it is a core +# route regex, and pinning those here would make an unrelated routing change +# fail the telemetry conformance test. What is pinned instead is that the name +# is exactly the method, a space, and the span's own `http.route` value - the +# `{method} {route}` shape semantic conventions specify. The workload below +# only issues GETs, so a change that stopped clamping the method, or that +# started naming the span after the path, fails here. +EXPECTED_HTTP_SPAN_NAME = "{http.request.method} {http.route}" +EXPECTED_HTTP_METHOD_NAMES = {"GET"} EXPECTED_HTTP_ATTRIBUTES = { "http.request.method", + "http.route", "url.path", "url.scheme", "server.address", @@ -205,10 +212,12 @@ async def exercise(): @pytest_asyncio.fixture async def emitted(otel_spans): """ - Every (span name, span kind, attribute keys) triple a broad workload emits. + Every (span name, span kind, attributes) triple a broad workload emits. The kind is carried because the request span's name is composed at - runtime, so `span_for()` resolves it by kind instead. + runtime, so `span_for()` resolves it by kind instead. The attributes are + carried as a mapping rather than a set of keys because the request span's + name has to be checked against its own `http.route` value. """ # otel_spans has already cleared the exporter, and nothing is cleared # after this point: the workload's own startup emits datasette.startup. @@ -222,7 +231,7 @@ async def emitted(otel_spans): ( str(span.name), span.kind, - frozenset(str(key) for key in span.attributes or {}), + {str(key): value for key, value in (span.attributes or {}).items()}, ) for span in spans ) @@ -239,8 +248,8 @@ def _partition(emitted): def _keys_by_span(records): by_span = {} - for name, _kind, keys in records: - by_span.setdefault(name, set()).update(keys) + for name, _kind, attributes in records: + by_span.setdefault(name, set()).update(attributes) return by_span @@ -258,11 +267,22 @@ async def test_workload_emits_exactly_the_expected_names(emitted): assert by_span == EXPECTED_ATTRIBUTES assert server, "the workload made HTTP requests but no SERVER span was emitted" - server_keys = _keys_by_span(server) - assert set(server_keys) == EXPECTED_HTTP_SPAN_NAMES union = set() - for keys in server_keys.values(): - union |= keys + methods = set() + for name, _kind, attributes in server: + union |= set(attributes) + route = attributes.get("http.route") + # Every request in the workload matches a route, so every one of these + # names must be `{method} {route}`. A 404 would be a bare method - the + # http_route tests cover that case with a real request. + assert route, f"the request span {name!r} carries no http.route" + method, _, name_route = name.partition(" ") + assert name_route == route, ( + f"the request span is named {name!r}, which is not the " + f"`{{method}} {{route}}` of {method!r} and {route!r}" + ) + methods.add(method) + assert methods == EXPECTED_HTTP_METHOD_NAMES assert union == EXPECTED_HTTP_ATTRIBUTES From 4aaf20355bae6b04db119dffaebe078c1c3ad4df Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Wed, 2 Sep 2026 11:27:06 -0700 Subject: [PATCH 16/24] Review polish: changelog entry, comment dedupe, semconv note, 404 route test - Add the missing changelog bullet for the HTTP request span - the most user-visible signal in the stack had no entry. - State the deliberate deviation on server.address: it is the verbatim Host header including any :port, not the semconv address/port split. - Deduplicate the is_recording()-vs-is_valid rationale: the middleware fast path keeps the full telling, request_span() now points at it. - Trim the http.route regex rationale in the docs intro to a pointer at the attribute description, halve the httpx raw_path digression, drop a comment that restated the call below it, and leave client-IP policy to the privacy section instead of the user_agent description. - Assert the routed 404 still carries http.route and an enriched span name - route enrichment must not be gated on a successful response. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012U7coQfVu8nK2R4q2mCULA --- datasette/telemetry.py | 21 +++++++-------------- datasette/telemetry_registry.py | 8 ++++---- docs/changelog.rst | 1 + docs/internals.rst | 6 +++--- tests/test_http_span.py | 3 +++ 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/datasette/telemetry.py b/datasette/telemetry.py index 53193eb0..1f74a3fb 100644 --- a/datasette/telemetry.py +++ b/datasette/telemetry.py @@ -194,13 +194,11 @@ def _url_path(scope): contain encoded slashes, which `scope["path"]` has already collapsed. The split on "?" is not decoration. The ASGI spec's `raw_path` excludes - the query string, and uvicorn honours that, but the name is used the - other way round elsewhere in this same dependency tree: httpx's - `URL.raw_path` is documented as "raw bytes of both the path and query". - A server that followed that reading would hand us `?sql=...` here, and - Datasette's query strings carry user-supplied SQL, which core never - records. A literal "?" cannot appear unencoded in a path, so the split - costs nothing when the server is well behaved. + the query string, but the name is read both ways in the wild - httpx's + own `raw_path` includes the query - and Datasette's query strings carry + user-supplied SQL, which core never records. A literal "?" cannot appear + unencoded in a path, so the defensive split costs nothing when the server + is well behaved. """ raw_path = scope.get("raw_path") if raw_path: @@ -235,10 +233,8 @@ def request_span(scope): span = scope.get(REQUEST_SPAN_SCOPE_KEY) if span is None: span = otel_trace.get_current_span() - # is_recording(), not `get_span_context().is_valid`: with no provider but - # an inbound `traceparent`, the API's NoOpTracer hands back a - # NonRecordingSpan carrying the *remote* context, which is perfectly valid - # and still records nothing. + # is_recording(), not `get_span_context().is_valid` - see the fast-path + # comment in TelemetryMiddleware for why valid is not the same as recording. return span if span.is_recording() else None @@ -327,9 +323,6 @@ class TelemetryMiddleware: escaped = False try: - # Positional (scope, receive, send) throughout this codebase - - # `wrapped_send` is the third argument. `receive` is passed - # through unwrapped. await self.app(scope, receive, wrapped_send) except BaseException as exception: # BaseException, not Exception: `route_path` turns almost diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py index 469a0191..0850a4ff 100644 --- a/datasette/telemetry_registry.py +++ b/datasette/telemetry_registry.py @@ -125,15 +125,15 @@ URL_PATH = Attribute( URL_SCHEME = Attribute("url.scheme", "``http`` or ``https``.") SERVER_ADDRESS = Attribute( "server.address", - "The ``Host`` header. Client-controlled, so treat it as untrusted input " + "The ``Host`` header, verbatim - including any ``:port`` suffix, a " + "deliberate deviation from semantic conventions' ``server.address`` / " + "``server.port`` split. Client-controlled, so treat it as untrusted input " "rather than as the identity of the server.", optional=True, ) USER_AGENT_ORIGINAL = Attribute( "user_agent.original", - "The ``User-Agent`` header, verbatim. Omitted if the client sent none. " - "The client's IP address is deliberately not recorded: core records no " - "identifier that would tie a span to a person.", + "The ``User-Agent`` header, verbatim. Omitted if the client sent none.", optional=True, ) ERROR_TYPE = Attribute( diff --git a/docs/changelog.rst b/docs/changelog.rst index 1488f3a7..68fd6a7e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,7 @@ Unreleased - Datasette's database layer now emits `OpenTelemetry `__ spans: one per query, covering the full round trip including time spent waiting for a SQL worker thread, plus separate child spans for the execution itself and for time spent in the write queue. Callback-style calls - :ref:`db.execute_fn() `, :ref:`db.execute_write_fn() ` and ``db.execute_isolated_fn()``, the documented way for plugins to run arbitrary SQL - are covered too, carrying ``datasette.callback`` in place of the SQL text. Datasette core depends on ``opentelemetry-api`` only and never installs an SDK provider, an exporter or a sampler, so there is no effect and no measurable overhead unless tracing is switched on externally - normally with the standard ``opentelemetry-instrument`` agent. See :ref:`internals_telemetry`. (:issue:`1730`) - :ref:`db.execute(sql, ..., table=None) ` has a new optional ``table=`` parameter, naming the table a query is about so it can be recorded on that query's OpenTelemetry span. It has no effect on query execution, and Datasette never derives it from the SQL. (:issue:`1730`) +- Every HTTP request now gets an OpenTelemetry ``SERVER`` span, named after the request method and matched route, carrying ``http.route``, the response status and W3C trace context extracted from inbound headers - so every database span has a request to belong to, and Datasette joins distributed traces started by a proxy or calling service. The query string is never recorded. See :ref:`internals_telemetry_requests`. (:issue:`1730`) Nothing is removed by the OpenTelemetry work: the ``?_trace=1`` query string parameter, the ``trace_debug`` setting and the :ref:`internals_tracer` module all continue to work as before. diff --git a/docs/internals.rst b/docs/internals.rst index 1283b899..c780a34e 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2375,7 +2375,7 @@ The request span's name is the only one that is not a fixed string - it is compo GET /(?P[^\/\.]+)/(?P
[^\/\.]+)(\.(?P\w+))?$ -That is the route's compiled regular expression, not a prettified ``/{database}/{table}`` template. It is deliberate: Datasette routes with compiled patterns and the route table is fixed when the app is built, so the pattern is exact, bounded and needs no parsing, while transforming it into something prettier accretes edge cases. Django's own instrumentation ships regex-flavoured routes for the same reason. +That is the route's compiled regular expression, not a prettified ``/{database}/{table}`` template - see the ``http.route`` attribute below for why. Django's own instrumentation ships regex-flavoured routes for the same reason. .. [[[cog from telemetry_doc import spans @@ -2393,8 +2393,8 @@ That is the route's compiled regular expression, not a prettified ``/{database}/ - ``http.route`` *(optional)* - The route the request matched, as the compiled regular expression pattern Datasette routes with - for example ``/(?P[^\/\.]+)/(?P
[^\/\.]+)(\.(?P\w+))?$`` for a table page. It is deliberately the pattern rather than a prettified ``/{database}/{table}`` template: the route table is fixed when the app is built, so the pattern is exact, bounded and needs no parsing, whereas the transform into something prettier accretes edge cases. Unlike ``url.path`` this is low cardinality, so it is the attribute to group by. Omitted when no route matched - a 404 - which is also when the span name falls back to the bare method. - ``url.path`` - The path portion of the URL. The query string is deliberately **not** recorded, on this or any other span: Datasette puts user-supplied SQL in ``?sql=`` and canned query parameters in the query string, so exporting it by default would export exactly the data the rest of this instrumentation is careful with. - ``url.scheme`` - ``http`` or ``https``. - - ``server.address`` *(optional)* - The ``Host`` header. Client-controlled, so treat it as untrusted input rather than as the identity of the server. - - ``user_agent.original`` *(optional)* - The ``User-Agent`` header, verbatim. Omitted if the client sent none. The client's IP address is deliberately not recorded: core records no identifier that would tie a span to a person. + - ``server.address`` *(optional)* - The ``Host`` header, verbatim - including any ``:port`` suffix, a deliberate deviation from semantic conventions' ``server.address`` / ``server.port`` split. Client-controlled, so treat it as untrusted input rather than as the identity of the server. + - ``user_agent.original`` *(optional)* - The ``User-Agent`` header, verbatim. Omitted if the client sent none. - ``http.response.status_code`` *(optional)* - The status of the response, read from the ASGI ``http.response.start`` message rather than from a :ref:`internals_response` object - several views, including static files, file downloads and streaming CSV, send that message themselves and never build one. Omitted if the connection closed before anything was sent. - ``error.type`` *(optional)* - Set when the request failed: the exception class name if one escaped the application, otherwise the status code as a string for a 5xx response. A 4xx does **not** set this and does not set an error status - per semantic conventions a client error is not a server span's failure. diff --git a/tests/test_http_span.py b/tests/test_http_span.py index ff792489..a69713cb 100644 --- a/tests/test_http_span.py +++ b/tests/test_http_span.py @@ -354,6 +354,9 @@ async def test_a_404_is_not_an_error(ds, otel_spans): assert server[0].attributes["http.response.status_code"] == 404 assert "error.type" not in server[0].attributes assert server[0].status.status_code is StatusCode.UNSET + # Route enrichment must not be gated on a successful response. + assert "http.route" in server[0].attributes + assert server[0].name != "GET" @pytest.mark.asyncio From 1e44c3dbec39fbc16adb8c90414e4f9216da59a2 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Wed, 2 Sep 2026 14:20:32 -0700 Subject: [PATCH 17/24] Mark in-process datasette.client requests on their SERVER span An internal datasette.client request runs the full ASGI stack, so it emits a second SERVER span nested inside the outer request's - which double-counts requests in any dashboard that counts by span kind. Rather than downgrading the inner span to INTERNAL (which would diverge from how httpx-ASGI instrumentation behaves and break the registry's kind-based dynamic-name matching), the span now carries an optional datasette.internal_client=True attribute for dashboards to filter on. The in_datasette_client ContextVar moves to telemetry.py so the middleware can read it without a circular import; its writers and the in_client() accessor stay in app.py. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012U7coQfVu8nK2R4q2mCULA --- datasette/app.py | 6 ++++-- datasette/telemetry.py | 12 ++++++++++++ datasette/telemetry_registry.py | 11 +++++++++++ docs/internals.rst | 1 + tests/test_http_span.py | 31 +++++++++++++++++++++++++++++++ tests/test_telemetry_registry.py | 1 + 6 files changed, 60 insertions(+), 2 deletions(-) diff --git a/datasette/app.py b/datasette/app.py index 37c4e882..6a06e091 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -51,6 +51,7 @@ from .renderer import json_renderer from .resources import DatabaseResource, TableResource from .telemetry import ( TelemetryMiddleware, + _in_datasette_client, clamp_http_method, request_span, tracer, @@ -171,8 +172,9 @@ app_root = Path(__file__).parent.parent logger = logging.getLogger(__name__) -# Context variable to track when code is executing within a datasette.client request -_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) +# _in_datasette_client itself lives in telemetry.py so the request span +# middleware can read it without a circular import; its writers +# (_DatasetteClientContext) and reader (in_client()) both live here. class _DatasetteClientContext: diff --git a/datasette/telemetry.py b/datasette/telemetry.py index 1f74a3fb..19462585 100644 --- a/datasette/telemetry.py +++ b/datasette/telemetry.py @@ -14,6 +14,7 @@ run-to-run variation. Installing an SDK provider is what costs something measurable. """ +import contextvars import re from opentelemetry import trace as otel_trace @@ -25,6 +26,7 @@ from .telemetry_registry import ( ERROR_TYPE, HTTP_REQUEST_METHOD, HTTP_RESPONSE_STATUS_CODE, + INTERNAL_CLIENT, SERVER_ADDRESS, URL_PATH, URL_SCHEME, @@ -32,6 +34,14 @@ from .telemetry_registry import ( ) from .version import __version__ +# True while code is executing within a datasette.client request. Defined +# here rather than in app.py (which owns its writers and the in_client() +# accessor) so TelemetryMiddleware can read it without a circular import: +# an in-process sub-request runs the full ASGI stack, so it emits a second, +# nested SERVER span - datasette.internal_client marks those so kind-based +# dashboards can filter the double-count out. +_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False) + # The semantic-convention version whose spellings this instrumentation # actually emits. Deliberately NOT the latest release. # @@ -301,6 +311,8 @@ class TelemetryMiddleware: user_agent = _first_header(headers, b"user-agent") if user_agent: span.set_attribute(USER_AGENT_ORIGINAL, user_agent) + if _in_datasette_client.get(): + span.set_attribute(INTERNAL_CLIENT, True) # A copy, not a mutation: the scope belongs to the server, and # every other layer in Datasette extends it the same way. diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py index 0850a4ff..283b8886 100644 --- a/datasette/telemetry_registry.py +++ b/datasette/telemetry_registry.py @@ -136,6 +136,16 @@ USER_AGENT_ORIGINAL = Attribute( "The ``User-Agent`` header, verbatim. Omitted if the client sent none.", optional=True, ) +INTERNAL_CLIENT = Attribute( + "datasette.internal_client", + "``True`` when the request was made in-process through " + "``datasette.client`` rather than arriving over the network. Such a " + "sub-request runs the full ASGI stack, so it emits its own nested " + "``SERVER`` span inside the outer request's - filter on this attribute " + "to keep kind-based dashboards from double-counting requests. Omitted " + "for real inbound requests.", + optional=True, +) ERROR_TYPE = Attribute( "error.type", "Set when the request failed: the exception class name if one escaped the " @@ -283,6 +293,7 @@ HTTP_REQUEST = SpanName( USER_AGENT_ORIGINAL, HTTP_RESPONSE_STATUS_CODE, ERROR_TYPE, + INTERNAL_CLIENT, ), dynamic=True, kind=SpanKind.SERVER, diff --git a/docs/internals.rst b/docs/internals.rst index c780a34e..6465ea49 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2397,6 +2397,7 @@ That is the route's compiled regular expression, not a prettified ``/{database}/ - ``user_agent.original`` *(optional)* - The ``User-Agent`` header, verbatim. Omitted if the client sent none. - ``http.response.status_code`` *(optional)* - The status of the response, read from the ASGI ``http.response.start`` message rather than from a :ref:`internals_response` object - several views, including static files, file downloads and streaming CSV, send that message themselves and never build one. Omitted if the connection closed before anything was sent. - ``error.type`` *(optional)* - Set when the request failed: the exception class name if one escaped the application, otherwise the status code as a string for a 5xx response. A 4xx does **not** set this and does not set an error status - per semantic conventions a client error is not a server span's failure. + - ``datasette.internal_client`` *(optional)* - ``True`` when the request was made in-process through ``datasette.client`` rather than arriving over the network. Such a sub-request runs the full ASGI stack, so it emits its own nested ``SERVER`` span inside the outer request's - filter on this attribute to keep kind-based dashboards from double-counting requests. Omitted for real inbound requests. ``db.query`` A SQL operation issued by Datasette, covering the full round trip including any time spent queued for a thread. Callback-style calls - ``execute_fn()``, ``execute_write_fn()`` and ``execute_isolated_fn()`` - appear here too, distinguished by ``datasette.callback`` in place of ``db.query.text``. diff --git a/tests/test_http_span.py b/tests/test_http_span.py index a69713cb..ca2239d5 100644 --- a/tests/test_http_span.py +++ b/tests/test_http_span.py @@ -856,3 +856,34 @@ def test_no_provider_takes_the_fast_path(): ) # Same fast path, other observable: nothing is stashed in the scope either. assert report["scope_keys"] == [False, False] + + +@pytest.mark.asyncio +async def test_internal_client_requests_are_marked(ds, otel_spans): + """ + An in-process `datasette.client` request runs the full ASGI stack, so it + emits its own SERVER span - `datasette.internal_client` marks those so + kind-based dashboards can filter the double-count out. A request that + arrives through the raw ASGI app (the shape of a real inbound request, + without the DatasetteClient wrapper setting the ContextVar) must not + carry the attribute. + """ + otel_spans.clear() + assert (await ds.client.get("/")).status_code == 200 + server = _server_spans(otel_spans) + assert server + assert all( + span.attributes.get("datasette.internal_client") is True for span in server + ) + + import httpx + + transport = httpx.ASGITransport(app=ds.app()) + async with httpx.AsyncClient( + transport=transport, base_url="http://localhost" + ) as client: + otel_spans.clear() + assert (await client.get("/")).status_code == 200 + server = _server_spans(otel_spans) + assert server + assert all("datasette.internal_client" not in span.attributes for span in server) diff --git a/tests/test_telemetry_registry.py b/tests/test_telemetry_registry.py index f37dc8f0..242d64d3 100644 --- a/tests/test_telemetry_registry.py +++ b/tests/test_telemetry_registry.py @@ -93,6 +93,7 @@ EXPECTED_HTTP_ATTRIBUTES = { "user_agent.original", "http.response.status_code", "error.type", + "datasette.internal_client", } # The registry's own name for the request span is that template, not anything From 441ce905353458141bb4d8b154f11c3a44c654b7 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 09:43:30 -0700 Subject: [PATCH 18/24] Add OpenTelemetry metrics for SQL thread pool saturation and query latency Spans describe requests that have finished. They structurally cannot answer "am I saturating my 3 SQL threads right now", because that is a level rather than an event - and with num_sql_threads defaulting to 3, it is usually the first thing worth knowing about a busy Datasette. This adds the metrics that answer it. Five observable gauges, computed only when something is collecting, so an instance with no MeterProvider installed does no work for them at all: datasette.sql.threads.limit num_sql_threads datasette.sql.threads.queue_depth queries waiting for a free thread datasette.sql.queries.pending in-flight reads, by db.namespace datasette.write.queue_depth writes behind the single write thread datasette.connections.open tracked file connections Three instruments recorded inline, which matters because metrics survive trace sampling and spans do not - an operator sampling 1% of traces still gets 100% of the latency distribution: db.client.operation.duration semconv histogram, with error.type datasette.write.queue_wait the metric twin of the existing span datasette.sql.queries.interrupted sql_time_limit_ms kills The interrupted counter closes a gap the plan called out as unanswerable: "how often are we killing queries at the limit" is a rate, and a rate cannot be recovered from sampled spans. Core still creates no provider of any kind, so the architecture is unchanged; `grep -rn 'opentelemetry.sdk' datasette/` stays empty. One real difference from tracing is worth recording: _ProxyMeter and its instruments forward to a provider installed after they were created, whereas ProxyTracer permanently caches the first concrete tracer it resolves. Module-level instruments are therefore safe and the test fixture has no ordering constraint. Live instances are tracked in a lock-guarded WeakSet so instrumenting an instance never keeps it alive. The pool gauges carry no attribute saying which Datasette produced them: production runs one instance per process, and adding an id to disambiguate the test suite's hundreds of instances would buy unbounded attribute cardinality to fix a case that does not occur. The collision is documented instead, and the gauge callbacks are plain generator functions so tests can assert exact values by calling them directly rather than through the SDK's last-value aggregation. demos/otel/metrics_demo.py fires 12 concurrent 40ms queries at a 3-thread pool and samples the gauges mid-flight: queue_depth peaks at exactly 9, and the duration histogram reads max=0.1695s for a query whose work is 40ms. That gap is the queue, and it is the thing traces alone will not show you. Also corrects the demo README's privacy section, which still claimed parameter values are never recorded - that stopped being unconditionally true when trace_sql_parameters landed. Co-Authored-By: Claude Opus 5 (cherry picked from 6ef0dd8c and adapted to the rebuilt phase-1 stack: attribute names now come from telemetry_registry where entries exist, the meter carries the instrumentation-scope version and schema URL, and the interrupted-queries counter skips expected timeouts - callers that opted into a deliberately short budget, like facet suggestion - matching how those are excluded from span error status. The internals.rst reference lands with the registry commit that follows.) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01F2h9ANGZ7paWSpqs5DUAcG --- datasette/app.py | 10 + datasette/database.py | 48 +++-- datasette/telemetry.py | 264 +++++++++++++++++++++++++- docs/changelog.rst | 1 + tests/conftest.py | 100 ++++++++++ tests/test_telemetry_metrics.py | 323 ++++++++++++++++++++++++++++++++ 6 files changed, 730 insertions(+), 16 deletions(-) create mode 100644 tests/test_telemetry_metrics.py diff --git a/datasette/app.py b/datasette/app.py index 6a06e091..8b9555c4 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -53,8 +53,10 @@ from .telemetry import ( TelemetryMiddleware, _in_datasette_client, clamp_http_method, + register_datasette, request_span, tracer, + unregister_datasette, ) from .telemetry_registry import HTTP_ROUTE, STARTUP from .tokens import TokenInvalid @@ -647,6 +649,10 @@ class Datasette: self.root_enabled = False self.default_deny = default_deny self.client = DatasetteClient(self) + # Last, so that the observable-gauge callbacks - which may fire on the + # SDK's collection thread the instant this returns - never see a + # half-built instance. + register_datasette(self) async def apply_metadata_json(self): # Apply any metadata entries from metadata.json to the internal tables @@ -986,6 +992,10 @@ class Datasette: if self._closed: return self._closed = True + # Stop reporting gauges before tearing anything down, so a collection + # cycle landing mid-close cannot observe a half-closed instance. The + # WeakSet would drop it eventually anyway; this makes it immediate. + unregister_datasette(self) first_exception = None dbs = list(self.databases.values()) + [self._internal_database] for db in dbs: diff --git a/datasette/database.py b/datasette/database.py index 6cad4889..c5589881 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -17,7 +17,15 @@ from opentelemetry import context as otel_context_api from opentelemetry.trace import Link, Status, StatusCode, get_current_span from .inspect import inspect_hash -from .telemetry import callback_name, sql_attribute, sql_operation_name, tracer +from .telemetry import ( + callback_name, + record_operation_duration, + record_query_interrupted, + record_write_queue_wait, + sql_attribute, + sql_operation_name, + tracer, +) from .telemetry_registry import ( CALLBACK, DB_COLLECTION_NAME, @@ -300,9 +308,10 @@ class Database: span.set_attribute(DB_OPERATION_NAME, operation_name) if params: span.set_attribute(PARAM_COUNT, len(params)) - results = await self._execute_write_fn( - _inner, block=block, request=request, transaction=transaction - ) + with record_operation_duration(self.name, "write"): + results = await self._execute_write_fn( + _inner, block=block, request=request, transaction=transaction + ) return results async def execute_write_script(self, sql, block=True, request=None): @@ -324,9 +333,10 @@ class Database: span.set_attribute(DB_NAMESPACE, self.name) span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql)) span.set_attribute(EXECUTESCRIPT, True) - results = await self._execute_write_fn( - _inner, block=block, transaction=False, request=request - ) + with record_operation_duration(self.name, "write"): + results = await self._execute_write_fn( + _inner, block=block, transaction=False, request=request + ) return results async def execute_write_many(self, sql, params_seq, block=True, request=None): @@ -357,9 +367,10 @@ class Database: operation_name = sql_operation_name(sql) if operation_name: span.set_attribute(DB_OPERATION_NAME, operation_name) - results, count = await self._execute_write_fn( - _inner, block=block, request=request - ) + with record_operation_duration(self.name, "write"): + results, count = await self._execute_write_fn( + _inner, block=block, request=request + ) # count is the number of parameter *sets* consumed by # executemany(), not a row count - executemany returns no rows. span.set_attribute(PARAM_SETS, count) @@ -640,11 +651,15 @@ class Database: # this span's duration is the time the task actually spent # waiting in the queue (enqueue -> dequeue), not the near- # zero time spent constructing/ending the span object here. + dequeued_at_ns = time.time_ns() tracer.start_span( DB_WRITE_QUEUE_WAIT, start_time=task.enqueued_at_ns, **write_span_kwargs, - ).end(end_time=time.time_ns()) + ).end(end_time=dequeued_at_ns) + record_write_queue_wait( + self.name, dequeued_at_ns - task.enqueued_at_ns + ) if conn_exception is not None: # fn never runs in this branch, so there is nothing to # wrap in a db.write.execute span. @@ -902,7 +917,8 @@ class Database: if params: span.set_attribute(PARAM_COUNT, len(params)) try: - results = await self._execute_fn(sql_operation_in_thread) + with record_operation_duration(self.name, "read"): + results = await self._execute_fn(sql_operation_in_thread) except QueryInterrupted as e: # datasette.interrupted is set either way - it is the # signal worth having. Only the ERROR status is @@ -911,6 +927,14 @@ class Database: if not timeout_expected: span.set_status(Status(StatusCode.ERROR, str(e))) span.record_exception(e) + # A counter rather than only a span, because this is the + # one thing an operator wants a rate and an alert on, and + # spans under a 1% sampler cannot provide either. An + # expected timeout - a caller that opted into a shorter + # budget, like facet suggestion - is not counted, for the + # same reason it is not a span error: it fires routinely + # by design and would drown the signal this exists for. + record_query_interrupted(self.name) raise except Exception as e: # log_sql_errors=False means the caller is probing and diff --git a/datasette/telemetry.py b/datasette/telemetry.py index 19462585..bf4d8e65 100644 --- a/datasette/telemetry.py +++ b/datasette/telemetry.py @@ -2,27 +2,36 @@ OpenTelemetry integration for Datasette core. Core depends on `opentelemetry-api` only. It never creates a -`TracerProvider`, never configures an exporter, and never touches -sampling - that is the responsibility of whoever is running Datasette -(an `opentelemetry-instrument` agent, a future plugin, or a test +`TracerProvider` or a `MeterProvider`, never configures an exporter, and +never touches sampling - that is the responsibility of whoever is running +Datasette (an `opentelemetry-instrument` agent, a future plugin, or a test harness). With no provider installed every span produced here is a `NonRecordingSpan`. That is not free - a table page emits ~100 spans - but end-to-end page benchmarks put the overhead below their own run-to-run variation. Installing an SDK provider is what costs -something measurable. +something measurable. Every metric instrument is likewise a no-op +without a provider, and the observable-gauge callbacks are never +invoked at all. """ import contextvars import re +import threading +import time +import weakref +from contextlib import contextmanager +from opentelemetry import metrics as otel_metrics from opentelemetry import trace as otel_trace from opentelemetry.propagate import extract from opentelemetry.propagators.textmap import Getter from opentelemetry.trace import SpanKind, Status, StatusCode from .telemetry_registry import ( + DB_NAMESPACE, + DB_SYSTEM, ERROR_TYPE, HTTP_REQUEST_METHOD, HTTP_RESPONSE_STATUS_CODE, @@ -64,6 +73,7 @@ _in_datasette_client = contextvars.ContextVar("in_datasette_client", default=Fal SCHEMA_URL = "https://opentelemetry.io/schemas/1.29.0" tracer = otel_trace.get_tracer("datasette", __version__, schema_url=SCHEMA_URL) +meter = otel_metrics.get_meter("datasette", __version__, schema_url=SCHEMA_URL) MAX_SQL_LENGTH = 2048 @@ -359,3 +369,249 @@ class TelemetryMiddleware: if status >= 500 and not escaped: span.set_status(Status(StatusCode.ERROR)) span.set_attribute(ERROR_TYPE, str(status)) + + +# --- Metrics -------------------------------------------------------------- +# +# Spans answer "what happened during this request". They cannot answer "am I +# saturating my 3 SQL threads right now", because that is a gauge: a level +# sampled at collection time, not an event with a duration. It is also the +# single most useful operational question about a Datasette deployment, since +# num_sql_threads defaults to 3 and every read query in the process competes +# for those threads. +# +# Two shapes are used here: +# +# Observable gauges - a callback the SDK invokes on its own collection +# cycle. Nothing is computed unless something is collecting, so the default +# no-provider install pays literally nothing for them. +# +# Synchronous histograms/counters - recorded inline on the query path. These +# survive trace sampling, which spans do not: an operator sampling 1% of +# traces still gets 100% of the latency distribution and the interrupted +# count. +# +# Note a real difference from tracing: `_ProxyMeter` and its instruments +# forward to a provider installed *after* they were created, whereas +# `ProxyTracer` permanently caches the concrete tracer it first resolves. So +# module-level instruments here are safe, and tests do not need a provider +# installed before this module is imported. + + +def _duration_attributes(database_name, operation): + return { + DB_SYSTEM: "sqlite", + DB_NAMESPACE: database_name, + "datasette.operation": operation, + } + + +sql_operation_duration = meter.create_histogram( + "db.client.operation.duration", + unit="s", + description="Duration of a SQL operation issued by Datasette", +) + +write_queue_wait = meter.create_histogram( + "datasette.write.queue_wait", + unit="s", + description=( + "Time a write spent queued behind the single write thread for its database" + ), +) + +queries_interrupted = meter.create_counter( + "datasette.sql.queries.interrupted", + unit="{query}", + description=( + "Queries cancelled for exceeding sql_time_limit_ms. Not derivable from " + "spans under sampling, and the signal that a time limit is too tight" + ), +) + + +@contextmanager +def record_operation_duration(database_name, operation): + """ + Record `db.client.operation.duration` for one SQL operation. + + `error.type` is set from the exception class on failure, per semconv, so a + latency distribution can be split by success and failure. For a + `block=False` write this measures the enqueue, not the write - the same + caveat that applies to the surrounding span. + """ + attributes = _duration_attributes(database_name, operation) + started = time.perf_counter() + try: + yield + except BaseException as exception: + attributes[ERROR_TYPE] = type(exception).__qualname__ + raise + finally: + sql_operation_duration.record(time.perf_counter() - started, attributes) + + +def record_write_queue_wait(database_name, waited_ns): + write_queue_wait.record(waited_ns / 1e9, {DB_NAMESPACE: database_name}) + + +def record_query_interrupted(database_name): + queries_interrupted.add(1, {DB_NAMESPACE: database_name}) + + +# Live Datasette instances, weakly held so that instrumenting an instance +# never keeps it alive. Guarded by a lock because the gauge callbacks run on +# the SDK's collection thread while the event loop may be building or closing +# a Datasette. +# +# Known limitation: the pool gauges below carry no attribute identifying which +# Datasette produced them, so if a single process runs more than one instance +# their observations collide and last-one-wins. Production runs one instance +# per process; adding an instance id to make the test suite's hundreds of +# instances distinguishable would mean unbounded attribute cardinality in +# exchange for fixing a case that does not occur in production. +_live_datasettes = weakref.WeakSet() +_live_datasettes_lock = threading.Lock() + + +def register_datasette(ds): + "Start reporting pool/queue gauges for this Datasette instance." + with _live_datasettes_lock: + _live_datasettes.add(ds) + + +def unregister_datasette(ds): + "Stop reporting gauges for an instance that has been closed." + with _live_datasettes_lock: + _live_datasettes.discard(ds) + + +def _live_instances(): + with _live_datasettes_lock: + return list(_live_datasettes) + + +def _databases_of(ds): + """ + Every Database attached to an instance, including the internal database. + + The internal database is deliberately included: permission checks run SQL + against it on essentially every request, so its queue depth and connection + count are as operationally interesting as any user database's. + """ + databases = list(ds.databases.values()) + internal = getattr(ds, "_internal_database", None) + if internal is not None: + databases.append(internal) + return databases + + +# Each callback is a plain generator function so it can be unit-tested +# directly, without standing up an SDK provider and a metric reader. + + +def observe_sql_thread_limit(options=None): + "Size of the shared read-query thread pool (the num_sql_threads setting)." + for ds in _live_instances(): + if ds.executor is None: + # num_sql_threads=0 - queries run on the event loop, no pool. + continue + yield otel_metrics.Observation(ds.setting("num_sql_threads"), {}) + + +def observe_sql_thread_queue_depth(options=None): + """ + Read queries waiting for a free thread in the shared pool. + + This is the saturation signal: sustained above zero means requests are + queueing on num_sql_threads. `_work_queue` is a private attribute of + ThreadPoolExecutor, so its absence is tolerated rather than fatal - a + missing gauge is much better than a crashed collection cycle. + """ + for ds in _live_instances(): + if ds.executor is None: + continue + work_queue = getattr(ds.executor, "_work_queue", None) + if work_queue is None: + continue + yield otel_metrics.Observation(work_queue.qsize(), {}) + + +def observe_pending_queries(options=None): + """ + Read queries submitted to the pool and not yet finished, per database. + + Summed across databases and compared against the thread limit, this is the + utilisation half of the saturation picture. `len()` is deliberately taken + without `_pending_execute_futures_lock`: it is atomic, and taking a lock + held on the request path from the collection thread would let telemetry + add latency to queries. + """ + for ds in _live_instances(): + for db in _databases_of(ds): + yield otel_metrics.Observation( + len(db._pending_execute_futures), {DB_NAMESPACE: db.name} + ) + + +def observe_write_queue_depth(options=None): + """ + Writes queued behind the single write thread, per database. + + Every database serialises its writes through one thread, so this is + unbounded backpressure that no amount of num_sql_threads will relieve. + """ + for ds in _live_instances(): + for db in _databases_of(ds): + write_queue = db._write_queue + if write_queue is None: + # No write has ever been queued for this database. + continue + yield otel_metrics.Observation( + write_queue.qsize(), {DB_NAMESPACE: db.name} + ) + + +def observe_open_connections(options=None): + "Open SQLite file connections tracked for closing, per database." + for ds in _live_instances(): + for db in _databases_of(ds): + yield otel_metrics.Observation( + len(db._all_file_connections), {DB_NAMESPACE: db.name} + ) + + +sql_thread_limit_gauge = meter.create_observable_gauge( + "datasette.sql.threads.limit", + callbacks=[observe_sql_thread_limit], + unit="{thread}", + description="Maximum concurrent read queries (the num_sql_threads setting)", +) + +sql_thread_queue_depth_gauge = meter.create_observable_gauge( + "datasette.sql.threads.queue_depth", + callbacks=[observe_sql_thread_queue_depth], + unit="{query}", + description="Read queries waiting for a free thread in the shared SQL pool", +) + +pending_queries_gauge = meter.create_observable_gauge( + "datasette.sql.queries.pending", + callbacks=[observe_pending_queries], + unit="{query}", + description="Read queries submitted to the pool and not yet complete", +) + +write_queue_depth_gauge = meter.create_observable_gauge( + "datasette.write.queue_depth", + callbacks=[observe_write_queue_depth], + unit="{write}", + description="Writes queued behind a database's single write thread", +) + +open_connections_gauge = meter.create_observable_gauge( + "datasette.connections.open", + callbacks=[observe_open_connections], + unit="{connection}", + description="Open SQLite file connections tracked for closing", +) diff --git a/docs/changelog.rst b/docs/changelog.rst index 68fd6a7e..7a083d8f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,7 @@ Unreleased - Datasette's database layer now emits `OpenTelemetry `__ spans: one per query, covering the full round trip including time spent waiting for a SQL worker thread, plus separate child spans for the execution itself and for time spent in the write queue. Callback-style calls - :ref:`db.execute_fn() `, :ref:`db.execute_write_fn() ` and ``db.execute_isolated_fn()``, the documented way for plugins to run arbitrary SQL - are covered too, carrying ``datasette.callback`` in place of the SQL text. Datasette core depends on ``opentelemetry-api`` only and never installs an SDK provider, an exporter or a sampler, so there is no effect and no measurable overhead unless tracing is switched on externally - normally with the standard ``opentelemetry-instrument`` agent. See :ref:`internals_telemetry`. (:issue:`1730`) - :ref:`db.execute(sql, ..., table=None) ` has a new optional ``table=`` parameter, naming the table a query is about so it can be recorded on that query's OpenTelemetry span. It has no effect on query execution, and Datasette never derives it from the SQL. (:issue:`1730`) - Every HTTP request now gets an OpenTelemetry ``SERVER`` span, named after the request method and matched route, carrying ``http.route``, the response status and W3C trace context extracted from inbound headers - so every database span has a request to belong to, and Datasette joins distributed traces started by a proxy or calling service. The query string is never recorded. See :ref:`internals_telemetry_requests`. (:issue:`1730`) +- Datasette core now also emits OpenTelemetry **metrics** covering SQL thread pool saturation, per-database write queue depth, open connections, query latency and time-limit interruptions. These answer operational questions that spans structurally cannot - "am I saturating my :ref:`setting_num_sql_threads` threads?" is a level, not an event - and they survive trace sampling. As with spans, core installs no ``MeterProvider``, so there is no cost unless metrics are collected externally. See :ref:`internals_telemetry`. (:issue:`1730`) Nothing is removed by the OpenTelemetry work: the ``?_trace=1`` query string parameter, the ``trace_debug`` setting and the :ref:`internals_tracer` module all continue to work as before. diff --git a/tests/conftest.py b/tests/conftest.py index b2453133..e0c1cab1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -114,6 +114,106 @@ def otel_spans(): yield _otel_span_exporter +_otel_metric_reader = None + + +@pytest.fixture(scope="session", autouse=True) +def _otel_meter_provider(): + """ + Install a real OTel SDK MeterProvider + InMemoryMetricReader once per + process. + + Unlike the tracer, ordering is not load-bearing here: `_ProxyMeter` and + the `_ProxyInstrument`s it hands out forward to a provider installed + *after* they were created, whereas `ProxyTracer` permanently caches the + first concrete tracer it resolves. This fixture is still session-scoped + and autouse for symmetry, and so that a single reader collects for the + whole run. + + DELTA temporality is chosen for counters and histograms so that each + collection reports only what happened since the previous one. With the + SDK default of CUMULATIVE, every metrics test would see every query run + by every earlier test in the session. + """ + global _otel_metric_reader + try: + from opentelemetry import metrics as otel_metrics + from opentelemetry.sdk.metrics import Counter, Histogram, MeterProvider + from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + InMemoryMetricReader, + ) + except ImportError: + return + reader = InMemoryMetricReader( + preferred_temporality={ + Counter: AggregationTemporality.DELTA, + Histogram: AggregationTemporality.DELTA, + } + ) + otel_metrics.set_meter_provider(MeterProvider(metric_readers=[reader])) + _otel_metric_reader = reader + + +class MetricsCollector: + """ + Thin reader over an `InMemoryMetricReader`. + + `collect()` runs a collection cycle - which is what invokes the observable + gauge callbacks - and snapshots the result. Queries then run against that + snapshot rather than re-collecting, so a test that inspects several + metrics sees one consistent moment and does not drain delta state twice. + """ + + def __init__(self, reader): + self.reader = reader + self.snapshot = {} + + def collect(self): + self.snapshot = {} + data = self.reader.get_metrics_data() + if data is None: + return self.snapshot + for resource_metrics in data.resource_metrics: + for scope_metrics in resource_metrics.scope_metrics: + for metric in scope_metrics.metrics: + self.snapshot.setdefault(metric.name, []).extend( + metric.data.data_points + ) + return self.snapshot + + def points(self, name, attributes=None): + "Data points for `name` whose attributes are a superset of `attributes`." + found = [] + for point in self.snapshot.get(name, []): + point_attributes = dict(point.attributes or {}) + if all(point_attributes.get(k) == v for k, v in (attributes or {}).items()): + found.append(point) + return found + + def point(self, name, attributes=None): + "The single matching data point, asserting there is exactly one." + found = self.points(name, attributes) + assert len(found) == 1, ( + f"expected exactly one {name} point matching {attributes}, " + f"got {len(found)}: {found}" + ) + return found[0] + + +@pytest.fixture +def otel_metrics(): + """ + Function-scoped metrics collector. Drains any delta state accumulated by + earlier tests before yielding, so counts start from zero. + """ + pytest.importorskip("opentelemetry.sdk") + if _otel_metric_reader is None: + pytest.skip("OpenTelemetry SDK meter provider was not installed") + _otel_metric_reader.get_metrics_data() + yield MetricsCollector(_otel_metric_reader) + + @pytest.fixture def bare_ds(): """ diff --git a/tests/test_telemetry_metrics.py b/tests/test_telemetry_metrics.py new file mode 100644 index 00000000..6d1efd1f --- /dev/null +++ b/tests/test_telemetry_metrics.py @@ -0,0 +1,323 @@ +""" +Tests for the OpenTelemetry metrics Datasette core emits. + +Two layers are tested separately and deliberately: + +- The gauge callbacks are plain generator functions, so they are called + directly for exact-value assertions. Going through the SDK for those would + be unreliable: the pool gauges carry no attribute identifying which + Datasette produced them, and a pytest session has many live instances, so + the SDK's last-value aggregation would report whichever one happened to be + observed last. + +- The SDK pipeline (instrument -> reader -> data points) is tested through + the `otel_metrics` fixture, using metrics that carry `db.namespace` - a + uniquely named in-memory database is enough to isolate those from every + other instance alive in the session. +""" + +import asyncio + +import pytest + +from datasette import telemetry +from datasette.app import Datasette +from datasette.database import Database +from datasette.utils.sqlite import sqlite3 + +pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning") + + +def observations(callback, datasette=None): + """ + Run a gauge callback, optionally keeping only observations produced by one + Datasette's databases. Returns a list of (attributes dict, value). + """ + results = [] + names = None + if datasette is not None: + names = {db.name for db in telemetry._databases_of(datasette)} + for observation in callback(): + attributes = dict(observation.attributes or {}) + namespace = attributes.get("db.namespace") + if names is not None and namespace is not None and namespace not in names: + continue + results.append((attributes, observation.value)) + return results + + +@pytest.fixture +def metrics_ds(): + "A Datasette with a distinctive thread count and a uniquely named database." + ds = Datasette( + memory=True, + settings={"num_sql_threads": 7}, + ) + ds.add_memory_database("metrics_test_db") + try: + yield ds + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_sql_thread_limit_gauge_reports_num_sql_threads(metrics_ds): + values = [value for _, value in observations(telemetry.observe_sql_thread_limit)] + # Other instances are alive in this session, so assert membership rather + # than uniqueness - 7 is distinctive enough to only come from metrics_ds. + assert 7 in values + + +@pytest.mark.asyncio +async def test_no_thread_gauges_in_non_threaded_mode(): + """ + num_sql_threads=0 means there is no pool at all, so the pool gauges must + skip the instance rather than report a bogus limit of 0. + + The pool gauges carry no attributes, so the assertion is that adding this + instance produces no *additional* observations. + """ + before_limits = len(list(telemetry.observe_sql_thread_limit())) + before_depths = len(list(telemetry.observe_sql_thread_queue_depth())) + ds = Datasette(memory=True, settings={"num_sql_threads": 0}) + try: + assert ds.executor is None + assert len(list(telemetry.observe_sql_thread_limit())) == before_limits + assert len(list(telemetry.observe_sql_thread_queue_depth())) == before_depths + # Per-database gauges are unaffected - they do not depend on the pool. + assert observations(telemetry.observe_pending_queries, ds) + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_pending_queries_gauge_tracks_in_flight_queries(metrics_ds): + db = metrics_ds.get_database("metrics_test_db") + attributes = {"db.namespace": "metrics_test_db"} + + def value(): + points = [ + v + for a, v in observations(telemetry.observe_pending_queries, metrics_ds) + if a == attributes + ] + assert len(points) == 1 + return points[0] + + assert value() == 0 + + # sqlite3.sleep is not a thing, so block the worker thread on an event we + # control from the event loop and sample the gauge while it is held. + release = asyncio.Event() + loop = asyncio.get_running_loop() + entered = asyncio.Event() + + def blocking_fn(conn): + loop.call_soon_threadsafe(entered.set) + asyncio.run_coroutine_threadsafe(release.wait(), loop).result() + return "done" + + task = asyncio.ensure_future(db.execute_fn(blocking_fn)) + await entered.wait() + assert value() == 1, "a query occupying a pool thread must be counted as pending" + release.set() + assert await task == "done" + assert value() == 0, "the count must drop once the query completes" + + +@pytest.mark.asyncio +async def test_write_queue_depth_gauge(metrics_ds): + db = metrics_ds.get_database("metrics_test_db") + attributes = {"db.namespace": "metrics_test_db"} + + def depths(): + return [ + v + for a, v in observations(telemetry.observe_write_queue_depth, metrics_ds) + if a == attributes + ] + + # No write has ever been queued, so there is no queue and no observation - + # rather than a fabricated zero for a queue that does not exist. + assert depths() == [] + + await db.execute_write("create table t (id integer primary key)") + assert depths() == [0], "an idle write queue reports zero, not nothing" + + +@pytest.mark.asyncio +async def test_open_connections_gauge(metrics_ds, tmp_path): + path = str(tmp_path / "conns.db") + sqlite3.connect(path).execute("create table t (id integer primary key)") + db = metrics_ds.add_database(Database(metrics_ds, path=path), name="conns_db") + attributes = {"db.namespace": "conns_db"} + + def open_connections(): + points = [ + v + for a, v in observations(telemetry.observe_open_connections, metrics_ds) + if a == attributes + ] + assert len(points) == 1 + return points[0] + + assert open_connections() == 0 + await db.execute("select 1") + assert open_connections() >= 1, "executing a query opens a tracked file connection" + + +@pytest.mark.asyncio +async def test_operation_duration_histogram_read(otel_metrics): + ds = Datasette(memory=True) + ds.add_memory_database("duration_read_db") + try: + db = ds.get_database("duration_read_db") + await db.execute("select 1") + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_read_db", "datasette.operation": "read"}, + ) + assert point.count == 1 + assert point.sum > 0 + assert dict(point.attributes)["db.system"] == "sqlite" + assert "error.type" not in dict(point.attributes) + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_operation_duration_histogram_write(otel_metrics): + ds = Datasette(memory=True) + ds.add_memory_database("duration_write_db") + try: + db = ds.get_database("duration_write_db") + await db.execute_write("create table t (id integer primary key)") + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_write_db", "datasette.operation": "write"}, + ) + assert point.count == 1 + assert point.sum > 0 + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_operation_duration_records_error_type(otel_metrics): + "A failed query is still timed, and is separable from a successful one." + ds = Datasette(memory=True) + ds.add_memory_database("duration_error_db") + try: + db = ds.get_database("duration_error_db") + with pytest.raises(sqlite3.OperationalError): + await db.execute("select * from nope") + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_error_db", "datasette.operation": "read"}, + ) + assert point.count == 1 + assert dict(point.attributes)["error.type"] == "OperationalError" + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_write_queue_wait_histogram(otel_metrics): + ds = Datasette(memory=True) + ds.add_memory_database("queue_wait_db") + try: + db = ds.get_database("queue_wait_db") + await db.execute_write("create table t (id integer primary key)") + await db.execute_write("insert into t (id) values (1)") + otel_metrics.collect() + point = otel_metrics.point( + "datasette.write.queue_wait", {"db.namespace": "queue_wait_db"} + ) + assert point.count == 2, "one measurement per write dequeued" + assert point.sum >= 0 + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_interrupted_queries_counter(otel_metrics): + "The count of time-limit kills, which sampled traces cannot provide." + ds = Datasette(memory=True, settings={"sql_time_limit_ms": 1}) + ds.add_memory_database("interrupted_db") + try: + db = ds.get_database("interrupted_db") + from datasette.database import QueryInterrupted + + with pytest.raises(QueryInterrupted): + await db.execute(""" + with recursive counter(x) as ( + select 0 union all select x + 1 from counter + ) + select * from counter + """) + otel_metrics.collect() + point = otel_metrics.point( + "datasette.sql.queries.interrupted", {"db.namespace": "interrupted_db"} + ) + assert point.value == 1 + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_metrics_are_reported_through_the_sdk_for_gauges(otel_metrics): + "End-to-end: a gauge callback reaches the reader as a data point." + ds = Datasette(memory=True) + ds.add_memory_database("gauge_pipeline_db") + try: + await ds.get_database("gauge_pipeline_db").execute("select 1") + otel_metrics.collect() + point = otel_metrics.point( + "datasette.sql.queries.pending", {"db.namespace": "gauge_pipeline_db"} + ) + assert point.value == 0 + assert otel_metrics.points("datasette.sql.threads.limit") + finally: + ds.close() + + +def test_closed_datasette_stops_being_observed(): + ds = Datasette(memory=True) + ds.add_memory_database("closed_db") + assert observations(telemetry.observe_pending_queries, ds) + ds.close() + names = [ + attributes.get("db.namespace") + for attributes, _ in observations(telemetry.observe_pending_queries) + ] + assert "closed_db" not in names + + +def test_registry_holds_instances_weakly(): + """ + Registering an instance must never be the thing that keeps it alive. + + A stand-in object is used rather than a real Datasette because a Datasette + with a temp-disk internal database is pinned for the life of the process + by `Database.__init__`'s `atexit.register(self._cleanup_temp_file)`, which + holds the Database, which holds the Datasette. That is pre-existing and + unrelated to telemetry; what is tested here is that this registry adds no + reference of its own. + """ + import gc + import weakref + + class FakeDatasette: + pass + + fake = FakeDatasette() + telemetry.register_datasette(fake) + assert fake in telemetry._live_instances() + ref = weakref.ref(fake) + del fake + gc.collect() + assert ref() is None + assert not any(isinstance(ds, FakeDatasette) for ds in telemetry._live_instances()) From 4a849df3f5034b8a626417aa08d3bb23665cf6f9 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Tue, 1 Sep 2026 12:24:01 -0700 Subject: [PATCH 19/24] Register metrics and give histograms bucket boundaries suited to seconds Both histograms declared unit="s" but inherited OpenTelemetry's default boundaries, which are tuned for milliseconds - so every SQLite query landed in the single (0, 5] second bucket and every quantile query returned noise. The boundaries are the semantic conventions' recommended set for db.client.operation.duration plus 0.0001 and 0.0005 at the bottom, since SQLite is in-process and many real queries take tens of microseconds. (Adapted from 024f2029: that commit assumed the metrics were already in telemetry_registry.py, which on this lineage held spans only - so this commit also brings the MetricName registry machinery, the registry entries for all eight phase-3 metrics, the cog-generated Metric reference in internals.rst, and the datasette.operation attribute. The template and facet histograms it also touched belong to phase 5 and are not included.) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01F2h9ANGZ7paWSpqs5DUAcG --- datasette/telemetry.py | 45 ++++++---- datasette/telemetry_registry.py | 138 +++++++++++++++++++++++++++++++ docs/internals.rst | 81 +++++++++++++++++- docs/telemetry_doc.py | 19 +++++ tests/test_telemetry_metrics.py | 64 ++++++++++++++ tests/test_telemetry_registry.py | 23 ++++++ 6 files changed, 352 insertions(+), 18 deletions(-) diff --git a/datasette/telemetry.py b/datasette/telemetry.py index bf4d8e65..f1570713 100644 --- a/datasette/telemetry.py +++ b/datasette/telemetry.py @@ -36,6 +36,15 @@ from .telemetry_registry import ( HTTP_REQUEST_METHOD, HTTP_RESPONSE_STATUS_CODE, INTERNAL_CLIENT, + M_CONNECTIONS_OPEN, + M_OPERATION_DURATION, + M_QUERIES_INTERRUPTED, + M_QUERIES_PENDING, + M_THREADS_LIMIT, + M_THREADS_QUEUE_DEPTH, + M_WRITE_QUEUE_DEPTH, + M_WRITE_QUEUE_WAIT, + OPERATION, SERVER_ADDRESS, URL_PATH, URL_SCHEME, @@ -402,27 +411,29 @@ def _duration_attributes(database_name, operation): return { DB_SYSTEM: "sqlite", DB_NAMESPACE: database_name, - "datasette.operation": operation, + OPERATION: operation, } sql_operation_duration = meter.create_histogram( - "db.client.operation.duration", - unit="s", + M_OPERATION_DURATION, + unit=M_OPERATION_DURATION.unit, description="Duration of a SQL operation issued by Datasette", + explicit_bucket_boundaries_advisory=M_OPERATION_DURATION.buckets, ) write_queue_wait = meter.create_histogram( - "datasette.write.queue_wait", - unit="s", + M_WRITE_QUEUE_WAIT, + unit=M_WRITE_QUEUE_WAIT.unit, description=( "Time a write spent queued behind the single write thread for its database" ), + explicit_bucket_boundaries_advisory=M_WRITE_QUEUE_WAIT.buckets, ) queries_interrupted = meter.create_counter( - "datasette.sql.queries.interrupted", - unit="{query}", + M_QUERIES_INTERRUPTED, + unit=M_QUERIES_INTERRUPTED.unit, description=( "Queries cancelled for exceeding sql_time_limit_ms. Not derivable from " "spans under sampling, and the signal that a time limit is too tight" @@ -582,36 +593,36 @@ def observe_open_connections(options=None): sql_thread_limit_gauge = meter.create_observable_gauge( - "datasette.sql.threads.limit", + M_THREADS_LIMIT, callbacks=[observe_sql_thread_limit], - unit="{thread}", + unit=M_THREADS_LIMIT.unit, description="Maximum concurrent read queries (the num_sql_threads setting)", ) sql_thread_queue_depth_gauge = meter.create_observable_gauge( - "datasette.sql.threads.queue_depth", + M_THREADS_QUEUE_DEPTH, callbacks=[observe_sql_thread_queue_depth], - unit="{query}", + unit=M_THREADS_QUEUE_DEPTH.unit, description="Read queries waiting for a free thread in the shared SQL pool", ) pending_queries_gauge = meter.create_observable_gauge( - "datasette.sql.queries.pending", + M_QUERIES_PENDING, callbacks=[observe_pending_queries], - unit="{query}", + unit=M_QUERIES_PENDING.unit, description="Read queries submitted to the pool and not yet complete", ) write_queue_depth_gauge = meter.create_observable_gauge( - "datasette.write.queue_depth", + M_WRITE_QUEUE_DEPTH, callbacks=[observe_write_queue_depth], - unit="{write}", + unit=M_WRITE_QUEUE_DEPTH.unit, description="Writes queued behind a database's single write thread", ) open_connections_gauge = meter.create_observable_gauge( - "datasette.connections.open", + M_CONNECTIONS_OPEN, callbacks=[observe_open_connections], - unit="{connection}", + unit=M_CONNECTIONS_OPEN.unit, description="Open SQLite file connections tracked for closing", ) diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py index 283b8886..3c775a7b 100644 --- a/datasette/telemetry_registry.py +++ b/datasette/telemetry_registry.py @@ -79,6 +79,33 @@ class SpanName(str): return f"SpanName({str(self)!r})" +class MetricName(str): + "A metric name, carrying its instrument kind, unit and attributes." + + __slots__ = ("attributes", "buckets", "description", "kind", "unit") + + def __new__(cls, name, kind, unit, description, attributes=(), buckets=None): + self = super().__new__(cls, name) + self.kind = kind + self.unit = unit + self.description = description + self.attributes = tuple(attributes) + # Explicit histogram bucket boundaries, for histograms only. Passed to + # create_histogram() as explicit_bucket_boundaries_advisory and + # published in the generated docs, since an operator writing a + # histogram_quantile() query needs to know them. + self.buckets = tuple(buckets) if buckets is not None else None + return self + + def __repr__(self): + return f"MetricName({str(self)!r})" + + +COUNTER = "Counter" +HISTOGRAM = "Histogram" +GAUGE = "Observable gauge" + + # --- Attributes ----------------------------------------------------------- # # Shared attributes are defined once and referenced by every span that sets @@ -157,6 +184,7 @@ ERROR_TYPE = Attribute( DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.") DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.") +OPERATION = Attribute("datasette.operation", "``read`` or ``write``.") DB_QUERY_TEXT = Attribute( "db.query.text", "The SQL, truncated to 2048 characters. Never the parameter values. " @@ -404,3 +432,113 @@ def attribute_allowed(span, emitted_key): if span is None: return False return emitted_key in span.attributes + + +# --- Metrics -------------------------------------------------------------- + +# Every duration histogram here is in seconds, and OpenTelemetry's default +# bucket boundaries are tuned for milliseconds - their first non-zero boundary +# is 5, so without explicit boundaries every SQLite query lands in the single +# (0, 5] second bucket and every quantile query returns noise. +# +# These are the OpenTelemetry semantic conventions' recommended boundaries for +# db.client.operation.duration, in seconds, plus 0.0001 and 0.0005 at the +# bottom. The deviation is deliberate: those boundaries assume a network +# database client, whereas SQLite is in-process and a large fraction of real +# queries run in 30-80us, which would otherwise all pile into the first +# bucket and be indistinguishable from each other. +# +# One shared list is used for every duration histogram rather than a tailored +# list each, so that dashboards stay comparable and a queue wait can be read +# against the query duration it delays. It already spans 100us to 10s, which +# covers both a fast in-process read and a write queued behind contention. +DURATION_BUCKETS = (0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10) + +M_OPERATION_DURATION = MetricName( + "db.client.operation.duration", + HISTOGRAM, + "s", + "Duration of a SQL operation. The standard OpenTelemetry semantic " + "convention metric, and the one that survives trace sampling.", + (DB_SYSTEM, DB_NAMESPACE, OPERATION, ERROR_TYPE), + buckets=DURATION_BUCKETS, +) + +M_WRITE_QUEUE_WAIT = MetricName( + "datasette.write.queue_wait", + HISTOGRAM, + "s", + "Time each write waited in its database's write queue. The metric " + "counterpart of the ``db.write.queue_wait`` span.", + (DB_NAMESPACE,), + buckets=DURATION_BUCKETS, +) + +M_QUERIES_INTERRUPTED = MetricName( + "datasette.sql.queries.interrupted", + COUNTER, + "{query}", + "Queries cancelled for exceeding :ref:`setting_sql_time_limit_ms`. Worth " + "alerting on: a rising rate means the limit is too tight or a table has " + "outgrown its queries. A caller that opted into a deliberately shorter " + "budget - facet suggestion, for example - is not counted, for the same " + "reason its timeout is not a span error.", + (DB_NAMESPACE,), +) + +M_THREADS_LIMIT = MetricName( + "datasette.sql.threads.limit", + GAUGE, + "{thread}", + "Maximum concurrent read queries - the :ref:`setting_num_sql_threads` " + "value. Not reported when ``num_sql_threads`` is ``0``, since then queries " + "run on the event loop and there is no pool.", +) + +M_THREADS_QUEUE_DEPTH = MetricName( + "datasette.sql.threads.queue_depth", + GAUGE, + "{query}", + "Read queries waiting for a free thread. **This is the saturation " + "signal** - sustained above zero means requests are queueing on " + "``num_sql_threads``.", +) + +M_QUERIES_PENDING = MetricName( + "datasette.sql.queries.pending", + GAUGE, + "{query}", + "Read queries submitted to the pool and not yet complete. Summed across " + "databases and compared against the thread limit, this is pool " + "utilisation.", + (DB_NAMESPACE,), +) + +M_WRITE_QUEUE_DEPTH = MetricName( + "datasette.write.queue_depth", + GAUGE, + "{write}", + "Writes queued behind a database's single write thread. Backpressure that " + "raising ``num_sql_threads`` cannot relieve. Not reported for a database " + "that has never been written to.", + (DB_NAMESPACE,), +) + +M_CONNECTIONS_OPEN = MetricName( + "datasette.connections.open", + GAUGE, + "{connection}", + "Open SQLite file connections currently tracked for closing.", + (DB_NAMESPACE,), +) + +METRICS = ( + M_OPERATION_DURATION, + M_WRITE_QUEUE_WAIT, + M_QUERIES_INTERRUPTED, + M_THREADS_LIMIT, + M_THREADS_QUEUE_DEPTH, + M_QUERIES_PENDING, + M_WRITE_QUEUE_DEPTH, + M_CONNECTIONS_OPEN, +) diff --git a/docs/internals.rst b/docs/internals.rst index 6465ea49..19040cc4 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2360,7 +2360,7 @@ A few things catch people out the first time: - **Always set** ``OTEL_SERVICE_NAME``. Without it the SDK's default resource reports a ``service.name`` of ``unknown_service``, and your traces will be filed under that instead of under a name you can search for. -- **Setting** ``OTEL_METRICS_EXPORTER=none`` **and** ``OTEL_LOGS_EXPORTER=none`` is worth doing unless your backend accepts those signals too - ``opentelemetry-distro`` defaults every signal to OTLP, and a traces-only backend will reject the other two noisily. Datasette itself emits no metrics and no logs through OpenTelemetry. +- **Setting** ``OTEL_LOGS_EXPORTER=none`` is worth doing unless your backend accepts logs too - ``opentelemetry-distro`` defaults every signal to OTLP, and a backend that does not take a signal will reject it noisily. Datasette emits no logs through OpenTelemetry; it does emit metrics (see :ref:`internals_telemetry_metrics`), so set ``OTEL_METRICS_EXPORTER=none`` only if your backend does not accept them. Span reference -------------- @@ -2447,6 +2447,85 @@ That is the route's compiled regular expression, not a prettified ``/{database}/ .. [[[end]]] +.. _internals_telemetry_metrics: + +Metric reference +---------------- + +Spans describe events; metrics describe levels and rates. "Am I saturating my :ref:`setting_num_sql_threads` threads right now?" cannot be answered by any span, because it is a level sampled at collection time - and it is usually the first thing worth knowing about a busy Datasette, since ``num_sql_threads`` defaults to ``3``. Metrics also survive trace sampling: an operator keeping 1% of traces still gets 100% of every histogram and counter below. + +As with spans, core emits these through the OpenTelemetry API only. Without a ``MeterProvider`` every instrument is a no-op, and the observable-gauge callbacks are never invoked at all, so an uninstrumented install pays nothing for them. + +Every duration histogram is in **seconds**, with explicit bucket boundaries chosen for an in-process database - OpenTelemetry's default boundaries are tuned for milliseconds and would file every SQLite query into a single bucket, making quantile queries meaningless. The boundaries are listed with each histogram because a ``histogram_quantile()`` query is only as good as the buckets underneath it. + +This reference is generated from ``datasette/telemetry_registry.py``, like the span reference above. + +.. [[[cog + from telemetry_doc import metrics + metrics(cog) +.. ]]] + +``db.client.operation.duration`` + Histogram, unit ``s``. Duration of a SQL operation. The standard OpenTelemetry semantic convention metric, and the one that survives trace sampling. + + Bucket boundaries: ``0.0001``, ``0.0005``, ``0.001``, ``0.005``, ``0.01``, ``0.05``, ``0.1``, ``0.5``, ``1``, ``5``, ``10``. + + Attributes: + + - ``db.system`` - Always ``sqlite``. + - ``db.namespace`` - Name of the database being queried. + - ``datasette.operation`` - ``read`` or ``write``. + - ``error.type`` - Set when the request failed: the exception class name if one escaped the application, otherwise the status code as a string for a 5xx response. A 4xx does **not** set this and does not set an error status - per semantic conventions a client error is not a server span's failure. + +``datasette.write.queue_wait`` + Histogram, unit ``s``. Time each write waited in its database's write queue. The metric counterpart of the ``db.write.queue_wait`` span. + + Bucket boundaries: ``0.0001``, ``0.0005``, ``0.001``, ``0.005``, ``0.01``, ``0.05``, ``0.1``, ``0.5``, ``1``, ``5``, ``10``. + + Attributes: + + - ``db.namespace`` - Name of the database being queried. + +``datasette.sql.queries.interrupted`` + Counter, unit ``{query}``. Queries cancelled for exceeding :ref:`setting_sql_time_limit_ms`. Worth alerting on: a rising rate means the limit is too tight or a table has outgrown its queries. A caller that opted into a deliberately shorter budget - facet suggestion, for example - is not counted, for the same reason its timeout is not a span error. + + Attributes: + + - ``db.namespace`` - Name of the database being queried. + +``datasette.sql.threads.limit`` + Observable gauge, unit ``{thread}``. Maximum concurrent read queries - the :ref:`setting_num_sql_threads` value. Not reported when ``num_sql_threads`` is ``0``, since then queries run on the event loop and there is no pool. + + No attributes. + +``datasette.sql.threads.queue_depth`` + Observable gauge, unit ``{query}``. Read queries waiting for a free thread. **This is the saturation signal** - sustained above zero means requests are queueing on ``num_sql_threads``. + + No attributes. + +``datasette.sql.queries.pending`` + Observable gauge, unit ``{query}``. Read queries submitted to the pool and not yet complete. Summed across databases and compared against the thread limit, this is pool utilisation. + + Attributes: + + - ``db.namespace`` - Name of the database being queried. + +``datasette.write.queue_depth`` + Observable gauge, unit ``{write}``. Writes queued behind a database's single write thread. Backpressure that raising ``num_sql_threads`` cannot relieve. Not reported for a database that has never been written to. + + Attributes: + + - ``db.namespace`` - Name of the database being queried. + +``datasette.connections.open`` + Observable gauge, unit ``{connection}``. Open SQLite file connections currently tracked for closing. + + Attributes: + + - ``db.namespace`` - Name of the database being queried. + +.. [[[end]]] + .. _internals_telemetry_requests: Requests and inbound trace context diff --git a/docs/telemetry_doc.py b/docs/telemetry_doc.py index d0af5dbd..725cb968 100644 --- a/docs/telemetry_doc.py +++ b/docs/telemetry_doc.py @@ -34,3 +34,22 @@ def spans(cog): if span.kind != SpanKind.INTERNAL: cog.out(f" Kind: ``{span.kind.name}``.\n\n") _attribute_lines(cog, span.attributes) + + +def metrics(cog): + from datasette.telemetry_registry import METRICS + + cog.out("\n") + for metric in METRICS: + cog.out(f"``{metric}``\n") + cog.out(f" {metric.kind}, unit ``{metric.unit}``. {metric.description}\n\n") + if metric.buckets: + boundaries = ", ".join(f"``{boundary}``" for boundary in metric.buckets) + cog.out(f" Bucket boundaries: {boundaries}.\n\n") + if metric.attributes: + cog.out(" Attributes:\n\n") + for attribute in metric.attributes: + cog.out(f" - ``{attribute}`` - {attribute.description}\n") + cog.out("\n") + else: + cog.out(" No attributes.\n\n") diff --git a/tests/test_telemetry_metrics.py b/tests/test_telemetry_metrics.py index 6d1efd1f..1422fbcd 100644 --- a/tests/test_telemetry_metrics.py +++ b/tests/test_telemetry_metrics.py @@ -321,3 +321,67 @@ def test_registry_holds_instances_weakly(): gc.collect() assert ref() is None assert not any(isinstance(ds, FakeDatasette) for ds in telemetry._live_instances()) + + +HISTOGRAM_PROBES = [ + # (instrument attribute on telemetry, metric name, isolating attributes) + ( + "sql_operation_duration", + "db.client.operation.duration", + {"db.namespace": "bucket_probe_operation"}, + ), + ( + "write_queue_wait", + "datasette.write.queue_wait", + {"db.namespace": "bucket_probe_queue_wait"}, + ), +] + +# One value inside each of six distinct registry buckets. Under OpenTelemetry's +# default boundaries - [0, 5, 10, 25, ...], meant for milliseconds - the first +# five of these all land in (0, 5] and only 7.0 lands elsewhere, so the +# "occupies six buckets" assertion below fails if the advisory is ever dropped. +SPREAD = [0.00005, 0.0003, 0.002, 0.03, 0.8, 7.0] + + +@pytest.mark.parametrize( + "instrument_name,metric_name,attributes", + HISTOGRAM_PROBES, + ids=[metric for _, metric, _ in HISTOGRAM_PROBES], +) +def test_histograms_spread_values_across_buckets( + otel_metrics, instrument_name, metric_name, attributes +): + """ + The registry's boundaries reach the SDK, and a realistic spread of + seconds-scale durations occupies more than one bucket. + + Recording onto the instrument directly rather than driving a workload is + deliberate: real durations here are all tens of microseconds and would + share a bucket no matter what the boundaries were, which is exactly the + situation this test exists to detect. + + `explicit_bounds` is compared against the registry rather than against the + instrument's own configuration - the instrument is built *from* the + registry, so that comparison would be a value against itself. What is + checked here is that the advisory survived the trip through the SDK. + """ + from datasette.telemetry_registry import METRICS + + metric = next(m for m in METRICS if m == metric_name) + instrument = getattr(telemetry, instrument_name) + for value in SPREAD: + instrument.record(value, attributes) + + otel_metrics.collect() + point = otel_metrics.point(metric_name, attributes) + + assert ( + tuple(point.explicit_bounds) == metric.buckets + ), "the registry's boundaries did not reach the SDK" + assert point.count == len(SPREAD) + occupied = [count for count in point.bucket_counts if count] + assert len(occupied) == len(SPREAD), ( + f"expected each of {SPREAD} in its own bucket, got bucket counts " + f"{list(point.bucket_counts)} for bounds {list(point.explicit_bounds)}" + ) diff --git a/tests/test_telemetry_registry.py b/tests/test_telemetry_registry.py index 242d64d3..7103a53f 100644 --- a/tests/test_telemetry_registry.py +++ b/tests/test_telemetry_registry.py @@ -395,6 +395,29 @@ def test_registry_entries_are_usable_as_plain_strings(): assert f"{reg.DB_QUERY}.execute" == "db.query.execute" +def test_every_histogram_declares_bucket_boundaries(): + """ + Every histogram must carry explicit boundaries, and only histograms may. + + OpenTelemetry's default boundaries start at 5 and are meant for + milliseconds, so a seconds-valued histogram that inherits them records + everything into one bucket. This is a registry self-consistency check, not + a check that the boundaries reached the SDK - for that see + `test_histograms_spread_values_across_buckets` in test_telemetry_metrics.py. + """ + for metric in reg.METRICS: + if metric.kind == reg.HISTOGRAM: + assert metric.buckets, f"{metric} is a histogram with no boundaries" + assert list(metric.buckets) == sorted( + set(metric.buckets) + ), f"{metric} boundaries must be ascending and unique" + assert metric.buckets[0] > 0, f"{metric} has a non-positive boundary" + else: + assert ( + metric.buckets is None + ), f"{metric} is a {metric.kind} and cannot have bucket boundaries" + + def test_dynamic_span_lookup(): """ `dynamic=True` matching, which is how the request span resolves. From 9ad983710f04838f28117c6847f5fbff3c9972ce Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Tue, 1 Sep 2026 12:25:38 -0700 Subject: [PATCH 20/24] Check metric attributes in the registry conformance test Span attributes were checked in both directions; metric attributes were not checked at all, so the generated reference could publish an incomplete list with nothing to catch it. The metric workload lives in an `emitted_metrics` fixture, mirroring the span side, and error.type is checked like every other attribute rather than exempted for being optional - the workload reaches it two separate ways. (Adapted from b30c5341: the old workload's facet-timeout probe belongs to phase 5 and is dropped, and the interrupted counter now needs a query that exceeds the *configured* time limit - custom short budgets are excluded from the count on this lineage - so the fixture runs one against a second instance configured with sql_time_limit_ms=5.) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01F2h9ANGZ7paWSpqs5DUAcG --- tests/test_telemetry_registry.py | 129 +++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/tests/test_telemetry_registry.py b/tests/test_telemetry_registry.py index 7103a53f..418b99ff 100644 --- a/tests/test_telemetry_registry.py +++ b/tests/test_telemetry_registry.py @@ -445,3 +445,132 @@ def test_span_and_attribute_lookup(): assert not reg.attribute_allowed(reg.DB_QUERY, "db.namespace.extra") assert not reg.attribute_allowed(reg.DB_QUERY, "datasette.isolated_connection") assert not reg.attribute_allowed(None, "db.namespace") + + +# --- Metric conformance ---------------------------------------------------- + + +@pytest_asyncio.fixture +async def emitted_metrics(otel_metrics): + """ + Every (metric name, attribute key) pair produced by a broad workload, + plus the raw set of metric names - the metric-side counterpart of the + `emitted` span fixture above. + + Metrics use DELTA temporality (see `_otel_meter_provider`), and the + function-scoped `otel_metrics` fixture drains any state left by an + earlier test before yielding, so this collection is not polluted by + other tests in the session - only by other *instances*, which is why the + checks below key everything off attribute names rather than values. + """ + # The span workload already reaches every synchronous metric except the + # interrupted counter: reads and writes drive db.client.operation.duration + # and datasette.write.queue_wait, and both the suppressed-error probe and + # the custom_time_limit interrupt raise through record_operation_duration, + # setting error.type. + ds = await exercise() + + # datasette.sql.queries.interrupted counts only queries that exceed the + # *configured* limit - a caller opting into a deliberately short budget + # via custom_time_limit (as exercise() does) is excluded by design. So a + # second instance whose configured limit is tiny provides the real thing. + slow_name = _unique("registry_metrics_slow") + slow = Datasette(memory=True, settings={"sql_time_limit_ms": 5}) + slow.add_memory_database(slow_name) + await slow.invoke_startup() + slow_db = slow.get_database(slow_name) + with pytest.raises(QueryInterrupted): + await slow_db.execute( + "with recursive c(x) as (select 0 union all select x+1 from c) " + "select * from c" + ) + + # Collect while both instances are still registered, so the observable + # gauges - which observe live instances at collection time - report. + otel_metrics.collect() + snapshot = otel_metrics.snapshot + assert snapshot, "no metrics captured - the fixture is not exercising anything" + pairs = set() + for metric_name, points in snapshot.items(): + for point in points: + for key in point.attributes or {}: + pairs.add((metric_name, key)) + ds.close() + slow.close() + return {"names": set(snapshot), "pairs": pairs} + + +@pytest.mark.asyncio +async def test_every_registered_metric_is_emitted(emitted_metrics): + "The both-ways name check for metrics." + names = emitted_metrics["names"] + missing = sorted(str(m) for m in reg.METRICS if m not in names) + assert not missing, f"documented but never emitted: {missing}" + + unregistered = sorted( + name for name in names if name not in {str(m) for m in reg.METRICS} + ) + assert not unregistered, f"emitted but not registered: {unregistered}" + + +@pytest.mark.asyncio +async def test_every_emitted_metric_attribute_is_registered(emitted_metrics): + """ + An attribute added to a metric without a registry entry would be missing + from the docs - the metric-side counterpart of + `test_every_emitted_attribute_is_registered`. + """ + metric_for = {str(m): m for m in reg.METRICS} + unregistered = sorted( + f"{metric_name} -> {key}" + for metric_name, key in emitted_metrics["pairs"] + # A metric name with no registry entry at all is already reported by + # test_every_registered_metric_is_emitted; do not double-report it + # here, and do not crash attribute_allowed() on a None metric. + if metric_name in metric_for + and not reg.attribute_allowed(metric_for[metric_name], key) + ) + assert ( + not unregistered + ), "these metric attributes are emitted but not registered: " + "\n".join( + unregistered + ) + + +@pytest.mark.asyncio +async def test_every_registered_metric_attribute_is_emitted(emitted_metrics): + """ + The direction nothing else catches: the docs must not describe a metric + attribute that no longer exists. + + Unlike the span-side attribute check, this does not skip `optional` + attributes. The only optional metric attribute is `error.type` on + `db.client.operation.duration`, and the workload reaches it from two + independent directions: the suppressed-error probe and the + custom_time_limit interrupt in `exercise()`, both of which raise through + `record_operation_duration`. So it is checked like any other attribute + rather than exempted; marking something optional here would opt it out of + verification entirely. + + Gauges with no registered attributes (`datasette.sql.threads.limit` and + `.queue_depth`) fall out correctly with no special case: their + `metric.attributes` is empty, so the inner loop makes no assertion. + """ + emitted_keys_by_metric = {} + for metric_name, key in emitted_metrics["pairs"]: + emitted_keys_by_metric.setdefault(metric_name, set()).add(key) + + missing = [] + for metric in reg.METRICS: + if str(metric) not in emitted_metrics["names"]: + # Not emitted at all - already reported by + # test_every_registered_metric_is_emitted; do not double-report. + continue + emitted_keys = emitted_keys_by_metric.get(str(metric), set()) + for attribute in metric.attributes: + if attribute not in emitted_keys: + missing.append(f"{metric} -> {attribute}") + assert not missing, ( + "these metric attributes are documented but never emitted by the " + "test workload: " + ", ".join(sorted(missing)) + ) From 89ce2759a73592e92f9be905c6ef44bba91859e3 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Thu, 30 Jul 2026 12:39:03 -0700 Subject: [PATCH 21/24] Document metric exemplars, which Datasette already emits Histograms recorded inside a sampled span carry trace IDs automatically, so a latency spike links to a trace that caused it. Nothing said so. Two things are documented because they were measured rather than assumed: an exemplar is kept per histogram bucket, so the bucket boundaries fixed earlier in this stack took the same workload from one reachable trace to four; and the pinned opentelemetry-exporter-prometheus drops exemplars entirely, so the path that works is an OTLP collector rather than Datasette's Prometheus exporter. Co-Authored-By: Claude Opus 5 (cherry picked from 9d066255; section numbering and cross-references adjusted to this branch's demo README, and the exemplar reference placed as a subsection of the new Metric reference in internals.rst.) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01F2h9ANGZ7paWSpqs5DUAcG --- docs/internals.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/internals.rst b/docs/internals.rst index 19040cc4..303857da 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2526,6 +2526,36 @@ This reference is generated from ``datasette/telemetry_registry.py``, like the s .. [[[end]]] +Exemplars +~~~~~~~~~ + +An OpenTelemetry `exemplar `__ attaches a trace ID and span ID to one sample backing a histogram measurement. Where a spike in ``db.client.operation.duration`` alone tells you "queries were slow sometime in this minute", the exemplar attached to one of the samples in that spike gives you the trace ID of an actual slow query to open. + +Datasette needs no configuration to produce these. Every histogram measurement on the query path is recorded while a span for that operation is active, and the OpenTelemetry SDK's default exemplar filter attaches the current trace ID and span ID to any measurement recorded inside a sampled span - this is SDK behaviour that Datasette's instrumentation does not need to opt into. Four queries of increasing cost, each inside its own span, produced one exemplar per query on ``db.client.operation.duration``: + +.. code-block:: text + + db.client.operation.duration count=4 + exemplars: 4 + value=0.001564s trace_id=ddfaf45fd4e14913497d7efeac95f381 span_id=fd5792bdbb01e533 + value=0.006320s trace_id=34aea775ade11a3c5f716695731000fe span_id=25ed9e29dd84dbee + value=0.045253s trace_id=a65cb58d1460a179f0d04046ff51ed0d span_id=7f34d6378c85d062 + value=0.305240s trace_id=6089f4c515c221c0ca7bb53667b37ac8 span_id=0516f4a6641eaa0b + +Exemplars are kept per histogram bucket - the SDK's default reservoir for an explicit-bucket histogram holds one exemplar per bucket - so the bucket boundaries above decide how many distinct traces a metric can point at. The same four queries, run against an earlier set of bucket boundaries under which every one of them fell into a single ``(0, 5]`` second bucket, produced one exemplar instead of four: + +.. code-block:: text + + db.client.operation.duration count=4 + exemplars: 1 + value=0.305349s trace_id=cd40f9af396ad1e1d71a8832d70ac84a span_id=8a8c288609731fea + +Correcting the bucket boundaries had a second effect beyond fixing the quantiles: it also multiplied the number of traces reachable from this metric, one to four for this workload. + +Which export path you use matters here. The OTLP exporter carries exemplars through unchanged. The pinned ``opentelemetry-exporter-prometheus`` (``0.65b0``) does not: the string ``exemplar`` does not appear anywhere in its source, and an OpenMetrics scrape of the workload above through that exporter contained zero exemplar markers - even though ``prometheus-client`` (``0.26.0``), the library it depends on for OpenMetrics output, supports the syntax. If exemplars need to reach Prometheus, the path that works is an OTLP collector writing to Prometheus, not Datasette's own Prometheus exporter. On that path, the Prometheus server needs `--enable-feature=exemplar-storage `__, and the scrape itself must use the OpenMetrics exposition format - Prometheus's default text format has no syntax for exemplars at all. Grafana then needs the Prometheus data source's `exemplar configuration `__ (``exemplarTraceIdDestinations``) pointed at a tracing data source before it will draw an exemplar as a clickable point rather than an ordinary sample. + +An exemplar can only exist for a trace that was sampled. The SDK's default exemplar filter only records one when the measurement happens inside a sampled span, and produces no exemplar at all rather than a link to a trace that was never kept. Verified: with the tracer provider's sampler set to ``ALWAYS_OFF``, the same four-query workload produced ``exemplars: 0`` on every data point. At 1% head sampling, 99% of measurements contribute no exemplar - but every exemplar you do get is guaranteed to resolve to a trace that exists. + .. _internals_telemetry_requests: Requests and inbound trace context From ed6419e4f65e571f4745988950de45dc66e165b9 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Tue, 1 Sep 2026 17:09:43 -0700 Subject: [PATCH 22/24] Apply black to the metrics additions Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01F2h9ANGZ7paWSpqs5DUAcG --- datasette/database.py | 4 +--- datasette/telemetry.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index c5589881..2ba89923 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -657,9 +657,7 @@ class Database: start_time=task.enqueued_at_ns, **write_span_kwargs, ).end(end_time=dequeued_at_ns) - record_write_queue_wait( - self.name, dequeued_at_ns - task.enqueued_at_ns - ) + record_write_queue_wait(self.name, dequeued_at_ns - task.enqueued_at_ns) if conn_exception is not None: # fn never runs in this branch, so there is nothing to # wrap in a db.write.execute span. diff --git a/datasette/telemetry.py b/datasette/telemetry.py index f1570713..581787fe 100644 --- a/datasette/telemetry.py +++ b/datasette/telemetry.py @@ -578,9 +578,7 @@ def observe_write_queue_depth(options=None): if write_queue is None: # No write has ever been queued for this database. continue - yield otel_metrics.Observation( - write_queue.qsize(), {DB_NAMESPACE: db.name} - ) + yield otel_metrics.Observation(write_queue.qsize(), {DB_NAMESPACE: db.name}) def observe_open_connections(options=None): From 721cc6d9f969aa9d68b956266f643ca29e0a5e09 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Wed, 2 Sep 2026 11:31:12 -0700 Subject: [PATCH 23/24] Review polish: fix stale exemplar context, dedupe rationales, close test gaps - The exemplars docs described "the pinned opentelemetry-exporter-prometheus" and "Datasette's own Prometheus exporter" - context from demo/plugin work that is no longer part of this stack. Reworded to stand alone. - Saturate a num_sql_threads=1 pool and assert the queue-depth gauge reads above zero - the headline alerting metric previously only had an absence test, and this also pins the private ThreadPoolExecutor._work_queue attribute it depends on. - Pin error.type on the write path of db.client.operation.duration - the write wrappers time a different code path than the read one already tested. - Isolate the non-threaded-mode gauge test from other live instances instead of comparing global observation counts, which a GC pass could shift. - Halve the metrics banner, point conftest's meter note at it, compact the interrupted-counter call-site comment to a registry pointer, note why instrument and registry descriptions are separate strings, and stop calling the metric dimension a "later phase" now that metrics shipped. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012U7coQfVu8nK2R4q2mCULA --- datasette/database.py | 10 ++-- datasette/telemetry.py | 37 ++++++-------- docs/internals.rst | 2 +- tests/conftest.py | 10 ++-- tests/test_telemetry_metrics.py | 87 ++++++++++++++++++++++++++++++--- 5 files changed, 104 insertions(+), 42 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index 2ba89923..ec83ecad 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -925,13 +925,9 @@ class Database: if not timeout_expected: span.set_status(Status(StatusCode.ERROR, str(e))) span.record_exception(e) - # A counter rather than only a span, because this is the - # one thing an operator wants a rate and an alert on, and - # spans under a 1% sampler cannot provide either. An - # expected timeout - a caller that opted into a shorter - # budget, like facet suggestion - is not counted, for the - # same reason it is not a span error: it fires routinely - # by design and would drown the signal this exists for. + # Expected timeouts (a caller that opted into a shorter + # budget, like facet suggestion) are not counted - see + # the M_QUERIES_INTERRUPTED registry entry for why. record_query_interrupted(self.name) raise except Exception as e: diff --git a/datasette/telemetry.py b/datasette/telemetry.py index 581787fe..4693e1fb 100644 --- a/datasette/telemetry.py +++ b/datasette/telemetry.py @@ -109,11 +109,11 @@ def callback_name(fn) -> str: # fixed allowlist - deliberately not a parse. # # This runs against arbitrary user-supplied SQL (the `?sql=` query string, -# canned queries, anything typed into the query editor), and the attribute is -# a candidate dimension on a query-duration metric in a later phase. A metric -# series is keyed by its attribute values, so echoing back an arbitrary first -# token would let one visitor's typo mint a new, permanent series. The -# allowlist bounds that at a fixed, small set regardless of what anyone sends. +# canned queries, anything typed into the query editor), and the attribute +# has to stay safe to use as a metric dimension. A metric series is keyed by +# its attribute values, so echoing back an arbitrary first token would let +# one visitor's typo mint a new, permanent series. The allowlist bounds that +# at a fixed, small set regardless of what anyone sends. DB_OPERATION_ALLOWLIST = frozenset( { "SELECT", @@ -382,23 +382,12 @@ class TelemetryMiddleware: # --- Metrics -------------------------------------------------------------- # -# Spans answer "what happened during this request". They cannot answer "am I -# saturating my 3 SQL threads right now", because that is a gauge: a level -# sampled at collection time, not an event with a duration. It is also the -# single most useful operational question about a Datasette deployment, since -# num_sql_threads defaults to 3 and every read query in the process competes -# for those threads. -# -# Two shapes are used here: -# -# Observable gauges - a callback the SDK invokes on its own collection -# cycle. Nothing is computed unless something is collecting, so the default -# no-provider install pays literally nothing for them. -# -# Synchronous histograms/counters - recorded inline on the query path. These -# survive trace sampling, which spans do not: an operator sampling 1% of -# traces still gets 100% of the latency distribution and the interrupted -# count. +# Two shapes. Observable gauges - a callback the SDK invokes on its own +# collection cycle, so a no-provider install never runs them - answer level +# questions no span can, like "am I saturating my SQL threads right now". +# Synchronous histograms/counters are recorded inline on the query path and +# survive trace sampling: 1% of traces still means 100% of the latency +# distribution. Why each metric exists is documented on its registry entry. # # Note a real difference from tracing: `_ProxyMeter` and its instruments # forward to a provider installed *after* they were created, whereas @@ -415,6 +404,10 @@ def _duration_attributes(database_name, operation): } +# Each instrument passes the SDK a short plain-text description; the registry +# entry for the same metric carries a longer RST one for the generated docs +# (it can use `:ref:` roles, which an exported description string cannot). + sql_operation_duration = meter.create_histogram( M_OPERATION_DURATION, unit=M_OPERATION_DURATION.unit, diff --git a/docs/internals.rst b/docs/internals.rst index 303857da..3b1ae285 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2552,7 +2552,7 @@ Exemplars are kept per histogram bucket - the SDK's default reservoir for an exp Correcting the bucket boundaries had a second effect beyond fixing the quantiles: it also multiplied the number of traces reachable from this metric, one to four for this workload. -Which export path you use matters here. The OTLP exporter carries exemplars through unchanged. The pinned ``opentelemetry-exporter-prometheus`` (``0.65b0``) does not: the string ``exemplar`` does not appear anywhere in its source, and an OpenMetrics scrape of the workload above through that exporter contained zero exemplar markers - even though ``prometheus-client`` (``0.26.0``), the library it depends on for OpenMetrics output, supports the syntax. If exemplars need to reach Prometheus, the path that works is an OTLP collector writing to Prometheus, not Datasette's own Prometheus exporter. On that path, the Prometheus server needs `--enable-feature=exemplar-storage `__, and the scrape itself must use the OpenMetrics exposition format - Prometheus's default text format has no syntax for exemplars at all. Grafana then needs the Prometheus data source's `exemplar configuration `__ (``exemplarTraceIdDestinations``) pointed at a tracing data source before it will draw an exemplar as a clickable point rather than an ordinary sample. +Which export path you use matters here. The OTLP exporter carries exemplars through unchanged. ``opentelemetry-exporter-prometheus`` (as of ``0.65b0``) does not: the string ``exemplar`` does not appear anywhere in its source, and an OpenMetrics scrape of the workload above through that exporter contained zero exemplar markers - even though ``prometheus-client`` (``0.26.0``), the library it depends on for OpenMetrics output, supports the syntax. If exemplars need to reach Prometheus, the path that works is an OTLP collector writing to Prometheus, not a plugin scraping through that exporter. On that path, the Prometheus server needs `--enable-feature=exemplar-storage `__, and the scrape itself must use the OpenMetrics exposition format - Prometheus's default text format has no syntax for exemplars at all. Grafana then needs the Prometheus data source's `exemplar configuration `__ (``exemplarTraceIdDestinations``) pointed at a tracing data source before it will draw an exemplar as a clickable point rather than an ordinary sample. An exemplar can only exist for a trace that was sampled. The SDK's default exemplar filter only records one when the measurement happens inside a sampled span, and produces no exemplar at all rather than a link to a trace that was never kept. Verified: with the tracer provider's sampler set to ``ALWAYS_OFF``, the same four-query workload produced ``exemplars: 0`` on every data point. At 1% head sampling, 99% of measurements contribute no exemplar - but every exemplar you do get is guaranteed to resolve to a trace that exists. diff --git a/tests/conftest.py b/tests/conftest.py index e0c1cab1..ff33da98 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -123,12 +123,10 @@ def _otel_meter_provider(): Install a real OTel SDK MeterProvider + InMemoryMetricReader once per process. - Unlike the tracer, ordering is not load-bearing here: `_ProxyMeter` and - the `_ProxyInstrument`s it hands out forward to a provider installed - *after* they were created, whereas `ProxyTracer` permanently caches the - first concrete tracer it resolves. This fixture is still session-scoped - and autouse for symmetry, and so that a single reader collects for the - whole run. + Unlike the tracer, ordering is not load-bearing here - see the metrics + banner in `datasette/telemetry.py` for the `_ProxyMeter`-vs-`ProxyTracer` + difference. This fixture is still session-scoped and autouse for + symmetry, and so that a single reader collects for the whole run. DELTA temporality is chosen for counters and histograms so that each collection reports only what happened since the previous one. With the diff --git a/tests/test_telemetry_metrics.py b/tests/test_telemetry_metrics.py index 1422fbcd..f1ac4a56 100644 --- a/tests/test_telemetry_metrics.py +++ b/tests/test_telemetry_metrics.py @@ -17,6 +17,8 @@ Two layers are tested separately and deliberately: """ import asyncio +import threading +import weakref import pytest @@ -74,22 +76,71 @@ async def test_no_thread_gauges_in_non_threaded_mode(): num_sql_threads=0 means there is no pool at all, so the pool gauges must skip the instance rather than report a bogus limit of 0. - The pool gauges carry no attributes, so the assertion is that adding this - instance produces no *additional* observations. + The pool gauges carry no attributes, so the live-instance registry is + narrowed to just this instance for the assertion - counting global + observations instead would let an unrelated instance being garbage + collected mid-test shift the baseline. """ - before_limits = len(list(telemetry.observe_sql_thread_limit())) - before_depths = len(list(telemetry.observe_sql_thread_queue_depth())) ds = Datasette(memory=True, settings={"num_sql_threads": 0}) try: assert ds.executor is None - assert len(list(telemetry.observe_sql_thread_limit())) == before_limits - assert len(list(telemetry.observe_sql_thread_queue_depth())) == before_depths + original = telemetry._live_datasettes + telemetry._live_datasettes = weakref.WeakSet([ds]) + try: + assert list(telemetry.observe_sql_thread_limit()) == [] + assert list(telemetry.observe_sql_thread_queue_depth()) == [] + finally: + telemetry._live_datasettes = original # Per-database gauges are unaffected - they do not depend on the pool. assert observations(telemetry.observe_pending_queries, ds) finally: ds.close() +@pytest.mark.asyncio +async def test_thread_queue_depth_gauge_reports_saturation(): + """ + The headline alerting metric must actually read above zero when reads + queue behind num_sql_threads. This also pins the private + `ThreadPoolExecutor._work_queue` attribute the callback depends on: if a + stdlib rename ever removes it, this fails instead of the metric silently + vanishing (the callback tolerates its absence at collection time). + """ + ds = Datasette(memory=True, settings={"num_sql_threads": 1}) + db = ds.add_memory_database("metrics_saturation_db") + entered = threading.Event() + release = threading.Event() + + def blocker(conn): + entered.set() + assert release.wait(timeout=10) + return 1 + + try: + first = asyncio.ensure_future(db.execute_fn(blocker)) + # Wait until the blocker owns the pool's only thread. + await asyncio.get_running_loop().run_in_executor(None, entered.wait, 10) + second = asyncio.ensure_future(db.execute_fn(lambda conn: 2)) + # The second submission lands in the executor's queue on the next + # event-loop turn; poll briefly rather than assume the timing. + depths = [] + for _ in range(500): + depths = [ + value + for _, value in observations(telemetry.observe_sql_thread_queue_depth) + ] + if any(value >= 1 for value in depths): + break + await asyncio.sleep(0.01) + assert any(value >= 1 for value in depths), depths + release.set() + assert await first == 1 + assert await second == 2 + finally: + release.set() + ds.close() + + @pytest.mark.asyncio async def test_pending_queries_gauge_tracks_in_flight_queries(metrics_ds): db = metrics_ds.get_database("metrics_test_db") @@ -224,6 +275,30 @@ async def test_operation_duration_records_error_type(otel_metrics): ds.close() +@pytest.mark.asyncio +async def test_operation_duration_records_write_error_type(otel_metrics): + """ + Same as the read-path error test, but the write wrappers time a different + code path - `execute_write_fn`, the write thread and its reply future - + so error propagation through them is pinned separately. + """ + ds = Datasette(memory=True) + ds.add_memory_database("duration_write_error_db") + try: + db = ds.get_database("duration_write_error_db") + with pytest.raises(sqlite3.OperationalError): + await db.execute_write("insert into nope values (1)") + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_write_error_db", "datasette.operation": "write"}, + ) + assert point.count == 1 + assert dict(point.attributes)["error.type"] == "OperationalError" + finally: + ds.close() + + @pytest.mark.asyncio async def test_write_queue_wait_histogram(otel_metrics): ds = Datasette(memory=True) From 9e5a4021fd03f2c1bf9e69c2084e786211afccd4 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Wed, 2 Sep 2026 11:51:03 -0700 Subject: [PATCH 24/24] Count callback-style calls in db.client.operation.duration The callback entry points gained db.query spans in the database-spans PR; this adds their other half - the duration histogram measurement, so a plugin's execute_fn/execute_write_fn work and the JSON write API's inserts and deletes stop being invisible to the one series that survives trace sampling. execute_isolated_fn records "write" when the database is mutable (the call blocks the write queue) and "read" when immutable (it runs on the read pool). error.type comes from the raised exception class, same as the SQL-string paths. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012U7coQfVu8nK2R4q2mCULA --- datasette/database.py | 45 +++++++++++---------- datasette/telemetry_registry.py | 4 +- docs/internals.rst | 2 +- tests/test_telemetry_metrics.py | 70 +++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 22 deletions(-) diff --git a/datasette/database.py b/datasette/database.py index ec83ecad..96d42a21 100644 --- a/datasette/database.py +++ b/datasette/database.py @@ -404,22 +404,25 @@ class Database: span.set_attribute(DB_SYSTEM, "sqlite") span.set_attribute(DB_NAMESPACE, self.name) span.set_attribute(CALLBACK, callback_name(fn)) - if self.ds.executor is None: - # non-threaded mode - return _run() - if not write: - # Immutable database - no writes can ever occur, so there is - # no write queue to block; run against a fresh read-only - # connection. copy_context() carries the caller's otel context - # onto the worker thread - see the notes in _execute_fn() for - # why it must be a fresh copy per submit and why carrying - # every ContextVar is safe. - ctx = contextvars.copy_context() - return await asyncio.get_running_loop().run_in_executor( - self.ds.executor, ctx.run, _run - ) - # Threaded mode - send to write thread - return await self._send_to_write_thread(fn, isolated_connection=True) + # "write" when mutable because the call blocks the write queue; + # "read" when immutable, where it runs on the read pool. + with record_operation_duration(self.name, "write" if write else "read"): + if self.ds.executor is None: + # non-threaded mode + return _run() + if not write: + # Immutable database - no writes can ever occur, so there + # is no write queue to block; run against a fresh + # read-only connection. copy_context() carries the + # caller's otel context onto the worker thread - see the + # notes in _execute_fn() for why it must be a fresh copy + # per submit and why carrying every ContextVar is safe. + ctx = contextvars.copy_context() + return await asyncio.get_running_loop().run_in_executor( + self.ds.executor, ctx.run, _run + ) + # Threaded mode - send to write thread + return await self._send_to_write_thread(fn, isolated_connection=True) async def analyze_sql(self, sql, params=None) -> SQLAnalysis: self._check_not_closed() @@ -449,9 +452,10 @@ class Database: span.set_attribute(DB_SYSTEM, "sqlite") span.set_attribute(DB_NAMESPACE, self.name) span.set_attribute(CALLBACK, name) - return await self._execute_write_fn( - fn, block=block, transaction=transaction, request=request - ) + with record_operation_duration(self.name, "write"): + return await self._execute_write_fn( + fn, block=block, transaction=transaction, request=request + ) async def _execute_write_fn(self, fn, block=True, transaction=True, request=None): self._check_not_closed() @@ -741,7 +745,8 @@ class Database: # Default exception handling applies, unlike execute(): there is # no log_sql_errors=False probing caller and no expected-timeout # budget on this path, so a raised exception is an error. - return await self._execute_fn(fn_in_execute_span) + with record_operation_duration(self.name, "read"): + return await self._execute_fn(fn_in_execute_span) async def _execute_fn(self, fn): self._check_not_closed() diff --git a/datasette/telemetry_registry.py b/datasette/telemetry_registry.py index 3c775a7b..e4082347 100644 --- a/datasette/telemetry_registry.py +++ b/datasette/telemetry_registry.py @@ -459,7 +459,9 @@ M_OPERATION_DURATION = MetricName( HISTOGRAM, "s", "Duration of a SQL operation. The standard OpenTelemetry semantic " - "convention metric, and the one that survives trace sampling.", + "convention metric, and the one that survives trace sampling. " + "Callback-style calls (``execute_fn()`` and friends) are counted " + "alongside the SQL-string methods.", (DB_SYSTEM, DB_NAMESPACE, OPERATION, ERROR_TYPE), buckets=DURATION_BUCKETS, ) diff --git a/docs/internals.rst b/docs/internals.rst index 3b1ae285..b82ebfbc 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -2466,7 +2466,7 @@ This reference is generated from ``datasette/telemetry_registry.py``, like the s .. ]]] ``db.client.operation.duration`` - Histogram, unit ``s``. Duration of a SQL operation. The standard OpenTelemetry semantic convention metric, and the one that survives trace sampling. + Histogram, unit ``s``. Duration of a SQL operation. The standard OpenTelemetry semantic convention metric, and the one that survives trace sampling. Callback-style calls (``execute_fn()`` and friends) are counted alongside the SQL-string methods. Bucket boundaries: ``0.0001``, ``0.0005``, ``0.001``, ``0.005``, ``0.01``, ``0.05``, ``0.1``, ``0.5``, ``1``, ``5``, ``10``. diff --git a/tests/test_telemetry_metrics.py b/tests/test_telemetry_metrics.py index f1ac4a56..5aade58b 100644 --- a/tests/test_telemetry_metrics.py +++ b/tests/test_telemetry_metrics.py @@ -460,3 +460,73 @@ def test_histograms_spread_values_across_buckets( f"expected each of {SPREAD} in its own bucket, got bucket counts " f"{list(point.bucket_counts)} for bounds {list(point.explicit_bounds)}" ) + + +@pytest.mark.asyncio +async def test_operation_duration_histogram_records_execute_fn(otel_metrics): + "Callback-style reads land in the same histogram as SQL-string reads." + ds = Datasette(memory=True) + ds.add_memory_database("duration_fn_db") + try: + db = ds.get_database("duration_fn_db") + + def read_one(conn): + return conn.execute("select 1").fetchone()[0] + + assert await db.execute_fn(read_one) == 1 + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_fn_db", "datasette.operation": "read"}, + ) + assert point.count == 1 + assert point.sum > 0 + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_operation_duration_histogram_records_execute_write_fn(otel_metrics): + "Callback-style writes - the JSON write API's whole diet - are counted too." + ds = Datasette(memory=True) + ds.add_memory_database("duration_write_fn_db") + try: + db = ds.get_database("duration_write_fn_db") + + def create_table(conn): + conn.execute("create table t (id integer primary key)") + + await db.execute_write_fn(create_table) + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_write_fn_db", "datasette.operation": "write"}, + ) + assert point.count == 1 + assert point.sum > 0 + finally: + ds.close() + + +@pytest.mark.asyncio +async def test_operation_duration_records_callback_error_type(otel_metrics): + "A callback that raises is still timed, with error.type from the exception." + ds = Datasette(memory=True) + ds.add_memory_database("duration_fn_error_db") + try: + db = ds.get_database("duration_fn_error_db") + + def boom(conn): + raise ValueError("callback failed") + + with pytest.raises(ValueError): + await db.execute_fn(boom) + otel_metrics.collect() + point = otel_metrics.point( + "db.client.operation.duration", + {"db.namespace": "duration_fn_error_db", "datasette.operation": "read"}, + ) + assert point.count == 1 + assert dict(point.attributes)["error.type"] == "ValueError" + finally: + ds.close()