remote/datasette
1
0
Fork 0
mirror of https://github.com/simonw/datasette.git synced 2026-09-09 01:54:15 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add a plugin telemetry kit: public registry API, linked_root_span_kwargs, test helpers, docs

Browse source 
A survey of five plugin OTel plans (datasette-paper, -agent, -litestream,
-accounts, -cron) found every one hand-copying the same core machinery:
the registry classes, the conformance-test harness, the pytest fixtures,
the bucket boundaries and the detached-root-with-Link recipe. This makes
that machinery importable instead:

- The registry classes are documented public API. Attribute gains
  values= (a closed enum the conformance helpers enforce - what makes an
  attribute safe as a metric dimension); SpanName gains prefix=True for
  span families like "chat {model}" whose names share a fixed prefix,
  matched by span_for() after exact names. span_for()/attribute helpers
  accept a spans= tuple so plugin registries can use them.
- datasette.telemetry.linked_root_span_kwargs(): the root-span-with-Link
  shape for work a request caused without containing - background jobs,
  scheduled ticks, block=False writes. Core's own write thread now uses
  it instead of building the kwargs inline.
- datasette.telemetry_testing: the session provider fixtures, otel_spans
  / otel_metrics, a two-way registry conformance checker (including enum
  and prefix handling, filtered by instrumentation scope) and an
  assert_package_never_imports_sdk() guard. Core's conftest now imports
  these instead of defining them, so the suite consumes the kit exactly
  as a plugin's would.
- New "Telemetry for plugin authors" docs page: scope discipline,
  registry usage, privacy/cardinality rules, named-callable guidance,
  request_span(), the background root-with-link convention (one root per
  tick, always emitted), provider-ordering facts and known caveats.
  request_span() is now documented public API.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012U7coQfVu8nK2R4q2mCULA
This commit is contained in:
Alex Garcia 2026-09-02 12:24:42 -07:00
commit f28db54eda
13 changed files with 785 additions and 196 deletions
Download patch file Download diff file Expand all files Collapse all files

161
tests/conftest.py
View file

@ -58,158 +58,15 @@ def find_free_port():
return sock.getsockname()[1]
_otel_span_exporter = None
@pytest.fixture(scope="session", autouse=True)
def _otel_provider():
"""
Install a real OTel SDK TracerProvider + InMemorySpanExporter exactly
once, before any span is ever created in this process.
This has to be session-scoped and autouse because
`opentelemetry.trace.set_tracer_provider()` is effectively
once-per-process: a second call logs a warning and is ignored. So the
install must happen exactly once, before anything asserts on spans.
`datasette.telemetry.tracer` is a module-level `ProxyTracer`. Once a
provider exists, the first span it starts resolves a concrete tracer
and caches it permanently. It does *not* cache the no-op tracer, so
any span started before this fixture runs is merely lost rather than
poisoning the tracer for the rest of the process. If the SDK isn't
installed, do nothing: core spans stay no-op `NonRecordingSpan`s and
the rest of the suite is unaffected.
"""
global _otel_span_exporter
try:
from opentelemetry import trace as otel_trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
except ImportError:
return
exporter = InMemorySpanExporter()
provider = TracerProvider()
# SimpleSpanProcessor exports synchronously on span end - no background
# batching thread, so assertions immediately after a request never race.
provider.add_span_processor(SimpleSpanProcessor(exporter))
otel_trace.set_tracer_provider(provider)
_otel_span_exporter = exporter
@pytest.fixture
def otel_spans():
"""
Function-scoped access to the finished-spans exporter: clears any spans
left over from previous tests, then yields the exporter so a test can
call `.get_finished_spans()` after making requests. Skips (rather than
fails) if the OTel SDK is not installed.
"""
pytest.importorskip("opentelemetry.sdk")
if _otel_span_exporter is None:
pytest.skip("OpenTelemetry SDK provider was not installed")
_otel_span_exporter.clear()
yield _otel_span_exporter
_otel_metric_reader = None
@pytest.fixture(scope="session", autouse=True)
def _otel_meter_provider():
"""
Install a real OTel SDK MeterProvider + InMemoryMetricReader once per
process.
Unlike the tracer, ordering is not load-bearing here - see the metrics
banner in `datasette/telemetry.py` for the `_ProxyMeter`-vs-`ProxyTracer`
difference. This fixture is still session-scoped and autouse for
symmetry, and so that a single reader collects for the whole run.
DELTA temporality is chosen for counters and histograms so that each
collection reports only what happened since the previous one. With the
SDK default of CUMULATIVE, every metrics test would see every query run
by every earlier test in the session.
"""
global _otel_metric_reader
try:
from opentelemetry import metrics as otel_metrics
from opentelemetry.sdk.metrics import Counter, Histogram, MeterProvider
from opentelemetry.sdk.metrics.export import (
AggregationTemporality,
InMemoryMetricReader,
)
except ImportError:
return
reader = InMemoryMetricReader(
preferred_temporality={
Counter: AggregationTemporality.DELTA,
Histogram: AggregationTemporality.DELTA,
}
)
otel_metrics.set_meter_provider(MeterProvider(metric_readers=[reader]))
_otel_metric_reader = reader
class MetricsCollector:
"""
Thin reader over an `InMemoryMetricReader`.
`collect()` runs a collection cycle - which is what invokes the observable
gauge callbacks - and snapshots the result. Queries then run against that
snapshot rather than re-collecting, so a test that inspects several
metrics sees one consistent moment and does not drain delta state twice.
"""
def __init__(self, reader):
self.reader = reader
self.snapshot = {}
def collect(self):
self.snapshot = {}
data = self.reader.get_metrics_data()
if data is None:
return self.snapshot
for resource_metrics in data.resource_metrics:
for scope_metrics in resource_metrics.scope_metrics:
for metric in scope_metrics.metrics:
self.snapshot.setdefault(metric.name, []).extend(
metric.data.data_points
)
return self.snapshot
def points(self, name, attributes=None):
"Data points for `name` whose attributes are a superset of `attributes`."
found = []
for point in self.snapshot.get(name, []):
point_attributes = dict(point.attributes or {})
if all(point_attributes.get(k) == v for k, v in (attributes or {}).items()):
found.append(point)
return found
def point(self, name, attributes=None):
"The single matching data point, asserting there is exactly one."
found = self.points(name, attributes)
assert len(found) == 1, (
f"expected exactly one {name} point matching {attributes}, "
f"got {len(found)}: {found}"
)
return found[0]
@pytest.fixture
def otel_metrics():
"""
Function-scoped metrics collector. Drains any delta state accumulated by
earlier tests before yielding, so counts start from zero.
"""
pytest.importorskip("opentelemetry.sdk")
if _otel_metric_reader is None:
pytest.skip("OpenTelemetry SDK meter provider was not installed")
_otel_metric_reader.get_metrics_data()
yield MetricsCollector(_otel_metric_reader)
# The otel fixtures moved to datasette.telemetry_testing, which is public
# plugin API - core's suite consumes it exactly the way a plugin's would.
from datasette.telemetry_testing import ( # noqa: F401, E402
MetricsCollector,
otel_metrics,
otel_meter_provider,
otel_provider,
otel_spans,
)
@pytest.fixture

2
tests/test_http_span.py
View file

@ -825,7 +825,7 @@ def test_no_provider_takes_the_fast_path():
install should pay essentially nothing for instrumentation it is not
using.
This has to run in a subprocess. The suite's `_otel_provider` fixture is
This has to run in a subprocess. The suite's `otel_provider` fixture is
session-scoped and autouse, and `set_tracer_provider()` is effectively
once-per-process, so in-process every span is recording and the fast path
is unreachable.

37
tests/test_telemetry_registry.py
View file

@ -457,7 +457,7 @@ async def emitted_metrics(otel_metrics):
plus the raw set of metric names - the metric-side counterpart of the
`emitted` span fixture above.
Metrics use DELTA temporality (see `_otel_meter_provider`), and the
Metrics use DELTA temporality (see `otel_meter_provider` in datasette.telemetry_testing), and the
function-scoped `otel_metrics` fixture drains any state left by an
earlier test before yielding, so this collection is not polluted by
other tests in the session - only by other *instances*, which is why the
@ -574,3 +574,38 @@ async def test_every_registered_metric_attribute_is_emitted(emitted_metrics):
"these metric attributes are documented but never emitted by the "
"test workload: " + ", ".join(sorted(missing))
)
def test_prefix_span_lookup():
"""
`prefix=True` matching, exercised directly.
Core registers no prefix spans - the flag exists for plugin registries
(e.g. a `chat {model}` span family) - so without this the branch in
`span_for()` would be untested code the conformance tests never reach.
"""
hook = reg.SpanName("myplugin.hook.", "A hypothetical span family", prefix=True)
spans = reg.SPANS + (hook,)
assert reg.span_for("myplugin.hook.render_cell", spans=spans) is hook
assert reg.span_for("myplugin.hook.anything", spans=spans) is hook
assert reg.span_for("myplugin.hookish", spans=spans) is None
assert reg.span_for("db.query", spans=spans) is reg.DB_QUERY
def test_exact_match_wins_over_prefix():
"A prefix family can never shadow a span with a registered exact name."
family = reg.SpanName("db.", "Greedy prefix", prefix=True)
spans = (family,) + reg.SPANS
assert reg.span_for("db.query", spans=spans) is reg.DB_QUERY
assert reg.span_for("db.anything-else", spans=spans) is family
def test_attribute_values_enum_enforced():
outcome = reg.Attribute("myplugin.outcome", "Enum.", values={"ok", "error"})
open_attr = reg.Attribute("myplugin.note", "Open value set.")
span = reg.SpanName("myplugin.job", "Test span", (outcome, open_attr))
assert reg.attribute_value_allowed(span, "myplugin.outcome", "ok")
assert not reg.attribute_value_allowed(span, "myplugin.outcome", "surprise")
assert reg.attribute_value_allowed(span, "myplugin.note", "anything at all")
assert not reg.attribute_value_allowed(span, "not.registered", "x")
assert not reg.attribute_value_allowed(None, "myplugin.outcome", "ok")

138
tests/test_telemetry_testing_kit.py Normal file
View file

@ -0,0 +1,138 @@
"""
The plugin telemetry kit (`datasette.telemetry_testing` plus the public
registry classes), exercised the way a third-party plugin would use it: a
toy plugin registry, a toy tracer scope, and the kit's own fixtures and
conformance helpers.
"""
import pytest
pytest.importorskip("opentelemetry.sdk")
from opentelemetry import trace as otel_trace
from datasette import telemetry_registry as reg
from datasette.telemetry import linked_root_span_kwargs
from datasette.telemetry_testing import (
assert_package_never_imports_sdk,
assert_registry_covered,
assert_spans_conform,
)
SCOPE = "toyplugin"
OUTCOME = reg.Attribute(
"toyplugin.outcome", "How the job ended.", values={"ok", "error"}
)
JOB_NAME = reg.Attribute("toyplugin.job", "The job's registered name.")
JOB = reg.SpanName("toyplugin.job.run", "One job execution.", (OUTCOME, JOB_NAME))
CHAT = reg.SpanName(
"toyplugin.chat ", "One model call, named `toyplugin.chat {model}`.", prefix=True
)
TOY_SPANS = (JOB, CHAT)
toy_tracer = otel_trace.get_tracer(SCOPE, "0.1")
def _toy_spans(otel_spans):
return [
span
for span in otel_spans.get_finished_spans()
if span.instrumentation_scope and span.instrumentation_scope.name == SCOPE
]
def _run_workload():
with toy_tracer.start_as_current_span(JOB) as span:
span.set_attribute(OUTCOME, "ok")
span.set_attribute(JOB_NAME, "nightly")
with toy_tracer.start_as_current_span("toyplugin.chat gpt-5"):
pass
def test_conformance_passes_for_a_conforming_workload(otel_spans):
_run_workload()
finished = otel_spans.get_finished_spans()
assert_spans_conform(TOY_SPANS, finished, scope_name=SCOPE)
# Coverage direction needs prefix families seen too - the chat span
# resolves to the CHAT entry despite its variable suffix.
assert_registry_covered(TOY_SPANS, finished, scope_name=SCOPE)
def test_conformance_catches_an_unregistered_span(otel_spans):
with toy_tracer.start_as_current_span("toyplugin.surprise"):
pass
with pytest.raises(AssertionError, match="unregistered span"):
assert_spans_conform(
TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE
)
def test_conformance_catches_an_unregistered_attribute(otel_spans):
with toy_tracer.start_as_current_span(JOB) as span:
span.set_attribute("toyplugin.stealth", 1)
with pytest.raises(AssertionError, match="unregistered attribute"):
assert_spans_conform(
TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE
)
def test_conformance_enforces_declared_enums(otel_spans):
with toy_tracer.start_as_current_span(JOB) as span:
span.set_attribute(OUTCOME, "surprise")
with pytest.raises(AssertionError, match="not in the declared enum"):
assert_spans_conform(
TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE
)
def test_coverage_catches_a_never_emitted_span(otel_spans):
with toy_tracer.start_as_current_span(JOB) as span:
span.set_attribute(OUTCOME, "ok")
span.set_attribute(JOB_NAME, "nightly")
# CHAT never emitted
with pytest.raises(AssertionError, match="never emitted"):
assert_registry_covered(
TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE
)
def test_scope_filter_ignores_other_scopes(otel_spans):
# Core's own spans are in the exporter too; a plugin's conformance run
# must not fail because of them.
other = otel_trace.get_tracer("someone-else", "1.0")
with other.start_as_current_span("not.in.the.toy.registry"):
pass
_run_workload()
assert_spans_conform(TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE)
def test_linked_root_span_kwargs_links_without_parenting(otel_spans):
with toy_tracer.start_as_current_span("toyplugin.cause") as cause:
cause_context = cause.get_span_context()
kwargs = linked_root_span_kwargs()
with toy_tracer.start_as_current_span("toyplugin.effect", **kwargs):
pass
effect = [
span for span in _toy_spans(otel_spans) if span.name == "toyplugin.effect"
][0]
assert effect.parent is None, "must be a root, not a child"
assert effect.context.trace_id != cause_context.trace_id
assert len(effect.links) == 1
assert effect.links[0].context.span_id == cause_context.span_id
def test_linked_root_span_kwargs_with_no_current_span(otel_spans):
kwargs = linked_root_span_kwargs()
assert kwargs["links"] == []
with toy_tracer.start_as_current_span("toyplugin.orphanless", **kwargs):
pass
span = _toy_spans(otel_spans)[0]
assert span.parent is None
assert span.links == ()
def test_kit_module_itself_never_imports_the_sdk():
# The kit imports the SDK lazily, so a plugin importing it at module
# level does not violate the api-only dependency rule.
assert_package_never_imports_sdk("datasette.telemetry_testing")