Extend the kit with metric-side conformance: kind, unit and enum checks

The five surveyed plugin plans all kept a hand-rolled metrics-vs-registry
diff because the kit's conformance helpers covered spans only. This adds
the metric side:

- metric_for() in the registry (the span_for analogue - no prefix/dynamic
  machinery, metric names are static), and the attribute helpers are
  documented as accepting MetricName entries.
- MetricsCollector.collect() now retains the instrumentation scope per
  collected metric, so a plugin is judged against its own meter only.
- assert_metrics_conform(): every collected metric in scope is registered,
  was created as the instrument kind and unit its registry entry declares
  (drift between the registry entry and the meter.create_*() call was
  previously caught by nothing, in core or any plugin), sets only
  registered attributes, and respects values= enums - the check that makes
  a metric dimension provably bounded.
- assert_metrics_covered(): every registered metric collected at least
  once with every non-optional attribute seen. Both *_covered helpers now
  exempt optional=True attributes, so a workload is not forced to
  manufacture every error path; pin those with targeted tests instead.
- datasette.operation declares values={"read", "write"} - core dogfoods
  the enum enforcement on the dimension where it matters most.
- Core's generic metric conformance tests are now calls to the kit
  helpers with scope_name="datasette"; the stricter literal-pinning and
  optional-attribute-coverage tests stay hand-written on purpose.
- The metric reference docs render attributes through the same helper as
  spans, so *(optional)* markers and enum values now appear there too.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012U7coQfVu8nK2R4q2mCULA

tests/test_telemetry_registry.py

@@ -34,6 +34,7 @@ from opentelemetry.trace import SpanKind
 
 from datasette import hookimpl
 from datasette import telemetry_registry as reg
+from datasette.telemetry_testing import assert_metrics_conform, assert_metrics_covered
 from datasette.app import Datasette
 from datasette.database import QueryInterrupted
 from datasette.utils.sqlite import sqlite3
@@ -497,43 +498,28 @@ async def emitted_metrics(otel_metrics):
                 pairs.add((metric_name, key))
     ds.close()
     slow.close()
-    return {"names": set(snapshot), "pairs": pairs}
+    return {"names": set(snapshot), "pairs": pairs, "collector": otel_metrics}
+
+
+@pytest.mark.asyncio
+async def test_metrics_conform_to_the_registry(emitted_metrics):
+    """
+    Emitted-but-unregistered, via the plugin kit's helper - consumed here
+    exactly the way a plugin's suite would. Beyond names and attribute keys,
+    this also asserts each instrument was created as the kind and unit its
+    registry entry declares, and that `datasette.operation` only ever takes
+    its declared enum values.
+    """
+    assert_metrics_conform(
+        reg.METRICS, emitted_metrics["collector"], scope_name="datasette"
+    )
 
 
 @pytest.mark.asyncio
 async def test_every_registered_metric_is_emitted(emitted_metrics):
-    "The both-ways name check for metrics."
-    names = emitted_metrics["names"]
-    missing = sorted(str(m) for m in reg.METRICS if m not in names)
-    assert not missing, f"documented but never emitted: {missing}"
-
-    unregistered = sorted(
-        name for name in names if name not in {str(m) for m in reg.METRICS}
-    )
-    assert not unregistered, f"emitted but not registered: {unregistered}"
-
-
-@pytest.mark.asyncio
-async def test_every_emitted_metric_attribute_is_registered(emitted_metrics):
-    """
-    An attribute added to a metric without a registry entry would be missing
-    from the docs - the metric-side counterpart of
-    `test_every_emitted_attribute_is_registered`.
-    """
-    metric_for = {str(m): m for m in reg.METRICS}
-    unregistered = sorted(
-        f"{metric_name} -> {key}"
-        for metric_name, key in emitted_metrics["pairs"]
-        # A metric name with no registry entry at all is already reported by
-        # test_every_registered_metric_is_emitted; do not double-report it
-        # here, and do not crash attribute_allowed() on a None metric.
-        if metric_name in metric_for
-        and not reg.attribute_allowed(metric_for[metric_name], key)
-    )
-    assert (
-        not unregistered
-    ), "these metric attributes are emitted but not registered: " + "\n".join(
-        unregistered
+    "Registered-but-never-collected, via the plugin kit's helper."
+    assert_metrics_covered(
+        reg.METRICS, emitted_metrics["collector"], scope_name="datasette"
     )
 
 

tests/test_telemetry_testing_kit.py

@@ -136,3 +136,127 @@ def test_kit_module_itself_never_imports_the_sdk():
     # The kit imports the SDK lazily, so a plugin importing it at module
     # level does not violate the api-only dependency rule.
     assert_package_never_imports_sdk("datasette.telemetry_testing")
+
+
+# --- Metric conformance helpers --------------------------------------------
+
+import itertools
+
+from opentelemetry import metrics as otel_metrics_api
+
+from datasette.telemetry_testing import (
+    assert_metrics_conform,
+    assert_metrics_covered,
+)
+
+toy_meter = otel_metrics_api.get_meter(SCOPE, "0.1")
+
+# Instrument names must be unique per meter for the SDK, so each test mints
+# its own via this counter rather than re-registering one name.
+_metric_ids = itertools.count()
+
+
+def _toy_metric_registry(name, kind="Counter", unit="{job}", attributes=None):
+    return (
+        reg.MetricName(
+            name,
+            kind,
+            unit,
+            "A toy metric.",
+            attributes if attributes is not None else (OUTCOME,),
+        ),
+    )
+
+
+def test_metrics_conform_passes_and_covers(otel_metrics):
+    name = f"toyplugin.jobs.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name)
+    counter = toy_meter.create_counter(name, unit="{job}", description="Jobs run")
+    counter.add(1, {OUTCOME: "ok"})
+    otel_metrics.collect()
+    assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+    assert_metrics_covered(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_conform_catches_unregistered_metric(otel_metrics):
+    name = f"toyplugin.stealth.{next(_metric_ids)}"
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1)
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="unregistered metric"):
+        assert_metrics_conform((), otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_conform_catches_kind_mismatch(otel_metrics):
+    name = f"toyplugin.kindclash.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name, kind="Histogram", unit="{job}")
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {OUTCOME: "ok"})
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="registry declares Histogram"):
+        assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_conform_catches_unit_mismatch(otel_metrics):
+    name = f"toyplugin.unitclash.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name, unit="s")
+    counter = toy_meter.create_counter(name, unit="ms")
+    counter.add(1, {OUTCOME: "ok"})
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="unit"):
+        assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_conform_catches_unregistered_attribute(otel_metrics):
+    name = f"toyplugin.attrclash.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name)
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {"toyplugin.stealth": "x"})
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="unregistered attribute"):
+        assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_conform_enforces_declared_enums(otel_metrics):
+    name = f"toyplugin.enumclash.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name)
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {OUTCOME: "surprise"})
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="not in the declared enum"):
+        assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_covered_catches_never_collected(otel_metrics):
+    registered_but_never_created = _toy_metric_registry(
+        f"toyplugin.ghost.{next(_metric_ids)}"
+    )
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="never collected"):
+        assert_metrics_covered(
+            registered_but_never_created, otel_metrics, scope_name=SCOPE
+        )
+
+
+def test_metrics_covered_skips_optional_attributes(otel_metrics):
+    name = f"toyplugin.optattr.{next(_metric_ids)}"
+    error_type = reg.Attribute("toyplugin.error", "Only on failure.", optional=True)
+    registry = _toy_metric_registry(name, attributes=(OUTCOME, error_type))
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {OUTCOME: "ok"})  # no error attribute - and that is fine
+    otel_metrics.collect()
+    assert_metrics_covered(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_scope_filter_ignores_other_scopes(otel_metrics):
+    # Core's own metrics are in the reader too; a plugin's conformance run
+    # must not fail because of them.
+    name = f"toyplugin.scoped.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name)
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {OUTCOME: "ok"})
+    other_meter = otel_metrics_api.get_meter("someone-else-metrics", "1.0")
+    stranger = other_meter.create_counter(f"stranger.{next(_metric_ids)}", unit="x")
+    stranger.add(1)
+    otel_metrics.collect()
+    assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)