Capstone review fixes: per-test reset, rename, provider guard, UpDownCounter, naming rules, privacy walk

Outcome of a whole-stack review with the kit visible as one system:

- otel_reset: an autouse fixture draining the span exporter and metric
  reader after every test. Without it a large suite accumulates hundreds
  of thousands of recorded spans in the session-scoped exporter - the
  likeliest amplifier of the slow-runner CI flakes - and plugins would
  inherit the same leak.
- assert_registry_covered renamed to assert_spans_covered: the old name
  read as covering the whole registry, which is exactly wrong next to
  assert_metrics_covered. Public API is forever; renamed before anything
  ships, no alias.
- The installers now verify their provider actually took: with a
  provider installed first (opentelemetry-instrument, an embedding app),
  set_*_provider() is silently ignored, and fixtures would assert
  against an exporter wired to nothing. They skip clearly instead.
- UPDOWN_COUNTER registry kind, mapped to Sum with monotonicity checked
  both ways - a Counter must collect monotonic, an UpDownCounter must
  not. Previously an UpDownCounter's kind check was silently skipped.
- The docs page now prescribes naming: scope = import package name
  (underscores), signal prefix = a name you own, never bare datasette.*;
  its own examples no longer teach the hyphenated outlier. Plus an
  observable-gauges pattern section and a prefix-overlap note.
- assert_no_forbidden_values(): the enforcement half of the privacy
  rules - plant sentinel secrets in a workload and assert they never
  appear in any span name, attribute, event, status description or
  metric attribute, across all scopes by default.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012U7coQfVu8nK2R4q2mCULA

tests/conftest.py

@@ -60,11 +60,12 @@ def find_free_port():
 
 # The otel fixtures moved to datasette.telemetry_testing, which is public
 # plugin API - core's suite consumes it exactly the way a plugin's would.
-from datasette.telemetry_testing import (  # noqa: F401, E402
+from datasette.telemetry_testing import (  # noqa: F401
     MetricsCollector,
-    otel_metrics,
     otel_meter_provider,
+    otel_metrics,
     otel_provider,
+    otel_reset,
     otel_spans,
 )
 

tests/test_telemetry_testing_kit.py

@@ -15,8 +15,8 @@ from datasette import telemetry_registry as reg
 from datasette.telemetry import linked_root_span_kwargs
 from datasette.telemetry_testing import (
     assert_package_never_imports_sdk,
-    assert_registry_covered,
     assert_spans_conform,
+    assert_spans_covered,
 )
 
 SCOPE = "toyplugin"
@@ -56,7 +56,7 @@ def test_conformance_passes_for_a_conforming_workload(otel_spans):
     assert_spans_conform(TOY_SPANS, finished, scope_name=SCOPE)
     # Coverage direction needs prefix families seen too - the chat span
     # resolves to the CHAT entry despite its variable suffix.
-    assert_registry_covered(TOY_SPANS, finished, scope_name=SCOPE)
+    assert_spans_covered(TOY_SPANS, finished, scope_name=SCOPE)
 
 
 def test_conformance_catches_an_unregistered_span(otel_spans):
@@ -92,7 +92,7 @@ def test_coverage_catches_a_never_emitted_span(otel_spans):
         span.set_attribute(JOB_NAME, "nightly")
     # CHAT never emitted
     with pytest.raises(AssertionError, match="never emitted"):
-        assert_registry_covered(
+        assert_spans_covered(
             TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE
         )
 
@@ -113,9 +113,9 @@ def test_linked_root_span_kwargs_links_without_parenting(otel_spans):
         kwargs = linked_root_span_kwargs()
     with toy_tracer.start_as_current_span("toyplugin.effect", **kwargs):
         pass
-    effect = [
+    effect = next(
         span for span in _toy_spans(otel_spans) if span.name == "toyplugin.effect"
-    ][0]
+    )
     assert effect.parent is None, "must be a root, not a child"
     assert effect.context.trace_id != cause_context.trace_id
     assert len(effect.links) == 1
@@ -268,3 +268,64 @@ def test_metrics_scope_filter_ignores_other_scopes(otel_metrics):
     stranger.add(1)
     otel_metrics.collect()
     assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+# --- UpDownCounter kind + privacy walk --------------------------------------
+
+from datasette.telemetry_testing import assert_no_forbidden_values
+
+
+def test_updown_counter_kind_passes(otel_metrics):
+    name = f"toyplugin.active.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name, kind=reg.UPDOWN_COUNTER, unit="{turn}")
+    updown = toy_meter.create_up_down_counter(name, unit="{turn}")
+    updown.add(1, {OUTCOME: "ok"})
+    otel_metrics.collect()
+    assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_counter_registered_as_updown_fails_on_monotonicity(otel_metrics):
+    name = f"toyplugin.monoclash.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name, kind=reg.UPDOWN_COUNTER, unit="{job}")
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {OUTCOME: "ok"})
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="is_monotonic"):
+        assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_forbidden_values_walk_catches_a_leak(otel_spans, otel_metrics):
+    secret = "sentinel-token-xyzzy"
+    with toy_tracer.start_as_current_span(JOB) as span:
+        span.set_attribute(OUTCOME, "ok")
+        span.set_attribute(JOB_NAME, f"job for {secret}")
+    with pytest.raises(AssertionError, match="sentinel-token-xyzzy"):
+        assert_no_forbidden_values(
+            {secret},
+            finished_spans=otel_spans.get_finished_spans(),
+            scope_name=SCOPE,
+        )
+
+
+def test_forbidden_values_walk_passes_a_clean_workload(otel_spans, otel_metrics):
+    _run_workload()
+    name = f"toyplugin.clean.{next(_metric_ids)}"
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {OUTCOME: "ok"})
+    otel_metrics.collect()
+    assert_no_forbidden_values(
+        {"sentinel-token-xyzzy", "alice@example.com", ""},
+        finished_spans=otel_spans.get_finished_spans(),
+        collector=otel_metrics,
+        scope_name=SCOPE,
+    )
+
+
+def test_forbidden_values_walk_checks_metric_attributes(otel_metrics):
+    secret = "leaky-metric-value"
+    name = f"toyplugin.leak.{next(_metric_ids)}"
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {"toyplugin.note": secret})
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="leaky-metric-value"):
+        assert_no_forbidden_values({secret}, collector=otel_metrics, scope_name=SCOPE)