datasette/datasette/telemetry_registry.py

"""
The single source of truth for every span and span attribute that Datasette
core emits.

Three things read this module, which is the point of it existing:

1. **The instrumentation itself.** `Attribute` and `SpanName` subclass `str`,
   so a registry entry *is* the string OpenTelemetry wants. Call sites pass
   `DB_NAMESPACE` where they used to pass `"db.namespace"` - no wrapper API
   over the OTel calls, no parallel structure to keep in step, and a typo is
   now an `ImportError` instead of a silently misnamed attribute.

2. **The documentation.** `docs/telemetry_doc.py` renders the span reference
   in `docs/internals.rst` from these definitions using cog, and
   `cog --check` runs in CI - so the docs cannot drift from the code.

3. **A conformance test.** `tests/test_telemetry_registry.py` makes real
   requests, collects every span and attribute actually emitted, and compares
   both directions: emitted-but-unregistered catches instrumentation added
   without documentation, registered-but-never-emitted catches documentation
   describing something that no longer exists. Neither the type system nor
   the generated docs can catch that second case.
"""

from opentelemetry.trace import SpanKind


class Attribute(str):
    """
    A span attribute key, carrying its own documentation.

    Subclasses `str` so it can be handed straight to `set_attribute()`.

    Part of Datasette's public plugin API - plugins declare their own
    telemetry registries with these classes. See the "Telemetry for plugin
    authors" documentation.
    """

    __slots__ = ("description", "optional", "values")

    def __new__(cls, name, description, optional=False, values=None):
        self = super().__new__(cls, name)
        self.description = description
        self.optional = optional
        # A closed enum vocabulary for the attribute's values, or None for
        # an open value set. Declaring one does two things: the conformance
        # helpers assert every emitted value is a member, and it marks the
        # attribute as bounded - safe to use as a metric dimension, where an
        # open value set would be a cardinality hazard.
        self.values = frozenset(values) if values is not None else None
        return self

    def __reduce__(self):
        # Copies and pickles collapse to a plain str. Without this, `copy` has
        # to reconstruct a str subclass through `cls.__new__(cls)`, which these
        # classes reject - their `__new__` requires the metadata arguments. It
        # is not a theoretical problem: the SDK's ConsoleMetricExporter renders
        # data points with `dataclasses.asdict()`, which deepcopies mappings,
        # and registry entries are used as metric attribute keys - so every
        # console metrics dump would crash. Collapsing is also the honest
        # answer, not a workaround. On the wire and in a copy an entry *is*
        # its string; the description, values and buckets describe the single
        # registered instance in this module, and nothing reads them off a
        # copy.
        return (str, (str(self),))

    def __repr__(self):
        return f"Attribute({str(self)!r})"


class SpanName(str):
    """A span name, carrying its documentation and the attributes it may set.

    Part of Datasette's public plugin API, like `Attribute`.
    """

    __slots__ = ("attributes", "description", "dynamic", "kind", "prefix")

    def __new__(
        cls,
        name,
        description,
        attributes=(),
        prefix=False,
        dynamic=False,
        kind=SpanKind.INTERNAL,
    ):
        self = super().__new__(cls, name)
        self.description = description
        self.attributes = tuple(attributes)
        # True for a span family whose emitted names carry a variable suffix
        # after a fixed prefix - e.g. a plugin's `chat {model}` registered as
        # SpanName("chat ", ..., prefix=True) - so `span_for()` matches by
        # prefix rather than equality. Core registers none itself; the flag
        # exists for plugin registries.
        self.prefix = prefix
        # True when the emitted name is composed at runtime and shares no
        # fixed prefix with the registry entry - the HTTP request span, whose
        # name is the request method followed by the matched route. There is
        # no substring of the entry that could be matched against the wire, so
        # `span_for()` resolves these by span kind instead, and the entry's own
        # string is a template written for a human reading the generated
        # reference.
        self.dynamic = dynamic
        # SpanKind.INTERNAL by default - every span Datasette emits describes
        # its own internal work. db.query is the one exception: it is a real
        # database call, so semantic conventions (and trace UIs, which key
        # their database styling off this) expect SpanKind.CLIENT.
        self.kind = kind
        return self

    def __reduce__(self):
        # See Attribute.__reduce__.
        return (str, (str(self),))

    def __repr__(self):
        return f"SpanName({str(self)!r})"


class MetricName(str):
    "A metric name, carrying its instrument kind, unit and attributes."

    __slots__ = ("attributes", "buckets", "description", "kind", "unit")

    def __new__(cls, name, kind, unit, description, attributes=(), buckets=None):
        self = super().__new__(cls, name)
        self.kind = kind
        self.unit = unit
        self.description = description
        self.attributes = tuple(attributes)
        # Explicit histogram bucket boundaries, for histograms only. Passed to
        # create_histogram() as explicit_bucket_boundaries_advisory and
        # published in the generated docs, since an operator writing a
        # histogram_quantile() query needs to know them.
        self.buckets = tuple(buckets) if buckets is not None else None
        return self

    def __reduce__(self):
        # See Attribute.__reduce__.
        return (str, (str(self),))

    def __repr__(self):
        return f"MetricName({str(self)!r})"


COUNTER = "Counter"
UPDOWN_COUNTER = "UpDownCounter"
HISTOGRAM = "Histogram"
GAUGE = "Observable gauge"


# --- Attributes -----------------------------------------------------------
#
# Shared attributes are defined once and referenced by every span that sets
# them, so "which spans carry db.namespace?" is answerable by grep.

HTTP_REQUEST_METHOD = Attribute(
    "http.request.method",
    "The HTTP method, clamped to the nine methods RFC 9110 and RFC 5789 "
    "define. Anything else is reported as ``_OTHER``: the method is a "
    "client-controlled string, so echoing it back unbounded would be a "
    "cardinality hazard.",
)
HTTP_RESPONSE_STATUS_CODE = Attribute(
    "http.response.status_code",
    "The status of the response, read from the ASGI ``http.response.start`` "
    "message rather than from a :ref:`internals_response` object - several "
    "views, including static files, file downloads and streaming CSV, send "
    "that message themselves and never build one. Omitted if the connection "
    "closed before anything was sent.",
    optional=True,
)
HTTP_ROUTE = Attribute(
    "http.route",
    "The route the request matched, as the compiled regular expression "
    "pattern Datasette routes with - for example "
    "``/(?P<database>[^\\/\\.]+)/(?P<table>[^\\/\\.]+)(\\.(?P<format>\\w+))?$`` "
    "for a table page. It is deliberately the pattern rather than a prettified "
    "``/{database}/{table}`` template: the route table is fixed when the app "
    "is built, so the pattern is exact, bounded and needs no parsing, whereas "
    "the transform into something prettier accretes edge cases. Unlike "
    "``url.path`` this is low cardinality, so it is the attribute to group by. "
    "Omitted when no route matched - a 404 - which is also when the span name "
    "falls back to the bare method.",
    optional=True,
)
URL_PATH = Attribute(
    "url.path",
    "The path portion of the URL. The query string is deliberately **not** "
    "recorded, on this or any other span: Datasette puts user-supplied SQL in "
    "``?sql=`` and canned query parameters in the query string, so exporting "
    "it by default would export exactly the data the rest of this "
    "instrumentation is careful with.",
)
URL_SCHEME = Attribute("url.scheme", "``http`` or ``https``.")
SERVER_ADDRESS = Attribute(
    "server.address",
    "The ``Host`` header, verbatim - including any ``:port`` suffix, a "
    "deliberate deviation from semantic conventions' ``server.address`` / "
    "``server.port`` split. Client-controlled, so treat it as untrusted input "
    "rather than as the identity of the server.",
    optional=True,
)
USER_AGENT_ORIGINAL = Attribute(
    "user_agent.original",
    "The ``User-Agent`` header, verbatim. Omitted if the client sent none.",
    optional=True,
)
INTERNAL_CLIENT = Attribute(
    "datasette.internal_client",
    "``True`` when the request was made in-process through "
    "``datasette.client`` rather than arriving over the network. Such a "
    "sub-request runs the full ASGI stack, so it emits its own nested "
    "``SERVER`` span inside the outer request's - filter on this attribute "
    "to keep kind-based dashboards from double-counting requests. Omitted "
    "for real inbound requests.",
    optional=True,
)
ERROR_TYPE = Attribute(
    "error.type",
    "Set when the request failed: the exception class name if one escaped the "
    "application, otherwise the status code as a string for a 5xx response. "
    "A 4xx does **not** set this and does not set an error status - per "
    "semantic conventions a client error is not a server span's failure.",
    optional=True,
)

DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.")
DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.")
OPERATION = Attribute(
    "datasette.operation",
    "Whether the operation was a read or a write.",
    values={"read", "write"},
)
DB_QUERY_TEXT = Attribute(
    "db.query.text",
    "The SQL, truncated to 2048 characters. Never the parameter values. "
    "Absent for a callback-style call (``execute_fn()`` and friends), where "
    "there is no SQL string to record - ``datasette.callback`` is set "
    "instead.",
    optional=True,
)
CALLBACK = Attribute(
    "datasette.callback",
    "The qualified name of the Python callable passed to ``execute_fn()``, "
    "``execute_write_fn()`` or ``execute_isolated_fn()`` - for example "
    "``TableInsertView.post.<locals>.insert_or_upsert_rows``. Set instead of "
    "``db.query.text``, which does not exist for a callback: the SQL is "
    "whatever the function chooses to run. A lambda reports ``<lambda>``, "
    "which is why callers wanting a recognisable span should pass a named "
    "function. Bounded cardinality: the set of callables is fixed by the "
    "installed code, not by request input.",
    optional=True,
)
DB_OPERATION_NAME = Attribute(
    "db.operation.name",
    "The statement's leading keyword - ``SELECT``, ``INSERT``, ``CREATE``, and "
    "so on - matched against a small fixed allowlist. Omitted rather than set "
    "to an arbitrary value: the attribute must stay safe to use as a metric "
    "dimension, and echoing an unrecognised first token from user-supplied "
    "SQL would be an unbounded-cardinality hazard. Also omitted for "
    "``execute_write_script()``, which runs multiple statements - per "
    "semantic conventions, the operation name should not be extracted from "
    "query text that can contain more than one operation. Note that a "
    "statement beginning with a CTE reports ``WITH``, not the operation "
    "inside it - a substantial share of Datasette's own reads take that "
    "form. Resolving it further would mean parsing.",
    optional=True,
)
DB_COLLECTION_NAME = Attribute(
    "db.collection.name",
    "The primary table, set only where the view already knows it - the table "
    "and row pages. Omitted for arbitrary ``?sql=`` queries, where determining "
    "the table would mean parsing the query.",
    optional=True,
)

PARAM_COUNT = Attribute(
    "datasette.param_count",
    "Number of bound parameters. Recorded instead of the values themselves.",
    optional=True,
)
PARAM_SETS = Attribute(
    "datasette.param_sets",
    "Number of parameter sets consumed by ``execute_write_many()``. Not a row "
    "count - ``executemany()`` returns no rows. The parameter values "
    "themselves are never recorded: that sequence can hold thousands of rows.",
    optional=True,
)
TIME_LIMIT_MS = Attribute(
    "datasette.time_limit_ms",
    "The :ref:`setting_sql_time_limit_ms` value this query ran under. Set on "
    "reads, which are the queries that time limit applies to.",
    optional=True,
)
ROWS_RETURNED = Attribute(
    "datasette.rows_returned",
    "Number of rows a read returned. Set on the read path only, and only when "
    "the read succeeded.",
    optional=True,
)
TRUNCATED = Attribute(
    "datasette.truncated",
    "True if the result was cut short by :ref:`setting_max_returned_rows`.",
    optional=True,
)
INTERRUPTED = Attribute(
    "datasette.interrupted",
    "True if the query was cancelled for exceeding the time limit. The span "
    "status is also set to ``ERROR``, unless the caller asked for a budget "
    "shorter than :ref:`setting_sql_time_limit_ms` - as table counts, facet "
    "suggestion and autocomplete all do - in which case running out of time "
    "is an expected answer rather than a failure and the status is left "
    "unset.",
    optional=True,
)
SQL_ERROR_SUPPRESSED = Attribute(
    "datasette.sql_error_suppressed",
    "True when the query failed but the caller passed ``log_sql_errors=False``, "
    "meaning it was probing and treats failure as an expected answer. Facet "
    "suggestion does this against every column.",
    optional=True,
)
EXECUTESCRIPT = Attribute(
    "datasette.executescript",
    "True for ``execute_write_script()``, which runs multiple statements.",
    optional=True,
)
EXECUTEMANY = Attribute(
    "datasette.executemany",
    "True for ``execute_write_many()``, which runs one statement against many "
    "parameter sets.",
    optional=True,
)
ISOLATED_CONNECTION = Attribute(
    "datasette.isolated_connection",
    "True if the write ran on its own connection rather than the shared write "
    "connection.",
)
TRANSACTION = Attribute(
    "datasette.transaction",
    "False for statements such as ``VACUUM`` that cannot run inside a transaction.",
)


# --- Spans ----------------------------------------------------------------

HTTP_REQUEST = SpanName(
    "{http.request.method} {http.route}",
    "One span per HTTP request, created by the outermost layer of the ASGI "
    "stack - so plugin ``asgi_wrapper()`` middleware, CSRF protection and "
    "every database span raised while serving the request all nest inside "
    "it. Without it each of those would be its own root trace. The span name "
    "is not a fixed string: it is the method followed by the matched route, "
    "and just the method for a request that matched no route. The span starts "
    "at the ASGI edge, before routing has happened, so it is named for the "
    "method there and renamed once the route is known. "
    "W3C ``traceparent`` and ``baggage`` headers are extracted using the "
    "global propagator, so a request arriving from an already-traced caller "
    "continues that trace; set ``OTEL_PROPAGATORS=none`` to turn that off, "
    "and strip those headers at your proxy if your instance is public.",
    (
        HTTP_REQUEST_METHOD,
        HTTP_ROUTE,
        URL_PATH,
        URL_SCHEME,
        SERVER_ADDRESS,
        USER_AGENT_ORIGINAL,
        HTTP_RESPONSE_STATUS_CODE,
        ERROR_TYPE,
        INTERNAL_CLIENT,
    ),
    dynamic=True,
    kind=SpanKind.SERVER,
)

DB_QUERY = SpanName(
    "db.query",
    "A SQL operation issued by Datasette, covering the full round trip "
    "including any time spent queued for a thread. Callback-style calls - "
    "``execute_fn()``, ``execute_write_fn()`` and ``execute_isolated_fn()`` - "
    "appear here too, distinguished by ``datasette.callback`` in place of "
    "``db.query.text``.",
    (
        DB_SYSTEM,
        DB_NAMESPACE,
        DB_QUERY_TEXT,
        CALLBACK,
        DB_OPERATION_NAME,
        DB_COLLECTION_NAME,
        PARAM_COUNT,
        PARAM_SETS,
        TIME_LIMIT_MS,
        ROWS_RETURNED,
        TRUNCATED,
        INTERRUPTED,
        SQL_ERROR_SUPPRESSED,
        EXECUTESCRIPT,
        EXECUTEMANY,
    ),
    kind=SpanKind.CLIENT,
)

DB_QUERY_EXECUTE = SpanName(
    "db.query.execute",
    "The read executing inside a SQL worker thread. Child of ``db.query``; the "
    "gap between the two is time spent waiting for a thread.",
)

DB_WRITE_QUEUE_WAIT = SpanName(
    "db.write.queue_wait",
    "Time a write spent waiting in its database's write queue before the write "
    "thread picked it up. Child of ``db.query`` for a ``block=True`` write, "
    "where the caller awaits the write and containment is accurate. For a "
    "``block=False`` write the caller does not await it - the enqueueing "
    "request *caused* the write without *containing* it, and the write's "
    "spans can outlive the request's own - so this is a root span instead, "
    "carrying an OpenTelemetry link back to the enqueueing span rather than "
    "a parent. A link records causation without asserting containment, which "
    "is exactly the distinction here.",
)

DB_WRITE_EXECUTE = SpanName(
    "db.write.execute",
    "The write executing on the write thread. Child of ``db.query`` for a "
    "``block=True`` write; for ``block=False`` a root span with a link back "
    "to the enqueueing span instead - see ``db.write.queue_wait`` above.",
    (ISOLATED_CONNECTION, TRANSACTION),
)

STARTUP = SpanName(
    "datasette.startup",
    "``invoke_startup()`` running: ``register_events``, ``register_actions``, "
    "``register_column_types``, ``prepare_jinja2_environment``, internal-database "
    "schema catalog refresh (including the ``prepare_connection`` warm-up this "
    "triggers for each database touched for the first time), saved queries, "
    "column type config and the ``startup`` hook. Runs once per process, before "
    "any request exists, so without this span every child it creates would be "
    "its own orphan root trace. A connection warmed later - lazily, the first "
    "time a *request* touches a new database or thread - nests under that "
    "request's own span instead, not under this one, since this span has "
    "already ended by then.",
)

SPANS = (
    HTTP_REQUEST,
    DB_QUERY,
    DB_QUERY_EXECUTE,
    DB_WRITE_QUEUE_WAIT,
    DB_WRITE_EXECUTE,
    STARTUP,
)


def span_for(emitted_name, kind=None, spans=None):
    """
    Resolve an emitted span name to its registry entry, or None.

    Handles the two entry kinds whose emitted names are not knowable in
    advance:

    - `prefix=True` - the name carries a variable suffix after a fixed
      prefix, matched by prefix. Core registers none; plugin registries use
      it for names like ``chat {model}``.
    - `dynamic=True` - the name has no fixed part at all, so it is matched
      on `kind` instead and the caller has to supply one.

    Exact matches win over prefix matches, and both win over dynamic, so a
    looser entry can never shadow a span with a registered name.

    `spans` defaults to core's own registry; the plugin testing kit passes a
    plugin's tuple instead.
    """
    if spans is None:
        spans = SPANS
    for span in spans:
        if span.dynamic:
            continue
        if emitted_name == span:
            return span
    for span in spans:
        if span.prefix and emitted_name.startswith(span):
            return span
    if kind is not None:
        for span in spans:
            if span.dynamic and span.kind == kind:
                return span
    return None


def metric_for(emitted_name, metrics=None):
    """
    Resolve an emitted metric name to its registry entry, or None.

    The `span_for()` analogue - simpler, because metric names are always
    static strings. `metrics` defaults to core's own registry; the plugin
    testing kit passes a plugin's tuple instead.
    """
    if metrics is None:
        metrics = METRICS
    for metric in metrics:
        if emitted_name == metric:
            return metric
    return None


def attribute_allowed(entry, emitted_key):
    """
    Whether `emitted_key` is a registered attribute of `entry`.

    `entry` is a `SpanName` or a `MetricName` - both carry `.attributes`.
    """
    if entry is None:
        return False
    return emitted_key in entry.attributes


def attribute_value_allowed(entry, emitted_key, value):
    """
    Whether `value` is permitted for `emitted_key` on `entry` (a `SpanName`
    or a `MetricName`).

    True for any value when the attribute declares no `values=` enum; when it
    does, membership is enforced - that is what makes a declared enum a real
    cardinality bound rather than documentation. On a metric entry this is
    where the bound matters most: a metric series is keyed by its attribute
    values.
    """
    if entry is None:
        return False
    for attribute in entry.attributes:
        if attribute == emitted_key:
            return attribute.values is None or value in attribute.values
    return False


# --- Metrics --------------------------------------------------------------

# Every duration histogram here is in seconds, and OpenTelemetry's default
# bucket boundaries are tuned for milliseconds - their first non-zero boundary
# is 5, so without explicit boundaries every SQLite query lands in the single
# (0, 5] second bucket and every quantile query returns noise.
#
# These are the OpenTelemetry semantic conventions' recommended boundaries for
# db.client.operation.duration, in seconds, plus 0.0001 and 0.0005 at the
# bottom. The deviation is deliberate: those boundaries assume a network
# database client, whereas SQLite is in-process and a large fraction of real
# queries run in 30-80us, which would otherwise all pile into the first
# bucket and be indistinguishable from each other.
#
# One shared list is used for every duration histogram rather than a tailored
# list each, so that dashboards stay comparable and a queue wait can be read
# against the query duration it delays. It already spans 100us to 10s, which
# covers both a fast in-process read and a write queued behind contention.
DURATION_BUCKETS = (0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10)

M_OPERATION_DURATION = MetricName(
    "db.client.operation.duration",
    HISTOGRAM,
    "s",
    "Duration of a SQL operation. The standard OpenTelemetry semantic "
    "convention metric, and the one that survives trace sampling. "
    "Callback-style calls (``execute_fn()`` and friends) are counted "
    "alongside the SQL-string methods.",
    (DB_SYSTEM, DB_NAMESPACE, OPERATION, ERROR_TYPE),
    buckets=DURATION_BUCKETS,
)

M_WRITE_QUEUE_WAIT = MetricName(
    "datasette.write.queue_wait",
    HISTOGRAM,
    "s",
    "Time each write waited in its database's write queue. The metric "
    "counterpart of the ``db.write.queue_wait`` span.",
    (DB_NAMESPACE,),
    buckets=DURATION_BUCKETS,
)

M_QUERIES_INTERRUPTED = MetricName(
    "datasette.sql.queries.interrupted",
    COUNTER,
    "{query}",
    "Queries cancelled for exceeding :ref:`setting_sql_time_limit_ms`. Worth "
    "alerting on: a rising rate means the limit is too tight or a table has "
    "outgrown its queries. A caller that opted into a deliberately shorter "
    "budget - facet suggestion, for example - is not counted, for the same "
    "reason its timeout is not a span error.",
    (DB_NAMESPACE,),
)

M_THREADS_LIMIT = MetricName(
    "datasette.sql.threads.limit",
    GAUGE,
    "{thread}",
    "Maximum concurrent read queries - the :ref:`setting_num_sql_threads` "
    "value. Not reported when ``num_sql_threads`` is ``0``, since then queries "
    "run on the event loop and there is no pool.",
)

M_THREADS_QUEUE_DEPTH = MetricName(
    "datasette.sql.threads.queue_depth",
    GAUGE,
    "{query}",
    "Read queries waiting for a free thread. **This is the saturation "
    "signal** - sustained above zero means requests are queueing on "
    "``num_sql_threads``.",
)

M_QUERIES_PENDING = MetricName(
    "datasette.sql.queries.pending",
    GAUGE,
    "{query}",
    "Read queries submitted to the pool and not yet complete. Summed across "
    "databases and compared against the thread limit, this is pool "
    "utilisation.",
    (DB_NAMESPACE,),
)

M_WRITE_QUEUE_DEPTH = MetricName(
    "datasette.write.queue_depth",
    GAUGE,
    "{write}",
    "Writes queued behind a database's single write thread. Backpressure that "
    "raising ``num_sql_threads`` cannot relieve. Not reported for a database "
    "that has never been written to.",
    (DB_NAMESPACE,),
)

M_CONNECTIONS_OPEN = MetricName(
    "datasette.connections.open",
    GAUGE,
    "{connection}",
    "Open SQLite connections currently tracked for closing.",
    (DB_NAMESPACE,),
)

METRICS = (
    M_OPERATION_DURATION,
    M_WRITE_QUEUE_WAIT,
    M_QUERIES_INTERRUPTED,
    M_THREADS_LIMIT,
    M_THREADS_QUEUE_DEPTH,
    M_QUERIES_PENDING,
    M_WRITE_QUEUE_DEPTH,
    M_CONNECTIONS_OPEN,
)