mirror of
https://github.com/simonw/datasette.git
synced 2026-11-20 00:58:31 +01:00
Register metrics and give histograms bucket boundaries suited to seconds
Both histograms declared unit="s" but inherited OpenTelemetry's default boundaries, which are tuned for milliseconds - so every SQLite query landed in the single (0, 5] second bucket and every quantile query returned noise. The boundaries are the semantic conventions' recommended set for db.client.operation.duration plus 0.0001 and 0.0005 at the bottom, since SQLite is in-process and many real queries take tens of microseconds. (Adapted from 024f2029: that commit assumed the metrics were already in telemetry_registry.py, which on this lineage held spans only - so this commit also brings the MetricName registry machinery, the registry entries for all eight phase-3 metrics, the cog-generated Metric reference in internals.rst, and the datasette.operation attribute. The template and facet histograms it also touched belong to phase 5 and are not included.) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01F2h9ANGZ7paWSpqs5DUAcG
This commit is contained in:
parent
90b727db73
commit
383fa1ad0a
6 changed files with 352 additions and 18 deletions
|
|
@ -36,6 +36,15 @@ from .telemetry_registry import (
|
|||
HTTP_REQUEST_METHOD,
|
||||
HTTP_RESPONSE_STATUS_CODE,
|
||||
INTERNAL_CLIENT,
|
||||
M_CONNECTIONS_OPEN,
|
||||
M_OPERATION_DURATION,
|
||||
M_QUERIES_INTERRUPTED,
|
||||
M_QUERIES_PENDING,
|
||||
M_THREADS_LIMIT,
|
||||
M_THREADS_QUEUE_DEPTH,
|
||||
M_WRITE_QUEUE_DEPTH,
|
||||
M_WRITE_QUEUE_WAIT,
|
||||
OPERATION,
|
||||
SERVER_ADDRESS,
|
||||
URL_PATH,
|
||||
URL_SCHEME,
|
||||
|
|
@ -402,27 +411,29 @@ def _duration_attributes(database_name, operation):
|
|||
return {
|
||||
DB_SYSTEM: "sqlite",
|
||||
DB_NAMESPACE: database_name,
|
||||
"datasette.operation": operation,
|
||||
OPERATION: operation,
|
||||
}
|
||||
|
||||
|
||||
sql_operation_duration = meter.create_histogram(
|
||||
"db.client.operation.duration",
|
||||
unit="s",
|
||||
M_OPERATION_DURATION,
|
||||
unit=M_OPERATION_DURATION.unit,
|
||||
description="Duration of a SQL operation issued by Datasette",
|
||||
explicit_bucket_boundaries_advisory=M_OPERATION_DURATION.buckets,
|
||||
)
|
||||
|
||||
write_queue_wait = meter.create_histogram(
|
||||
"datasette.write.queue_wait",
|
||||
unit="s",
|
||||
M_WRITE_QUEUE_WAIT,
|
||||
unit=M_WRITE_QUEUE_WAIT.unit,
|
||||
description=(
|
||||
"Time a write spent queued behind the single write thread for its database"
|
||||
),
|
||||
explicit_bucket_boundaries_advisory=M_WRITE_QUEUE_WAIT.buckets,
|
||||
)
|
||||
|
||||
queries_interrupted = meter.create_counter(
|
||||
"datasette.sql.queries.interrupted",
|
||||
unit="{query}",
|
||||
M_QUERIES_INTERRUPTED,
|
||||
unit=M_QUERIES_INTERRUPTED.unit,
|
||||
description=(
|
||||
"Queries cancelled for exceeding sql_time_limit_ms. Not derivable from "
|
||||
"spans under sampling, and the signal that a time limit is too tight"
|
||||
|
|
@ -582,36 +593,36 @@ def observe_open_connections(options=None):
|
|||
|
||||
|
||||
sql_thread_limit_gauge = meter.create_observable_gauge(
|
||||
"datasette.sql.threads.limit",
|
||||
M_THREADS_LIMIT,
|
||||
callbacks=[observe_sql_thread_limit],
|
||||
unit="{thread}",
|
||||
unit=M_THREADS_LIMIT.unit,
|
||||
description="Maximum concurrent read queries (the num_sql_threads setting)",
|
||||
)
|
||||
|
||||
sql_thread_queue_depth_gauge = meter.create_observable_gauge(
|
||||
"datasette.sql.threads.queue_depth",
|
||||
M_THREADS_QUEUE_DEPTH,
|
||||
callbacks=[observe_sql_thread_queue_depth],
|
||||
unit="{query}",
|
||||
unit=M_THREADS_QUEUE_DEPTH.unit,
|
||||
description="Read queries waiting for a free thread in the shared SQL pool",
|
||||
)
|
||||
|
||||
pending_queries_gauge = meter.create_observable_gauge(
|
||||
"datasette.sql.queries.pending",
|
||||
M_QUERIES_PENDING,
|
||||
callbacks=[observe_pending_queries],
|
||||
unit="{query}",
|
||||
unit=M_QUERIES_PENDING.unit,
|
||||
description="Read queries submitted to the pool and not yet complete",
|
||||
)
|
||||
|
||||
write_queue_depth_gauge = meter.create_observable_gauge(
|
||||
"datasette.write.queue_depth",
|
||||
M_WRITE_QUEUE_DEPTH,
|
||||
callbacks=[observe_write_queue_depth],
|
||||
unit="{write}",
|
||||
unit=M_WRITE_QUEUE_DEPTH.unit,
|
||||
description="Writes queued behind a database's single write thread",
|
||||
)
|
||||
|
||||
open_connections_gauge = meter.create_observable_gauge(
|
||||
"datasette.connections.open",
|
||||
M_CONNECTIONS_OPEN,
|
||||
callbacks=[observe_open_connections],
|
||||
unit="{connection}",
|
||||
unit=M_CONNECTIONS_OPEN.unit,
|
||||
description="Open SQLite connections tracked for closing",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -79,6 +79,33 @@ class SpanName(str):
|
|||
return f"SpanName({str(self)!r})"
|
||||
|
||||
|
||||
class MetricName(str):
|
||||
"A metric name, carrying its instrument kind, unit and attributes."
|
||||
|
||||
__slots__ = ("attributes", "buckets", "description", "kind", "unit")
|
||||
|
||||
def __new__(cls, name, kind, unit, description, attributes=(), buckets=None):
|
||||
self = super().__new__(cls, name)
|
||||
self.kind = kind
|
||||
self.unit = unit
|
||||
self.description = description
|
||||
self.attributes = tuple(attributes)
|
||||
# Explicit histogram bucket boundaries, for histograms only. Passed to
|
||||
# create_histogram() as explicit_bucket_boundaries_advisory and
|
||||
# published in the generated docs, since an operator writing a
|
||||
# histogram_quantile() query needs to know them.
|
||||
self.buckets = tuple(buckets) if buckets is not None else None
|
||||
return self
|
||||
|
||||
def __repr__(self):
|
||||
return f"MetricName({str(self)!r})"
|
||||
|
||||
|
||||
COUNTER = "Counter"
|
||||
HISTOGRAM = "Histogram"
|
||||
GAUGE = "Observable gauge"
|
||||
|
||||
|
||||
# --- Attributes -----------------------------------------------------------
|
||||
#
|
||||
# Shared attributes are defined once and referenced by every span that sets
|
||||
|
|
@ -157,6 +184,7 @@ ERROR_TYPE = Attribute(
|
|||
|
||||
DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.")
|
||||
DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.")
|
||||
OPERATION = Attribute("datasette.operation", "``read`` or ``write``.")
|
||||
DB_QUERY_TEXT = Attribute(
|
||||
"db.query.text",
|
||||
"The SQL, truncated to 2048 characters. Never the parameter values. "
|
||||
|
|
@ -395,3 +423,113 @@ def attribute_allowed(span, emitted_key):
|
|||
if span is None:
|
||||
return False
|
||||
return emitted_key in span.attributes
|
||||
|
||||
|
||||
# --- Metrics --------------------------------------------------------------
|
||||
|
||||
# Every duration histogram here is in seconds, and OpenTelemetry's default
|
||||
# bucket boundaries are tuned for milliseconds - their first non-zero boundary
|
||||
# is 5, so without explicit boundaries every SQLite query lands in the single
|
||||
# (0, 5] second bucket and every quantile query returns noise.
|
||||
#
|
||||
# These are the OpenTelemetry semantic conventions' recommended boundaries for
|
||||
# db.client.operation.duration, in seconds, plus 0.0001 and 0.0005 at the
|
||||
# bottom. The deviation is deliberate: those boundaries assume a network
|
||||
# database client, whereas SQLite is in-process and a large fraction of real
|
||||
# queries run in 30-80us, which would otherwise all pile into the first
|
||||
# bucket and be indistinguishable from each other.
|
||||
#
|
||||
# One shared list is used for every duration histogram rather than a tailored
|
||||
# list each, so that dashboards stay comparable and a queue wait can be read
|
||||
# against the query duration it delays. It already spans 100us to 10s, which
|
||||
# covers both a fast in-process read and a write queued behind contention.
|
||||
DURATION_BUCKETS = (0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10)
|
||||
|
||||
M_OPERATION_DURATION = MetricName(
|
||||
"db.client.operation.duration",
|
||||
HISTOGRAM,
|
||||
"s",
|
||||
"Duration of a SQL operation. The standard OpenTelemetry semantic "
|
||||
"convention metric, and the one that survives trace sampling.",
|
||||
(DB_SYSTEM, DB_NAMESPACE, OPERATION, ERROR_TYPE),
|
||||
buckets=DURATION_BUCKETS,
|
||||
)
|
||||
|
||||
M_WRITE_QUEUE_WAIT = MetricName(
|
||||
"datasette.write.queue_wait",
|
||||
HISTOGRAM,
|
||||
"s",
|
||||
"Time each write waited in its database's write queue. The metric "
|
||||
"counterpart of the ``db.write.queue_wait`` span.",
|
||||
(DB_NAMESPACE,),
|
||||
buckets=DURATION_BUCKETS,
|
||||
)
|
||||
|
||||
M_QUERIES_INTERRUPTED = MetricName(
|
||||
"datasette.sql.queries.interrupted",
|
||||
COUNTER,
|
||||
"{query}",
|
||||
"Queries cancelled for exceeding :ref:`setting_sql_time_limit_ms`. Worth "
|
||||
"alerting on: a rising rate means the limit is too tight or a table has "
|
||||
"outgrown its queries. A caller that opted into a deliberately shorter "
|
||||
"budget - facet suggestion, for example - is not counted, for the same "
|
||||
"reason its timeout is not a span error.",
|
||||
(DB_NAMESPACE,),
|
||||
)
|
||||
|
||||
M_THREADS_LIMIT = MetricName(
|
||||
"datasette.sql.threads.limit",
|
||||
GAUGE,
|
||||
"{thread}",
|
||||
"Maximum concurrent read queries - the :ref:`setting_num_sql_threads` "
|
||||
"value. Not reported when ``num_sql_threads`` is ``0``, since then queries "
|
||||
"run on the event loop and there is no pool.",
|
||||
)
|
||||
|
||||
M_THREADS_QUEUE_DEPTH = MetricName(
|
||||
"datasette.sql.threads.queue_depth",
|
||||
GAUGE,
|
||||
"{query}",
|
||||
"Read queries waiting for a free thread. **This is the saturation "
|
||||
"signal** - sustained above zero means requests are queueing on "
|
||||
"``num_sql_threads``.",
|
||||
)
|
||||
|
||||
M_QUERIES_PENDING = MetricName(
|
||||
"datasette.sql.queries.pending",
|
||||
GAUGE,
|
||||
"{query}",
|
||||
"Read queries submitted to the pool and not yet complete. Summed across "
|
||||
"databases and compared against the thread limit, this is pool "
|
||||
"utilisation.",
|
||||
(DB_NAMESPACE,),
|
||||
)
|
||||
|
||||
M_WRITE_QUEUE_DEPTH = MetricName(
|
||||
"datasette.write.queue_depth",
|
||||
GAUGE,
|
||||
"{write}",
|
||||
"Writes queued behind a database's single write thread. Backpressure that "
|
||||
"raising ``num_sql_threads`` cannot relieve. Not reported for a database "
|
||||
"that has never been written to.",
|
||||
(DB_NAMESPACE,),
|
||||
)
|
||||
|
||||
M_CONNECTIONS_OPEN = MetricName(
|
||||
"datasette.connections.open",
|
||||
GAUGE,
|
||||
"{connection}",
|
||||
"Open SQLite connections currently tracked for closing.",
|
||||
(DB_NAMESPACE,),
|
||||
)
|
||||
|
||||
METRICS = (
|
||||
M_OPERATION_DURATION,
|
||||
M_WRITE_QUEUE_WAIT,
|
||||
M_QUERIES_INTERRUPTED,
|
||||
M_THREADS_LIMIT,
|
||||
M_THREADS_QUEUE_DEPTH,
|
||||
M_QUERIES_PENDING,
|
||||
M_WRITE_QUEUE_DEPTH,
|
||||
M_CONNECTIONS_OPEN,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue