From 1326042b45acc0221aad9167c1695f3375dd5b0a Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Mon, 28 Sep 2026 14:01:27 -0700 Subject: [PATCH 1/3] Telemetry scaffolding: registry, helpers, kit fixtures Declares every span, metric and attribute in telemetry_registry.py using Datasette 1.0a41's plugin telemetry kit. No instrumentation yet. Co-Authored-By: Claude Opus 5.5 (1M context) --- datasette_enrichments/telemetry.py | 449 ++++++++++++++++++++ datasette_enrichments/telemetry_registry.py | 277 ++++++++++++ pyproject.toml | 3 + tests/conftest.py | 27 ++ tests/test_telemetry.py | 65 +++ 5 files changed, 821 insertions(+) create mode 100644 datasette_enrichments/telemetry.py create mode 100644 datasette_enrichments/telemetry_registry.py create mode 100644 tests/test_telemetry.py diff --git a/datasette_enrichments/telemetry.py b/datasette_enrichments/telemetry.py new file mode 100644 index 0000000..65314b0 --- /dev/null +++ b/datasette_enrichments/telemetry.py @@ -0,0 +1,449 @@ +""" +OpenTelemetry integration for datasette-enrichments. + +Depends on ``opentelemetry-api`` only, like Datasette core: this module never +creates a ``TracerProvider`` or ``MeterProvider``, never configures an +exporter, and must never import ``opentelemetry.sdk`` (a test imports the +package in a fresh subprocess and checks). With no provider installed every +span is a ``NonRecordingSpan`` and every instrument a no-op - turning +telemetry on is the operator's move (``opentelemetry-instrument datasette +...``), exactly as with core. + +The tracer and meter live under their own ``datasette_enrichments`` +instrumentation scope, versioned with the plugin. Every wire name comes from +``telemetry_registry``. + +Instruments are module-level: OpenTelemetry's ``_ProxyMeter`` forwards to a +provider installed later. The ``ProxyTracer`` resolves and caches a concrete +tracer on first use, so the test harness installs its provider in a +session-scoped autouse fixture before any span is created. + +Privacy: spans are started with ``record_exception=False`` and +``set_status_on_exception=False``. Exception messages can contain row data or +API responses, so only the exception class name (``error.type``) is ever +recorded, and error statuses carry no description. +""" + +import asyncio +import time +from contextlib import contextmanager +from importlib import metadata + +from datasette.telemetry import linked_root_span_kwargs +from opentelemetry import metrics as otel_metrics +from opentelemetry import trace as otel_trace +from opentelemetry.trace import Status, StatusCode + +from . import telemetry_registry as reg + + +def _version(): + try: + return metadata.version("datasette-enrichments") + except metadata.PackageNotFoundError: + return None + + +# No schema URL: every attribute here is plugin-specific, and a wrong schema +# URL is worse than none. +tracer = otel_trace.get_tracer("datasette_enrichments", _version()) +meter = otel_metrics.get_meter("datasette_enrichments", _version()) + + +# --- Small helpers -------------------------------------------------------- + + +def clamp(value, allowed, default): + """Clamp an attribute value to its registry enum. + + Enum attributes declare a closed ``values=`` set in the registry; this is + the call-site half of that promise - anything unexpected becomes + ``default`` instead of minting a new metric series. + """ + return value if value in allowed else default + + +def error_type(exception): + "The ``error.type`` value for an exception: its class name, never the message." + return type(exception).__qualname__ + + +def set_attrs(span, mapping): + "Set each attribute in ``mapping`` whose value is not None, if the span records." + if not span.is_recording(): + return + for key, value in mapping.items(): + if value is not None: + span.set_attribute(key, value) + + +def _is_interruption(exception): + # CancelledError is process shutdown (or a test cancelling the job task); + # GeneratorExit is the coroutine being closed without finishing. Neither + # is a failure of the enrichment. + return isinstance(exception, (asyncio.CancelledError, GeneratorExit)) + + +def _start_span(name, **kwargs): + return tracer.start_as_current_span( + name, record_exception=False, set_status_on_exception=False, **kwargs + ) + + +def _mark_error(span): + if span.is_recording(): + span.set_status(Status(StatusCode.ERROR)) + + +# --- Instruments ---------------------------------------------------------- +# +# Unit and buckets always come off the registry entry so the create_*() call +# cannot drift from it (the kit's assert_metrics_conform checks them). + + +def _histogram(entry): + return meter.create_histogram( + entry, + unit=entry.unit, + description=entry.description, + explicit_bucket_boundaries_advisory=entry.buckets, + ) + + +def _counter(entry): + return meter.create_counter(entry, unit=entry.unit, description=entry.description) + + +def _up_down_counter(entry): + return meter.create_up_down_counter( + entry, unit=entry.unit, description=entry.description + ) + + +runs = _counter(reg.RUNS) +runs_active = _up_down_counter(reg.RUNS_ACTIVE) +run_duration = _histogram(reg.RUN_DURATION) +batch_duration = _histogram(reg.BATCH_DURATION) +rows = _counter(reg.ROWS) +cost = _counter(reg.COST) + + +# --- Job runs ------------------------------------------------------------- + + +class RunRecorder: + """What one ``enrichments.job.run`` span learns as the run progresses. + + Handed out by ``job_run_span()``. The call site reports how the run ended + with ``set_outcome()``, ``stopped_with_status()`` or ``fail()``; + ``batch_span(run)`` accumulates ``batches`` and ``rows``. + """ + + def __init__(self, enrichment, job_id): + self.enrichment = enrichment + self.job_id = job_id + self.outcome = None + self.error_type = None + self.batches = 0 + self.rows = 0 + self.batch_spans_started = 0 + + def set_outcome(self, outcome): + "One of ``enrichments.run.outcome``'s values; anything else becomes ``stopped``." + self.outcome = outcome + + def stopped_with_status(self, status): + """The run stopped because the job's status is not ``running``: + ``paused`` and ``cancelled`` map to themselves, anything else + (including ``None`` for a missing row) to ``stopped``.""" + self.outcome = status if status in ("paused", "cancelled") else "stopped" + + def fail(self, exception): + """Classify an exception that ended the run - one the call site + caught itself, or one escaping the ``with`` block.""" + if _is_interruption(exception): + self.outcome = "interrupted" + self.error_type = None + else: + self.outcome = "error" + self.error_type = error_type(exception) + + +@contextmanager +def job_run_span(enrichment, job, trigger, link_kwargs=None): + """``enrichments.job.run`` around one run of a job's batch loop, plus the + ``runs``, ``runs.active`` and ``run.duration`` metrics. + + ``enrichment`` is the enrichment's slug, ``job`` the ``_enrichment_jobs`` + row as a dict, ``trigger`` one of ``enqueue``/``resume``/``restart``. + ``link_kwargs`` is ``datasette.telemetry.linked_root_span_kwargs()``, + captured where the run was scheduled, so the run becomes a root span + linked to its cause. + + Yields a ``RunRecorder``. An exception escaping the block is classified by + ``fail()`` and re-raised. On every exit path the outcome (clamped; + ``stopped`` when none was reported), batch and row totals are set on the + span, status ``ERROR`` is set for ``error`` only, and ``runs.active`` + goes back down. + """ + trigger = clamp(trigger, reg.TRIGGER.values, "enqueue") + recorder = RunRecorder(enrichment, job["id"]) + active_attributes = {reg.ENRICHMENT: enrichment} + started = time.perf_counter() + runs_active.add(1, active_attributes) + try: + with _start_span(reg.JOB_RUN, **(link_kwargs or {})) as span: + set_attrs( + span, + { + reg.ENRICHMENT: enrichment, + reg.JOB_ID: job["id"], + reg.DB_NAMESPACE: job.get("database_name"), + reg.TRIGGER: trigger, + reg.ROW_COUNT: job.get("row_count"), + }, + ) + try: + yield recorder + except BaseException as exception: + recorder.fail(exception) + raise + finally: + recorder.outcome = clamp( + recorder.outcome, reg.RUN_OUTCOME.values, "stopped" + ) + set_attrs( + span, + { + reg.RUN_OUTCOME: recorder.outcome, + reg.RUN_BATCHES: recorder.batches, + reg.RUN_ROWS: recorder.rows, + reg.ERROR_TYPE: recorder.error_type, + }, + ) + if recorder.outcome == "error": + _mark_error(span) + finally: + outcome = clamp(recorder.outcome, reg.RUN_OUTCOME.values, "stopped") + runs.add( + 1, + { + reg.ENRICHMENT: enrichment, + reg.TRIGGER: trigger, + reg.RUN_OUTCOME: outcome, + }, + ) + run_duration.record( + time.perf_counter() - started, + {reg.ENRICHMENT: enrichment, reg.RUN_OUTCOME: outcome}, + ) + runs_active.add(-1, active_attributes) + + +# --- Batches -------------------------------------------------------------- + + +class BatchRecorder: + """What one ``enrichments.batch`` span learns. + + Call ``fetched(len(rows))`` once the rows are in, then + ``succeeded(success_count)`` when ``enrich_batch()`` returns, + ``set_outcome("paused" | "cancelled")`` for Pause/Cancel, or + ``fail(exception)`` for an exception the call site caught itself. + """ + + def __init__(self): + self.rows = None + self.success_count = None + self.outcome = None + self.error_type = None + self.interrupted = False + + def fetched(self, row_count): + self.rows = row_count + + def succeeded(self, success_count): + self.success_count = success_count + self.outcome = "ok" + + def set_outcome(self, outcome): + "One of ``enrichments.batch.outcome``'s values; anything else becomes ``ok``." + self.outcome = outcome + + def fail(self, exception): + if _is_interruption(exception): + # enrichments.batch.outcome has no value for this: the attribute + # and the batch metrics are left off an interrupted batch + self.interrupted = True + else: + self.outcome = "error" + self.error_type = error_type(exception) + + +def _record_rows(enrichment, recorder): + if not recorder.rows: + return + if recorder.outcome == "ok": + success = recorder.success_count + if success is None: + success = recorder.rows + # A buggy enrich_batch() could return more than len(rows), or a + # negative number + success = min(max(success, 0), recorder.rows) + failed = recorder.rows - success + elif recorder.outcome == "error": + success, failed = 0, recorder.rows + else: + # paused / cancelled: those rows are processed again on resume + return + if success: + rows.add(success, {reg.ENRICHMENT: enrichment, reg.ROW_RESULT: "success"}) + if failed: + rows.add(failed, {reg.ENRICHMENT: enrichment, reg.ROW_RESULT: "error"}) + + +@contextmanager +def batch_span(run, index=None): + """``enrichments.batch`` around one batch of ``run`` (a ``RunRecorder``), + plus the ``batch.duration`` and ``rows`` metrics. + + ``index`` defaults to the number of batch spans already started in this + run. Yields a ``BatchRecorder``. On exit the outcome is clamped (``ok`` + when none was reported), status ``ERROR`` is set for ``error`` only, rows + are counted as ``success``/``error`` for ``ok`` and ``error`` batches, + and a batch with rows that ended ``ok`` or ``error`` adds to the run's + ``batches`` and ``rows`` totals. + """ + if index is None: + index = run.batch_spans_started + run.batch_spans_started += 1 + recorder = BatchRecorder() + started = time.perf_counter() + with _start_span(reg.BATCH) as span: + set_attrs( + span, + { + reg.ENRICHMENT: run.enrichment, + reg.JOB_ID: run.job_id, + reg.BATCH_INDEX: index, + }, + ) + try: + yield recorder + except BaseException as exception: + recorder.fail(exception) + raise + finally: + elapsed = time.perf_counter() - started + if recorder.interrupted: + set_attrs(span, {reg.BATCH_ROWS: recorder.rows}) + else: + outcome = clamp(recorder.outcome, reg.BATCH_OUTCOME.values, "ok") + recorder.outcome = outcome + set_attrs( + span, + { + reg.BATCH_ROWS: recorder.rows, + reg.BATCH_SUCCESS: recorder.success_count, + reg.BATCH_OUTCOME: outcome, + reg.ERROR_TYPE: recorder.error_type, + }, + ) + if outcome == "error": + _mark_error(span) + batch_duration.record( + elapsed, + {reg.ENRICHMENT: run.enrichment, reg.BATCH_OUTCOME: outcome}, + ) + _record_rows(run.enrichment, recorder) + if recorder.rows and outcome in ("ok", "error"): + run.batches += 1 + run.rows += recorder.rows + + +# --- initialize() / finalize() ------------------------------------------- + + +class SpanRecorder: + "Handed out by ``initialize_span()`` / ``finalize_span()``." + + def __init__(self, span): + self.span = span + + def fail(self, exception): + """Mark the span failed for an exception the call site caught itself. + An exception escaping the ``with`` block does this automatically.""" + if _is_interruption(exception): + return + set_attrs(self.span, {reg.ERROR_TYPE: error_type(exception)}) + _mark_error(self.span) + + +@contextmanager +def _call_span(name, attributes): + with _start_span(name) as span: + set_attrs(span, attributes) + recorder = SpanRecorder(span) + try: + yield recorder + except BaseException as exception: + recorder.fail(exception) + raise + + +def initialize_span(enrichment, database): + """``enrichments.initialize`` around the enrichment's ``initialize()`` + call. ``enrichment`` is the slug, ``database`` the database name.""" + return _call_span( + reg.INITIALIZE, {reg.ENRICHMENT: enrichment, reg.DB_NAMESPACE: database} + ) + + +def finalize_span(enrichment, job_id): + """``enrichments.finalize`` around the enrichment's ``finalize()`` call. + ``enrichment`` is the slug.""" + return _call_span(reg.FINALIZE, {reg.ENRICHMENT: enrichment, reg.JOB_ID: job_id}) + + +# --- Restart -------------------------------------------------------------- + + +class RestartRecorder: + "Counts for ``enrichments.restart``: increment ``jobs`` and ``unknown``." + + def __init__(self): + self.jobs = 0 + self.unknown = 0 + + +@contextmanager +def restart_span(): + """``enrichments.restart`` around the task that resumes jobs left + ``running``. Always a root span linked to whatever is current when it + starts: on hosts without ASGI lifespan that is the first request, which + the restart must not become a child of. Yields a ``RestartRecorder``.""" + recorder = RestartRecorder() + with _start_span(reg.RESTART, **linked_root_span_kwargs()) as span: + try: + yield recorder + except BaseException as exception: + if not _is_interruption(exception): + _mark_error(span) + raise + finally: + set_attrs( + span, + { + reg.RESTART_JOBS: recorder.jobs, + reg.RESTART_UNKNOWN: recorder.unknown, + }, + ) + + +# --- Cost ----------------------------------------------------------------- + + +def record_cost(enrichment, amount): + "Add ``amount`` hundredths of a cent to ``enrichments.cost`` for this slug." + if amount is not None and amount > 0: + cost.add(amount, {reg.ENRICHMENT: enrichment}) diff --git a/datasette_enrichments/telemetry_registry.py b/datasette_enrichments/telemetry_registry.py new file mode 100644 index 0000000..b43ec5a --- /dev/null +++ b/datasette_enrichments/telemetry_registry.py @@ -0,0 +1,277 @@ +""" +Every span, metric and attribute datasette-enrichments emits. + +Built from Datasette's plugin telemetry kit (``datasette.telemetry_registry``). +``Attribute`` / ``SpanName`` / ``MetricName`` subclass ``str``, so a registry +entry *is* the name handed to OpenTelemetry, and a typo is an ``ImportError`` +rather than a silently misnamed signal. Call sites import these constants +and never hard-code wire names. + +Scope: ``datasette_enrichments``. Prefix: ``enrichments.*`` (``datasette.*`` is +reserved for core). Shared names reused from core: ``db.namespace``, +``error.type``. + +Privacy: no enrichment config, filter querystrings, row or primary-key +values, cursors, actor ids, exception messages or Pause/Cancel reasons are +ever recorded - counts, sizes and closed enums instead. Job ids and database +names ride on spans only, never on a metric. +""" + +from datasette.telemetry_registry import ( + COUNTER, + DB_NAMESPACE, + ERROR_TYPE, + HISTOGRAM, + UPDOWN_COUNTER, + Attribute, + MetricName, + SpanName, +) +from opentelemetry.trace import SpanKind + +# --- Histogram bucket boundaries ------------------------------------------ +# +# Core's DURATION_BUCKETS top out at 10s - right for SQL, wrong here: an LLM +# batch takes tens of seconds and a job run can take hours. +BATCH_DURATION_BUCKETS = (0.01, 0.05, 0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600) +RUN_DURATION_BUCKETS = (1, 5, 10, 30, 60, 300, 600, 1800, 3600, 7200, 21600, 86400) + +# --- Attributes ----------------------------------------------------------- + +ENRICHMENT = Attribute( + "enrichments.enrichment", + "The enrichment's slug, e.g. ``openai``. Bounded by the installed " + "enrichment plugins, so safe to use as a metric dimension.", +) +JOB_ID = Attribute( + "enrichments.job_id", + "The job's id in the database's ``_enrichment_jobs`` table. Spans only, " + "never on a metric.", +) +TRIGGER = Attribute( + "enrichments.trigger", + "What started this run of the job: ``enqueue`` (a new job submitted " + "through the enrichment form or API), ``resume`` (the Resume button on a " + "paused job) or ``restart`` (the startup task resuming a job a previous " + "process left ``running``).", + values={"enqueue", "resume", "restart"}, +) +RUN_OUTCOME = Attribute( + "enrichments.run.outcome", + "How this run of the job ended. ``finished``: the last batch was " + "processed. ``paused`` / ``cancelled``: either ``enrich_batch()`` raised " + "``self.Pause`` / ``self.Cancel``, or the loop saw the job's status " + "changed to ``paused`` / ``cancelled`` (the Pause and Cancel buttons). " + "``stopped``: the loop saw some other non-``running`` status, or the job " + "row disappeared. ``interrupted``: the run was cancelled with " + "``asyncio.CancelledError``, which means the process is shutting down; " + "the job stays ``running`` and is resumed on the next start. ``error``: " + "an unexpected exception escaped the run, e.g. a failing ``finalize()``. " + "Only ``error`` sets span status ``ERROR``.", + values={"finished", "paused", "cancelled", "stopped", "interrupted", "error"}, +) +BATCH_OUTCOME = Attribute( + "enrichments.batch.outcome", + "How the batch ended. ``ok``: ``enrich_batch()`` returned, including " + "when it logged errors for some rows, and the trailing fetch that " + "returns no rows. ``error``: an exception escaped the batch, e.g. " + "``enrich_batch()`` raised; every row in the batch counts as an error " + "and the run carries on. ``paused`` / ``cancelled``: ``enrich_batch()`` " + "raised ``self.Pause`` / ``self.Cancel``; its rows are processed again " + "on resume. Only ``error`` sets span status ``ERROR``.", + values={"ok", "error", "paused", "cancelled"}, +) +ROW_RESULT = Attribute( + "enrichments.row.result", + "``success`` for rows ``enrich_batch()`` counted as enriched, ``error`` " + "for the rest of the batch - rows it logged errors for, or every row of " + "a batch that raised.", + values={"success", "error"}, +) +ROW_COUNT = Attribute( + "enrichments.job.row_count", + "Total rows the job targets, counted when the job was enqueued " + "(``_enrichment_jobs.row_count``).", +) +RUN_BATCHES = Attribute( + "enrichments.run.batches", + "Batches this run processed: batches with at least one row that ended " + "``ok`` or ``error``. A resumed job's earlier runs are not included.", +) +RUN_ROWS = Attribute( + "enrichments.run.rows", + "Rows this run processed, successful or not, across the batches counted " + "by ``enrichments.run.batches``.", +) +BATCH_INDEX = Attribute( + "enrichments.batch.index", + "0-based index of the batch within this run - it restarts at 0 when a " + "job is resumed.", +) +BATCH_ROWS = Attribute( + "enrichments.batch.rows", + "Rows fetched for this batch. ``0`` for the trailing fetch that finds " + "no rows left.", +) +BATCH_SUCCESS = Attribute( + "enrichments.batch.success_count", + "Rows ``enrich_batch()`` reported as enriched (its return value, or " + "every row when it returns ``None``). Set when ``enrich_batch()`` " + "returned.", +) +RESTART_JOBS = Attribute( + "enrichments.restart.jobs", + "Jobs left ``running`` by a previous process that the restart task resumed.", +) +RESTART_UNKNOWN = Attribute( + "enrichments.restart.unknown", + "Jobs left ``running`` whose enrichment slug matches no installed " + "enrichment, so they could not be resumed.", +) + +ATTRIBUTES = ( + ENRICHMENT, + JOB_ID, + TRIGGER, + RUN_OUTCOME, + BATCH_OUTCOME, + ROW_RESULT, + ROW_COUNT, + RUN_BATCHES, + RUN_ROWS, + BATCH_INDEX, + BATCH_ROWS, + BATCH_SUCCESS, + RESTART_JOBS, + RESTART_UNKNOWN, + DB_NAMESPACE, + ERROR_TYPE, +) + +# --- Spans ---------------------------------------------------------------- + +JOB_RUN = SpanName( + "enrichments.job.run", + "One run of an enrichment job's batch loop, from claiming the job to " + "finishing, pausing, cancelling or failing. A job has several runs if it " + "is paused and resumed, or resumed after a restart. Each run is a **root " + "span in its own trace** with a link back to whatever started it - the " + "enqueue or resume request, or the ``enrichments.restart`` span - " + "because a run outlives its cause. Status is ``ERROR`` only for " + "``enrichments.run.outcome=error``.", + ( + ENRICHMENT, + JOB_ID, + DB_NAMESPACE, + TRIGGER, + ROW_COUNT, + RUN_OUTCOME, + RUN_BATCHES, + RUN_ROWS, + ERROR_TYPE, + ), + kind=SpanKind.INTERNAL, +) +BATCH = SpanName( + "enrichments.batch", + "One batch within a run: fetching the rows, ``enrich_batch()`` and " + "recording progress. A child of ``enrichments.job.run``; the row fetch's " + "internal request and ``db.query`` spans, and any spans the enrichment " + "itself emits (e.g. an instrumented HTTP client), nest under it. One " + "span per batch, never one per row. Status is ``ERROR`` only for " + "``enrichments.batch.outcome=error``; the run carries on.", + ( + ENRICHMENT, + JOB_ID, + BATCH_INDEX, + BATCH_ROWS, + BATCH_SUCCESS, + BATCH_OUTCOME, + ERROR_TYPE, + ), + kind=SpanKind.INTERNAL, +) +INITIALIZE = SpanName( + "enrichments.initialize", + "The enrichment's ``initialize()`` call, made while handling the request " + "that submits a new job - a child of that request's span. Status " + "``ERROR`` if it raised.", + (ENRICHMENT, DB_NAMESPACE, ERROR_TYPE), + kind=SpanKind.INTERNAL, +) +FINALIZE = SpanName( + "enrichments.finalize", + "The enrichment's ``finalize()`` call once a job's last batch is done - " + "a child of ``enrichments.job.run``. Status ``ERROR`` if it raised, " + "which also makes the run's outcome ``error``.", + (ENRICHMENT, JOB_ID, ERROR_TYPE), + kind=SpanKind.INTERNAL, +) +RESTART = SpanName( + "enrichments.restart", + "The startup task that resumes jobs a previous process left " + "``running``. A root span with a link to whatever was current when it " + "started (on hosts without ASGI lifespan, the first request). Each job " + "it resumes gets its own ``enrichments.job.run`` root span linked back " + "to this one.", + (RESTART_JOBS, RESTART_UNKNOWN), + kind=SpanKind.INTERNAL, +) + +SPANS = (JOB_RUN, BATCH, INITIALIZE, FINALIZE, RESTART) + +# --- Metrics -------------------------------------------------------------- +# +# Only bounded attributes: ENRICHMENT and the enum attributes. Never JOB_ID +# or DB_NAMESPACE. + +RUNS = MetricName( + "enrichments.job.runs", + COUNTER, + "{run}", + "Job runs that ended, by enrichment, trigger and outcome.", + (ENRICHMENT, TRIGGER, RUN_OUTCOME), +) +RUNS_ACTIVE = MetricName( + "enrichments.job.runs.active", + UPDOWN_COUNTER, + "{run}", + "Job runs in progress right now, by enrichment.", + (ENRICHMENT,), +) +RUN_DURATION = MetricName( + "enrichments.job.run.duration", + HISTOGRAM, + "s", + "Duration of one job run, by enrichment and outcome. A job that is " + "paused and resumed records one measurement per run.", + (ENRICHMENT, RUN_OUTCOME), + buckets=RUN_DURATION_BUCKETS, +) +BATCH_DURATION = MetricName( + "enrichments.batch.duration", + HISTOGRAM, + "s", + "Duration of one batch - fetch, ``enrich_batch()`` and progress writes - " + "by enrichment and outcome.", + (ENRICHMENT, BATCH_OUTCOME), + buckets=BATCH_DURATION_BUCKETS, +) +ROWS = MetricName( + "enrichments.rows", + COUNTER, + "{row}", + "Rows processed, by enrichment and result. Rows of a paused or cancelled " + "batch are not counted: they are processed again on resume.", + (ENRICHMENT, ROW_RESULT), +) +COST = MetricName( + "enrichments.cost", + COUNTER, + "{hundredth_cent}", + "Cost reported by enrichments through ``increment_cost()``, in " + "hundredths of a cent, by enrichment.", + (ENRICHMENT,), +) + +METRICS = (RUNS, RUNS_ACTIVE, RUN_DURATION, BATCH_DURATION, ROWS, COST) diff --git a/pyproject.toml b/pyproject.toml index 0726ff2..9defca6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,8 @@ classifiers = [ requires-python = ">=3.10" dependencies = [ "datasette>=1.0a41", + # Declared explicitly because the package imports it directly + "opentelemetry-api>=1.37", "WTForms", "datasette-secrets>=0.2" ] @@ -20,6 +22,7 @@ dev = [ "pytest", "pytest-asyncio", "packaging", + "opentelemetry-sdk>=1.37", # Pinned: ruff's default rule set changes between releases "ruff==0.16.9", ] diff --git a/tests/conftest.py b/tests/conftest.py index 361a341..cb1a55d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,10 +9,37 @@ from datasette.app import Datasette from datasette.database import Database from datasette.plugins import pm + +# Importing the fixture names registers them: session-scoped autouse +# tracer/meter providers with in-memory export (skipped when the SDK is not +# installed), and a per-test reset that drains them. Tests take otel_spans / +# otel_metrics. +from datasette.telemetry_testing import ( # noqa: F401 + MetricsCollector, + otel_meter_provider, + otel_metrics, + otel_provider, + otel_reset, + otel_spans, +) from wtforms import Form, SelectField, StringField from wtforms.widgets import CheckboxInput, ListWidget +def pytest_collection_modifyitems(items): + # These tests shell out to a fresh interpreter, and the kit documents a + # macOS/CPython 3.13 fork+exec crash (SIGBUS) when subprocess-spawning + # tests run late in a thread-heavy process - so run them first, as + # Datasette's own conftest does. + subprocess_tests = { + "test_package_never_imports_the_sdk", + "test_import_without_provider_is_noop", + } + front = [item for item in items if item.name in subprocess_tests] + for item in reversed(front): + items.insert(0, items.pop(items.index(item))) + + class MultiCheckboxField(SelectField): widget = ListWidget(prefix_label=False) option_widget = CheckboxInput() diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py new file mode 100644 index 0000000..08aa33b --- /dev/null +++ b/tests/test_telemetry.py @@ -0,0 +1,65 @@ +import subprocess +import sys +import textwrap + +import pytest + +pytest.importorskip("opentelemetry.sdk") + +from datasette.telemetry_testing import assert_package_never_imports_sdk + + +def test_package_never_imports_the_sdk(): + # Front-loaded by conftest's pytest_collection_modifyitems + assert_package_never_imports_sdk( + "datasette_enrichments", + "datasette_enrichments.telemetry", + "datasette_enrichments.telemetry_registry", + ) + + +def test_import_without_provider_is_noop(): + """With no provider installed every span is non-recording and every + instrument a no-op, and the helpers must not raise. Runs in a fresh + interpreter because the test session has a provider installed. + Front-loaded by conftest, like the SDK-import test.""" + script = textwrap.dedent( + """ + from opentelemetry import trace + + import datasette_enrichments # noqa: F401 + from datasette_enrichments import telemetry + + assert type(trace.get_tracer_provider()).__name__ == "ProxyTracerProvider" + + job = {"id": 1, "database_name": "data", "row_count": 2} + with telemetry.job_run_span("uppercasedemo", job, "enqueue") as run: + assert not trace.get_current_span().is_recording() + with telemetry.batch_span(run) as batch: + assert not trace.get_current_span().is_recording() + batch.fetched(2) + batch.succeeded(2) + with telemetry.batch_span(run) as batch: + batch.fetched(2) + batch.fail(ValueError("secret row data")) + with telemetry.finalize_span("uppercasedemo", 1): + pass + run.set_outcome("finished") + with telemetry.initialize_span("uppercasedemo", "data"): + pass + with telemetry.restart_span() as restart: + restart.jobs += 1 + telemetry.record_cost("uppercasedemo", 5) + assert (run.batches, run.rows, run.outcome) == (2, 4, "finished") + print("NO_PROVIDER_OK") + """ + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=120, + check=False, + ) + assert result.returncode == 0, result.stderr + assert "NO_PROVIDER_OK" in result.stdout From 458edd311cd3ab8f1434e4792b8aca6683617fd0 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Mon, 28 Sep 2026 14:10:17 -0700 Subject: [PATCH 2/3] Trace enrichment runs and batches, record job metrics Each run is a new trace linked to the request (or restart) that started it, with a child span per batch. Adds run/batch/row/cost metrics. Co-Authored-By: Claude Opus 5.5 (1M context) --- datasette_enrichments/__init__.py | 283 +++++--- datasette_enrichments/telemetry.py | 34 +- datasette_enrichments/telemetry_registry.py | 13 +- datasette_enrichments/views.py | 25 +- tests/conftest.py | 12 + tests/test_telemetry.py | 680 ++++++++++++++++++++ 6 files changed, 934 insertions(+), 113 deletions(-) diff --git a/datasette_enrichments/__init__.py b/datasette_enrichments/__init__.py index 8b9526b..42a8dc7 100644 --- a/datasette_enrichments/__init__.py +++ b/datasette_enrichments/__init__.py @@ -14,6 +14,7 @@ from datasette.permissions import Action from datasette.plugins import pm from datasette.resources import DatabaseResource +from datasette.telemetry import linked_root_span_kwargs from datasette.utils import ( async_call_with_supported_arguments, await_me_maybe, @@ -25,7 +26,7 @@ from wtforms import PasswordField from wtforms.validators import DataRequired -from . import hookspecs, views +from . import hookspecs, telemetry, views from .utils import TERMINAL_STATUSES, mark_job_complete, pks_for_rows if TYPE_CHECKING: @@ -372,6 +373,7 @@ async def increment_cost( """, (total_cost_rounded_up, job_id), ) + telemetry.record_cost(self.slug, total_cost_rounded_up) async def enqueue( self, @@ -390,7 +392,9 @@ async def enqueue( table_path = datasette.urls.table(db.name, table) response = await datasette.client.get( - table_path + ".json" + "?" + qs, skip_permission_checks=True + table_path + ".json" + "?" + qs, + headers=telemetry.trace_headers(), + skip_permission_checks=True, ) filtered_data = response.json() if "count" in filtered_data: @@ -425,12 +429,20 @@ def _insert(conn): return cursor.lastrowid job_id = await db.execute_write_fn(_insert) + # trigger defaults to "enqueue"; not passed, so subclasses overriding + # start_enrichment_in_process() without it keep working await self.start_enrichment_in_process(datasette, db, job_id) return job_id async def start_enrichment_in_process( - self, datasette: "Datasette", db: "Database", job_id: int + self, + datasette: "Datasette", + db: "Database", + job_id: int, + trigger: str = "enqueue", ): + # trigger says what started this run - "enqueue", "resume" or + # "restart" - and is only used for telemetry job_row = ( await db.execute("select * from _enrichment_jobs where id = ?", (job_id,)) ).first() @@ -443,10 +455,12 @@ async def start_enrichment_in_process( return registry = _job_tasks(datasette) key = (db.name, job_id) + # Captured here, in the caller's context and before create_task(), so + # the run's root span links to whatever started it: the enqueue or + # resume request, or the restart span + link_kwargs = linked_root_span_kwargs() - async def complete(row_count: int): - # Mark the job finished, then run finalize(). If finalize() raises, - # run_enrichment() turns the job into 'error'. + async def mark_finished(row_count: int): await db.execute_write( """ update _enrichment_jobs @@ -458,15 +472,19 @@ async def complete(row_count: int): """, (row_count, job["id"]), ) - await async_call_with_supported_arguments( - self.finalize, - datasette=datasette, - db=db, - table=job["table_name"], - config=json.loads(job["config"]), - ) - async def run_batches() -> str: + async def finalize(): + # If finalize() raises, run_enrichment() turns the job into 'error' + with telemetry.finalize_span(self.slug, job_id): + await async_call_with_supported_arguments( + self.finalize, + datasette=datasette, + db=db, + table=job["table_name"], + config=json.loads(job["config"]), + ) + + async def run_batches(run: telemetry.RunRecorder) -> str: # Returns "finished" (no rows left, including a fetch that returns # no rows at all) or "stopped" (status no longer 'running', or the # enrichment raised Pause/Cancel) @@ -494,105 +512,150 @@ def claim(conn): ) ).first() if not job_row or job_row[0] != "running": + # Record the status seen here: a Resume may set it back + # to 'running' before run_enrichment() re-reads it + run.stopped_with_status(job_row[0] if job_row else None) return "stopped" - # Get next batch - table_path = datasette.urls.table( - job["database_name"], job["table_name"], format="json" - ) - qs = job["filter_querystring"] - if next_cursor: - qs += f"&_next={next_cursor}" - qs += f"&_size={self.batch_size}&_shape=objects" - response = await datasette.client.get( - table_path + "?" + qs, skip_permission_checks=True - ) - rows = response.json()["rows"] - if not rows: - # Filter matched nothing, or the remaining rows were deleted - # (or stopped matching) since the previous batch - await complete(0) - return "finished" - # Enrich batch - pks = await db.primary_keys(job["table_name"]) - try: - success_count = await async_call_with_supported_arguments( - self.enrich_batch, - datasette=datasette, - db=db, - table=job["table_name"], - rows=rows, - pks=pks or ["rowid"], - config=json.loads(job["config"]), - job_id=job_id, - actor_id=job["actor_id"], + # One span per batch: the fetch, enrich_batch() and the writes + with telemetry.batch_span(run) as batch: + # Get next batch + table_path = datasette.urls.table( + job["database_name"], job["table_name"], format="json" ) - if success_count is None: - success_count = len(rows) - await record_progress(db, job_id, success_count, 0) - except self.Cancel as ex: - await set_job_status(db, job_id, "cancelled", message=str(ex)) - return "stopped" - except self.Pause as ex: - await set_job_status(db, job_id, "paused", message=str(ex)) - return "stopped" - except Exception as ex: # noqa: BLE001 - await self.log_error(db, job_id, pks_for_rows(rows, pks), str(ex)) - # Update next_cursor - next_cursor = response.json()["next"] - if next_cursor: - await db.execute_write( - """ - update _enrichment_jobs - set - next_cursor = ?, - done_count = done_count + ? - where id = ? - """, - (next_cursor, len(rows), job["id"]), + qs = job["filter_querystring"] + if next_cursor: + qs += f"&_next={next_cursor}" + qs += f"&_size={self.batch_size}&_shape=objects" + response = await datasette.client.get( + table_path + "?" + qs, + headers=telemetry.trace_headers(), + skip_permission_checks=True, ) - else: - await complete(len(rows)) + rows = response.json()["rows"] + batch.fetched(len(rows)) + if not rows: + # Filter matched nothing, or the remaining rows were + # deleted (or stopped matching) since the previous batch + batch.set_outcome("ok") + await mark_finished(0) + finished = True + else: + # Enrich batch + pks = await db.primary_keys(job["table_name"]) + try: + success_count = await async_call_with_supported_arguments( + self.enrich_batch, + datasette=datasette, + db=db, + table=job["table_name"], + rows=rows, + pks=pks or ["rowid"], + config=json.loads(job["config"]), + job_id=job_id, + actor_id=job["actor_id"], + ) + if success_count is None: + success_count = len(rows) + await record_progress(db, job_id, success_count, 0) + batch.succeeded(success_count) + except self.Cancel as ex: + batch.set_outcome("cancelled") + run.set_outcome("cancelled") + await set_job_status( + db, job_id, "cancelled", message=str(ex) + ) + return "stopped" + except self.Pause as ex: + batch.set_outcome("paused") + run.set_outcome("paused") + await set_job_status(db, job_id, "paused", message=str(ex)) + return "stopped" + except Exception as ex: # noqa: BLE001 + # The batch failed; the run carries on + batch.fail(ex) + await self.log_error( + db, job_id, pks_for_rows(rows, pks), str(ex) + ) + # Update next_cursor + next_cursor = response.json()["next"] + if next_cursor: + await db.execute_write( + """ + update _enrichment_jobs + set + next_cursor = ?, + done_count = done_count + ? + where id = ? + """, + (next_cursor, len(rows), job["id"]), + ) + finished = False + else: + await mark_finished(len(rows)) + finished = True + if finished: + # Outside the batch span: finalize() is a child of the run + await finalize() return "finished" async def run_enrichment(): # asyncio.CancelledError (shutdown) is deliberately not caught: the - # job stays 'running' in the DB and is resumed on the next start - try: - outcome = await run_batches() - except Exception as ex: - # Anything that escaped the per-batch error handling, e.g. a DB - # error or a failing finalize(). Policy: mark the job 'error' - # even if it was already set to 'finished' before finalize() - # ran, so waiters wake up and the failure is visible in the UI. - logger.exception( - "Enrichment job %s in database %s failed", job_id, db.name - ) + # job stays 'running' in the DB and is resumed on the next start. + # job_run_span() records it as an 'interrupted' run and re-raises. + with telemetry.job_run_span(self.slug, job, trigger, link_kwargs) as run: try: - # If this write fails too, that exception is logged again by - # the done-callback and the status stays 'running'; acceptable - await set_job_status(db, job_id, "error", message=type(ex).__name__) - finally: - await mark_job_complete(datasette, job_id, db.name) - return - if outcome == "finished": - await mark_job_complete(datasette, job_id, db.name) - elif outcome == "stopped": - # We stopped because the status was not 'running'. A Resume may - # have set it back to 'running' after we checked, but seen this - # task still registered and so not started a new loop. Close - # that window: deregister first, then re-read the status. - _forget_job_task(registry, key, asyncio.current_task()) - row = ( - await db.execute( - "select status from _enrichment_jobs where id = ?", (job_id,) + outcome = await run_batches(run) + except Exception as ex: + # Anything that escaped the per-batch error handling, e.g. a + # DB error or a failing finalize(). Policy: mark the job + # 'error' even if it was already set to 'finished' before + # finalize() ran, so waiters wake up and the failure is + # visible in the UI. + run.fail(ex) + logger.exception( + "Enrichment job %s in database %s failed", job_id, db.name ) - ).first() - status = row[0] if row else None - if status == "running": - await self.start_enrichment_in_process(datasette, db, job_id) - elif status in TERMINAL_STATUSES: - # e.g. cancelled - wake anything in wait_for_job() + try: + # If this write fails too, that exception is logged again + # by the done-callback and the status stays 'running'; + # acceptable + await set_job_status( + db, job_id, "error", message=type(ex).__name__ + ) + finally: + await mark_job_complete(datasette, job_id, db.name) + return + if outcome == "finished": + run.set_outcome("finished") await mark_job_complete(datasette, job_id, db.name) + elif outcome == "stopped": + # We stopped because the status was not 'running'. A Resume + # may have set it back to 'running' after we checked, but + # seen this task still registered and so not started a new + # loop. Close that window: deregister first, then re-read + # the status. + _forget_job_task(registry, key, asyncio.current_task()) + row = ( + await db.execute( + "select status from _enrichment_jobs where id = ?", + (job_id,), + ) + ).first() + status = row[0] if row else None + if run.outcome is None: + # Only when claim() failed: the status check and a + # Pause/Cancel from the enrichment set it themselves + run.stopped_with_status(status) + if status == "running": + # Only reached when a Resume raced this stopping loop, + # so the new run is a resume. Its span links to this + # run's span, which is current here. + await self.start_enrichment_in_process( + datasette, db, job_id, trigger="resume" + ) + elif status in TERMINAL_STATUSES: + # e.g. cancelled - wake anything in wait_for_job() + await mark_job_complete(datasette, job_id, db.name) # The live-task check and the registry insert must not have an await # between them, or two concurrent callers (Resume vs. restart, Resume @@ -1041,6 +1104,14 @@ class JobProgress extends HTMLElement { async def _restart_running_jobs_task(datasette): + # A linked root span of its own: on hosts without ASGI lifespan this task + # is launched from inside the first request, which it must not become a + # child of. Each job it resumes gets a run span linked back to it. + with telemetry.restart_span() as restart: + await _restart_running_jobs(datasette, restart) + + +async def _restart_running_jobs(datasette, restart: telemetry.RestartRecorder): # For each database known to Datasette, look for running jobs for database_name in datasette.databases: db = datasette.get_database(database_name) @@ -1073,8 +1144,12 @@ async def _restart_running_jobs_task(datasette): if enrichment_slug in all_enrichments: enrichment = all_enrichments[enrichment_slug] # Resume from wherever it left off - await enrichment.start_enrichment_in_process(datasette, db, job_id) + await enrichment.start_enrichment_in_process( + datasette, db, job_id, trigger="restart" + ) + restart.jobs += 1 else: + restart.unknown += 1 logger.warning( "Cannot resume job %s: unknown enrichment %s", job_id, diff --git a/datasette_enrichments/telemetry.py b/datasette_enrichments/telemetry.py index 65314b0..38560d3 100644 --- a/datasette_enrichments/telemetry.py +++ b/datasette_enrichments/telemetry.py @@ -32,6 +32,7 @@ from datasette.telemetry import linked_root_span_kwargs from opentelemetry import metrics as otel_metrics from opentelemetry import trace as otel_trace +from opentelemetry.propagate import inject from opentelemetry.trace import Status, StatusCode from . import telemetry_registry as reg @@ -77,6 +78,25 @@ def set_attrs(span, mapping): span.set_attribute(key, value) +def trace_headers(): + """Headers carrying the current trace context, for ``datasette.client`` + calls whose internal request span should nest under the current span. + + Workaround for a Datasette core bug: ``TelemetryMiddleware`` extracts the + parent from request headers into an empty context, so an internal request + with no ``traceparent`` header starts a new trace instead of nesting - + https://github.com/simonw/datasette/issues/2969 . Remove this once the + minimum Datasette version includes the fix. + + Uses the global propagator: with ``OTEL_PROPAGATORS=none`` it injects + nothing and internal requests show up as their own traces again. With no + provider installed the current span is invalid and nothing is injected. + """ + headers = {} + inject(headers) + return headers + + def _is_interruption(exception): # CancelledError is process shutdown (or a test cancelling the job task); # GeneratorExit is the coroutine being closed without finishing. Neither @@ -263,7 +283,13 @@ def fetched(self, row_count): self.rows = row_count def succeeded(self, success_count): - self.success_count = success_count + # enrich_batch()'s return value is only loosely typed (SQLite coerces + # it for the progress table): telemetry must never crash the job, so + # anything that is not a number counts as "every row" + try: + self.success_count = int(success_count) + except (TypeError, ValueError): + self.success_count = None self.outcome = "ok" def set_outcome(self, outcome): @@ -271,6 +297,11 @@ def set_outcome(self, outcome): self.outcome = outcome def fail(self, exception): + if self.outcome == "ok": + # enrich_batch() already returned: its rows were enriched, even + # if a later write (cursor, finished) failed. That failure is the + # run's outcome, not this batch's. + return if _is_interruption(exception): # enrichments.batch.outcome has no value for this: the attribute # and the batch metrics are left off an interrupted batch @@ -428,6 +459,7 @@ def restart_span(): yield recorder except BaseException as exception: if not _is_interruption(exception): + set_attrs(span, {reg.ERROR_TYPE: error_type(exception)}) _mark_error(span) raise finally: diff --git a/datasette_enrichments/telemetry_registry.py b/datasette_enrichments/telemetry_registry.py index b43ec5a..78cee25 100644 --- a/datasette_enrichments/telemetry_registry.py +++ b/datasette_enrichments/telemetry_registry.py @@ -65,7 +65,10 @@ "``stopped``: the loop saw some other non-``running`` status, or the job " "row disappeared. ``interrupted``: the run was cancelled with " "``asyncio.CancelledError``, which means the process is shutting down; " - "the job stays ``running`` and is resumed on the next start. ``error``: " + "the job usually stays ``running`` and is resumed on the next start - " + "unless the cancellation landed once the job was being marked " + "``finished`` or during ``finalize()``, in which case it is not run " + "again. ``error``: " "an unexpected exception escaped the run, e.g. a failing ``finalize()``. " "Only ``error`` sets span status ``ERROR``.", values={"finished", "paused", "cancelled", "stopped", "interrupted", "error"}, @@ -121,7 +124,9 @@ ) RESTART_JOBS = Attribute( "enrichments.restart.jobs", - "Jobs left ``running`` by a previous process that the restart task resumed.", + "Jobs left ``running`` by a previous process that the restart task " + "tried to resume. A job whose loop is already running in this process is " + "counted but left alone.", ) RESTART_UNKNOWN = Attribute( "enrichments.restart.unknown", @@ -213,8 +218,8 @@ "``running``. A root span with a link to whatever was current when it " "started (on hosts without ASGI lifespan, the first request). Each job " "it resumes gets its own ``enrichments.job.run`` root span linked back " - "to this one.", - (RESTART_JOBS, RESTART_UNKNOWN), + "to this one. Status ``ERROR`` if the task raised.", + (RESTART_JOBS, RESTART_UNKNOWN, ERROR_TYPE), kind=SpanKind.INTERNAL, ) diff --git a/datasette_enrichments/views.py b/datasette_enrichments/views.py index df54b0d..7d503b1 100644 --- a/datasette_enrichments/views.py +++ b/datasette_enrichments/views.py @@ -3,6 +3,7 @@ from datasette import Forbidden, NotFound, Response from datasette.resources import DatabaseResource +from datasette.telemetry import request_span from datasette.utils import ( MultiParams, async_call_with_supported_arguments, @@ -10,6 +11,8 @@ tilde_decode, ) +from . import telemetry +from . import telemetry_registry as reg from .utils import TERMINAL_STATUSES @@ -300,9 +303,14 @@ async def enrich_data_post(datasette, request, enrichment, filtered_data): config = {field.name: field.data for field in form} # Call initialize method, which can create tables etc - await async_call_with_supported_arguments( - enrichment.initialize, datasette=datasette, db=db, table=table, config=config - ) + with telemetry.initialize_span(enrichment.slug, db.name): + await async_call_with_supported_arguments( + enrichment.initialize, + datasette=datasette, + db=db, + table=table, + config=config, + ) job_id = await enrichment.enqueue( datasette, @@ -312,6 +320,10 @@ async def enrich_data_post(datasette, request, enrichment, filtered_data): config, request.actor.get("id") if request.actor else None, ) + span = request_span(request.scope) + if span is not None: + span.set_attribute(reg.ENRICHMENT, enrichment.slug) + span.set_attribute(reg.JOB_ID, job_id) # Set message and redirect to table datasette.add_message( @@ -409,7 +421,9 @@ async def resume_job(datasette, db, job_id, message): enrichment = all_enrichments[job["enrichment"]] # No-op if this job's previous loop is still running (e.g. paused mid-batch): # that loop sees 'running' at its next status check and carries on - await enrichment.start_enrichment_in_process(datasette, db, job_id) + await enrichment.start_enrichment_in_process( + datasette, db, job_id, trigger="resume" + ) async def cancel_job(db, job_id, message): @@ -432,6 +446,9 @@ async def update_job_status_view(datasette, request, action): job_id = int(request.url_vars["job_id"]) if request.method != "POST": return Response("POST required", status=400) + span = request_span(request.scope) + if span is not None: + span.set_attribute(reg.JOB_ID, job_id) try: if action == "pause": await pause_job(db, job_id, message) diff --git a/tests/conftest.py b/tests/conftest.py index cb1a55d..14ad23d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -319,6 +319,17 @@ async def enrich_batch(self, rows: list[dict]): async def finalize(self, datasette, db, table, config): raise RuntimeError("finalize() failed") + class CostDemo(Enrichment): + # Reports a cost of 5 hundredths of a cent for every batch + name = "Cost demo" + slug = "costdemo" + description = "Calls increment_cost() once per batch" + batch_size = 10 + cost_per_batch = 5 + + async def enrich_batch(self, db: Database, rows: list[dict], job_id: int): + await self.increment_cost(db, job_id, self.cost_per_batch) + class EnrichmentsDemoPlugin: __name__ = "EnrichmentsDemoPlugin" @@ -332,6 +343,7 @@ def register_enrichments(self): QueueControlledEnrichment(), CountBatches(), FinalizeRaises(), + CostDemo(), ] pm.register(EnrichmentsDemoPlugin(), name="undo_EnrichmentsDemoPlugin") diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 08aa33b..204274b 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,12 +1,19 @@ +import asyncio +import inspect import subprocess import sys import textwrap +import time import pytest pytest.importorskip("opentelemetry.sdk") +from datasette import telemetry_registry as core from datasette.telemetry_testing import assert_package_never_imports_sdk +from opentelemetry.trace import SpanKind, StatusCode + +from datasette_enrichments import telemetry_registry as reg def test_package_never_imports_the_sdk(): @@ -63,3 +70,676 @@ def test_import_without_provider_is_noop(): ) assert result.returncode == 0, result.stderr assert "NO_PROVIDER_OK" in result.stdout + + +# --- Instrumented jobs ---------------------------------------------------- + +SCOPE = "datasette_enrichments" + + +def ours(otel_spans, name=None): + "Finished spans from this plugin's scope, optionally just those named ``name``." + return [ + s + for s in otel_spans.get_finished_spans() + if s.instrumentation_scope.name == SCOPE and (name is None or s.name == name) + ] + + +def for_job(job_id): + "Span predicate: the span carries ``enrichments.job_id == job_id``." + return lambda span: (span.attributes or {}).get(reg.JOB_ID) == job_id + + +async def wait_until(condition, description, timeout=5.0): + "Poll ``condition`` (sync or async) until it returns something truthy." + deadline = time.monotonic() + timeout + while True: + result = condition() + if inspect.isawaitable(result): + result = await result + if result: + return result + if time.monotonic() > deadline: + raise AssertionError( + f"Timed out after {timeout}s waiting for {description}" + ) + await asyncio.sleep(0.01) + + +async def wait_for_span(otel_spans, name, predicate=lambda s: True, timeout=5.0): + """The first finished span of ours named ``name`` matching ``predicate``. + + wait_for_job() returns when mark_job_complete() fires, which is BEFORE the + run span's ``with`` block exits - so poll for the finished span instead. + The run's metrics are recorded synchronously right after its span ends, so + once the span is visible here they are recorded too.""" + found = await wait_until( + lambda: [s for s in ours(otel_spans, name) if predicate(s)], + f"a finished {name} span", + timeout, + ) + return found[0] + + +def job_spans(otel_spans, name, job_id): + return [s for s in ours(otel_spans, name) if for_job(job_id)(s)] + + +def batches_of(otel_spans, run): + "The run's batch spans, ordered by index." + return sorted( + ( + s + for s in ours(otel_spans, reg.BATCH) + if s.parent is not None and s.parent.span_id == run.context.span_id + ), + key=lambda s: s.attributes[reg.BATCH_INDEX], + ) + + +def server_spans(otel_spans, method, path): + return [ + s + for s in otel_spans.get_finished_spans() + if s.kind == SpanKind.SERVER + and s.attributes.get(core.HTTP_REQUEST_METHOD) == method + and s.attributes.get(core.URL_PATH) == path + ] + + +def assert_linked_to(span, cause): + "``span`` is a root in its own trace, with a single link to ``cause``." + assert span.parent is None + assert span.context.trace_id != cause.context.trace_id + assert len(span.links) == 1 + link = span.links[0].context + assert (link.trace_id, link.span_id) == ( + cause.context.trace_id, + cause.context.span_id, + ) + + +def assert_text_nowhere(spans, text): + "``text`` appears in no attribute, event or status description." + for span in spans: + assert span.status.description is None or text not in span.status.description + for value in (span.attributes or {}).values(): + assert text not in str(value), (span.name, value) + for event in span.events: + assert text not in event.name + for value in (event.attributes or {}).values(): + assert text not in str(value), (span.name, event.name, value) + + +def active_runs(otel_metrics, slug): + """Current ``runs.active`` for one enrichment, from the last collect(). + + The UpDownCounter is CUMULATIVE, so its series persist across tests: always + filter by enrichment and compare with the value from before the workload.""" + points = otel_metrics.points(reg.RUNS_ACTIVE, {reg.ENRICHMENT: slug}) + assert len(points) <= 1 + return points[0].value if points else 0 + + +def root_cookies(datasette): + return {"ds_actor": datasette.sign({"a": {"id": "root"}}, "actor")} + + +async def enqueue(datasette, table, slug, data=None, query=""): + "Submit a job through the enrichment form; returns the new job's id." + response = await datasette.client.post( + f"/-/enrich/data/{table}/{slug}{query}", + cookies=root_cookies(datasette), + data=data or {}, + ) + assert response.status_code == 302 + return int(response.headers["location"].split("=")[-1]) + + +async def job_action(datasette, job_id, action): + response = await datasette.client.post( + f"/-/enrich/data/-/jobs/{job_id}/{action}", + cookies=root_cookies(datasette), + data={}, + ) + assert response.status_code == 302 + + +def first_batch_in_flight(datasette): + # The countbatches enrichment is blocked on datasette._enrich_gate + return getattr(datasette, "_enrich_in_flight", 0) == 1 + + +@pytest.mark.asyncio +async def test_job_run_is_linked_root_of_enqueue_request(datasette, otel_spans): + job_id = await enqueue(datasette, "t", "uppercasedemo", {"columns": "s"}) + run = await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + (post,) = server_spans(otel_spans, "POST", "/-/enrich/data/t/uppercasedemo") + assert_linked_to(run, post) + assert run.attributes[reg.ENRICHMENT] == "uppercasedemo" + assert run.attributes[reg.TRIGGER] == "enqueue" + assert run.attributes[reg.RUN_OUTCOME] == "finished" + assert run.attributes[reg.RUN_ROWS] == 2 + assert run.attributes[reg.RUN_BATCHES] == 1 + assert run.attributes[reg.ROW_COUNT] == 2 + assert run.attributes[reg.DB_NAMESPACE] == "data" + assert run.attributes[reg.JOB_ID] == job_id + assert reg.ERROR_TYPE not in run.attributes + assert run.status.status_code == StatusCode.UNSET + assert len(job_spans(otel_spans, reg.JOB_RUN, job_id)) == 1 + + +@pytest.mark.asyncio +async def test_batch_spans_are_children_of_run(datasette, otel_spans): + job_id = await enqueue(datasette, "has_50_rows", "countbatches") + run = await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + batches = batches_of(otel_spans, run) + # Datasette returns no next cursor for the last full page of 10, so there + # is no trailing empty batch here + assert [b.attributes[reg.BATCH_INDEX] for b in batches] == [0, 1, 2, 3, 4] + for batch in batches: + assert batch.context.trace_id == run.context.trace_id + assert batch.attributes[reg.JOB_ID] == job_id + assert batch.attributes[reg.ENRICHMENT] == "countbatches" + assert batch.attributes[reg.BATCH_ROWS] == 10 + assert batch.attributes[reg.BATCH_SUCCESS] == 10 + assert batch.attributes[reg.BATCH_OUTCOME] == "ok" + assert batch.status.status_code == StatusCode.UNSET + assert run.attributes[reg.RUN_BATCHES] == 5 + assert run.attributes[reg.RUN_ROWS] == 50 + + # The row fetch's internal request nests under its batch, in the run's + # trace - thanks to the injected trace headers + everything = otel_spans.get_finished_spans() + in_trace = [s for s in everything if s.context.trace_id == run.context.trace_id] + internal = [ + s + for s in in_trace + if s.kind == SpanKind.SERVER and s.attributes.get(core.INTERNAL_CLIENT) + ] + assert sorted(s.parent.span_id for s in internal) == sorted( + b.context.span_id for b in batches + ) + children = {} + for span in in_trace: + if span.parent is not None: + children.setdefault(span.parent.span_id, []).append(span) + + def descendants(span): + for child in children.get(span.context.span_id, []): + yield child + yield from descendants(child) + + for request in internal: + assert any(d.name == core.DB_QUERY for d in descendants(request)) + # finalize() is a child of the run, not of the last batch + (finalize,) = job_spans(otel_spans, reg.FINALIZE, job_id) + assert finalize.parent.span_id == run.context.span_id + + +@pytest.mark.asyncio +async def test_trailing_empty_batch(datasette, otel_spans): + # The first batch returns a next cursor, but the remaining rows are gone by + # the time the loop fetches again: a trailing batch with no rows + datasette._enrich_gate = asyncio.Event() + job_id = await enqueue(datasette, "has_50_rows", "countbatches") + await wait_until(lambda: first_batch_in_flight(datasette), "first batch") + with datasette._test_db: + datasette._test_db.execute("delete from has_50_rows where id > 10") + datasette._enrich_gate.set() + run = await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + first, empty = batches_of(otel_spans, run) + assert first.attributes[reg.BATCH_ROWS] == 10 + assert empty.attributes[reg.BATCH_INDEX] == 1 + assert empty.attributes[reg.BATCH_ROWS] == 0 + assert empty.attributes[reg.BATCH_OUTCOME] == "ok" + assert reg.BATCH_SUCCESS not in empty.attributes + assert run.attributes[reg.RUN_OUTCOME] == "finished" + # Only batches with rows count towards the run's totals + assert run.attributes[reg.RUN_BATCHES] == 1 + assert run.attributes[reg.RUN_ROWS] == 10 + + +@pytest.mark.asyncio +async def test_initialize_span_and_request_span_attributes(datasette, otel_spans): + job_id = await enqueue(datasette, "t", "uppercasedemo", {"columns": "s"}) + (post,) = server_spans(otel_spans, "POST", "/-/enrich/data/t/uppercasedemo") + assert post.attributes[reg.ENRICHMENT] == "uppercasedemo" + assert post.attributes[reg.JOB_ID] == job_id + (initialize,) = ours(otel_spans, reg.INITIALIZE) + assert initialize.parent.span_id == post.context.span_id + assert initialize.attributes[reg.ENRICHMENT] == "uppercasedemo" + assert initialize.attributes[reg.DB_NAMESPACE] == "data" + assert initialize.status.status_code == StatusCode.UNSET + # enqueue()'s internal count request nests under the POST too + counts = [ + s + for s in otel_spans.get_finished_spans() + if s.kind == SpanKind.SERVER + and s.attributes.get(core.INTERNAL_CLIENT) + and s.parent is not None + and s.parent.span_id == post.context.span_id + ] + assert len(counts) == 1 + # Let the job finish inside this test + await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + + +@pytest.mark.asyncio +async def test_batch_error_marks_batch_not_run(datasette, otel_spans, otel_metrics): + datasette._trigger_enrich_batch_error = True + job_id = await enqueue(datasette, "t", "uppercasedemo", {"columns": "s"}) + run = await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + batches = batches_of(otel_spans, run) + assert batches + for batch in batches: + assert batch.attributes[reg.BATCH_OUTCOME] == "error" + assert batch.attributes[reg.ERROR_TYPE] == "Exception" + assert batch.status.status_code == StatusCode.ERROR + assert reg.BATCH_SUCCESS not in batch.attributes + assert run.attributes[reg.RUN_OUTCOME] == "finished" + assert run.attributes[reg.RUN_BATCHES] == 1 + assert run.attributes[reg.RUN_ROWS] == 2 + assert reg.ERROR_TYPE not in run.attributes + assert run.status.status_code == StatusCode.UNSET + assert_text_nowhere(otel_spans.get_finished_spans(), "Error in enrich_batch") + otel_metrics.collect() + labels = {reg.ENRICHMENT: "uppercasedemo"} + assert otel_metrics.point(reg.ROWS, {**labels, reg.ROW_RESULT: "error"}).value == 2 + assert not otel_metrics.points(reg.ROWS, {**labels, reg.ROW_RESULT: "success"}) + assert ( + otel_metrics.point( + reg.BATCH_DURATION, {**labels, reg.BATCH_OUTCOME: "error"} + ).count + == 1 + ) + + +@pytest.mark.asyncio +async def test_partial_row_errors(datasette, otel_spans, otel_metrics): + job_id = await enqueue(datasette, "has_50_rows", "haserrors") + run = await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + batches = batches_of(otel_spans, run) + assert len(batches) == 5 + for batch in batches: + assert batch.attributes[reg.BATCH_OUTCOME] == "ok" + assert batch.attributes[reg.BATCH_ROWS] == 10 + assert batch.attributes[reg.BATCH_SUCCESS] == 8 + assert batch.status.status_code == StatusCode.UNSET + assert run.attributes[reg.RUN_OUTCOME] == "finished" + otel_metrics.collect() + labels = {reg.ENRICHMENT: "haserrors"} + # len(rows) - success_count = 2 errors per batch + assert otel_metrics.point(reg.ROWS, {**labels, reg.ROW_RESULT: "error"}).value == ( + 2 * 5 + ) + assert ( + otel_metrics.point(reg.ROWS, {**labels, reg.ROW_RESULT: "success"}).value + == 8 * 5 + ) + + +@pytest.mark.asyncio +async def test_pause_and_cancel_exceptions_outcomes(datasette, otel_spans): + job_id = await enqueue(datasette, "has_50_rows", "queue") + await wait_until( + lambda: hasattr(datasette, "enrichment_queue"), "initialize() to run" + ) + queue = datasette.enrichment_queue + await queue.put("0") + await queue.put("pause") + paused = await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + assert paused.attributes[reg.TRIGGER] == "enqueue" + assert paused.attributes[reg.RUN_OUTCOME] == "paused" + assert paused.status.status_code == StatusCode.UNSET + assert [ + b.attributes[reg.BATCH_OUTCOME] for b in batches_of(otel_spans, paused) + ] == [ + "ok", + "paused", + ] + # Paused rows are processed again on resume, so they do not count + assert paused.attributes[reg.RUN_BATCHES] == 1 + assert paused.attributes[reg.RUN_ROWS] == 1 + + await job_action(datasette, job_id, "resume") + await queue.put("cancel") + cancelled = await wait_for_span( + otel_spans, + reg.JOB_RUN, + lambda s: for_job(job_id)(s) and s.attributes[reg.TRIGGER] == "resume", + ) + assert cancelled.attributes[reg.RUN_OUTCOME] == "cancelled" + assert cancelled.status.status_code == StatusCode.UNSET + (batch,) = batches_of(otel_spans, cancelled) + assert batch.attributes[reg.BATCH_INDEX] == 0 + assert batch.attributes[reg.BATCH_OUTCOME] == "cancelled" + assert batch.status.status_code == StatusCode.UNSET + spans = otel_spans.get_finished_spans() + assert_text_nowhere(spans, "pause message") + assert_text_nowhere(spans, "cancel message") + + +@pytest.mark.asyncio +async def test_ui_pause_then_resume(datasette, otel_spans): + datasette._enrich_gate = asyncio.Event() + job_id = await enqueue(datasette, "has_50_rows", "countbatches") + await wait_until(lambda: first_batch_in_flight(datasette), "first batch") + await job_action(datasette, job_id, "pause") + # Let the batch finish: the loop then sees 'paused' and stops + datasette._enrich_gate.set() + paused = await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + assert paused.attributes[reg.TRIGGER] == "enqueue" + assert paused.attributes[reg.RUN_OUTCOME] == "paused" + assert paused.attributes[reg.RUN_BATCHES] == 1 + assert paused.status.status_code == StatusCode.UNSET + (pause_post,) = server_spans( + otel_spans, "POST", f"/-/enrich/data/-/jobs/{job_id}/pause" + ) + assert pause_post.attributes[reg.JOB_ID] == job_id + + await job_action(datasette, job_id, "resume") + resumed = await wait_for_span( + otel_spans, + reg.JOB_RUN, + lambda s: for_job(job_id)(s) and s.attributes[reg.TRIGGER] == "resume", + ) + (resume_post,) = server_spans( + otel_spans, "POST", f"/-/enrich/data/-/jobs/{job_id}/resume" + ) + assert_linked_to(resumed, resume_post) + assert resumed.context.trace_id != paused.context.trace_id + assert resumed.attributes[reg.RUN_OUTCOME] == "finished" + assert resumed.attributes[reg.RUN_BATCHES] == 4 + assert resumed.attributes[reg.RUN_ROWS] == 40 + # Batch indexes restart at 0 in the new run + assert [b.attributes[reg.BATCH_INDEX] for b in batches_of(otel_spans, resumed)] == [ + 0, + 1, + 2, + 3, + ] + assert len(job_spans(otel_spans, reg.JOB_RUN, job_id)) == 2 + + +async def _insert_running_job(datasette, enrichment): + # Straight into the database: a datasette.client request would launch the + # background tasks, including the restart scan, too early + from datasette_enrichments import ensure_tables + + db = datasette.get_database("data") + await ensure_tables(db) + return ( + await db.execute_write( + """ + insert into _enrichment_jobs ( + status, enrichment, database_name, table_name, filter_querystring, + config, next_cursor, row_count, done_count, error_count + ) values ('running', ?, 'data', 'has_50_rows', '', '{}', '40', 50, 40, 0) + """, + (enrichment,), + ) + ).lastrowid + + +@pytest.mark.asyncio +async def test_restart_span_and_restarted_run(datasette, otel_spans): + job_id = await _insert_running_job(datasette, "countbatches") + await _insert_running_job(datasette, "no-such-enrichment") + await datasette.start_background_tasks() + run = await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + (restart,) = ours(otel_spans, reg.RESTART) + assert restart.parent is None + assert restart.attributes[reg.RESTART_JOBS] == 1 + assert restart.attributes[reg.RESTART_UNKNOWN] == 1 + assert restart.status.status_code == StatusCode.UNSET + assert_linked_to(run, restart) + assert run.attributes[reg.TRIGGER] == "restart" + assert run.attributes[reg.RUN_OUTCOME] == "finished" + # Resumed from the cursor: rows 41-50 + assert run.attributes[reg.RUN_ROWS] == 10 + + +@pytest.mark.asyncio +async def test_finalize_span(datasette, otel_spans): + job_id = await enqueue(datasette, "t", "uppercasedemo", {"columns": "s"}) + run = await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + (finalize,) = job_spans(otel_spans, reg.FINALIZE, job_id) + assert finalize.parent.span_id == run.context.span_id + assert finalize.attributes[reg.ENRICHMENT] == "uppercasedemo" + assert reg.ERROR_TYPE not in finalize.attributes + assert finalize.status.status_code == StatusCode.UNSET + + +@pytest.mark.asyncio +async def test_finalize_error_makes_run_error(datasette, otel_spans): + job_id = await enqueue(datasette, "t", "finalizeraises") + run = await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + (finalize,) = job_spans(otel_spans, reg.FINALIZE, job_id) + assert finalize.parent.span_id == run.context.span_id + assert finalize.attributes[reg.ERROR_TYPE] == "RuntimeError" + assert finalize.status.status_code == StatusCode.ERROR + assert run.attributes[reg.RUN_OUTCOME] == "error" + assert run.attributes[reg.ERROR_TYPE] == "RuntimeError" + assert run.status.status_code == StatusCode.ERROR + # The batch itself succeeded + (batch,) = batches_of(otel_spans, run) + assert batch.attributes[reg.BATCH_OUTCOME] == "ok" + assert_text_nowhere(otel_spans.get_finished_spans(), "finalize() failed") + + +@pytest.mark.asyncio +async def test_interrupted_run(datasette, otel_spans, otel_metrics): + otel_metrics.collect() + before = active_runs(otel_metrics, "countbatches") + datasette._enrich_gate = asyncio.Event() # never set + job_id = await enqueue(datasette, "has_50_rows", "countbatches") + await wait_until(lambda: first_batch_in_flight(datasette), "first batch") + otel_metrics.collect() + assert active_runs(otel_metrics, "countbatches") == before + 1 + + (task,) = datasette._enrichment_job_tasks.values() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert task.cancelled() + + run = await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + assert run.attributes[reg.RUN_OUTCOME] == "interrupted" + assert reg.ERROR_TYPE not in run.attributes + assert run.status.status_code == StatusCode.UNSET + (batch,) = batches_of(otel_spans, run) + # No batch outcome has a value for an interrupted batch, so none is set + assert reg.BATCH_OUTCOME not in batch.attributes + assert batch.attributes[reg.BATCH_ROWS] == 10 + assert batch.status.status_code == StatusCode.UNSET + otel_metrics.collect() + assert active_runs(otel_metrics, "countbatches") == before + assert ( + otel_metrics.point( + reg.RUNS, + { + reg.ENRICHMENT: "countbatches", + reg.TRIGGER: "enqueue", + reg.RUN_OUTCOME: "interrupted", + }, + ).value + == 1 + ) + + +@pytest.mark.asyncio +async def test_run_metrics(datasette, otel_spans, otel_metrics): + slug = "countbatches" + otel_metrics.collect() + before = active_runs(otel_metrics, slug) + job_id = await enqueue(datasette, "has_50_rows", slug) + await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + + otel_metrics.collect() + labels = {reg.ENRICHMENT: slug} + finished = {reg.RUN_OUTCOME: "finished"} + assert ( + otel_metrics.point( + reg.RUNS, {**labels, **finished, reg.TRIGGER: "enqueue"} + ).value + == 1 + ) + assert otel_metrics.point(reg.RUN_DURATION, {**labels, **finished}).count == 1 + assert ( + otel_metrics.point( + reg.BATCH_DURATION, {**labels, reg.BATCH_OUTCOME: "ok"} + ).count + == 5 + ) + assert ( + otel_metrics.point(reg.ROWS, {**labels, reg.ROW_RESULT: "success"}).value == 50 + ) + assert not otel_metrics.points(reg.ROWS, {**labels, reg.ROW_RESULT: "error"}) + assert active_runs(otel_metrics, slug) == before + # Job ids and database names never become metric dimensions + for name in reg.METRICS: + for point in otel_metrics.snapshot.get(name, []): + assert reg.JOB_ID not in point.attributes + assert reg.DB_NAMESPACE not in point.attributes + + +@pytest.mark.asyncio +async def test_cost_metric(datasette, otel_spans, otel_metrics): + job_id = await enqueue(datasette, "has_50_rows", "costdemo") + await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + otel_metrics.collect() + assert otel_metrics.point(reg.COST, {reg.ENRICHMENT: "costdemo"}).value == 5 * 5 + # The same total the job row records + assert ( + datasette._test_db.execute( + "select cost_100ths_cent from _enrichment_jobs where id = ?", (job_id,) + ).fetchone()[0] + == 25 + ) + + +@pytest.mark.asyncio +async def test_resume_racing_a_stopping_loop_keeps_paused( + datasette, otel_spans, monkeypatch +): + # The loop sees 'paused' and stops; before it deregisters, a Resume sets + # 'running' but does nothing more because the task is still registered. + # The stopping run is still 'paused', and the loop restarts the job itself + # as a resume. + import datasette_enrichments + + real_forget = datasette_enrichments._forget_job_task + resumed = [] + + def forget_after_resume(registry, key, task): + if not resumed: + with datasette._test_db: + datasette._test_db.execute( + "update _enrichment_jobs set status = 'running'" + ) + resumed.append(task) + return real_forget(registry, key, task) + + monkeypatch.setattr(datasette_enrichments, "_forget_job_task", forget_after_resume) + datasette._enrich_gate = asyncio.Event() + job_id = await enqueue(datasette, "has_50_rows", "countbatches") + await wait_until(lambda: first_batch_in_flight(datasette), "first batch") + await job_action(datasette, job_id, "pause") + datasette._enrich_gate.set() + first = await wait_for_span( + otel_spans, + reg.JOB_RUN, + lambda s: for_job(job_id)(s) and s.attributes[reg.TRIGGER] == "enqueue", + ) + second = await wait_for_span( + otel_spans, + reg.JOB_RUN, + lambda s: for_job(job_id)(s) and s.attributes[reg.TRIGGER] == "resume", + ) + assert resumed + assert first.attributes[reg.RUN_OUTCOME] == "paused" + assert_linked_to(second, first) + assert second.attributes[reg.RUN_OUTCOME] == "finished" + assert len(job_spans(otel_spans, reg.JOB_RUN, job_id)) == 2 + + +@pytest.mark.asyncio +async def test_restart_from_first_request_is_linked_root(datasette, otel_spans): + # On hosts without ASGI lifespan, background tasks launch from inside the + # first request's span. The restart span must link to it, not be its child. + job_id = await _insert_running_job(datasette, "countbatches") + response = await datasette.client.get("/-/versions.json") + assert response.status_code == 200 + run = await wait_for_span(otel_spans, reg.JOB_RUN, for_job(job_id)) + (restart,) = ours(otel_spans, reg.RESTART) + (first_request,) = server_spans(otel_spans, "GET", "/-/versions.json") + assert_linked_to(restart, first_request) + assert restart.attributes[reg.RESTART_JOBS] == 1 + assert_linked_to(run, restart) + assert run.attributes[reg.TRIGGER] == "restart" + + +def test_non_numeric_success_count_does_not_crash(otel_spans, otel_metrics): + from datasette_enrichments import telemetry + + job = {"id": 1, "database_name": "data", "row_count": 20} + with telemetry.job_run_span("stringcount", job, "enqueue") as run: + with telemetry.batch_span(run) as batch: + batch.fetched(10) + batch.succeeded("8") # coerced, as SQLite would + with telemetry.batch_span(run) as batch: + batch.fetched(10) + batch.succeeded("lots") # not a number: every row succeeded + run.set_outcome("finished") + first, second = ours(otel_spans, reg.BATCH) + assert first.attributes[reg.BATCH_SUCCESS] == 8 + assert reg.BATCH_SUCCESS not in second.attributes + assert second.attributes[reg.BATCH_OUTCOME] == "ok" + otel_metrics.collect() + labels = {reg.ENRICHMENT: "stringcount"} + assert otel_metrics.point( + reg.ROWS, {**labels, reg.ROW_RESULT: "success"} + ).value == (18) + assert otel_metrics.point(reg.ROWS, {**labels, reg.ROW_RESULT: "error"}).value == 2 + + +def test_failure_after_enrich_batch_keeps_batch_ok(otel_spans, otel_metrics): + # e.g. the next_cursor write fails after enrich_batch() returned: the rows + # were enriched, so the batch stays ok and the run records the error + from datasette_enrichments import telemetry + + job = {"id": 1, "database_name": "data", "row_count": 10} + with ( + pytest.raises(RuntimeError), + telemetry.job_run_span("latefailure", job, "enqueue") as run, + telemetry.batch_span(run) as batch, + ): + batch.fetched(10) + batch.succeeded(10) + raise RuntimeError("cursor write failed") + (batch_span,) = ours(otel_spans, reg.BATCH) + (run_span,) = ours(otel_spans, reg.JOB_RUN) + assert batch_span.attributes[reg.BATCH_OUTCOME] == "ok" + assert reg.ERROR_TYPE not in batch_span.attributes + assert batch_span.status.status_code == StatusCode.UNSET + assert run_span.attributes[reg.RUN_OUTCOME] == "error" + assert run_span.attributes[reg.ERROR_TYPE] == "RuntimeError" + assert run_span.status.status_code == StatusCode.ERROR + otel_metrics.collect() + labels = {reg.ENRICHMENT: "latefailure"} + assert ( + otel_metrics.point(reg.ROWS, {**labels, reg.ROW_RESULT: "success"}).value == 10 + ) + assert not otel_metrics.points(reg.ROWS, {**labels, reg.ROW_RESULT: "error"}) + + +def test_restart_span_failure_records_error_type(otel_spans): + from datasette_enrichments import telemetry + + with pytest.raises(KeyError), telemetry.restart_span(): + raise KeyError("secret-ish") + (restart,) = ours(otel_spans, reg.RESTART) + assert restart.attributes[reg.ERROR_TYPE] == "KeyError" + assert restart.status.status_code == StatusCode.ERROR + assert_text_nowhere([restart], "secret-ish") From bfa1fbe061a74039b79d57401bae9250470885a7 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Mon, 28 Sep 2026 14:31:58 -0700 Subject: [PATCH 3/3] Telemetry conformance and privacy tests, docs/telemetry.md Registry conformance via Datasette's telemetry_testing kit, sentinel privacy checks, and a cog-generated span/metric reference checked in CI. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/test.yml | 3 + datasette_enrichments/telemetry_registry.py | 8 +- docs/developing.md | 12 + docs/index.md | 1 + docs/telemetry.md | 264 ++++++++++++++++++ pyproject.toml | 2 + tests/conftest.py | 13 +- tests/test_telemetry.py | 290 +++++++++++++++++++- tests/test_telemetry_registry.py | 129 +++++++++ 9 files changed, 705 insertions(+), 17 deletions(-) create mode 100644 docs/telemetry.md create mode 100644 tests/test_telemetry_registry.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 611da85..85b0f72 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,3 +44,6 @@ jobs: - name: Run ruff format run: | ruff format --check . + - name: Check docs are up to date (cog) + run: | + cog --check docs/*.md diff --git a/datasette_enrichments/telemetry_registry.py b/datasette_enrichments/telemetry_registry.py index 78cee25..49ef571 100644 --- a/datasette_enrichments/telemetry_registry.py +++ b/datasette_enrichments/telemetry_registry.py @@ -77,9 +77,11 @@ "enrichments.batch.outcome", "How the batch ended. ``ok``: ``enrich_batch()`` returned, including " "when it logged errors for some rows, and the trailing fetch that " - "returns no rows. ``error``: an exception escaped the batch, e.g. " - "``enrich_batch()`` raised; every row in the batch counts as an error " - "and the run carries on. ``paused`` / ``cancelled``: ``enrich_batch()`` " + "returns no rows. ``error``: an exception escaped the batch and every " + "row in the batch counts as an error. If ``enrich_batch()`` raised it, " + "the error is logged and the run carries on; any other exception, e.g. " + "a failed row fetch, also ends the run with ``error``. " + "``paused`` / ``cancelled``: ``enrich_batch()`` " "raised ``self.Pause`` / ``self.Cancel``; its rows are processed again " "on resume. Only ``error`` sets span status ``ERROR``.", values={"ok", "error", "paused", "cancelled"}, diff --git a/docs/developing.md b/docs/developing.md index 4c07f48..50b4d91 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -263,6 +263,18 @@ api_key = await self.get_secret(datasette, config) ``` You must pass both the `datasette` and the `config` arguments that were passed to those methods. +## Telemetry + +Datasette 1.0a41 and later can emit OpenTelemetry traces and metrics, and so can this plugin: see {doc}`telemetry` for what a job looks like and how to switch it on. Here is what that means for enrichment authors. + +- **Your code is already traced.** Everything your `enrich_batch()` does runs inside that batch's `enrichments.batch` span. Your `db.execute*()` calls show up beneath it as core's `db.query` spans. Calls made through an HTTP client that has OpenTelemetry instrumentation installed show up beneath it as that client's own spans. You don't need to do anything for this. +- **Adding your own spans is optional.** If you do, create your own tracer with `opentelemetry.trace.get_tracer()` using your package name as the instrumentation scope, and prefix span, metric and attribute names with your package name, such as `enrichments_opencage.geocode`. Don't use the `enrichments.*` or `datasette.*` prefixes, which belong to this plugin and to Datasette. Depend on `opentelemetry-api` only, and never configure a provider or an exporter: that is the operator's job. +- **Pass named functions to `db.execute_write_fn()`.** Core records the function's `__qualname__` as the span's `datasette.callback` attribute, and a lambda shows up as just ``. +- **Never put row data, prompts, API responses or other user data in span attributes, events or status descriptions.** Telemetry is often exported to a third-party service. Record counts, sizes and fixed sets of values instead, and use the exception's class name rather than its message. Pass values to SQL as parameters: core records the text of every query, but never its parameters. +- **Keep row data out of database exceptions.** When a database call fails, core records the exception's message and traceback on its `db.query` span. Don't put row data in exceptions raised inside `execute_write_fn()` callbacks, and don't build SQL that could fail with row values in the error message. + +See [Telemetry for plugin authors](https://docs.datasette.io/en/latest/plugin_telemetry.html) in the Datasette documentation for how to declare, test and document your own spans and metrics. + ## Writing tests for enrichments Take a look at the [test suite for datasette-enrichments-opencage](https://github.com/datasette/datasette-enrichments-opencage/blob/main/tests/test_enrichments_opencage.py) for an example of how to test an enrichment. diff --git a/docs/index.md b/docs/index.md index af9ee28..ef672f8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,4 +25,5 @@ setup usage permissions developing +telemetry ``` \ No newline at end of file diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000..3fd319f --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,264 @@ +# Telemetry + +Datasette 1.0a41 and later can emit [OpenTelemetry](https://opentelemetry.io/) traces and metrics. It instruments every HTTP request, every `db.query` and the write queue, as described in [Datasette's telemetry documentation](https://docs.datasette.io/en/latest/internals.html#internals-telemetry). Datasette Enrichments adds spans and metrics that describe enrichment jobs: which enrichment ran, how long each batch took, how many rows failed, why a job stopped and how much it cost. + +Everything described here lives in the `datasette_enrichments` instrumentation scope, uses the `enrichments.*` prefix, and is declared in `datasette_enrichments/telemetry_registry.py`. A conformance test holds the plugin to that registry in both directions, and the {ref}`reference ` at the end of this page is generated from it. + +## Turning it on + +The plugin depends on `opentelemetry-api` only, just like Datasette core. It never installs a provider or an exporter. With no OpenTelemetry SDK configured every span is a no-op and every instrument does nothing, so telemetry costs nothing until you switch it on. + +To switch it on, install the SDK, an exporter and the instrumentation agent, then run Datasette through `opentelemetry-instrument`: + +```bash +pip install opentelemetry-distro opentelemetry-exporter-otlp + +OTEL_SERVICE_NAME=datasette \ +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \ +OTEL_TRACES_EXPORTER=otlp \ +OTEL_METRICS_EXPORTER=otlp \ +OTEL_LOGS_EXPORTER=none \ + opentelemetry-instrument datasette mydb.db +``` + +The enrichment spans and metrics then show up alongside core's. See [Turning tracing on](https://docs.datasette.io/en/latest/internals.html#internals-telemetry-turning-on) in the Datasette documentation for more options, including printing telemetry to the console and viewing traces locally, and [Telemetry for plugin authors](https://docs.datasette.io/en/latest/plugin_telemetry.html) for the conventions this plugin's spans and metrics follow. + +## What a job looks like + +An enrichment job runs in the background, long after the request that started it has returned. Each run of a job is therefore a **root span in a trace of its own**, with a [span link](https://opentelemetry.io/docs/concepts/signals/traces/#span-links) back to whatever started it. That is the shape Datasette recommends for background work. + +Submitting a job through the enrichment form produces two traces: + +``` +POST /-/enrich/{database}/{table}/{enrichment} SERVER (core) +│ + enrichments.enrichment, enrichments.job_id +├── db.query ×N (permission check, config form) (core) +├── enrichments.initialize +├── GET /{database}/{table}.json (counts the rows) SERVER (core), datasette.internal_client=true +│ └── db.query ×N (core) +├── db.query ×N (create tables, insert the job) (core) +└── db.query (read the job back) (core) + ┆ + ┆ link + ▼ +enrichments.job.run trigger=enqueue root of a new trace +├── db.query (claim the job: status = 'running') (core) +├── db.query (status check) (core) +├── enrichments.batch batch.index=0 +│ ├── GET /{database}/{table}.json (fetch the rows) SERVER (core), datasette.internal_client=true +│ │ └── db.query ×N (core) +│ ├── db.query (primary keys) (core) +│ ├── ... your enrich_batch(): its own db.query spans, instrumented HTTP clients, ... +│ ├── db.query (record progress) (core) +│ └── db.query (save the cursor, or mark the job finished) +├── db.query (status check) (core) +├── enrichments.batch batch.index=1 +│ └── ... +└── enrichments.finalize +``` + +Writes also carry core's `db.write.queue_wait` and `db.write.execute` children, left out above. + +The internal `GET ….json` requests nest under the POST and under each batch only because this plugin injects W3C trace context headers (`traceparent`) into its own `datasette.client` calls. This works around [a Datasette bug](https://github.com/simonw/datasette/issues/2969) where internal requests start a new trace. It uses the global propagator, so if you set `OTEL_PROPAGATORS=none` nothing is injected and each internal request shows up as a trace of its own, with its `db.query` spans, instead of inside the batch. + +`finalize()` runs after the last batch's span has closed, so `enrichments.finalize` is a child of the run, not of the last batch. + +### Resumed and restarted jobs + +A job has one run span per run. Pausing and resuming a job, or restarting Datasette while a job is running, produces several runs, each in a new trace linked to its cause: + +``` +POST /-/enrich/{database}/-/jobs/{job_id}/resume SERVER (core), + enrichments.job_id +└── db.query ×N (update the status, read the job) (core) + ┆ link + ▼ +enrichments.job.run trigger=resume root of a new trace +└── ... batches as above, carrying on from the saved cursor + + +enrichments.restart restart.jobs=1 restart.unknown=0 root of a new trace +├── db.query ×N (find jobs left 'running') (core) +└── db.query (read the job) (core) + ┆ link + ▼ +enrichments.job.run trigger=restart root of a new trace +└── ... batches as above, carrying on from the saved cursor +``` + +- `enrichments.restart` runs once when Datasette starts, and resumes every job a previous process left `running`. It is a root span too. On hosts without ASGI lifespan support, Datasette starts it from inside the first request, and the span links to that request rather than becoming its child. Jobs whose enrichment is no longer installed are counted in `enrichments.restart.unknown` and left alone. +- The Pause and Cancel buttons are plain requests whose spans carry `enrichments.job_id`. The running job notices the new status at its next status check, and its run ends with outcome `paused` or `cancelled`. +- If Resume is pressed while the paused run is still stopping, that run starts the next run itself as it exits. The new run still has `trigger=resume`, and it links to the run that stopped rather than to the Resume request. +- `enrichments.batch.index` starts again at 0 in every run. + +### Details worth knowing + +- **Empty fetches complete the job.** If a fetch returns no rows, the batch records `enrichments.batch.rows=0` and outcome `ok`, and the run ends `finished`. A filter that matches nothing gives a single batch like this. Datasette returns no `next` cursor for the last full page, so a trailing empty batch only happens when rows are deleted, or stop matching the filter, while the job is running. +- **`enrichments.run.batches` and `enrichments.run.rows` count only batches that had rows and ended `ok` or `error`.** Paused, cancelled and interrupted batches and empty fetches are not counted, and neither are earlier runs of the same job. +- **A batch that has succeeded stays `ok`.** Once `enrich_batch()` has returned, the batch's outcome is `ok` even if saving the cursor or marking the job finished then fails. That failure ends the run with outcome `error` instead. +- **An interrupted batch has no outcome.** If the process shuts down mid-batch, the batch span ends without `enrichments.batch.outcome`, and records no `enrichments.batch.duration` or `enrichments.rows`. The run ends with outcome `interrupted`. + +## Outcomes + +Every run ends with an `enrichments.run.outcome`: + +| Outcome | Meaning | Span status | +|---|---|---| +| `finished` | The last batch was processed and `finalize()` returned. | `UNSET` | +| `paused` | `enrich_batch()` raised `self.Pause`, or the loop saw the job's status set to `paused` (the Pause button). | `UNSET` | +| `cancelled` | `enrich_batch()` raised `self.Cancel`, or the loop saw the job's status set to `cancelled` (the Cancel button). | `UNSET` | +| `stopped` | The loop saw some other status that is not `running`, or the job's row disappeared. | `UNSET` | +| `interrupted` | The run was cancelled, which means the process is shutting down. The job stays `running` and is resumed on the next start, unless the cancellation arrived while the job was being marked finished or during `finalize()`. | `UNSET` | +| `error` | An unexpected exception escaped the run, for example from a failing `finalize()`, a failed row fetch or a database error. The job's status becomes `error`. `error.type` records the exception class. | `ERROR` | + +Every batch ends with an `enrichments.batch.outcome`, apart from an interrupted batch, which gets none: + +| Outcome | Meaning | Span status | +|---|---|---| +| `ok` | `enrich_batch()` returned. That includes batches where it logged errors for some rows: `enrichments.batch.success_count` and the `enrichments.rows` metric show how many rows succeeded. | `UNSET` | +| `error` | An exception escaped the batch. `error.type` records the exception class and every row in the batch counts as an error. If `enrich_batch()` raised it, the error is logged and the run carries on with the next batch. Anything else, such as a failed row fetch, also ends the run with outcome `error`. | `ERROR` | +| `paused` / `cancelled` | `enrich_batch()` raised `self.Pause` / `self.Cancel`. Its rows are processed again on resume, so they are not counted in `enrichments.rows`. | `UNSET` | + +A batch error raised by `enrich_batch()` never sets the run's status to `ERROR`: only an exception that escapes the batch some other way, such as a failed row fetch, does. `enrichments.initialize`, `enrichments.finalize` and `enrichments.restart` get status `ERROR` and `error.type` if the code they wrap raises. A failing `finalize()` also ends the run with outcome `error`. Error statuses on this plugin's spans never carry a description. + +(telemetry_privacy)= +## Privacy + +Enrichment jobs handle some of the most sensitive data in a Datasette instance, and telemetry is often exported to a third-party service. This plugin's own spans and metrics never record any of the following, in a span name, attribute, event or status, or in a metric attribute. Datasette core's spans have their own rules, listed further down. + +- the enrichment's configuration: prompts, templates, and API keys or other secrets +- the filter that selected the rows +- row values or primary key values, including the cursor a job resumes from +- actor IDs +- exception messages or tracebacks: only the exception class name, as `error.type` +- the messages passed to `self.Pause()` or `self.Cancel()`, and the note saved when someone presses Pause, Resume or Cancel, which names them + +Job IDs and database names are recorded on spans only, never as metric attributes. A test plants a unique string in each of these, runs jobs that exercise every span and metric, and checks that none of the strings appear anywhere in the telemetry, including in Datasette core's own spans. + +Some things Datasette core records that you should know about. See [Privacy and safety](https://docs.datasette.io/en/latest/internals.html#internals-telemetry-privacy) in the Datasette documentation for the full list. + +- **`_where=` filters are recorded.** Core's `db.query` spans record SQL text as `db.query.text`, with literals intact. Normal column filters such as `?name__contains=…` and the job's cursor are bound as parameters, which are never recorded. But a `_where=` clause becomes part of the SQL text of the query that fetches each batch, so any values typed into it are exported. +- **SQL text is recorded, including the SQL your enrichment runs.** It shows table and column names. Values passed as parameters are not recorded. +- **URL paths are recorded,** so database names, table names, enrichment slugs and job IDs appear on request spans. Query strings are not recorded. +- **Failed database calls record the exception message and traceback.** When a query or a write fails, core's `db.query` span gets status `ERROR` with the exception message as its description, plus an `exception` event carrying the message and traceback. That covers `execute()`, `execute_fn()` and every `execute_write*()` method, including exceptions raised inside your own `execute_write_fn()` callbacks. SQLite error messages can quote values from your SQL, and an exception your callback raises carries whatever message you gave it. +- **Exception messages can be recorded on a request span.** If an exception escapes Datasette's router, core's request span can carry its message in the status description. The router normally turns exceptions into a 500 response with no description. +- On a public instance, strip incoming `traceparent` headers at your proxy if callers should not be able to attach their own trace context. + +(telemetry_reference)= +## Reference + +Generated from `datasette_enrichments/telemetry_registry.py`. + + +### Spans + +| Span | Kind | Attributes | Description | +|---|---|---|---| +| `enrichments.job.run` | `INTERNAL` | `enrichments.enrichment`, `enrichments.job_id`, `db.namespace`, `enrichments.trigger`, `enrichments.job.row_count`, `enrichments.run.outcome`, `enrichments.run.batches`, `enrichments.run.rows`, `error.type` (optional) | One run of an enrichment job's batch loop, from claiming the job to finishing, pausing, cancelling or failing. A job has several runs if it is paused and resumed, or resumed after a restart. Each run is a **root span in its own trace** with a link back to whatever started it - the enqueue or resume request, or the ``enrichments.restart`` span - because a run outlives its cause. Status is ``ERROR`` only for ``enrichments.run.outcome=error``. | +| `enrichments.batch` | `INTERNAL` | `enrichments.enrichment`, `enrichments.job_id`, `enrichments.batch.index`, `enrichments.batch.rows`, `enrichments.batch.success_count`, `enrichments.batch.outcome`, `error.type` (optional) | One batch within a run: fetching the rows, ``enrich_batch()`` and recording progress. A child of ``enrichments.job.run``; the row fetch's internal request and ``db.query`` spans, and any spans the enrichment itself emits (e.g. an instrumented HTTP client), nest under it. One span per batch, never one per row. Status is ``ERROR`` only for ``enrichments.batch.outcome=error``; the run carries on. | +| `enrichments.initialize` | `INTERNAL` | `enrichments.enrichment`, `db.namespace`, `error.type` (optional) | The enrichment's ``initialize()`` call, made while handling the request that submits a new job - a child of that request's span. Status ``ERROR`` if it raised. | +| `enrichments.finalize` | `INTERNAL` | `enrichments.enrichment`, `enrichments.job_id`, `error.type` (optional) | The enrichment's ``finalize()`` call once a job's last batch is done - a child of ``enrichments.job.run``. Status ``ERROR`` if it raised, which also makes the run's outcome ``error``. | +| `enrichments.restart` | `INTERNAL` | `enrichments.restart.jobs`, `enrichments.restart.unknown`, `error.type` (optional) | The startup task that resumes jobs a previous process left ``running``. A root span with a link to whatever was current when it started (on hosts without ASGI lifespan, the first request). Each job it resumes gets its own ``enrichments.job.run`` root span linked back to this one. Status ``ERROR`` if the task raised. | + +### Metrics + +Metric attributes are limited to the enrichment slug and closed sets of values, so the number of series stays bounded. + +| Metric | Instrument | Unit | Attributes | Description | +|---|---|---|---|---| +| `enrichments.job.runs` | Counter | `{run}` | `enrichments.enrichment`, `enrichments.trigger`, `enrichments.run.outcome` | Job runs that ended, by enrichment, trigger and outcome. | +| `enrichments.job.runs.active` | UpDownCounter | `{run}` | `enrichments.enrichment` | Job runs in progress right now, by enrichment. | +| `enrichments.job.run.duration` | Histogram | `s` | `enrichments.enrichment`, `enrichments.run.outcome` | Duration of one job run, by enrichment and outcome. A job that is paused and resumed records one measurement per run. Buckets: 1, 5, 10, 30, 60, 300, 600, 1800, 3600, 7200, 21600, 86400. | +| `enrichments.batch.duration` | Histogram | `s` | `enrichments.enrichment`, `enrichments.batch.outcome` | Duration of one batch - fetch, ``enrich_batch()`` and progress writes - by enrichment and outcome. Buckets: 0.01, 0.05, 0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600. | +| `enrichments.rows` | Counter | `{row}` | `enrichments.enrichment`, `enrichments.row.result` | Rows processed, by enrichment and result. Rows of a paused or cancelled batch are not counted: they are processed again on resume. | +| `enrichments.cost` | Counter | `{hundredth_cent}` | `enrichments.enrichment` | Cost reported by enrichments through ``increment_cost()``, in hundredths of a cent, by enrichment. | + +### Attributes + +`db.namespace` and `error.type` are shared with Datasette core; everything else is specific to this plugin. Besides the spans and metrics listed here, `enrichments.enrichment` and `enrichments.job_id` are added to core's request span for the request that submits a job, and `enrichments.job_id` to the request spans of the Pause, Resume and Cancel buttons. + +| Attribute | Values | Recorded on | Description | +|---|---|---|---| +| `enrichments.enrichment` | - | `enrichments.job.run`, `enrichments.batch`, `enrichments.initialize`, `enrichments.finalize`, `enrichments.job.runs`, `enrichments.job.runs.active`, `enrichments.job.run.duration`, `enrichments.batch.duration`, `enrichments.rows`, `enrichments.cost` | The enrichment's slug, e.g. ``openai``. Bounded by the installed enrichment plugins, so safe to use as a metric dimension. | +| `enrichments.job_id` | - | `enrichments.job.run`, `enrichments.batch`, `enrichments.finalize` | The job's id in the database's ``_enrichment_jobs`` table. Spans only, never on a metric. | +| `enrichments.trigger` | `enqueue`, `restart`, `resume` | `enrichments.job.run`, `enrichments.job.runs` | What started this run of the job: ``enqueue`` (a new job submitted through the enrichment form or API), ``resume`` (the Resume button on a paused job) or ``restart`` (the startup task resuming a job a previous process left ``running``). | +| `enrichments.run.outcome` | `cancelled`, `error`, `finished`, `interrupted`, `paused`, `stopped` | `enrichments.job.run`, `enrichments.job.runs`, `enrichments.job.run.duration` | How this run of the job ended. ``finished``: the last batch was processed. ``paused`` / ``cancelled``: either ``enrich_batch()`` raised ``self.Pause`` / ``self.Cancel``, or the loop saw the job's status changed to ``paused`` / ``cancelled`` (the Pause and Cancel buttons). ``stopped``: the loop saw some other non-``running`` status, or the job row disappeared. ``interrupted``: the run was cancelled with ``asyncio.CancelledError``, which means the process is shutting down; the job usually stays ``running`` and is resumed on the next start - unless the cancellation landed once the job was being marked ``finished`` or during ``finalize()``, in which case it is not run again. ``error``: an unexpected exception escaped the run, e.g. a failing ``finalize()``. Only ``error`` sets span status ``ERROR``. | +| `enrichments.batch.outcome` | `cancelled`, `error`, `ok`, `paused` | `enrichments.batch`, `enrichments.batch.duration` | How the batch ended. ``ok``: ``enrich_batch()`` returned, including when it logged errors for some rows, and the trailing fetch that returns no rows. ``error``: an exception escaped the batch and every row in the batch counts as an error. If ``enrich_batch()`` raised it, the error is logged and the run carries on; any other exception, e.g. a failed row fetch, also ends the run with ``error``. ``paused`` / ``cancelled``: ``enrich_batch()`` raised ``self.Pause`` / ``self.Cancel``; its rows are processed again on resume. Only ``error`` sets span status ``ERROR``. | +| `enrichments.row.result` | `error`, `success` | `enrichments.rows` | ``success`` for rows ``enrich_batch()`` counted as enriched, ``error`` for the rest of the batch - rows it logged errors for, or every row of a batch that raised. | +| `enrichments.job.row_count` | - | `enrichments.job.run` | Total rows the job targets, counted when the job was enqueued (``_enrichment_jobs.row_count``). | +| `enrichments.run.batches` | - | `enrichments.job.run` | Batches this run processed: batches with at least one row that ended ``ok`` or ``error``. A resumed job's earlier runs are not included. | +| `enrichments.run.rows` | - | `enrichments.job.run` | Rows this run processed, successful or not, across the batches counted by ``enrichments.run.batches``. | +| `enrichments.batch.index` | - | `enrichments.batch` | 0-based index of the batch within this run - it restarts at 0 when a job is resumed. | +| `enrichments.batch.rows` | - | `enrichments.batch` | Rows fetched for this batch. ``0`` for the trailing fetch that finds no rows left. | +| `enrichments.batch.success_count` | - | `enrichments.batch` | Rows ``enrich_batch()`` reported as enriched (its return value, or every row when it returns ``None``). Set when ``enrich_batch()`` returned. | +| `enrichments.restart.jobs` | - | `enrichments.restart` | Jobs left ``running`` by a previous process that the restart task tried to resume. A job whose loop is already running in this process is counted but left alone. | +| `enrichments.restart.unknown` | - | `enrichments.restart` | Jobs left ``running`` whose enrichment slug matches no installed enrichment, so they could not be resumed. | +| `db.namespace` | - | `enrichments.job.run`, `enrichments.initialize` | Name of the database being queried. | +| `error.type` | - | `enrichments.job.run`, `enrichments.batch`, `enrichments.initialize`, `enrichments.finalize`, `enrichments.restart` | The exception class name for a failed operation. On HTTP spans, also set to the status code as a string for 5xx responses. A 4xx response alone does not set this attribute or an error status. | + diff --git a/pyproject.toml b/pyproject.toml index 9defca6..e19bd8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,8 @@ dev = [ "opentelemetry-sdk>=1.37", # Pinned: ruff's default rule set changes between releases "ruff==0.16.9", + # CI checks the cog-generated parts of docs/*.md are up to date + "cogapp", ] [build-system] diff --git a/tests/conftest.py b/tests/conftest.py index 14ad23d..68bb455 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -259,10 +259,15 @@ async def enrich_batch( ): row = rows[0] result = await datasette.enrichment_queue.get() - if result == "pause": - raise self.Pause("pause message") - if result == "cancel": - raise self.Cancel("cancel message") + # "pause", "cancel" or "raise", optionally followed by + # ":" to set the exception's message + action, _, message = result.partition(":") + if action == "pause": + raise self.Pause(message or "pause message") + if action == "cancel": + raise self.Cancel(message or "cancel message") + if action == "raise": + raise Exception(message or "raise message") # noqa: TRY002 datasette.enrichment_processed_count += 1 wheres = " and ".join(f'"{pk}" = ?' for pk in pks) await db.execute_write( diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 204274b..9451ca8 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -1,16 +1,27 @@ import asyncio import inspect +import json +import sqlite3 import subprocess import sys import textwrap import time import pytest +import pytest_asyncio pytest.importorskip("opentelemetry.sdk") from datasette import telemetry_registry as core -from datasette.telemetry_testing import assert_package_never_imports_sdk +from datasette.app import Datasette +from datasette.telemetry_testing import ( + assert_metrics_conform, + assert_metrics_covered, + assert_no_forbidden_values, + assert_package_never_imports_sdk, + assert_spans_conform, + assert_spans_covered, +) from opentelemetry.trace import SpanKind, StatusCode from datasette_enrichments import telemetry_registry as reg @@ -182,25 +193,28 @@ def active_runs(otel_metrics, slug): return points[0].value if points else 0 -def root_cookies(datasette): - return {"ds_actor": datasette.sign({"a": {"id": "root"}}, "actor")} +def actor_cookies(datasette, actor_id="root"): + return {"ds_actor": datasette.sign({"a": {"id": actor_id}}, "actor")} -async def enqueue(datasette, table, slug, data=None, query=""): +async def enqueue( + datasette, table, slug, data=None, query="", database="data", actor_id="root" +): "Submit a job through the enrichment form; returns the new job's id." response = await datasette.client.post( - f"/-/enrich/data/{table}/{slug}{query}", - cookies=root_cookies(datasette), + f"/-/enrich/{database}/{table}/{slug}{query}", + cookies=actor_cookies(datasette, actor_id), data=data or {}, ) - assert response.status_code == 302 + assert response.status_code == 302, response.text return int(response.headers["location"].split("=")[-1]) -async def job_action(datasette, job_id, action): +async def job_action(datasette, job_id, action, database="data", actor_id="root"): + "Press the Pause, Resume or Cancel button on a job." response = await datasette.client.post( - f"/-/enrich/data/-/jobs/{job_id}/{action}", - cookies=root_cookies(datasette), + f"/-/enrich/{database}/-/jobs/{job_id}/{action}", + cookies=actor_cookies(datasette, actor_id), data={}, ) assert response.status_code == 302 @@ -743,3 +757,259 @@ def test_restart_span_failure_records_error_type(otel_spans): assert restart.attributes[reg.ERROR_TYPE] == "KeyError" assert restart.status.status_code == StatusCode.ERROR assert_text_nowhere([restart], "secret-ish") + + +# --- Registry conformance and privacy ------------------------------------- +# +# One broad workload that makes every registered span and metric fire, with a +# distinct sentinel string planted in every piece of user data a job touches. +# Then a single collect() - the counters and histograms are DELTA, so an +# earlier collect() would hide what it drained - and the kit's conformance +# assertions in both directions, plus the privacy walk across EVERY scope: +# a sentinel leaking through core's request or db.query spans is still a leak. + +SENTINEL_CONFIG = "SENTINEL_CONFIG_7f3a" +SENTINEL_FILTER = "SENTINEL_FILTER_2b9e" +SENTINEL_ROW = "SENTINEL_ROW_5c1d" +SENTINEL_PK = "SENTINEL_PK_8e4f" +SENTINEL_ACTOR = "SENTINEL_ACTOR_3a6b" +SENTINEL_EXC = "SENTINEL_EXC_9d2c" +SENTINEL_PAUSE = "SENTINEL_PAUSE_1f7e" +SENTINEL_CANCEL = "SENTINEL_CANCEL_6b3a" +SENTINEL_SECRET = "SENTINEL_SECRET_4e8d" +FORBIDDEN = { + SENTINEL_CONFIG, + SENTINEL_FILTER, + SENTINEL_ROW, + SENTINEL_PK, + SENTINEL_ACTOR, + SENTINEL_EXC, + SENTINEL_PAUSE, + SENTINEL_CANCEL, + SENTINEL_SECRET, +} +# Every row matches this filter; a normal column filter, so core binds the +# value as a parameter. (A _where= literal would reach core's db.query.text - +# documented in docs/telemetry.md, deliberately not planted here.) +SENTINEL_QUERY = f"?s__not={SENTINEL_FILTER}" +PEOPLE_ROWS = 25 + + +def person_id(i): + # A text primary key, so batch cursors (next_cursor, _next=) and the ids + # passed to log_error() carry the sentinel + return f"{SENTINEL_PK}_{i:03}" + + +@pytest_asyncio.fixture +async def private_datasette(tmpdir, monkeypatch): + # The secretreplace enrichment's API key, read from the environment + monkeypatch.setenv("DATASETTE_SECRETS_STRING_SECRET", SENTINEL_SECRET) + path = str(tmpdir / "private.db") + conn = sqlite3.connect(path) + with conn: + conn.execute("create table people (id text primary key, s text)") + for i in range(PEOPLE_ROWS): + # secretreplace swaps SENTINEL_CONFIG for the secret in these + conn.execute( + "insert into people (id, s) values (?, ?)", + (person_id(i), f"{SENTINEL_ROW} {i} {SENTINEL_CONFIG}"), + ) + datasette = Datasette( + [path], + # Only the sentinel actor may run enrichments + config={"permissions": {"enrichments": {"id": SENTINEL_ACTOR}}}, + ) + datasette._test_db = conn + await datasette.invoke_startup() + yield datasette + tasks = list(getattr(datasette, "_enrichment_job_tasks", {}).values()) + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + conn.close() + + +async def _insert_private_running_job(datasette, enrichment, next_cursor): + # A job a previous process left 'running', carrying every sentinel it can + from datasette_enrichments import ensure_tables + + db = datasette.get_database("private") + await ensure_tables(db) + return ( + await db.execute_write( + """ + insert into _enrichment_jobs ( + status, enrichment, database_name, table_name, + filter_querystring, config, next_cursor, row_count, + done_count, error_count, actor_id + ) values ( + 'running', :enrichment, 'private', 'people', :filter, :config, + :next_cursor, :row_count, 0, 0, :actor_id + ) + """, + { + "enrichment": enrichment, + "filter": SENTINEL_QUERY[1:], + "config": json.dumps({"prompt": SENTINEL_CONFIG}), + "next_cursor": next_cursor, + "row_count": PEOPLE_ROWS, + "actor_id": SENTINEL_ACTOR, + }, + ) + ).lastrowid + + +def run_of(job_id, trigger=None): + "Span predicate: the run span of ``job_id``, optionally with ``trigger``." + return lambda span: ( + for_job(job_id)(span) + and (trigger is None or span.attributes[reg.TRIGGER] == trigger) + ) + + +@pytest.mark.asyncio +async def test_registry_conformance_and_privacy( + private_datasette, otel_spans, otel_metrics +): + datasette = private_datasette + + async def submit(slug, data=None): + return await enqueue( + datasette, + "people", + slug, + data, + query=SENTINEL_QUERY, + database="private", + actor_id=SENTINEL_ACTOR, + ) + + async def press(job_id, action): + await job_action( + datasette, job_id, action, database="private", actor_id=SENTINEL_ACTOR + ) + + # 1. Restart: a running job with a known slug, resuming from a sentinel + # cursor, and one with an unknown slug. Inserted BEFORE any + # datasette.client request: the first request launches the background + # tasks, including the restart scan, which would otherwise find nothing. + restarted = await _insert_private_running_job( + datasette, "countbatches", person_id(PEOPLE_ROWS - 6) + ) + await _insert_private_running_job(datasette, "no-such-enrichment", None) + await datasette.start_background_tasks() + restart = await wait_for_span(otel_spans, reg.RESTART) + assert restart.attributes[reg.RESTART_JOBS] == 1 + assert restart.attributes[reg.RESTART_UNKNOWN] == 1 + run = await wait_for_span(otel_spans, reg.JOB_RUN, run_of(restarted, "restart")) + assert run.attributes[reg.RUN_ROWS] == 5 + + # 2. Batches that succeed, with the config, filter, rows, primary keys, + # actor and secret all carrying sentinels + secret_job = await submit( + "secretreplace", {"column": "s", "string": SENTINEL_CONFIG} + ) + await wait_for_span(otel_spans, reg.JOB_RUN, run_of(secret_job)) + # The secret really was used + (row,) = datasette._test_db.execute( + "select s from people where id = ?", (person_id(0),) + ).fetchall() + assert row == (f"{SENTINEL_ROW} 0 {SENTINEL_SECRET}",) + # The config and actor reached the job row through the form POST, not + # just through the directly inserted restart fixture + config, actor_id = datasette._test_db.execute( + "select config, actor_id from _enrichment_jobs where id = ?", (secret_job,) + ).fetchone() + assert SENTINEL_CONFIG in config + assert actor_id == SENTINEL_ACTOR + + # 3. A batch that raises, then a Pause exception; resumed through the UI, + # then a Cancel exception - every message a sentinel + queue_job = await submit("queue") + queue = datasette.enrichment_queue + await queue.put("0") + await queue.put(f"raise:{SENTINEL_EXC}") + await queue.put(f"pause:{SENTINEL_PAUSE}") + await wait_for_span(otel_spans, reg.JOB_RUN, run_of(queue_job, "enqueue")) + await press(queue_job, "resume") + await queue.put(f"cancel:{SENTINEL_CANCEL}") + await wait_for_span(otel_spans, reg.JOB_RUN, run_of(queue_job, "resume")) + + # 4. Partial row errors, logged by the enrichment itself + errors_job = await submit("haserrors") + await wait_for_span(otel_spans, reg.JOB_RUN, run_of(errors_job)) + + # 5. Pause from the UI mid-batch, then Resume + datasette._enrich_gate = asyncio.Event() + paused_job = await submit("countbatches") + await wait_until(lambda: first_batch_in_flight(datasette), "first batch") + await press(paused_job, "pause") + datasette._enrich_gate.set() + await wait_for_span(otel_spans, reg.JOB_RUN, run_of(paused_job, "enqueue")) + await press(paused_job, "resume") + await wait_for_span(otel_spans, reg.JOB_RUN, run_of(paused_job, "resume")) + + # 6. finalize() raises + finalize_job = await submit("finalizeraises") + await wait_for_span(otel_spans, reg.JOB_RUN, run_of(finalize_job)) + + # 7. Cost + cost_job = await submit("costdemo") + await wait_for_span(otel_spans, reg.JOB_RUN, run_of(cost_job)) + + # 8. A run interrupted mid-batch, as at shutdown + datasette._enrich_gate = asyncio.Event() # never set + interrupted_job = await submit("countbatches") + await wait_until(lambda: first_batch_in_flight(datasette), "first batch") + task = datasette._enrichment_job_tasks[("private", interrupted_job)] + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await wait_for_span(otel_spans, reg.JOB_RUN, run_of(interrupted_job)) + + # Every job loop has exited, so no span is still open - and so unchecked - + # when the spans are gathered + await wait_until( + lambda: all(t.done() for t in datasette._enrichment_job_tasks.values()), + "every job task to finish", + ) + finished = otel_spans.get_finished_spans() + otel_metrics.collect() + + # Every sentinel really was planted: the database holds each one (the job + # rows, progress messages, error log and enriched rows), so the privacy + # walk below cannot pass vacuously + stored = "\n".join(datasette._test_db.iterdump()) + assert {sentinel for sentinel in FORBIDDEN if sentinel not in stored} == set() + + # The workload reached every outcome it claims to + runs = ours(otel_spans, reg.JOB_RUN) + assert {s.attributes[reg.RUN_OUTCOME] for s in runs} == { + "finished", + "paused", + "cancelled", + "interrupted", + "error", + } + assert {s.attributes[reg.TRIGGER] for s in runs} == {"enqueue", "resume", "restart"} + assert { + s.attributes[reg.BATCH_OUTCOME] + for s in ours(otel_spans, reg.BATCH) + if reg.BATCH_OUTCOME in s.attributes + } == {"ok", "error", "paused", "cancelled"} + # haserrors logs 2 errors in each full batch of 10: 25 rows -> 4 errors + assert ( + otel_metrics.point( + reg.ROWS, {reg.ENRICHMENT: "haserrors", reg.ROW_RESULT: "error"} + ).value + == 4 + ) + + assert_spans_conform(reg.SPANS, finished, scope_name=SCOPE) + assert_spans_covered(reg.SPANS, finished, scope_name=SCOPE) + assert_metrics_conform(reg.METRICS, otel_metrics, scope_name=SCOPE) + assert_metrics_covered(reg.METRICS, otel_metrics, scope_name=SCOPE) + # No scope_name: core's request, db.query and write spans are checked too + assert_no_forbidden_values( + FORBIDDEN, finished_spans=finished, collector=otel_metrics + ) diff --git a/tests/test_telemetry_registry.py b/tests/test_telemetry_registry.py new file mode 100644 index 0000000..9e4ab6e --- /dev/null +++ b/tests/test_telemetry_registry.py @@ -0,0 +1,129 @@ +"""Static checks on the telemetry registry. These need no OpenTelemetry SDK. + +The dynamic half - one broad workload checked against the registry in both +directions, plus the sentinel privacy walk - is +``test_registry_conformance_and_privacy`` in ``test_telemetry.py``. +""" + +from itertools import pairwise + +from datasette.telemetry_registry import DB_NAMESPACE, ERROR_TYPE, HISTOGRAM + +from datasette_enrichments.telemetry_registry import ( + ATTRIBUTES, + ENRICHMENT, + JOB_ID, + METRICS, + SPANS, +) + +PREFIX = "enrichments." +# Names shared with Datasette core rather than prefixed, so backends line +# them up with core's own spans +CORE_SHARED = {str(DB_NAMESPACE), str(ERROR_TYPE)} + +# --- Wire names pinned as literals ---------------------------------------- +# +# Renaming a signal breaks every dashboard and alert built on it, so a +# rename has to be a deliberate edit here - together with the docs +# reference, which cog regenerates from the registry. + +EXPECTED_SPANS = { + "enrichments.job.run", + "enrichments.batch", + "enrichments.initialize", + "enrichments.finalize", + "enrichments.restart", +} + +EXPECTED_METRICS = { + "enrichments.job.runs", + "enrichments.job.runs.active", + "enrichments.job.run.duration", + "enrichments.batch.duration", + "enrichments.rows", + "enrichments.cost", +} + +EXPECTED_ATTRIBUTES = { + "enrichments.enrichment", + "enrichments.job_id", + "enrichments.trigger", + "enrichments.run.outcome", + "enrichments.batch.outcome", + "enrichments.row.result", + "enrichments.job.row_count", + "enrichments.run.batches", + "enrichments.run.rows", + "enrichments.batch.index", + "enrichments.batch.rows", + "enrichments.batch.success_count", + "enrichments.restart.jobs", + "enrichments.restart.unknown", + "db.namespace", + "error.type", +} + + +def test_wire_names_are_pinned(): + assert {str(span) for span in SPANS} == EXPECTED_SPANS + assert {str(metric) for metric in METRICS} == EXPECTED_METRICS + assert {str(attribute) for attribute in ATTRIBUTES} == EXPECTED_ATTRIBUTES + + +def test_every_name_is_prefixed(): + # datasette.* is reserved for core; everything of ours lives under + # enrichments.*, apart from the two names deliberately shared with core + for entry in (*SPANS, *METRICS): + assert str(entry).startswith(PREFIX), entry + for attribute in ATTRIBUTES: + assert str(attribute).startswith(PREFIX) or str(attribute) in CORE_SHARED, ( + attribute + ) + for entry in (*SPANS, *METRICS, *ATTRIBUTES): + assert not str(entry).startswith("datasette."), entry + + +def test_span_and_metric_attributes_are_registered(): + registered = {str(attribute) for attribute in ATTRIBUTES} + for entry in (*SPANS, *METRICS): + for attribute in entry.attributes: + assert str(attribute) in registered, f"{entry}: unlisted {attribute}" + + +def test_metric_dimensions_are_bounded(): + # Every metric attribute is the enrichment slug (bounded by the installed + # plugins) or a closed enum. Job ids and database names would mint a new + # series per job or per database, so they ride on spans only. + for metric in METRICS: + for attribute in metric.attributes: + assert str(attribute) == str(ENRICHMENT) or attribute.values, ( + f"{metric}: {attribute} is neither the enrichment nor an enum" + ) + assert str(attribute) not in {str(JOB_ID), str(DB_NAMESPACE)}, ( + f"{metric}: {attribute} must never be a metric dimension" + ) + + +def test_every_histogram_declares_buckets(): + for metric in METRICS: + if metric.kind == HISTOGRAM: + assert metric.buckets, f"{metric} declares no buckets" + assert all(a < b for a, b in pairwise(metric.buckets)), ( + f"{metric} buckets are not strictly increasing: {metric.buckets}" + ) + else: + assert metric.buckets is None, f"{metric} is not a histogram" + + +def test_every_entry_is_documented(): + for entry in (*SPANS, *METRICS, *ATTRIBUTES): + assert entry.description and entry.description.strip(), ( + f"{entry} has no description" + ) + + +def test_no_duplicate_names(): + for group in (SPANS, METRICS, ATTRIBUTES): + names = [str(entry) for entry in group] + assert len(names) == len(set(names)), f"duplicates in {names}"