From 6bf841e466d1345aefc6ee816dd14645b3e9ea9f Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 10:22:24 -0700 Subject: [PATCH 01/28] Add diagnostic provenance scanner plan --- .../phase-4-provenance-scanner.md | 216 +++++++++++++----- docs/github-issues/174-zppy-links/plan.md | 155 ++++++++----- 2 files changed, 265 insertions(+), 106 deletions(-) diff --git a/docs/github-issues/174-zppy-links/phase-4-provenance-scanner.md b/docs/github-issues/174-zppy-links/phase-4-provenance-scanner.md index e62d50cb..b66c2cde 100644 --- a/docs/github-issues/174-zppy-links/phase-4-provenance-scanner.md +++ b/docs/github-issues/174-zppy-links/phase-4-provenance-scanner.md @@ -1,73 +1,174 @@ -# Phase 4 Plan: Provenance Scanner and Ops Docs +# Phase 4 Plan: Provenance Scanner and Operations ## Task -Add standalone scanner that discovers zppy provenance cfg files from configured NERSC roots, verifies completion markers, and calls internal diagnostics-link API with service-account auth. +Add a standalone scanner that discovers zppy provenance under each configured +site diagnostics archive, verifies that diagnostics have published output, and +links them to cases with service-account authentication. + +The scanner follows the zppy SimBoard publishing contract: diagnostics live +under `diagnostics_archive//`, optionally grouped by +`/`, and each case directory may contain multiple timestamped +provenance files. The newest timestamped provenance is authoritative. ## Scope ### In scope - New script `backend/app/scripts/ingestion/diagnostics_link_scanner.py` -- State-file persistence and retry behavior -- Provenance cfg parsing and completion checks -- Script test coverage -- Script documentation and env example updates +- Static diagnostics-archive site registry under `backend/app/scripts/ingestion/` +- Provenance discovery, settings parsing, validation, and published-output checks +- Database-backed provenance state, dry-run behavior, and transient-failure retries +- Scanner-specific diagnostics state API, schema, and migration +- Site-wrapper configuration, scanner tests, and operational documentation ### Out of scope -- zppy repo changes -- Historical backfill tooling beyond normal scanner behavior -- New backend endpoint behavior outside Phase 2 contract +- Mache runtime/config retrieval +- zppy publishing or provenance-contract changes +- Historical backfill beyond normal scanner operation +- Changes to the existing `POST /api/v1/diagnostics/link` contract or frontend +- Diagnostics content ingestion or interpretation ## Approach -1. Mirror existing operational script structure. - - Base new script on patterns from `backend/app/scripts/ingestion/nersc_upload_archive_ingestor.py`. - - Reuse same style for config parsing, structured logs, dry-run handling, retry/backoff, and state persistence. - -2. Discover provenance cfg files. - - Recursively search configured roots from required env var `ZPPY_PROVENANCE_ROOTS`. - - Accept files matching `provenance*.cfg`. - -3. Parse required fields from each cfg. - - Require `case_name`, `machine`, `hpc_username`, `diagnostic_url`, and `output`. - - Also extract `www` from the provenance cfg for preserved diagnostics provenance context. - - Do not derive `diagnostic_url` from `www` in MVP; continue treating explicit `diagnostic_url` as authoritative. - - Treat missing required fields as terminal skip with structured log. - -4. Verify diagnostics completion before linking. - - Require `/index.html` to exist. - - Require every filename listed in env var `DIAGNOSTICS_REQUIRED_STATUS_FILES` to exist under ``. - - Skip incomplete diagnostics without calling API. - -5. Call internal diagnostics-link API. - - Send bearer token from `SIMBOARD_API_TOKEN`. - - POST one diagnostics-link request per eligible cfg to `POST /api/v1/diagnostics/link`. - - Use one diagnostics item for MVP: `name="zppy diagnostics"`, `url=diagnostic_url`, `kind="diagnostic"`. - -6. Persist scanner state. - - Store state in `DIAGNOSTICS_STATE_PATH`. - - Key by provenance file path. - - Persist cfg fingerprint, last outcome, and timestamp. - - Reprocess only when cfg fingerprint changes. - -7. Document operational config. - - Update `backend/app/scripts/README.md` with purpose, env vars, and example invocation. - - Add placeholders to `.envs/example/backend.env.example` only for operator-provided values required by this script. +1. Resolve archive locations from a static internal site registry. + - Add a module under `backend/app/scripts/ingestion/` containing a + `DIAGNOSTICS_ARCHIVES_BY_MACHINE` dictionary. Each entry contains the + complete diagnostics archive root and matching public archive base URL. + - Seed the checked-in dictionary by parsing Mache's + `mache/machines/*.cfg` files during development, retaining machines with + non-empty `[web_portal] base_path` and `base_url`, then appending the + `diagnostics_archive` path component. This is a deliberate development or + maintenance operation, never a scanner runtime dependency. + - Key the registry by SimBoard's accepted machine names and aliases, not + solely Mache cfg filenames. Map machine aliases with the same published + archive to one registry entry. + - Select the registry entry from `MACHINE_NAME`. Fail before scanning for an + unsupported machine; do not accept archive locations from environment + variables or fetch Mache configuration at runtime. + - Validate that the selected filesystem root is absolute and readable and + that the public base URL uses HTTP or HTTPS before scanning. + - Treat registry values as deliberate source-controlled site configuration: + refresh the dictionary through a reviewed SimBoard change when a site moves + its published archive. + +2. Discover case provenance within one bounded archive. + - Scan only the configured root's `production/` and `development/` + subdirectories. + - Support both `//` and + `///` layouts. + - Traverse only inside the configured archive root; do not follow symlinks + outside it. Use a normalized archive-relative case-directory path for + scanner-state identity. + - Find only `provenance..cfg` files with a valid zppy timestamp + in case directories; order candidates by that parsed timestamp, not file + modification time. + - Select the newest timestamp for each case and require a matching + `provenance..settings` file. + - If the newest pair is incomplete, defer that case instead of falling back + to older provenance. + +3. Parse and validate the selected provenance pair. + - Read `case_name`, `machine`, `hpc_username`, optional `case_group`, and + authoritative `diagnostics_url` only from the settings file. The cfg + exists only to establish a matching timestamped provenance pair. + - Parse settings as bounded UTF-8 `key = value` lines without evaluation; + reject malformed input and duplicate required keys. + - Require all case-identity fields needed by `POST /api/v1/diagnostics/link`. + - Verify the provenance case and optional case group agree with the archive + layout. + - Parse `diagnostics_url` and require an exact scheme, authority, and path + boundary under the configured `DIAGNOSTICS_ARCHIVE_BASE_URL`; never + derive or accept an unrelated URL. + - Log and skip malformed or unsafe provenance without terminating the full + scan. + +4. Verify published diagnostic output before linking. + - Require the published case directory to contain at least one + non-provenance diagnostic artifact or a non-empty diagnostic subdirectory. + - Do not inspect zppy status files: they are not published archive artifacts + and are not a reliable completion signal for one timestamped provenance + pair. + - Treat published-output presence as a readiness check, not proof that every + zppy task has completed. + +5. Read and update scanner state through a scanner-specific diagnostics API. + - Read the database-backed state for the configured machine before submitting + a candidate. Compare the selected settings filename and fingerprint with + the state for its archive-relative case-directory path. + - Skip a candidate whose selected settings filename and fingerprint already + match successful state. + - Build the scanner endpoint from `SIMBOARD_API_BASE_URL`. Keep the existing + `POST /api/v1/diagnostics/link` contract unchanged; add a separate internal + scanner endpoint that accepts the diagnostics-link payload plus provenance + source metadata. + - Authenticate with bearer token from `SIMBOARD_API_TOKEN`. + - Submit the provenance identity, one diagnostics item with + `name="zppy diagnostics"`, the authoritative `diagnostics_url`, and + `kind="diagnostic"`, plus the archive-relative case path, selected settings + filename, timestamp, and fingerprint. + - Treat HTTP 204 as success. + - Retry network failures, HTTP 408/429, and 5xx responses with bounded + backoff. Do not retry permanent 4xx responses within the same run. + - The scanner endpoint must atomically upsert the case-scoped link and its + successful provenance state. Leave failed and output-not-ready candidates + without successful state so a later scan retries them. + +6. Persist central successful provenance state. + - Add a `DiagnosticProvenanceState` record for each scanner-managed + diagnostic link. Key the record by canonical machine and normalized + archive-relative case-directory path, which includes the simulation type + and optional case group. + - Store the selected settings filename, parsed timestamp, content fingerprint, + linked URL, and successful submission timestamp. + - Link state to its scanner-managed `ExternalLink` with a unique foreign key + using `ON DELETE CASCADE`. Deleting that link removes or invalidates its + state; a later scan can recreate the still-published link. + - Use a database uniqueness constraint and one transaction for the link + upsert and state upsert so concurrent scanners are safe. + - Development and production directory paths may create distinct diagnostic + links for the same SimBoard case. The scanner never removes obsolete links; + operators remove them manually. + - In dry-run mode, state reads are allowed, but make no link or state writes. + +7. Document and expose site operation. + - Update `backend/app/scripts/README.md` with configuration, dry-run rollout, + retry behavior, database-state handling, and example scheduled invocation. + - Add scanner execution to supported site wrappers without moving scanning + logic into shell. + - Document required shared-archive permissions, including scanner read access + to provenance settings. + - Explain how maintainers add or refresh a machine entry in the static site + registry from Mache cfg data. ## Tests -- Add `backend/tests/features/ingestion/test_diagnostics_link_scanner.py` covering: - - provenance discovery - - cfg parsing success and failure - - `www` extraction when present - - missing required identity or URL - - completion-marker checks - - dry-run behavior - - retry behavior for transient API failures - - state dedup and retry-on-fingerprint-change - - API payload formatting +- Add `backend/tests/features/ingestion/test_diagnostics_link_scanner.py` + covering: + - static site-registry selection by canonical machine name and alias + - rejection of unsupported machines + - valid and invalid registry filesystem roots and public URLs + - registry generation from representative Mache cfg files, including skipped + files with missing `[web_portal]` values + - production and development discovery + - grouped and ungrouped case layouts + - newest-timestamp selection + - missing newest settings file without stale fallback + - settings-only identity and URL parsing, including malformed and duplicate + required settings keys + - case-directory and case-group mismatch rejection + - diagnostics URL scheme, authority, and path-boundary validation + - published diagnostic output, empty output, and provenance-only directories + - exact scanner API payload and bearer authentication + - transient retries and permanent response handling + - database-state lookup, successful-state deduplication, and changed-settings + reprocessing + - atomic link-and-state persistence, concurrent submissions, and cascade state + removal when a scanner-managed link is deleted + - retry after output-not-ready or failed submissions + - dry-run behavior with no link or state writes + - Run: - `make backend-test` - `make pre-commit-run` @@ -76,8 +177,15 @@ Add standalone scanner that discovers zppy provenance cfg files from configured - Risk score: 5 - Main failure modes: - - Completion-marker policy is too strict or too loose. - - State logic suppresses needed retries or replays unchanged cfgs. + - Static registry becomes stale after a site moves its published archive + location. + - zppy provenance settings format changes before its publishing contract is + finalized. + - Published output appears before every zppy task finishes; this MVP links + published diagnostics rather than proving complete zppy execution. + - Archive permissions prevent the scanner from reading provenance settings. + - State identity or transaction logic suppresses needed retries or records a + link without matching successful provenance state. ## Open Questions diff --git a/docs/github-issues/174-zppy-links/plan.md b/docs/github-issues/174-zppy-links/plan.md index 0c034abf..d45eae06 100644 --- a/docs/github-issues/174-zppy-links/plan.md +++ b/docs/github-issues/174-zppy-links/plan.md @@ -4,7 +4,7 @@ Replace manual diagnostics URL entry with automated linking from zppy diagnostics outputs to existing SimBoard simulation records. -MVP is NERSC-only. +MVP uses SimBoard's static diagnostics-archive registry for supported machines. ## Scope @@ -12,12 +12,12 @@ MVP is NERSC-only. - Add required zppy provenance fields: `case_name`, `machine`, `hpc_username` - Add required diagnostics URLs in zppy provenance -- Require standardized zppy diagnostics output locations for NERSC production runs -- Discover zppy diagnostics provenance files from configured NERSC production filesystem roots -- Confirm diagnostics completion from index page plus status files +- Require standardized zppy diagnostics archive locations for supported machines +- Discover newest paired zppy provenance from the static archive registry +- Require published diagnostic output before linking - Match diagnostics to SimBoard records using `(case_name, machine, hpc_username)` - Create idempotent case-scoped diagnostic links -- Maintain scanner state to avoid repeated processing +- Maintain database-backed scanner provenance state ### Out @@ -27,7 +27,7 @@ MVP is NERSC-only. - Diagnostics content ingestion or indexing - Public HTML directory scraping - Historical backfill beyond configured provenance roots -- Non-NERSC deployments +- Mache runtime/config retrieval ## Core Decisions @@ -45,9 +45,12 @@ All three fields are required. `case_name` alone is not globally safe, and `CASE Avoid public directory scraping. It is fragile, web-server-coupled, slow, and expands the SSRF/content-injection attack surface. -### Use zppy provenance cfg as the primary input +### Use paired zppy provenance files -SimBoard discovers zppy provenance files from configured NERSC filesystem roots. Newer zppy runs already emit provenance cfg files under diagnostics output paths, for example: +SimBoard discovers timestamped paired provenance files from the filesystem roots +selected by its static site registry. The cfg establishes that a matching +provenance pair exists; the paired settings file is the authoritative source of +case identity and diagnostics URL. For example: ```text post/scripts/provenance.20260303_230804_991619.cfg @@ -58,7 +61,7 @@ Reference example: - https://github.com/E3SM-Project/zppy/blob/main/examples/post.v3.LR.historical.zppy_v3.cfg - https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/zppy_example/v3.2.0/v3.LR.historical_0051/provenance.20260303_230804_991619.cfg -Current cfg examples expose useful fields: +Current cfg examples expose useful contextual fields: - `case`: case name - `input`: case run directory @@ -66,7 +69,7 @@ Current cfg examples expose useful fields: - `www`: public diagnostics root - `campaign`: optional campaign metadata -But current cfg is not yet an authoritative join source because it may lack: +The cfg is not an authoritative join source because it may lack: - `machine` - canonical simulation owner @@ -79,7 +82,8 @@ input path owner: ac.wlin output path owner: ac.zhang40 ``` -Therefore, zppy must enrich provenance cfg with required case identity copied from `/case_scripts/env_case.xml`: +Therefore, zppy must write required case identity to provenance settings, copied +from `/case_scripts/env_case.xml`: | XML field | Provenance field | | ---------- | ---------------- | @@ -87,39 +91,57 @@ Therefore, zppy must enrich provenance cfg with required case identity copied fr | `MACH` | `machine` | | `REALUSER` | `hpc_username` | -If any required field is missing, SimBoard skips the provenance file and logs it as invalid for linking. +If any required field is missing, SimBoard skips the provenance pair and logs it +as invalid for linking. Settings also provide the explicit `diagnostics_url`. -For MVP, zppy should reuse existing top-level cfg fields rather than emit a new versioned normalized block. +SimBoard selects the newest complete cfg/settings pair by parsed filename +timestamp and does not fall back to older provenance when the newest pair is +incomplete. -### Require standardized output locations for production runs +### Require standardized archive locations -For MVP, NERSC production runs must use standardized zppy diagnostics output locations. SimBoard relies on those known production roots for provenance discovery. +SimBoard keeps a checked-in `DIAGNOSTICS_ARCHIVES_BY_MACHINE` registry under +`backend/app/scripts/ingestion/`. Each entry supplies the complete filesystem +archive root and matching public archive URL for a supported SimBoard machine. +The registry is seeded or refreshed during development from Mache machine cfg +files, but the scanner never fetches or parses Mache at runtime. -Custom or ad hoc layouts do not block the overall design, but they are not the required path for MVP. +Custom or ad hoc layouts are not part of this MVP. ### Require explicit diagnostics URLs in provenance -For MVP, SimBoard should not derive diagnostics URLs from path conventions. zppy should emit explicit diagnostics URLs in provenance cfg. +For MVP, SimBoard should not derive diagnostics URLs from path conventions. zppy +should emit an explicit `diagnostics_url` in provenance settings. SimBoard +validates that URL against the configured public archive prefix. -### Use index page plus status files as completion signal +### Use published output as readiness signal -Treat diagnostics as complete only when the expected index page and zppy status files are present. +Treat a candidate as ready when its published case directory contains a +non-provenance diagnostic artifact or non-empty diagnostic subdirectory. Do not +inspect zppy status files: they are not published archive artifacts and do not +reliably identify one provenance run. This is a published-output readiness +check, not proof every zppy task has completed. ### Persist links, do not resolve at query time Create database rows when diagnostics are discovered. Frontend queries should not crawl filesystems or remote URLs. -Diagnostic links are case-scoped. For MVP, store them on `Case` by adding `case_id` to `ExternalLink`. Keep the existing manual-link rendering path where possible by surfacing case-scoped diagnostic links alongside current links. +Diagnostic links are case-scoped. Store scanner state centrally with each +scanner-managed link, keyed by canonical machine and archive-relative case path. +State is removed when its linked `ExternalLink` is deleted, allowing a later +scan to recreate a still-published link. ## Implementation -Implement in order: provenance contract -> scanner -> storage target -> resolver/API -> frontend verification. +Implement in order: provenance contract -> storage/API state -> scanner -> +frontend verification. ### zppy #### 1. Emit required provenance fields -For MVP, production runs must write diagnostics outputs and provenance cfg files to the standardized NERSC zppy output locations. +For MVP, runs must write diagnostics outputs and paired provenance +cfg/settings files to the standardized site diagnostics archive. | Field | Source | | -------------- | ------------------------- | @@ -135,9 +157,9 @@ Implementation note: Tests: -- uses standardized NERSC production output locations +- uses standardized diagnostics archive locations - emits `case_name`, `machine`, `hpc_username` -- emits explicit diagnostics URLs (`diagnostic_url`) +- emits explicit diagnostics URLs (`diagnostics_url`) - can construct explicit diagnostics URLs from cfg `www` plus `mache` machine metadata - parses values from `env_case.xml` - parses values from `env_build.xml` @@ -146,69 +168,95 @@ Tests: ### SimBoard -#### 1. Add diagnostics scanner +#### 1. Add diagnostics scanner and static site registry Add `diagnostics_link_scanner.py`. Responsibilities: -- scan configured NERSC production diagnostics roots for `provenance*.cfg` -- dedup with state file -- verify diagnostics completion from index page plus status files -- parse `case_name`, `machine`, `hpc_username` -- parse explicit diagnostics URLs (`diagnostic_url`) -- call internal API with service-account auth -- skip and log if full join key is unavailable +- add a checked-in machine-to-archive registry under `scripts/ingestion`, seeded + or refreshed from Mache cfg files during development only +- select a registry entry by accepted SimBoard machine name or alias; reject an + unsupported machine before scanning +- scan bounded production and development archive trees, including optional + case-group directories, without following symlinks outside the archive root +- select newest paired cfg/settings provenance by parsed filename timestamp +- parse identity and `diagnostics_url` only from settings; validate settings + syntax, archive layout, and URL scheme/authority/path boundary +- require published diagnostic output beyond provenance files; do not inspect + zppy status files +- read database-backed provenance state and skip an unchanged successful + settings filename/fingerprint +- call the scanner-specific internal diagnostics endpoint with service-account + auth, leaving `POST /api/v1/diagnostics/link` unchanged +- skip and log malformed, output-not-ready, or non-matching candidates Tests: -- discovers cfgs -- parses required cfg identity -- handles malformed cfgs +- selects static archive registry entries by canonical machine name and alias +- rejects unsupported machines and invalid registry roots or public URLs +- generates registry candidates from representative Mache cfg files while + skipping files without usable `[web_portal]` values +- discovers grouped and ungrouped cases in both archive classifications +- selects the newest paired provenance without stale fallback +- parses required settings identity and URL without parsing cfg identity +- handles malformed or unsafe provenance - skips missing identity -- checks index-plus-status completion marker -- dedups state +- checks published output and provenance-only directories without status files +- retries transient submissions and dedups central successful state - handles duplicate links idempotently -#### 2. Resolve link storage +#### 2. Resolve link storage and scanner state -Add `DiagnosticsLinkRequest` in `backend/app/features/simulation/schemas.py`. +Use the existing diagnostics-link request schema in +`backend/app/features/catalog/schemas.py`. -For MVP, add `case_id` to `ExternalLink` and store diagnostic links at case scope. -Add a partial unique index on `(case_id, kind, url)` where `case_id IS NOT NULL` so case-owned diagnostic links remain idempotent under repeated or concurrent writes. +Use the existing case-owned `ExternalLink` storage and partial uniqueness on +`(case_id, kind, url)` so repeated or concurrent submissions remain idempotent. + +Add `DiagnosticProvenanceState` for each scanner-managed diagnostic link. Store +the canonical machine, normalized archive-relative case path, settings filename, +timestamp, fingerprint, URL, and successful submission time. Give the state row +a unique foreign key to `ExternalLink` with cascade deletion, and add a unique +machine/path constraint. #### 3. Add matching resolver | Input | Match | | -------------- | ----------------------- | | `case_name` | `Case.name` | -| `machine` | joined case simulations | -| `hpc_username` | joined case simulations | +| `machine` | resolved `Case.machine_id` | +| `hpc_username` | `Case.hpc_username` | Outcomes: - 1 case match: create/update case-scoped links - 0 matches: `404` -- multiple matches: `409` +- case uniqueness makes multiple matches invalid Tests: - matching triple creates links - same case/machine under different user does not cross-link - no match returns `404` -- ambiguous match returns `409` +- case uniqueness prevents ambiguous matches -#### 4. Add internal API endpoint +#### 4. Add internal diagnostics APIs Endpoint: `POST /api/v1/diagnostics/link` Implementation note: -- Define the endpoint in `backend/app/features/simulation/api.py` using a dedicated `diagnostics_router` with prefix `/diagnostics`. +- Define the endpoint in `backend/app/features/catalog/api.py` using a dedicated `diagnostics_router` with prefix `/diagnostics`. - Register that router in `backend/app/main.py` with `API_BASE` so the public path remains exactly `/api/v1/diagnostics/link` instead of inheriting the `/simulations` prefix. Roles: `ADMIN`, `SERVICE_ACCOUNT` +Keep this endpoint's contract unchanged. Add scanner-specific state read and +link endpoints for service accounts. The scanner link endpoint accepts existing +diagnostics-link identity plus provenance metadata and atomically upserts the +case link and `DiagnosticProvenanceState` in one transaction. + Request: | Field | Required | @@ -232,6 +280,10 @@ Tests: - concurrent duplicate request is idempotent - invalid payload returns `422` - auth required +- scanner state lookup skips unchanged successful provenance +- scanner link submission atomically persists link and state +- concurrent scanner submissions remain safe +- deleting a scanner-managed link cascades state deletion #### 5. Keep frontend unchanged @@ -255,13 +307,12 @@ make backend-test && make pre-commit-run Mitigation: add `case_id` for MVP and keep migration/API behavior narrow. - **Missing identity**: SimBoard cannot link a provenance file without `case_name`, `machine`, and `hpc_username`. Mitigation: require zppy provenance enrichment; skip and log invalid files. -- **NERSC deployment variability**: zppy roots and public URL prefixes may still vary by campaign or user layout within NERSC. - Mitigation: use env-configured NERSC scanner roots and NERSC public-prefix mappings. +- **Static registry drift**: a site may move its published archive. + Mitigation: refresh the checked-in registry through a reviewed SimBoard change. - **Provenance drift**: cfg layout and required-field coverage may vary across zppy versions. Mitigation: add parser tests, schema/version detection, and a documented support window. ## Remaining Open Questions -1. **NERSC deployment scope:** Which NERSC scanner roots and public URL prefixes are supported in MVP? -2. **Retroactive linking:** Does MVP include historical backfill, or only provenance files with the required join key? -3. **Case identity hardening:** Is `(case_name, machine, hpc_username)` sufficient until issue #136 is resolved? +1. **Retroactive linking:** Does MVP include historical backfill, or only provenance files with the required join key? +2. **Case identity hardening:** Is `(case_name, machine, hpc_username)` sufficient until issue #136 is resolved? From ccfe8d771f34fdbe6a44ec791afb04029906aa0b Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 10:43:16 -0700 Subject: [PATCH 02/28] Add diagnostic provenance scanner state API --- backend/app/features/catalog/api.py | 119 ++++++++++++++++++ backend/app/features/catalog/models.py | 45 +++++++ backend/app/features/catalog/schemas.py | 27 ++++ ..._000000_add_diagnostic_provenance_state.py | 44 +++++++ 4 files changed, 235 insertions(+) create mode 100644 backend/migrations/versions/20260811_000000_add_diagnostic_provenance_state.py diff --git a/backend/app/features/catalog/api.py b/backend/app/features/catalog/api.py index f64f196d..40e5731e 100644 --- a/backend/app/features/catalog/api.py +++ b/backend/app/features/catalog/api.py @@ -20,6 +20,7 @@ from app.features.catalog.models import ( Artifact, Case, + DiagnosticProvenanceState, Execution, ExternalLink, MetadataChange, @@ -32,7 +33,9 @@ CaseSummaryOut, CaseUpdate, CatalogOverviewOut, + DiagnosticProvenanceStateOut, DiagnosticsLinkRequest, + DiagnosticsScannerLinkRequest, ExecutionCreate, ExecutionExternalLinkOut, ExecutionFilterOptionsOut, @@ -748,6 +751,108 @@ def link_case_diagnostics( ) +@diagnostics_router.get( + "/scanner-state", response_model=DiagnosticProvenanceStateOut | None +) +def get_diagnostics_scanner_state( + machine: str, + archive_relative_case_path: str, + db: Session = Depends(get_database_session), + user: User = Depends(current_active_user), +) -> DiagnosticProvenanceStateOut | None: + """Return successful scanner state for one machine/archive case path.""" + _require_diagnostics_scanner_role(user) + resolved_machine = resolve_machine_by_name(db, machine) + if resolved_machine is None: + raise HTTPException(status_code=404, detail="Unknown machine.") + state = ( + db.query(DiagnosticProvenanceState) + .filter(DiagnosticProvenanceState.machine_name == resolved_machine.name) + .filter( + DiagnosticProvenanceState.archive_relative_case_path + == archive_relative_case_path + ) + .one_or_none() + ) + return DiagnosticProvenanceStateOut.model_validate(state) if state else None + + +@diagnostics_router.post("/scanner/link", status_code=status.HTTP_204_NO_CONTENT) +def link_scanner_diagnostics( + payload: DiagnosticsScannerLinkRequest, + db: Session = Depends(get_database_session), + user: User = Depends(current_active_user), +) -> None: + """Atomically upsert one scanner-managed case diagnostic link and state.""" + _require_diagnostics_scanner_role(user) + if len(payload.diagnostics) != 1: + raise HTTPException( + status_code=422, detail="Scanner payload requires one diagnostic." + ) + if _unsafe_archive_relative_path(payload.provenance.archive_relative_case_path): + raise HTTPException( + status_code=422, detail="Invalid archive-relative case path." + ) + + machine = resolve_machine_by_name(db, payload.machine) + if machine is None: + raise HTTPException(status_code=404, detail="No matching case found.") + case_id = _resolve_case_id_for_diagnostics_link( + db=db, + case_name=payload.case_name, + machine_name=payload.machine, + hpc_username=payload.hpc_username, + ) + diagnostic = payload.diagnostics[0] + now = datetime.now(timezone.utc) + with transaction(db): + link_id = db.execute( + pg_insert(ExternalLink) + .values( + case_id=case_id, + kind=ExternalLinkKind.DIAGNOSTIC, + url=str(diagnostic.url), + label=diagnostic.name, + created_at=now, + updated_at=now, + ) + .on_conflict_do_update( + index_elements=[ + ExternalLink.case_id, + ExternalLink.kind, + ExternalLink.url, + ], + index_where=ExternalLink.case_id.is_not(None), + set_={"label": diagnostic.name, "updated_at": now}, + ) + .returning(ExternalLink.id) + ).scalar_one() + db.execute( + pg_insert(DiagnosticProvenanceState) + .values( + link_id=link_id, + machine_name=machine.name, + archive_relative_case_path=payload.provenance.archive_relative_case_path, + settings_filename=payload.provenance.settings_filename, + provenance_timestamp=payload.provenance.provenance_timestamp, + fingerprint=payload.provenance.fingerprint, + linked_url=str(diagnostic.url), + submitted_at=now, + ) + .on_conflict_do_update( + constraint="uq_diagnostic_provenance_states_machine_path", + set_={ + "link_id": link_id, + "settings_filename": payload.provenance.settings_filename, + "provenance_timestamp": payload.provenance.provenance_timestamp, + "fingerprint": payload.provenance.fingerprint, + "linked_url": str(diagnostic.url), + "submitted_at": now, + }, + ) + ) + + @execution_router.get( "", response_model=ExecutionPageOut, @@ -1219,6 +1324,20 @@ def _resolve_case_id_for_diagnostics_link( return match[0] +def _require_diagnostics_scanner_role(user: User) -> None: + if user.role not in (UserRole.ADMIN, UserRole.SERVICE_ACCOUNT): + raise HTTPException( + status_code=403, + detail="Scanner access requires an administrator or service account.", + ) + + +def _unsafe_archive_relative_path(value: str) -> bool: + return value.startswith("/") or any( + part in {"", ".", ".."} for part in value.split("/") + ) + + def _upsert_case_diagnostic_links( *, db: Session, diff --git a/backend/app/features/catalog/models.py b/backend/app/features/catalog/models.py index 77c3a6cb..47f8d2e4 100644 --- a/backend/app/features/catalog/models.py +++ b/backend/app/features/catalog/models.py @@ -322,3 +322,48 @@ class ExternalLink(Base, IDMixin, TimestampMixin): foreign_keys=[case_id], passive_deletes=True, ) + diagnostic_provenance_state: Mapped[DiagnosticProvenanceState | None] = ( + relationship( + back_populates="link", + cascade="all, delete-orphan", + passive_deletes=True, + uselist=False, + ) + ) + + +class DiagnosticProvenanceState(Base, IDMixin): + """Successful scanner submission state for one published diagnostics link.""" + + __tablename__ = "diagnostic_provenance_states" + __table_args__ = ( + UniqueConstraint( + "machine_name", + "archive_relative_case_path", + name="uq_diagnostic_provenance_states_machine_path", + ), + UniqueConstraint("link_id", name="uq_diagnostic_provenance_states_link_id"), + ) + + link_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("external_links.id", ondelete="CASCADE"), + nullable=False, + ) + machine_name: Mapped[str] = mapped_column(String(200), nullable=False) + archive_relative_case_path: Mapped[str] = mapped_column(Text, nullable=False) + settings_filename: Mapped[str] = mapped_column(String(255), nullable=False) + provenance_timestamp: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + fingerprint: Mapped[str] = mapped_column(String(128), nullable=False) + linked_url: Mapped[str] = mapped_column(String(1000), nullable=False) + submitted_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + + link: Mapped[ExternalLink] = relationship( + back_populates="diagnostic_provenance_state", + foreign_keys=[link_id], + passive_deletes=True, + ) diff --git a/backend/app/features/catalog/schemas.py b/backend/app/features/catalog/schemas.py index 37a5906e..c814bcbf 100644 --- a/backend/app/features/catalog/schemas.py +++ b/backend/app/features/catalog/schemas.py @@ -152,6 +152,33 @@ class DiagnosticsLinkRequest(CamelInBaseModel): ] +class DiagnosticProvenanceMetadata(CamelInBaseModel): + """Immutable provenance identity supplied by the diagnostics scanner.""" + + archive_relative_case_path: Annotated[ + str, Field(..., min_length=1, max_length=1000) + ] + settings_filename: Annotated[str, Field(..., min_length=1, max_length=255)] + provenance_timestamp: datetime + fingerprint: Annotated[str, Field(..., min_length=1, max_length=128)] + + +class DiagnosticsScannerLinkRequest(DiagnosticsLinkRequest): + """Scanner-only diagnostics link request with successful provenance state.""" + + provenance: DiagnosticProvenanceMetadata + + +class DiagnosticProvenanceStateOut(CamelOutBaseModel): + machine_name: str + archive_relative_case_path: str + settings_filename: str + provenance_timestamp: datetime + fingerprint: str + linked_url: str + submitted_at: datetime + + class ArtifactCreate(CamelInBaseModel): """Schema for creating a new Artifact.""" diff --git a/backend/migrations/versions/20260811_000000_add_diagnostic_provenance_state.py b/backend/migrations/versions/20260811_000000_add_diagnostic_provenance_state.py new file mode 100644 index 00000000..27ea92ca --- /dev/null +++ b/backend/migrations/versions/20260811_000000_add_diagnostic_provenance_state.py @@ -0,0 +1,44 @@ +"""Add successful diagnostics scanner provenance state. + +Revision ID: 20260811_000000 +Revises: 20260728_010000 +Create Date: 2026-08-11 00:00:00.000000 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "20260811_000000" +down_revision: Union[str, Sequence[str], None] = "20260728_010000" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "diagnostic_provenance_states", + sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("link_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("machine_name", sa.String(length=200), nullable=False), + sa.Column("archive_relative_case_path", sa.Text(), nullable=False), + sa.Column("settings_filename", sa.String(length=255), nullable=False), + sa.Column("provenance_timestamp", sa.DateTime(timezone=True), nullable=False), + sa.Column("fingerprint", sa.String(length=128), nullable=False), + sa.Column("linked_url", sa.String(length=1000), nullable=False), + sa.Column("submitted_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["link_id"], ["external_links.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("link_id", name="uq_diagnostic_provenance_states_link_id"), + sa.UniqueConstraint( + "machine_name", + "archive_relative_case_path", + name="uq_diagnostic_provenance_states_machine_path", + ), + ) + + +def downgrade() -> None: + op.drop_table("diagnostic_provenance_states") From 7c481b571c2259179fc6ade090b11e13aca00fc0 Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 10:45:45 -0700 Subject: [PATCH 03/28] Test diagnostic provenance scanner state API --- .../test_diagnostic_provenance_state.py | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 backend/tests/features/catalog/test_diagnostic_provenance_state.py diff --git a/backend/tests/features/catalog/test_diagnostic_provenance_state.py b/backend/tests/features/catalog/test_diagnostic_provenance_state.py new file mode 100644 index 00000000..3714ca30 --- /dev/null +++ b/backend/tests/features/catalog/test_diagnostic_provenance_state.py @@ -0,0 +1,136 @@ +from unittest.mock import patch +from uuid import uuid4 + +from sqlalchemy.orm import Session + +from app.api.version import API_BASE +from app.features.catalog.models import DiagnosticProvenanceState, ExternalLink +from app.features.machine.models import Machine +from tests.features.catalog.test_api import ( + _create_matching_execution, + _create_service_account_token, + use_real_auth, +) + + +def _payload(*, case_name: str, machine: str, path: str) -> dict: + return { + "caseName": case_name, + "machine": machine, + "hpcUsername": "scanner-user", + "diagnostics": [ + { + "name": "zppy diagnostics", + "url": "https://diagnostics.example.org/archive/case", + "kind": "diagnostic", + } + ], + "provenance": { + "archiveRelativeCasePath": path, + "settingsFilename": "provenance.20260811_120000_000000.settings", + "provenanceTimestamp": "2026-08-11T12:00:00Z", + "fingerprint": "a" * 64, + }, + } + + +def _matching_case(db: Session): + machine = db.query(Machine).first() + assert machine is not None + user, token = _create_service_account_token(db) + case, _ = _create_matching_execution( + db, + case_name=f"scanner-state-{uuid4()}", + machine_id=machine.id, + machine_name=machine.name, + user_id=user.id, + execution_id=f"scanner-{uuid4()}", + hpc_username="scanner-user", + source_reference=f"scanner-state-{uuid4()}", + ) + return machine, user, token, case + + +@use_real_auth +def test_scanner_link_is_idempotent_and_state_is_readable(client, db: Session) -> None: + machine, _, token, case = _matching_case(db) + payload = _payload( + case_name=case.name, machine=machine.name, path="production/e3sm/case" + ) + headers = {"Authorization": f"Bearer {token}"} + + assert ( + client.post( + f"{API_BASE}/diagnostics/scanner/link", json=payload, headers=headers + ).status_code + == 204 + ) + assert ( + client.post( + f"{API_BASE}/diagnostics/scanner/link", json=payload, headers=headers + ).status_code + == 204 + ) + + state = db.query(DiagnosticProvenanceState).one() + assert state.machine_name == machine.name + assert state.settings_filename == payload["provenance"]["settingsFilename"] + assert db.query(ExternalLink).filter(ExternalLink.case_id == case.id).count() == 1 + + response = client.get( + f"{API_BASE}/diagnostics/scanner-state", + params={ + "machine": machine.name, + "archiveRelativeCasePath": "production/e3sm/case", + }, + headers=headers, + ) + assert response.status_code == 200 + assert response.json()["fingerprint"] == "a" * 64 + + +@use_real_auth +def test_scanner_link_rolls_back_link_when_state_write_fails( + client, db: Session +) -> None: + machine, _, token, case = _matching_case(db) + payload = _payload( + case_name=case.name, machine=machine.name, path="production/e3sm/fail" + ) + headers = {"Authorization": f"Bearer {token}"} + + with patch.object( + db, + "execute", + side_effect=[ + type("Result", (), {"scalar_one": lambda self: uuid4()})(), + RuntimeError("state failure"), + ], + ): + response = client.post( + f"{API_BASE}/diagnostics/scanner/link", json=payload, headers=headers + ) + + assert response.status_code == 500 + assert db.query(ExternalLink).filter(ExternalLink.case_id == case.id).count() == 0 + + +@use_real_auth +def test_deleting_scanner_link_cascades_provenance_state(client, db: Session) -> None: + machine, _, token, case = _matching_case(db) + payload = _payload( + case_name=case.name, machine=machine.name, path="development/e3sm/case" + ) + assert ( + client.post( + f"{API_BASE}/diagnostics/scanner/link", + json=payload, + headers={"Authorization": f"Bearer {token}"}, + ).status_code + == 204 + ) + + link = db.query(ExternalLink).filter(ExternalLink.case_id == case.id).one() + db.delete(link) + db.commit() + assert db.query(DiagnosticProvenanceState).count() == 0 From b9ab70671fd3bd4a9b1eadbcf4f500d6138a7a55 Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 10:47:38 -0700 Subject: [PATCH 04/28] Fix diagnostic provenance rollback test --- .../test_diagnostic_provenance_state.py | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/backend/tests/features/catalog/test_diagnostic_provenance_state.py b/backend/tests/features/catalog/test_diagnostic_provenance_state.py index 3714ca30..1ebc9397 100644 --- a/backend/tests/features/catalog/test_diagnostic_provenance_state.py +++ b/backend/tests/features/catalog/test_diagnostic_provenance_state.py @@ -6,6 +6,8 @@ from app.api.version import API_BASE from app.features.catalog.models import DiagnosticProvenanceState, ExternalLink from app.features.machine.models import Machine +from app.features.user.manager import current_active_user +from app.main import app from tests.features.catalog.test_api import ( _create_matching_execution, _create_service_account_token, @@ -93,23 +95,23 @@ def test_scanner_link_is_idempotent_and_state_is_readable(client, db: Session) - def test_scanner_link_rolls_back_link_when_state_write_fails( client, db: Session ) -> None: - machine, _, token, case = _matching_case(db) + machine, service_user, _, case = _matching_case(db) payload = _payload( case_name=case.name, machine=machine.name, path="production/e3sm/fail" ) - headers = {"Authorization": f"Bearer {token}"} - - with patch.object( - db, - "execute", - side_effect=[ - type("Result", (), {"scalar_one": lambda self: uuid4()})(), - RuntimeError("state failure"), - ], - ): - response = client.post( - f"{API_BASE}/diagnostics/scanner/link", json=payload, headers=headers - ) + original_execute = db.execute + + def fail_only_state_insert(statement, *args, **kwargs): + if statement.table.name == "diagnostic_provenance_states": + raise RuntimeError("state failure") + return original_execute(statement, *args, **kwargs) + + app.dependency_overrides[current_active_user] = lambda: service_user + try: + with patch.object(db, "execute", side_effect=fail_only_state_insert): + response = client.post(f"{API_BASE}/diagnostics/scanner/link", json=payload) + finally: + app.dependency_overrides.pop(current_active_user, None) assert response.status_code == 500 assert db.query(ExternalLink).filter(ExternalLink.case_id == case.id).count() == 0 From dbffa34a9ac64f972c154ed0be01a25b4c7e1cd4 Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 10:48:08 -0700 Subject: [PATCH 05/28] Correct diagnostic provenance rollback test mock --- .../tests/features/catalog/test_diagnostic_provenance_state.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/tests/features/catalog/test_diagnostic_provenance_state.py b/backend/tests/features/catalog/test_diagnostic_provenance_state.py index 1ebc9397..decad85d 100644 --- a/backend/tests/features/catalog/test_diagnostic_provenance_state.py +++ b/backend/tests/features/catalog/test_diagnostic_provenance_state.py @@ -102,7 +102,8 @@ def test_scanner_link_rolls_back_link_when_state_write_fails( original_execute = db.execute def fail_only_state_insert(statement, *args, **kwargs): - if statement.table.name == "diagnostic_provenance_states": + table = getattr(statement, "table", None) + if table is not None and table.name == "diagnostic_provenance_states": raise RuntimeError("state failure") return original_execute(statement, *args, **kwargs) From e5829932bd91c4b0ccc8bceae90eb91cc600b95b Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 10:54:23 -0700 Subject: [PATCH 06/28] Add diagnostics provenance scanner --- .../scripts/ingestion/diagnostics_archives.py | 30 +++ .../ingestion/diagnostics_link_scanner.py | 190 ++++++++++++++++++ 2 files changed, 220 insertions(+) create mode 100644 backend/app/scripts/ingestion/diagnostics_archives.py create mode 100644 backend/app/scripts/ingestion/diagnostics_link_scanner.py diff --git a/backend/app/scripts/ingestion/diagnostics_archives.py b/backend/app/scripts/ingestion/diagnostics_archives.py new file mode 100644 index 00000000..09f56532 --- /dev/null +++ b/backend/app/scripts/ingestion/diagnostics_archives.py @@ -0,0 +1,30 @@ +"""Reviewed diagnostics archive locations; never populated at scanner runtime.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class DiagnosticsArchive: + root: str + public_base_url: str + + +# Refresh from Mache [web_portal] configuration in a reviewed change when sites move. +DIAGNOSTICS_ARCHIVES_BY_MACHINE: dict[str, DiagnosticsArchive] = { + "perlmutter": DiagnosticsArchive( + root="/global/cfs/cdirs/e3sm/diagnostic_output", + public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostic_output", + ), + "pm": DiagnosticsArchive( + root="/global/cfs/cdirs/e3sm/diagnostic_output", + public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostic_output", + ), + "pm-cpu": DiagnosticsArchive( + root="/global/cfs/cdirs/e3sm/diagnostic_output", + public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostic_output", + ), + "pm-gpu": DiagnosticsArchive( + root="/global/cfs/cdirs/e3sm/diagnostic_output", + public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostic_output", + ), +} diff --git a/backend/app/scripts/ingestion/diagnostics_link_scanner.py b/backend/app/scripts/ingestion/diagnostics_link_scanner.py new file mode 100644 index 00000000..0e138f5d --- /dev/null +++ b/backend/app/scripts/ingestion/diagnostics_link_scanner.py @@ -0,0 +1,190 @@ +"""Discover published zppy provenance and submit case diagnostics links.""" + +from __future__ import annotations + +import hashlib +import logging +import os +import re +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlparse + +import httpx + +from app.scripts.ingestion.diagnostics_archives import ( + DIAGNOSTICS_ARCHIVES_BY_MACHINE, + DiagnosticsArchive, +) + +LOGGER = logging.getLogger(__name__) +TIMESTAMP_RE = re.compile(r"^provenance\.(\d{8}_\d{6}_\d{6})\.cfg$") +REQUIRED_SETTINGS = {"case_name", "machine", "hpc_username", "diagnostics_url"} + + +@dataclass(frozen=True) +class Candidate: + path: Path + settings: Path + timestamp: datetime + values: dict[str, str] + fingerprint: str + + +def resolve_archive(machine_name: str) -> DiagnosticsArchive: + archive = DIAGNOSTICS_ARCHIVES_BY_MACHINE.get(machine_name.lower()) + if archive is None: + raise ValueError(f"Unsupported diagnostics scanner machine: {machine_name}") + root = Path(archive.root) + parsed = urlparse(archive.public_base_url) + if not root.is_absolute() or not root.is_dir() or not os.access(root, os.R_OK): + raise ValueError(f"Diagnostics archive is not readable: {root}") + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("Diagnostics archive public URL must be absolute HTTP(S)") + return archive + + +def parse_settings(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + if "=" not in line: + raise ValueError("Malformed provenance settings line") + key, value = (part.strip() for part in line.split("=", 1)) + if not key or not value or key in values: + raise ValueError("Malformed or duplicate provenance setting") + values[key] = value + if REQUIRED_SETTINGS - values.keys(): + raise ValueError("Missing required provenance settings") + return values + + +def _published_output(case_dir: Path) -> bool: + for entry in case_dir.iterdir(): + if entry.name.startswith("provenance."): + continue + if entry.is_file() or (entry.is_dir() and any(entry.iterdir())): + return True + return False + + +def discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 + base = urlparse(public_base_url) + candidates: list[Candidate] = [] + for tier in ("production", "development"): + tier_root = root / tier + if not tier_root.is_dir(): + continue + for cfg in tier_root.rglob("provenance.*.cfg"): + if cfg.is_symlink() or root not in cfg.resolve().parents: + continue + match = TIMESTAMP_RE.match(cfg.name) + if match is None: + continue + case_dir = cfg.parent + timestamp = datetime.strptime(match.group(1), "%Y%m%d_%H%M%S_%f").replace( + tzinfo=timezone.utc + ) + settings = cfg.with_suffix(".settings") + # newest cfg controls: missing paired settings must defer this case. + prior = next( + (item for item in candidates if item.path.parent == case_dir), None + ) + if prior and prior.timestamp >= timestamp: + continue + if prior: + candidates.remove(prior) + if not settings.is_file() or not _published_output(case_dir): + continue + try: + values = parse_settings(settings) + url = urlparse(values["diagnostics_url"]) + if (url.scheme, url.netloc) != ( + base.scheme, + base.netloc, + ) or not url.path.startswith(base.path.rstrip("/") + "/"): + raise ValueError( + "Diagnostics URL outside configured public archive" + ) + relative = case_dir.relative_to(root).as_posix() + if values.get("case_group") and values[ + "case_group" + ] not in relative.split("/"): + raise ValueError("Case group does not match archive layout") + digest = hashlib.sha256(settings.read_bytes()).hexdigest() + candidates.append(Candidate(cfg, settings, timestamp, values, digest)) + except (OSError, UnicodeError, ValueError) as exc: + LOGGER.warning("Skipping invalid provenance %s: %s", cfg, exc) + return candidates + + +def run() -> int: + machine = os.environ.get("MACHINE_NAME", "perlmutter") + archive = resolve_archive(machine) + root = Path(archive.root) + dry_run = os.environ.get("DRY_RUN", "true").lower() in {"1", "true", "yes"} + api_base = os.environ["SIMBOARD_API_BASE_URL"].rstrip("/") + token = os.environ["SIMBOARD_API_TOKEN"] + headers = {"Authorization": f"Bearer {token}"} + with httpx.Client(timeout=30) as client: + for candidate in discover(root, archive.public_base_url): + relative = candidate.path.parent.relative_to(root).as_posix() + if dry_run: + LOGGER.info("Would link diagnostics for %s", relative) + continue + state = client.get( + f"{api_base}/api/v1/diagnostics/scanner-state", + params={"machine": machine, "archive_relative_case_path": relative}, + headers=headers, + ) + if ( + state.status_code == 200 + and state.json() + and ( + state.json().get("settingsFilename") == candidate.settings.name + and state.json().get("fingerprint") == candidate.fingerprint + ) + ): + continue + payload = { + "caseName": candidate.values["case_name"], + "machine": candidate.values["machine"], + "hpcUsername": candidate.values["hpc_username"], + "diagnostics": [ + { + "name": "zppy diagnostics", + "url": candidate.values["diagnostics_url"], + "kind": "diagnostic", + } + ], + "provenance": { + "archiveRelativeCasePath": relative, + "settingsFilename": candidate.settings.name, + "provenanceTimestamp": candidate.timestamp.isoformat(), + "fingerprint": candidate.fingerprint, + }, + } + for attempt in range(3): + try: + response = client.post( + f"{api_base}/api/v1/diagnostics/scanner/link", + json=payload, + headers=headers, + ) + if ( + response.status_code == 204 + or response.status_code < 500 + and response.status_code not in {408, 429} + ): + break + except httpx.RequestError: + pass + time.sleep(2**attempt) + return 0 + + +if __name__ == "__main__": + raise SystemExit(run()) From ed5d4819333ec0e5c4f8f76feb8c27fb93b60c8d Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 13:10:57 -0700 Subject: [PATCH 07/28] Fix diagnostics provenance scanner validation --- .../ingestion/diagnostics_link_scanner.py | 94 ++++++++++++------- .../test_diagnostics_link_scanner.py | 62 ++++++++++++ 2 files changed, 121 insertions(+), 35 deletions(-) create mode 100644 backend/tests/features/ingestion/test_diagnostics_link_scanner.py diff --git a/backend/app/scripts/ingestion/diagnostics_link_scanner.py b/backend/app/scripts/ingestion/diagnostics_link_scanner.py index 0e138f5d..05dcf7cb 100644 --- a/backend/app/scripts/ingestion/diagnostics_link_scanner.py +++ b/backend/app/scripts/ingestion/diagnostics_link_scanner.py @@ -71,6 +71,26 @@ def _published_output(case_dir: Path) -> bool: return False +def _validate_layout(case_dir: Path, root: Path, values: dict[str, str]) -> None: + parts = case_dir.relative_to(root).parts + if len(parts) not in {3, 4} or parts[0] not in {"production", "development"}: + raise ValueError("Invalid diagnostics archive case layout") + if values["case_name"] != parts[-1]: + raise ValueError("Provenance case_name does not match archive layout") + expected_group = parts[-2] if len(parts) == 4 else None + if values.get("case_group") != expected_group: + raise ValueError("Provenance case_group does not match archive layout") + + +def _timestamp(cfg: Path) -> datetime | None: + match = TIMESTAMP_RE.match(cfg.name) + if match is None: + return None + return datetime.strptime(match.group(1), "%Y%m%d_%H%M%S_%f").replace( + tzinfo=timezone.utc + ) + + def discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 base = urlparse(public_base_url) candidates: list[Candidate] = [] @@ -78,25 +98,20 @@ def discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 tier_root = root / tier if not tier_root.is_dir(): continue + newest_by_case: dict[Path, tuple[Path, datetime]] = {} for cfg in tier_root.rglob("provenance.*.cfg"): if cfg.is_symlink() or root not in cfg.resolve().parents: continue - match = TIMESTAMP_RE.match(cfg.name) - if match is None: + timestamp = _timestamp(cfg) + if timestamp is None: continue case_dir = cfg.parent - timestamp = datetime.strptime(match.group(1), "%Y%m%d_%H%M%S_%f").replace( - tzinfo=timezone.utc - ) + prior = newest_by_case.get(case_dir) + if prior is None or timestamp > prior[1]: + newest_by_case[case_dir] = (cfg, timestamp) + + for case_dir, (cfg, timestamp) in newest_by_case.items(): settings = cfg.with_suffix(".settings") - # newest cfg controls: missing paired settings must defer this case. - prior = next( - (item for item in candidates if item.path.parent == case_dir), None - ) - if prior and prior.timestamp >= timestamp: - continue - if prior: - candidates.remove(prior) if not settings.is_file() or not _published_output(case_dir): continue try: @@ -109,11 +124,7 @@ def discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 raise ValueError( "Diagnostics URL outside configured public archive" ) - relative = case_dir.relative_to(root).as_posix() - if values.get("case_group") and values[ - "case_group" - ] not in relative.split("/"): - raise ValueError("Case group does not match archive layout") + _validate_layout(case_dir, root, values) digest = hashlib.sha256(settings.read_bytes()).hexdigest() candidates.append(Candidate(cfg, settings, timestamp, values, digest)) except (OSError, UnicodeError, ValueError) as exc: @@ -121,6 +132,23 @@ def discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 return candidates +def _request_with_retry(method, url: str, **kwargs) -> httpx.Response | None: + for attempt in range(3): + try: + response = method(url, **kwargs) + except httpx.RequestError: + response = None + if ( + response is not None + and response.status_code not in {408, 429} + and response.status_code < 500 + ): + return response + if attempt < 2: + time.sleep(2**attempt) + return response + + def run() -> int: machine = os.environ.get("MACHINE_NAME", "perlmutter") archive = resolve_archive(machine) @@ -135,11 +163,15 @@ def run() -> int: if dry_run: LOGGER.info("Would link diagnostics for %s", relative) continue - state = client.get( + state = _request_with_retry( + client.get, f"{api_base}/api/v1/diagnostics/scanner-state", params={"machine": machine, "archive_relative_case_path": relative}, headers=headers, ) + if state is None: + LOGGER.warning("State lookup failed for %s; deferring", relative) + continue if ( state.status_code == 200 and state.json() @@ -167,22 +199,14 @@ def run() -> int: "fingerprint": candidate.fingerprint, }, } - for attempt in range(3): - try: - response = client.post( - f"{api_base}/api/v1/diagnostics/scanner/link", - json=payload, - headers=headers, - ) - if ( - response.status_code == 204 - or response.status_code < 500 - and response.status_code not in {408, 429} - ): - break - except httpx.RequestError: - pass - time.sleep(2**attempt) + response = _request_with_retry( + client.post, + f"{api_base}/api/v1/diagnostics/scanner/link", + json=payload, + headers=headers, + ) + if response is None or response.status_code != 204: + LOGGER.warning("Diagnostics link submission failed for %s", relative) return 0 diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py new file mode 100644 index 00000000..aeb3ac87 --- /dev/null +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -0,0 +1,62 @@ +from pathlib import Path + +import httpx +import pytest + +from app.scripts.ingestion.diagnostics_link_scanner import ( + _request_with_retry, + discover, + parse_settings, +) + +BASE_URL = "https://diagnostics.example.org/archive" + + +def _case(root: Path, path: str, *, timestamp: str = "20260811_120000_000000") -> Path: + directory = root / path + directory.mkdir(parents=True) + cfg = directory / f"provenance.{timestamp}.cfg" + cfg.write_text("cfg", encoding="utf-8") + cfg.with_suffix(".settings").write_text( + "case_name = case\nmachine = perlmutter\nhpc_username = user\n" + "diagnostics_url = https://diagnostics.example.org/archive/case\n", + encoding="utf-8", + ) + (directory / "index.html").write_text("ready", encoding="utf-8") + return directory + + +def test_newest_missing_settings_defers_without_stale_fallback(tmp_path: Path) -> None: + directory = _case(tmp_path, "production/type/case") + (directory / "provenance.20260812_120000_000000.cfg").write_text("cfg") + assert discover(tmp_path, BASE_URL) == [] + + +def test_discovery_rejects_case_and_group_mismatches(tmp_path: Path) -> None: + directory = _case(tmp_path, "development/type/group/case") + settings = next(directory.glob("*.settings")) + settings.write_text( + settings.read_text().replace("case_name = case", "case_name = wrong") + ) + assert discover(tmp_path, BASE_URL) == [] + + +def test_parse_settings_rejects_duplicate_required_key(tmp_path: Path) -> None: + settings = tmp_path / "provenance.settings" + settings.write_text("case_name = one\ncase_name = two\n", encoding="utf-8") + with pytest.raises(ValueError): + parse_settings(settings) + + +def test_retry_helper_retries_transient_response( + monkeypatch: pytest.MonkeyPatch, +) -> None: + responses = [httpx.Response(503), httpx.Response(204)] + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner.time.sleep", lambda _: None + ) + response = _request_with_retry( + lambda *_args, **_kwargs: responses.pop(0), "https://x" + ) + assert response is not None + assert response.status_code == 204 From 797e8147fbb2c975955143f1129e96cd5c9bc9c9 Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 13:11:55 -0700 Subject: [PATCH 08/28] Constrain diagnostics scanner retries and symlinks --- .../app/scripts/ingestion/diagnostics_link_scanner.py | 9 +++++++-- .../ingestion/test_diagnostics_link_scanner.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/backend/app/scripts/ingestion/diagnostics_link_scanner.py b/backend/app/scripts/ingestion/diagnostics_link_scanner.py index 05dcf7cb..5405f3ac 100644 --- a/backend/app/scripts/ingestion/diagnostics_link_scanner.py +++ b/backend/app/scripts/ingestion/diagnostics_link_scanner.py @@ -112,7 +112,12 @@ def discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 for case_dir, (cfg, timestamp) in newest_by_case.items(): settings = cfg.with_suffix(".settings") - if not settings.is_file() or not _published_output(case_dir): + if ( + settings.is_symlink() + or root not in settings.resolve().parents + or not settings.is_file() + or not _published_output(case_dir) + ): continue try: values = parse_settings(settings) @@ -169,7 +174,7 @@ def run() -> int: params={"machine": machine, "archive_relative_case_path": relative}, headers=headers, ) - if state is None: + if state is None or state.status_code != 200: LOGGER.warning("State lookup failed for %s; deferring", relative) continue if ( diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index aeb3ac87..5d35048e 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -48,6 +48,16 @@ def test_parse_settings_rejects_duplicate_required_key(tmp_path: Path) -> None: parse_settings(settings) +def test_discovery_rejects_settings_symlink_outside_root(tmp_path: Path) -> None: + directory = _case(tmp_path, "production/type/case") + settings = next(directory.glob("*.settings")) + outside = tmp_path.parent / "outside.settings" + outside.write_text(settings.read_text(), encoding="utf-8") + settings.unlink() + settings.symlink_to(outside) + assert discover(tmp_path, BASE_URL) == [] + + def test_retry_helper_retries_transient_response( monkeypatch: pytest.MonkeyPatch, ) -> None: From 01eee96a213183ac88e75357ead7a0edf1e210e2 Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 13:18:38 -0700 Subject: [PATCH 09/28] Document diagnostics scanner operations --- backend/app/scripts/README.md | 29 ++++++++ .../sites/nersc-diagnostics-scanner.sh | 17 +++++ .../ingestion/sites/nersc.crontab.example | 3 + .../test_diagnostics_link_scanner.py | 67 +++++++++++++++++++ 4 files changed, 116 insertions(+) create mode 100755 backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh diff --git a/backend/app/scripts/README.md b/backend/app/scripts/README.md index 082ad006..de82f29b 100644 --- a/backend/app/scripts/README.md +++ b/backend/app/scripts/README.md @@ -209,6 +209,35 @@ the documented Chrysalis archive root, and records uploads under machine `ARCHIVE_YEAR_END` variables remain supported. `SCAN_MODE`, `ARCHIVE_YEAR_START`, and `MACHINE_NAME` are ignored because source site and scan scope are fixed. +## Diagnostics Provenance Scanner + +`diagnostics_link_scanner` discovers newest paired zppy provenance under the +reviewed, static diagnostics-archive registry and creates case-scoped diagnostic +links through the scanner API. It never reads Mache configuration at runtime. + +Run through the NERSC wrapper: + +```bash +SIMBOARD_API_TOKEN= \ +DRY_RUN=true \ +backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh +``` + +Required configuration: `SIMBOARD_API_BASE_URL`, `SIMBOARD_API_TOKEN`, and +`MACHINE_NAME`. The archive root and public URL come only from +`diagnostics_archives.py`. Start with `DRY_RUN=true`; it performs discovery and +state-safe planning without link/state writes. After log review, schedule the +provided cron example with `DRY_RUN=false`. + +Scanner account needs read/traverse access to archive `production/` and +`development/` trees, including provenance `.settings` files and published +diagnostic output. Transient network responses are retried; malformed, +output-not-ready, or failed candidates remain unstated and retry next scan. + +When a site archive moves, maintainers generate candidate values from Mache +`[web_portal]` cfg data during development, then update the checked-in registry +in a reviewed SimBoard change. Do not add runtime environment overrides for +archive roots or public URLs. ## HPC Upload Archive Ingestor diff --git a/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh b/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh new file mode 100755 index 00000000..b8020e03 --- /dev/null +++ b/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +BACKEND_DIR="$(cd -- "${SCRIPT_DIR}/../../../../" && pwd)" +PYTHON_BIN="${PYTHON_BIN:-${BACKEND_DIR}/.venv/bin/python}" + +[[ -x "${PYTHON_BIN}" ]] || { echo "Missing backend Python environment" >&2; exit 1; } +: "${SIMBOARD_API_TOKEN:?SIMBOARD_API_TOKEN must be set}" + +export SIMBOARD_API_BASE_URL="${SIMBOARD_API_BASE_URL:-https://simboard-dev-api.e3sm.org}" +export MACHINE_NAME="${MACHINE_NAME:-perlmutter}" +export DRY_RUN="${DRY_RUN:-true}" + +cd "${BACKEND_DIR}" +exec "${PYTHON_BIN}" -m app.scripts.ingestion.diagnostics_link_scanner "$@" diff --git a/backend/app/scripts/ingestion/sites/nersc.crontab.example b/backend/app/scripts/ingestion/sites/nersc.crontab.example index f154738c..dd6f80a1 100644 --- a/backend/app/scripts/ingestion/sites/nersc.crontab.example +++ b/backend/app/scripts/ingestion/sites/nersc.crontab.example @@ -24,3 +24,6 @@ OLD_PERF_ARCHIVE_ROOT=/global/cfs/projectdirs/e3sm/OLD_PERF # Archive scan: run daily at 03:15 UTC. Add ARCHIVE_YEAR_START / ARCHIVE_YEAR_END # here only when you want a scoped archive backfill. Values may use YYYY or YYYY-MM. 15 3 * * * cd ${REPO_DIR} && SCAN_MODE=archive ARCHIVE_YEAR_START=2025-01 ARCHIVE_YEAR_END=2025-03 ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc.sh.log 2>&1 + +# Diagnostics provenance scan: start dry-run, inspect logs, then set DRY_RUN=false. +20 * * * * cd ${REPO_DIR} && ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.log 2>&1 diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index 5d35048e..a9fb49e8 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -3,10 +3,12 @@ import httpx import pytest +from app.scripts.ingestion.diagnostics_archives import DiagnosticsArchive from app.scripts.ingestion.diagnostics_link_scanner import ( _request_with_retry, discover, parse_settings, + run, ) BASE_URL = "https://diagnostics.example.org/archive" @@ -70,3 +72,68 @@ def test_retry_helper_retries_transient_response( ) assert response is not None assert response.status_code == 204 + + +class _Client: + def __init__(self, get_response: httpx.Response) -> None: + self.get_response = get_response + self.get_calls: list[dict] = [] + self.post_calls: list[dict] = [] + + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def get(self, *_args, **kwargs): + self.get_calls.append(kwargs) + return self.get_response + + def post(self, *_args, **kwargs): + self.post_calls.append(kwargs) + return httpx.Response(204) + + +def test_run_submits_exact_payload_and_bearer_auth( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _case(tmp_path, "production/type/case") + client = _Client(httpx.Response(404)) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner.resolve_archive", + lambda _machine: DiagnosticsArchive(str(tmp_path), BASE_URL), + ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner.httpx.Client", + lambda **_kwargs: client, + ) + monkeypatch.setenv("SIMBOARD_API_BASE_URL", "https://api.example.org") + monkeypatch.setenv("SIMBOARD_API_TOKEN", "token") + monkeypatch.setenv("DRY_RUN", "false") + assert run() == 0 + assert client.post_calls[0]["headers"] == {"Authorization": "Bearer token"} + assert client.post_calls[0]["json"]["diagnostics"][0]["name"] == "zppy diagnostics" + + +def test_run_defers_after_exhausted_state_lookup( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _case(tmp_path, "production/type/case") + client = _Client(httpx.Response(503)) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner.time.sleep", lambda _: None + ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner.resolve_archive", + lambda _machine: DiagnosticsArchive(str(tmp_path), BASE_URL), + ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner.httpx.Client", + lambda **_kwargs: client, + ) + monkeypatch.setenv("SIMBOARD_API_BASE_URL", "https://api.example.org") + monkeypatch.setenv("SIMBOARD_API_TOKEN", "token") + monkeypatch.setenv("DRY_RUN", "false") + run() + assert client.post_calls == [] From 7376dd4810f362aabae9ada65358025a12ed7d27 Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 13:19:14 -0700 Subject: [PATCH 10/28] Correct diagnostics scanner payload test --- .../features/ingestion/test_diagnostics_link_scanner.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index a9fb49e8..ceb40d3c 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -99,7 +99,7 @@ def test_run_submits_exact_payload_and_bearer_auth( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: _case(tmp_path, "production/type/case") - client = _Client(httpx.Response(404)) + client = _Client(httpx.Response(200, json=None)) monkeypatch.setattr( "app.scripts.ingestion.diagnostics_link_scanner.resolve_archive", lambda _machine: DiagnosticsArchive(str(tmp_path), BASE_URL), @@ -112,6 +112,10 @@ def test_run_submits_exact_payload_and_bearer_auth( monkeypatch.setenv("SIMBOARD_API_TOKEN", "token") monkeypatch.setenv("DRY_RUN", "false") assert run() == 0 + assert client.get_calls[0]["params"] == { + "machine": "perlmutter", + "archive_relative_case_path": "production/type/case", + } assert client.post_calls[0]["headers"] == {"Authorization": "Bearer token"} assert client.post_calls[0]["json"]["diagnostics"][0]["name"] == "zppy diagnostics" From dc5f4498a32a0d62c1622dddeec5e337fa8b0e64 Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 13:20:46 -0700 Subject: [PATCH 11/28] Add Chrysalis diagnostics scanner settings --- backend/app/scripts/README.md | 5 +++++ .../scripts/ingestion/diagnostics_archives.py | 4 ++++ .../sites/chrysalis-diagnostics-scanner.sh | 17 +++++++++++++++++ .../ingestion/sites/nersc.crontab.example | 3 +++ .../ingestion/test_diagnostics_link_scanner.py | 14 +++++++++++++- 5 files changed, 42 insertions(+), 1 deletion(-) create mode 100755 backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh diff --git a/backend/app/scripts/README.md b/backend/app/scripts/README.md index de82f29b..874f7b0f 100644 --- a/backend/app/scripts/README.md +++ b/backend/app/scripts/README.md @@ -223,6 +223,11 @@ DRY_RUN=true \ backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh ``` +For LCRC, use `sites/chrysalis-diagnostics-scanner.sh`; it defaults +`MACHINE_NAME=chrysalis` and its reviewed registry entry maps +`/lcrc/group/e3sm/diagnostic_output` to +`https://web.lcrc.anl.gov/public/e3sm/diagnostic_output`. + Required configuration: `SIMBOARD_API_BASE_URL`, `SIMBOARD_API_TOKEN`, and `MACHINE_NAME`. The archive root and public URL come only from `diagnostics_archives.py`. Start with `DRY_RUN=true`; it performs discovery and diff --git a/backend/app/scripts/ingestion/diagnostics_archives.py b/backend/app/scripts/ingestion/diagnostics_archives.py index 09f56532..b0bad18a 100644 --- a/backend/app/scripts/ingestion/diagnostics_archives.py +++ b/backend/app/scripts/ingestion/diagnostics_archives.py @@ -27,4 +27,8 @@ class DiagnosticsArchive: root="/global/cfs/cdirs/e3sm/diagnostic_output", public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostic_output", ), + "chrysalis": DiagnosticsArchive( + root="/lcrc/group/e3sm/diagnostic_output", + public_base_url="https://web.lcrc.anl.gov/public/e3sm/diagnostic_output", + ), } diff --git a/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh b/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh new file mode 100755 index 00000000..0993ea58 --- /dev/null +++ b/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +BACKEND_DIR="$(cd -- "${SCRIPT_DIR}/../../../../" && pwd)" +PYTHON_BIN="${PYTHON_BIN:-${BACKEND_DIR}/.venv/bin/python}" + +[[ -x "${PYTHON_BIN}" ]] || { echo "Missing backend Python environment" >&2; exit 1; } +: "${SIMBOARD_API_TOKEN:?SIMBOARD_API_TOKEN must be set}" + +export SIMBOARD_API_BASE_URL="${SIMBOARD_API_BASE_URL:-https://simboard-dev-api.e3sm.org}" +export MACHINE_NAME="${MACHINE_NAME:-chrysalis}" +export DRY_RUN="${DRY_RUN:-true}" + +cd "${BACKEND_DIR}" +exec "${PYTHON_BIN}" -m app.scripts.ingestion.diagnostics_link_scanner "$@" diff --git a/backend/app/scripts/ingestion/sites/nersc.crontab.example b/backend/app/scripts/ingestion/sites/nersc.crontab.example index dd6f80a1..ebdf18f0 100644 --- a/backend/app/scripts/ingestion/sites/nersc.crontab.example +++ b/backend/app/scripts/ingestion/sites/nersc.crontab.example @@ -27,3 +27,6 @@ OLD_PERF_ARCHIVE_ROOT=/global/cfs/projectdirs/e3sm/OLD_PERF # Diagnostics provenance scan: start dry-run, inspect logs, then set DRY_RUN=false. 20 * * * * cd ${REPO_DIR} && ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.log 2>&1 + +# Chrysalis diagnostics provenance scan; use its local checkout path for REPO_DIR. +25 * * * * cd ${REPO_DIR} && ${REPO_DIR}/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.log 2>&1 diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index ceb40d3c..c3ec6d9f 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -3,7 +3,10 @@ import httpx import pytest -from app.scripts.ingestion.diagnostics_archives import DiagnosticsArchive +from app.scripts.ingestion.diagnostics_archives import ( + DIAGNOSTICS_ARCHIVES_BY_MACHINE, + DiagnosticsArchive, +) from app.scripts.ingestion.diagnostics_link_scanner import ( _request_with_retry, discover, @@ -14,6 +17,15 @@ BASE_URL = "https://diagnostics.example.org/archive" +def test_chrysalis_diagnostics_archive_settings() -> None: + archive = DIAGNOSTICS_ARCHIVES_BY_MACHINE["chrysalis"] + assert archive.root == "/lcrc/group/e3sm/diagnostic_output" + assert ( + archive.public_base_url + == "https://web.lcrc.anl.gov/public/e3sm/diagnostic_output" + ) + + def _case(root: Path, path: str, *, timestamp: str = "20260811_120000_000000") -> Path: directory = root / path directory.mkdir(parents=True) From e4db96e584e9d3561872db8eeb196af7e94deb57 Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 13:25:18 -0700 Subject: [PATCH 12/28] Re-order functions --- .../ingestion/diagnostics_link_scanner.py | 253 ++++++++++-------- .../test_diagnostics_link_scanner.py | 12 +- 2 files changed, 152 insertions(+), 113 deletions(-) diff --git a/backend/app/scripts/ingestion/diagnostics_link_scanner.py b/backend/app/scripts/ingestion/diagnostics_link_scanner.py index 5405f3ac..a44ff446 100644 --- a/backend/app/scripts/ingestion/diagnostics_link_scanner.py +++ b/backend/app/scripts/ingestion/diagnostics_link_scanner.py @@ -33,80 +33,115 @@ class Candidate: fingerprint: str -def resolve_archive(machine_name: str) -> DiagnosticsArchive: +def run() -> int: + machine = os.environ.get("MACHINE_NAME", "perlmutter") + archive = _resolve_archive(machine) + root = Path(archive.root) + dry_run = os.environ.get("DRY_RUN", "true").lower() in {"1", "true", "yes"} + api_base = os.environ["SIMBOARD_API_BASE_URL"].rstrip("/") + token = os.environ["SIMBOARD_API_TOKEN"] + headers = {"Authorization": f"Bearer {token}"} + + with httpx.Client(timeout=30) as client: + for candidate in _discover(root, archive.public_base_url): + relative = candidate.path.parent.relative_to(root).as_posix() + + if dry_run: + LOGGER.info("Would link diagnostics for %s", relative) + continue + + state = _request_with_retry( + client.get, + f"{api_base}/api/v1/diagnostics/scanner-state", + params={"machine": machine, "archive_relative_case_path": relative}, + headers=headers, + ) + + if state is None or state.status_code != 200: + LOGGER.warning("State lookup failed for %s; deferring", relative) + continue + + if ( + state.status_code == 200 + and state.json() + and ( + state.json().get("settingsFilename") == candidate.settings.name + and state.json().get("fingerprint") == candidate.fingerprint + ) + ): + continue + + payload = { + "caseName": candidate.values["case_name"], + "machine": candidate.values["machine"], + "hpcUsername": candidate.values["hpc_username"], + "diagnostics": [ + { + "name": "zppy diagnostics", + "url": candidate.values["diagnostics_url"], + "kind": "diagnostic", + } + ], + "provenance": { + "archiveRelativeCasePath": relative, + "settingsFilename": candidate.settings.name, + "provenanceTimestamp": candidate.timestamp.isoformat(), + "fingerprint": candidate.fingerprint, + }, + } + + response = _request_with_retry( + client.post, + f"{api_base}/api/v1/diagnostics/scanner/link", + json=payload, + headers=headers, + ) + + if response is None or response.status_code != 204: + LOGGER.warning("Diagnostics link submission failed for %s", relative) + + return 0 + + +def _resolve_archive(machine_name: str) -> DiagnosticsArchive: archive = DIAGNOSTICS_ARCHIVES_BY_MACHINE.get(machine_name.lower()) if archive is None: raise ValueError(f"Unsupported diagnostics scanner machine: {machine_name}") + root = Path(archive.root) parsed = urlparse(archive.public_base_url) if not root.is_absolute() or not root.is_dir() or not os.access(root, os.R_OK): raise ValueError(f"Diagnostics archive is not readable: {root}") + if parsed.scheme not in {"http", "https"} or not parsed.netloc: raise ValueError("Diagnostics archive public URL must be absolute HTTP(S)") - return archive - - -def parse_settings(path: Path) -> dict[str, str]: - values: dict[str, str] = {} - for line in path.read_text(encoding="utf-8").splitlines(): - if not line.strip() or line.lstrip().startswith("#"): - continue - if "=" not in line: - raise ValueError("Malformed provenance settings line") - key, value = (part.strip() for part in line.split("=", 1)) - if not key or not value or key in values: - raise ValueError("Malformed or duplicate provenance setting") - values[key] = value - if REQUIRED_SETTINGS - values.keys(): - raise ValueError("Missing required provenance settings") - return values - - -def _published_output(case_dir: Path) -> bool: - for entry in case_dir.iterdir(): - if entry.name.startswith("provenance."): - continue - if entry.is_file() or (entry.is_dir() and any(entry.iterdir())): - return True - return False - - -def _validate_layout(case_dir: Path, root: Path, values: dict[str, str]) -> None: - parts = case_dir.relative_to(root).parts - if len(parts) not in {3, 4} or parts[0] not in {"production", "development"}: - raise ValueError("Invalid diagnostics archive case layout") - if values["case_name"] != parts[-1]: - raise ValueError("Provenance case_name does not match archive layout") - expected_group = parts[-2] if len(parts) == 4 else None - if values.get("case_group") != expected_group: - raise ValueError("Provenance case_group does not match archive layout") - -def _timestamp(cfg: Path) -> datetime | None: - match = TIMESTAMP_RE.match(cfg.name) - if match is None: - return None - return datetime.strptime(match.group(1), "%Y%m%d_%H%M%S_%f").replace( - tzinfo=timezone.utc - ) + return archive -def discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 +def _discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 base = urlparse(public_base_url) candidates: list[Candidate] = [] + for tier in ("production", "development"): tier_root = root / tier + if not tier_root.is_dir(): continue + newest_by_case: dict[Path, tuple[Path, datetime]] = {} + for cfg in tier_root.rglob("provenance.*.cfg"): if cfg.is_symlink() or root not in cfg.resolve().parents: continue + timestamp = _timestamp(cfg) if timestamp is None: continue + case_dir = cfg.parent prior = newest_by_case.get(case_dir) + if prior is None or timestamp > prior[1]: newest_by_case[case_dir] = (cfg, timestamp) @@ -120,8 +155,9 @@ def discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 ): continue try: - values = parse_settings(settings) + values = _parse_settings(settings) url = urlparse(values["diagnostics_url"]) + if (url.scheme, url.netloc) != ( base.scheme, base.netloc, @@ -130,10 +166,12 @@ def discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 "Diagnostics URL outside configured public archive" ) _validate_layout(case_dir, root, values) + digest = hashlib.sha256(settings.read_bytes()).hexdigest() candidates.append(Candidate(cfg, settings, timestamp, values, digest)) except (OSError, UnicodeError, ValueError) as exc: LOGGER.warning("Skipping invalid provenance %s: %s", cfg, exc) + return candidates @@ -149,70 +187,71 @@ def _request_with_retry(method, url: str, **kwargs) -> httpx.Response | None: and response.status_code < 500 ): return response + if attempt < 2: time.sleep(2**attempt) + return response -def run() -> int: - machine = os.environ.get("MACHINE_NAME", "perlmutter") - archive = resolve_archive(machine) - root = Path(archive.root) - dry_run = os.environ.get("DRY_RUN", "true").lower() in {"1", "true", "yes"} - api_base = os.environ["SIMBOARD_API_BASE_URL"].rstrip("/") - token = os.environ["SIMBOARD_API_TOKEN"] - headers = {"Authorization": f"Bearer {token}"} - with httpx.Client(timeout=30) as client: - for candidate in discover(root, archive.public_base_url): - relative = candidate.path.parent.relative_to(root).as_posix() - if dry_run: - LOGGER.info("Would link diagnostics for %s", relative) - continue - state = _request_with_retry( - client.get, - f"{api_base}/api/v1/diagnostics/scanner-state", - params={"machine": machine, "archive_relative_case_path": relative}, - headers=headers, - ) - if state is None or state.status_code != 200: - LOGGER.warning("State lookup failed for %s; deferring", relative) - continue - if ( - state.status_code == 200 - and state.json() - and ( - state.json().get("settingsFilename") == candidate.settings.name - and state.json().get("fingerprint") == candidate.fingerprint - ) - ): - continue - payload = { - "caseName": candidate.values["case_name"], - "machine": candidate.values["machine"], - "hpcUsername": candidate.values["hpc_username"], - "diagnostics": [ - { - "name": "zppy diagnostics", - "url": candidate.values["diagnostics_url"], - "kind": "diagnostic", - } - ], - "provenance": { - "archiveRelativeCasePath": relative, - "settingsFilename": candidate.settings.name, - "provenanceTimestamp": candidate.timestamp.isoformat(), - "fingerprint": candidate.fingerprint, - }, - } - response = _request_with_retry( - client.post, - f"{api_base}/api/v1/diagnostics/scanner/link", - json=payload, - headers=headers, - ) - if response is None or response.status_code != 204: - LOGGER.warning("Diagnostics link submission failed for %s", relative) - return 0 +def _parse_settings(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + + if "=" not in line: + raise ValueError("Malformed provenance settings line") + + key, value = (part.strip() for part in line.split("=", 1)) + + if not key or not value or key in values: + raise ValueError("Malformed or duplicate provenance setting") + + values[key] = value + + if REQUIRED_SETTINGS - values.keys(): + raise ValueError("Missing required provenance settings") + + return values + + +def _published_output(case_dir: Path) -> bool: + for entry in case_dir.iterdir(): + if entry.name.startswith("provenance."): + continue + + if entry.is_file() or (entry.is_dir() and any(entry.iterdir())): + return True + + return False + + +def _validate_layout(case_dir: Path, root: Path, values: dict[str, str]) -> None: + parts = case_dir.relative_to(root).parts + + if len(parts) not in {3, 4} or parts[0] not in {"production", "development"}: + raise ValueError("Invalid diagnostics archive case layout") + + if values["case_name"] != parts[-1]: + raise ValueError("Provenance case_name does not match archive layout") + + expected_group = parts[-2] if len(parts) == 4 else None + + if values.get("case_group") != expected_group: + raise ValueError("Provenance case_group does not match archive layout") + + +def _timestamp(cfg: Path) -> datetime | None: + match = TIMESTAMP_RE.match(cfg.name) + + if match is None: + return None + + return datetime.strptime(match.group(1), "%Y%m%d_%H%M%S_%f").replace( + tzinfo=timezone.utc + ) if __name__ == "__main__": diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index c3ec6d9f..4740be3f 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -8,9 +8,9 @@ DiagnosticsArchive, ) from app.scripts.ingestion.diagnostics_link_scanner import ( + _discover, + _parse_settings, _request_with_retry, - discover, - parse_settings, run, ) @@ -43,7 +43,7 @@ def _case(root: Path, path: str, *, timestamp: str = "20260811_120000_000000") - def test_newest_missing_settings_defers_without_stale_fallback(tmp_path: Path) -> None: directory = _case(tmp_path, "production/type/case") (directory / "provenance.20260812_120000_000000.cfg").write_text("cfg") - assert discover(tmp_path, BASE_URL) == [] + assert _discover(tmp_path, BASE_URL) == [] def test_discovery_rejects_case_and_group_mismatches(tmp_path: Path) -> None: @@ -52,14 +52,14 @@ def test_discovery_rejects_case_and_group_mismatches(tmp_path: Path) -> None: settings.write_text( settings.read_text().replace("case_name = case", "case_name = wrong") ) - assert discover(tmp_path, BASE_URL) == [] + assert _discover(tmp_path, BASE_URL) == [] def test_parse_settings_rejects_duplicate_required_key(tmp_path: Path) -> None: settings = tmp_path / "provenance.settings" settings.write_text("case_name = one\ncase_name = two\n", encoding="utf-8") with pytest.raises(ValueError): - parse_settings(settings) + _parse_settings(settings) def test_discovery_rejects_settings_symlink_outside_root(tmp_path: Path) -> None: @@ -69,7 +69,7 @@ def test_discovery_rejects_settings_symlink_outside_root(tmp_path: Path) -> None outside.write_text(settings.read_text(), encoding="utf-8") settings.unlink() settings.symlink_to(outside) - assert discover(tmp_path, BASE_URL) == [] + assert _discover(tmp_path, BASE_URL) == [] def test_retry_helper_retries_transient_response( From 7529c60f2588a71414a30f2a34e2b9e064164c6f Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 13:27:25 -0700 Subject: [PATCH 13/28] Add source to mache --- backend/app/features/catalog/api.py | 7 +++++++ backend/app/scripts/ingestion/diagnostics_archives.py | 1 + 2 files changed, 8 insertions(+) diff --git a/backend/app/features/catalog/api.py b/backend/app/features/catalog/api.py index 40e5731e..b8573c7c 100644 --- a/backend/app/features/catalog/api.py +++ b/backend/app/features/catalog/api.py @@ -763,8 +763,10 @@ def get_diagnostics_scanner_state( """Return successful scanner state for one machine/archive case path.""" _require_diagnostics_scanner_role(user) resolved_machine = resolve_machine_by_name(db, machine) + if resolved_machine is None: raise HTTPException(status_code=404, detail="Unknown machine.") + state = ( db.query(DiagnosticProvenanceState) .filter(DiagnosticProvenanceState.machine_name == resolved_machine.name) @@ -774,6 +776,7 @@ def get_diagnostics_scanner_state( ) .one_or_none() ) + return DiagnosticProvenanceStateOut.model_validate(state) if state else None @@ -785,10 +788,12 @@ def link_scanner_diagnostics( ) -> None: """Atomically upsert one scanner-managed case diagnostic link and state.""" _require_diagnostics_scanner_role(user) + if len(payload.diagnostics) != 1: raise HTTPException( status_code=422, detail="Scanner payload requires one diagnostic." ) + if _unsafe_archive_relative_path(payload.provenance.archive_relative_case_path): raise HTTPException( status_code=422, detail="Invalid archive-relative case path." @@ -797,6 +802,7 @@ def link_scanner_diagnostics( machine = resolve_machine_by_name(db, payload.machine) if machine is None: raise HTTPException(status_code=404, detail="No matching case found.") + case_id = _resolve_case_id_for_diagnostics_link( db=db, case_name=payload.case_name, @@ -805,6 +811,7 @@ def link_scanner_diagnostics( ) diagnostic = payload.diagnostics[0] now = datetime.now(timezone.utc) + with transaction(db): link_id = db.execute( pg_insert(ExternalLink) diff --git a/backend/app/scripts/ingestion/diagnostics_archives.py b/backend/app/scripts/ingestion/diagnostics_archives.py index b0bad18a..8ef7fe14 100644 --- a/backend/app/scripts/ingestion/diagnostics_archives.py +++ b/backend/app/scripts/ingestion/diagnostics_archives.py @@ -10,6 +10,7 @@ class DiagnosticsArchive: # Refresh from Mache [web_portal] configuration in a reviewed change when sites move. +# Source: https://github.com/E3SM-Project/mache/tree/main/mache/machines DIAGNOSTICS_ARCHIVES_BY_MACHINE: dict[str, DiagnosticsArchive] = { "perlmutter": DiagnosticsArchive( root="/global/cfs/cdirs/e3sm/diagnostic_output", From 2afe4adf0b7ba657968f17efd2cc1628bd4fb85f Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 13:31:13 -0700 Subject: [PATCH 14/28] Harden diagnostics scanner archive reads --- backend/app/scripts/README.md | 40 ++++++++----------- .../ingestion/diagnostics_link_scanner.py | 28 ++++++++++--- .../test_diagnostics_link_scanner.py | 19 ++++++++- 3 files changed, 55 insertions(+), 32 deletions(-) diff --git a/backend/app/scripts/README.md b/backend/app/scripts/README.md index 874f7b0f..93d37001 100644 --- a/backend/app/scripts/README.md +++ b/backend/app/scripts/README.md @@ -18,10 +18,14 @@ scripts/ │ ├── archive_ingestor_core.py │ ├── archive_layout.py │ ├── archive_workflow.py +│ ├── diagnostics_archives.py +│ ├── diagnostics_link_scanner.py │ ├── hpc_upload_archive_ingestor.py │ ├── nersc_archive_ingestor.py │ ├── sites/ -│ └── nersc.sh +│ │ ├── chrysalis-diagnostics-scanner.sh +│ │ ├── nersc-diagnostics-scanner.sh +│ │ └── nersc.sh │ └── v3_data/ │ ├── __init__.py │ ├── lcrc-v3.env.example @@ -211,9 +215,8 @@ the documented Chrysalis archive root, and records uploads under machine scan scope are fixed. ## Diagnostics Provenance Scanner -`diagnostics_link_scanner` discovers newest paired zppy provenance under the -reviewed, static diagnostics-archive registry and creates case-scoped diagnostic -links through the scanner API. It never reads Mache configuration at runtime. +Scans newest paired zppy provenance from the reviewed static registry and creates +case-scoped diagnostic links. It never reads Mache configuration at runtime. Run through the NERSC wrapper: @@ -223,26 +226,15 @@ DRY_RUN=true \ backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh ``` -For LCRC, use `sites/chrysalis-diagnostics-scanner.sh`; it defaults -`MACHINE_NAME=chrysalis` and its reviewed registry entry maps -`/lcrc/group/e3sm/diagnostic_output` to -`https://web.lcrc.anl.gov/public/e3sm/diagnostic_output`. - -Required configuration: `SIMBOARD_API_BASE_URL`, `SIMBOARD_API_TOKEN`, and -`MACHINE_NAME`. The archive root and public URL come only from -`diagnostics_archives.py`. Start with `DRY_RUN=true`; it performs discovery and -state-safe planning without link/state writes. After log review, schedule the -provided cron example with `DRY_RUN=false`. - -Scanner account needs read/traverse access to archive `production/` and -`development/` trees, including provenance `.settings` files and published -diagnostic output. Transient network responses are retried; malformed, -output-not-ready, or failed candidates remain unstated and retry next scan. - -When a site archive moves, maintainers generate candidate values from Mache -`[web_portal]` cfg data during development, then update the checked-in registry -in a reviewed SimBoard change. Do not add runtime environment overrides for -archive roots or public URLs. +Use `sites/chrysalis-diagnostics-scanner.sh` at LCRC. Required: API base URL, +service-account token, and machine name. Roots and public URLs come only from +`diagnostics_archives.py`. + +Start with `DRY_RUN=true`; inspect logs, then schedule with `DRY_RUN=false`. +Scanner account needs read/traverse access to `production/` and `development/`, +provenance settings, and published output. Failed or not-ready candidates retry +next run. Refresh registry entries from Mache `[web_portal]` cfg data only in a +reviewed change; never add archive-path environment overrides. ## HPC Upload Archive Ingestor diff --git a/backend/app/scripts/ingestion/diagnostics_link_scanner.py b/backend/app/scripts/ingestion/diagnostics_link_scanner.py index a44ff446..95dfaaa5 100644 --- a/backend/app/scripts/ingestion/diagnostics_link_scanner.py +++ b/backend/app/scripts/ingestion/diagnostics_link_scanner.py @@ -22,6 +22,8 @@ LOGGER = logging.getLogger(__name__) TIMESTAMP_RE = re.compile(r"^provenance\.(\d{8}_\d{6}_\d{6})\.cfg$") REQUIRED_SETTINGS = {"case_name", "machine", "hpc_username", "diagnostics_url"} +MAX_SETTINGS_BYTES = 64 * 1024 +MAX_SETTINGS_LINES = 200 @dataclass(frozen=True) @@ -151,11 +153,12 @@ def _discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C90 settings.is_symlink() or root not in settings.resolve().parents or not settings.is_file() - or not _published_output(case_dir) + or not _published_output(case_dir, root) ): continue try: - values = _parse_settings(settings) + settings_bytes = _read_settings_bytes(settings) + values = _parse_settings_bytes(settings_bytes) url = urlparse(values["diagnostics_url"]) if (url.scheme, url.netloc) != ( @@ -167,7 +170,7 @@ def _discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C90 ) _validate_layout(case_dir, root, values) - digest = hashlib.sha256(settings.read_bytes()).hexdigest() + digest = hashlib.sha256(settings_bytes).hexdigest() candidates.append(Candidate(cfg, settings, timestamp, values, digest)) except (OSError, UnicodeError, ValueError) as exc: LOGGER.warning("Skipping invalid provenance %s: %s", cfg, exc) @@ -194,10 +197,20 @@ def _request_with_retry(method, url: str, **kwargs) -> httpx.Response | None: return response -def _parse_settings(path: Path) -> dict[str, str]: +def _read_settings_bytes(path: Path) -> bytes: + with path.open("rb") as settings_file: + content = settings_file.read(MAX_SETTINGS_BYTES + 1) + if len(content) > MAX_SETTINGS_BYTES: + raise ValueError("Provenance settings file is too large") + return content + + +def _parse_settings_bytes(content: bytes) -> dict[str, str]: values: dict[str, str] = {} - for line in path.read_text(encoding="utf-8").splitlines(): + for line_number, line in enumerate(content.decode("utf-8").splitlines(), start=1): + if line_number > MAX_SETTINGS_LINES: + raise ValueError("Provenance settings file has too many lines") if not line.strip() or line.lstrip().startswith("#"): continue @@ -217,11 +230,14 @@ def _parse_settings(path: Path) -> dict[str, str]: return values -def _published_output(case_dir: Path) -> bool: +def _published_output(case_dir: Path, root: Path) -> bool: for entry in case_dir.iterdir(): if entry.name.startswith("provenance."): continue + if entry.is_symlink() or root not in entry.resolve().parents: + continue + if entry.is_file() or (entry.is_dir() and any(entry.iterdir())): return True diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index 4740be3f..996c320e 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -9,7 +9,7 @@ ) from app.scripts.ingestion.diagnostics_link_scanner import ( _discover, - _parse_settings, + _parse_settings_bytes, _request_with_retry, run, ) @@ -59,7 +59,12 @@ def test_parse_settings_rejects_duplicate_required_key(tmp_path: Path) -> None: settings = tmp_path / "provenance.settings" settings.write_text("case_name = one\ncase_name = two\n", encoding="utf-8") with pytest.raises(ValueError): - _parse_settings(settings) + _parse_settings_bytes(settings.read_bytes()) + + +def test_parse_settings_rejects_oversized_file() -> None: + with pytest.raises(ValueError): + _parse_settings_bytes(b"x" * (64 * 1024 + 1)) def test_discovery_rejects_settings_symlink_outside_root(tmp_path: Path) -> None: @@ -72,6 +77,16 @@ def test_discovery_rejects_settings_symlink_outside_root(tmp_path: Path) -> None assert _discover(tmp_path, BASE_URL) == [] +def test_discovery_rejects_external_output_directory_symlink(tmp_path: Path) -> None: + directory = _case(tmp_path, "production/type/case") + (directory / "index.html").unlink() + outside = tmp_path.parent / "outside-output" + outside.mkdir(exist_ok=True) + (outside / "index.html").write_text("ready") + (directory / "output").symlink_to(outside, target_is_directory=True) + assert _discover(tmp_path, BASE_URL) == [] + + def test_retry_helper_retries_transient_response( monkeypatch: pytest.MonkeyPatch, ) -> None: From f5079d254ea119c7630cfc8319066843971f2439 Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 13:32:27 -0700 Subject: [PATCH 15/28] Fix diagnostics scanner safety tests --- .../ingestion/test_diagnostics_link_scanner.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index 996c320e..8e588d7e 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -10,6 +10,7 @@ from app.scripts.ingestion.diagnostics_link_scanner import ( _discover, _parse_settings_bytes, + _read_settings_bytes, _request_with_retry, run, ) @@ -62,9 +63,11 @@ def test_parse_settings_rejects_duplicate_required_key(tmp_path: Path) -> None: _parse_settings_bytes(settings.read_bytes()) -def test_parse_settings_rejects_oversized_file() -> None: +def test_settings_reader_rejects_oversized_file(tmp_path: Path) -> None: + settings = tmp_path / "provenance.settings" + settings.write_bytes(b"x" * (64 * 1024 + 1)) with pytest.raises(ValueError): - _parse_settings_bytes(b"x" * (64 * 1024 + 1)) + _read_settings_bytes(settings) def test_discovery_rejects_settings_symlink_outside_root(tmp_path: Path) -> None: @@ -128,7 +131,7 @@ def test_run_submits_exact_payload_and_bearer_auth( _case(tmp_path, "production/type/case") client = _Client(httpx.Response(200, json=None)) monkeypatch.setattr( - "app.scripts.ingestion.diagnostics_link_scanner.resolve_archive", + "app.scripts.ingestion.diagnostics_link_scanner._resolve_archive", lambda _machine: DiagnosticsArchive(str(tmp_path), BASE_URL), ) monkeypatch.setattr( @@ -156,7 +159,7 @@ def test_run_defers_after_exhausted_state_lookup( "app.scripts.ingestion.diagnostics_link_scanner.time.sleep", lambda _: None ) monkeypatch.setattr( - "app.scripts.ingestion.diagnostics_link_scanner.resolve_archive", + "app.scripts.ingestion.diagnostics_link_scanner._resolve_archive", lambda _machine: DiagnosticsArchive(str(tmp_path), BASE_URL), ) monkeypatch.setattr( From 980c8516ee4ece390a45f8096d501a6608a50654 Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 13:39:09 -0700 Subject: [PATCH 16/28] Allow credential-free diagnostics scanner dry runs --- backend/app/scripts/README.md | 3 ++- .../scripts/ingestion/diagnostics_link_scanner.py | 14 +++++++++----- .../sites/chrysalis-diagnostics-scanner.sh | 5 ++++- .../ingestion/sites/nersc-diagnostics-scanner.sh | 5 ++++- .../ingestion/test_diagnostics_link_scanner.py | 14 ++++++++++++++ 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/backend/app/scripts/README.md b/backend/app/scripts/README.md index 93d37001..4b6a761c 100644 --- a/backend/app/scripts/README.md +++ b/backend/app/scripts/README.md @@ -230,7 +230,8 @@ Use `sites/chrysalis-diagnostics-scanner.sh` at LCRC. Required: API base URL, service-account token, and machine name. Roots and public URLs come only from `diagnostics_archives.py`. -Start with `DRY_RUN=true`; inspect logs, then schedule with `DRY_RUN=false`. +Start with `DRY_RUN=true`; it needs no API URL or token. Inspect logs, then +schedule with `DRY_RUN=false`, which requires both API URL and service token. Scanner account needs read/traverse access to `production/` and `development/`, provenance settings, and published output. Failed or not-ready candidates retry next run. Refresh registry entries from Mache `[web_portal]` cfg data only in a diff --git a/backend/app/scripts/ingestion/diagnostics_link_scanner.py b/backend/app/scripts/ingestion/diagnostics_link_scanner.py index 95dfaaa5..725ed31f 100644 --- a/backend/app/scripts/ingestion/diagnostics_link_scanner.py +++ b/backend/app/scripts/ingestion/diagnostics_link_scanner.py @@ -40,18 +40,22 @@ def run() -> int: archive = _resolve_archive(machine) root = Path(archive.root) dry_run = os.environ.get("DRY_RUN", "true").lower() in {"1", "true", "yes"} + candidates = _discover(root, archive.public_base_url) + + if dry_run: + for candidate in candidates: + relative = candidate.path.parent.relative_to(root).as_posix() + LOGGER.info("Would link diagnostics for %s", relative) + return 0 + api_base = os.environ["SIMBOARD_API_BASE_URL"].rstrip("/") token = os.environ["SIMBOARD_API_TOKEN"] headers = {"Authorization": f"Bearer {token}"} with httpx.Client(timeout=30) as client: - for candidate in _discover(root, archive.public_base_url): + for candidate in candidates: relative = candidate.path.parent.relative_to(root).as_posix() - if dry_run: - LOGGER.info("Would link diagnostics for %s", relative) - continue - state = _request_with_retry( client.get, f"{api_base}/api/v1/diagnostics/scanner-state", diff --git a/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh b/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh index 0993ea58..35d154a9 100755 --- a/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh +++ b/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh @@ -7,11 +7,14 @@ BACKEND_DIR="$(cd -- "${SCRIPT_DIR}/../../../../" && pwd)" PYTHON_BIN="${PYTHON_BIN:-${BACKEND_DIR}/.venv/bin/python}" [[ -x "${PYTHON_BIN}" ]] || { echo "Missing backend Python environment" >&2; exit 1; } -: "${SIMBOARD_API_TOKEN:?SIMBOARD_API_TOKEN must be set}" export SIMBOARD_API_BASE_URL="${SIMBOARD_API_BASE_URL:-https://simboard-dev-api.e3sm.org}" export MACHINE_NAME="${MACHINE_NAME:-chrysalis}" export DRY_RUN="${DRY_RUN:-true}" +if [[ "${DRY_RUN,,}" != "true" && "${DRY_RUN}" != "1" && "${DRY_RUN,,}" != "yes" ]]; then + : "${SIMBOARD_API_TOKEN:?SIMBOARD_API_TOKEN must be set when DRY_RUN is false}" +fi + cd "${BACKEND_DIR}" exec "${PYTHON_BIN}" -m app.scripts.ingestion.diagnostics_link_scanner "$@" diff --git a/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh b/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh index b8020e03..60f88428 100755 --- a/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh +++ b/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh @@ -7,11 +7,14 @@ BACKEND_DIR="$(cd -- "${SCRIPT_DIR}/../../../../" && pwd)" PYTHON_BIN="${PYTHON_BIN:-${BACKEND_DIR}/.venv/bin/python}" [[ -x "${PYTHON_BIN}" ]] || { echo "Missing backend Python environment" >&2; exit 1; } -: "${SIMBOARD_API_TOKEN:?SIMBOARD_API_TOKEN must be set}" export SIMBOARD_API_BASE_URL="${SIMBOARD_API_BASE_URL:-https://simboard-dev-api.e3sm.org}" export MACHINE_NAME="${MACHINE_NAME:-perlmutter}" export DRY_RUN="${DRY_RUN:-true}" +if [[ "${DRY_RUN,,}" != "true" && "${DRY_RUN}" != "1" && "${DRY_RUN,,}" != "yes" ]]; then + : "${SIMBOARD_API_TOKEN:?SIMBOARD_API_TOKEN must be set when DRY_RUN is false}" +fi + cd "${BACKEND_DIR}" exec "${PYTHON_BIN}" -m app.scripts.ingestion.diagnostics_link_scanner "$@" diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index 8e588d7e..beb380d6 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -150,6 +150,20 @@ def test_run_submits_exact_payload_and_bearer_auth( assert client.post_calls[0]["json"]["diagnostics"][0]["name"] == "zppy diagnostics" +def test_dry_run_requires_no_api_configuration( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _case(tmp_path, "production/type/case") + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._resolve_archive", + lambda _machine: DiagnosticsArchive(str(tmp_path), BASE_URL), + ) + monkeypatch.delenv("SIMBOARD_API_BASE_URL", raising=False) + monkeypatch.delenv("SIMBOARD_API_TOKEN", raising=False) + monkeypatch.setenv("DRY_RUN", "true") + assert run() == 0 + + def test_run_defers_after_exhausted_state_lookup( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 25704257c171fc7c8afa7659da71eddb24bd4dae Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 13:46:23 -0700 Subject: [PATCH 17/28] Fix diagnostic archive paths for machines --- .../scripts/ingestion/diagnostics_archives.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/app/scripts/ingestion/diagnostics_archives.py b/backend/app/scripts/ingestion/diagnostics_archives.py index 8ef7fe14..c1124c32 100644 --- a/backend/app/scripts/ingestion/diagnostics_archives.py +++ b/backend/app/scripts/ingestion/diagnostics_archives.py @@ -13,23 +13,23 @@ class DiagnosticsArchive: # Source: https://github.com/E3SM-Project/mache/tree/main/mache/machines DIAGNOSTICS_ARCHIVES_BY_MACHINE: dict[str, DiagnosticsArchive] = { "perlmutter": DiagnosticsArchive( - root="/global/cfs/cdirs/e3sm/diagnostic_output", - public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostic_output", + root="/global/cfs/cdirs/e3sm/www", + public_base_url="https://portal.nersc.gov/cfs/e3sm/", ), "pm": DiagnosticsArchive( - root="/global/cfs/cdirs/e3sm/diagnostic_output", - public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostic_output", + root="/global/cfs/cdirs/e3sm/www", + public_base_url="https://portal.nersc.gov/cfs/e3sm/", ), "pm-cpu": DiagnosticsArchive( - root="/global/cfs/cdirs/e3sm/diagnostic_output", - public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostic_output", + root="/global/cfs/cdirs/e3sm/www", + public_base_url="https://portal.nersc.gov/cfs/e3sm/", ), "pm-gpu": DiagnosticsArchive( - root="/global/cfs/cdirs/e3sm/diagnostic_output", - public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostic_output", + root="/global/cfs/cdirs/e3sm/www", + public_base_url="https://portal.nersc.gov/cfs/e3sm/", ), "chrysalis": DiagnosticsArchive( - root="/lcrc/group/e3sm/diagnostic_output", + root="/lcrc/group/e3sm/public_html/diagnostic_output", public_base_url="https://web.lcrc.anl.gov/public/e3sm/diagnostic_output", ), } From 894befd7de7e9137db2ba6320bece737e2a9a611 Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 11 Aug 2026 13:48:03 -0700 Subject: [PATCH 18/28] Report diagnostics scanner dry-run summary --- backend/app/scripts/ingestion/diagnostics_link_scanner.py | 1 + .../features/ingestion/test_diagnostics_link_scanner.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/app/scripts/ingestion/diagnostics_link_scanner.py b/backend/app/scripts/ingestion/diagnostics_link_scanner.py index 725ed31f..8674b0b8 100644 --- a/backend/app/scripts/ingestion/diagnostics_link_scanner.py +++ b/backend/app/scripts/ingestion/diagnostics_link_scanner.py @@ -46,6 +46,7 @@ def run() -> int: for candidate in candidates: relative = candidate.path.parent.relative_to(root).as_posix() LOGGER.info("Would link diagnostics for %s", relative) + print(f"Dry run complete: {len(candidates)} diagnostics candidate(s) found.") return 0 api_base = os.environ["SIMBOARD_API_BASE_URL"].rstrip("/") diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index beb380d6..4dbf12b1 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -151,7 +151,7 @@ def test_run_submits_exact_payload_and_bearer_auth( def test_dry_run_requires_no_api_configuration( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: _case(tmp_path, "production/type/case") monkeypatch.setattr( @@ -162,6 +162,9 @@ def test_dry_run_requires_no_api_configuration( monkeypatch.delenv("SIMBOARD_API_TOKEN", raising=False) monkeypatch.setenv("DRY_RUN", "true") assert run() == 0 + assert ( + "Dry run complete: 1 diagnostics candidate(s) found." in capsys.readouterr().out + ) def test_run_defers_after_exhausted_state_lookup( From 8f1ed525d7c3d76904e7edf8126a01943c8bdbab Mon Sep 17 00:00:00 2001 From: Vo Date: Thu, 13 Aug 2026 16:31:56 -0700 Subject: [PATCH 19/28] Improve diagnostics scanner logging --- .../ingestion/diagnostics_link_scanner.py | 156 +++++++++++++++--- .../test_diagnostics_link_scanner.py | 116 ++++++++++++- docs/architecture/diagnostics-linkage.md | 32 ++++ docs/architecture/metadata-ingestion.md | 2 + docs/developer/README.md | 2 + 5 files changed, 283 insertions(+), 25 deletions(-) create mode 100644 docs/architecture/diagnostics-linkage.md diff --git a/backend/app/scripts/ingestion/diagnostics_link_scanner.py b/backend/app/scripts/ingestion/diagnostics_link_scanner.py index 8674b0b8..1228781d 100644 --- a/backend/app/scripts/ingestion/diagnostics_link_scanner.py +++ b/backend/app/scripts/ingestion/diagnostics_link_scanner.py @@ -3,23 +3,22 @@ from __future__ import annotations import hashlib -import logging import os import re import time from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from urllib.parse import urlparse +from urllib.parse import urlparse, urlunparse import httpx +from app.scripts.ingestion.archive_ingestor_core import _log_event from app.scripts.ingestion.diagnostics_archives import ( DIAGNOSTICS_ARCHIVES_BY_MACHINE, DiagnosticsArchive, ) -LOGGER = logging.getLogger(__name__) TIMESTAMP_RE = re.compile(r"^provenance\.(\d{8}_\d{6}_\d{6})\.cfg$") REQUIRED_SETTINGS = {"case_name", "machine", "hpc_username", "diagnostics_url"} MAX_SETTINGS_BYTES = 64 * 1024 @@ -40,13 +39,43 @@ def run() -> int: archive = _resolve_archive(machine) root = Path(archive.root) dry_run = os.environ.get("DRY_RUN", "true").lower() in {"1", "true", "yes"} + summary = { + "discovered_candidates": 0, + "dry_run_candidates": 0, + "unchanged_candidates": 0, + "deferred_state_lookups": 0, + "submitted_links": 0, + "failed_link_submissions": 0, + } + _log_event( + "diagnostics_scanner_startup_configuration", + { + "machine_name": machine, + "archive_root": str(root), + "public_base_url": _sanitize_url(archive.public_base_url), + "dry_run": dry_run, + "has_api_base_url": bool(os.environ.get("SIMBOARD_API_BASE_URL")), + "has_api_token": bool(os.environ.get("SIMBOARD_API_TOKEN")), + }, + ) candidates = _discover(root, archive.public_base_url) + summary["discovered_candidates"] = len(candidates) + _log_event("diagnostics_scanner_discovery_completed", summary.copy()) if dry_run: for candidate in candidates: relative = candidate.path.parent.relative_to(root).as_posix() - LOGGER.info("Would link diagnostics for %s", relative) - print(f"Dry run complete: {len(candidates)} diagnostics candidate(s) found.") + summary["dry_run_candidates"] += 1 + _log_event( + "diagnostics_scanner_dry_run_candidate", + { + "archive_relative_case_path": relative, + "settings_filename": candidate.settings.name, + "fingerprint": candidate.fingerprint, + }, + ) + _log_event("diagnostics_scanner_dry_run_completed", summary.copy()) + _log_event("diagnostics_scanner_completed", summary.copy()) return 0 api_base = os.environ["SIMBOARD_API_BASE_URL"].rstrip("/") @@ -63,19 +92,39 @@ def run() -> int: params={"machine": machine, "archive_relative_case_path": relative}, headers=headers, ) + _log_event( + "diagnostics_scanner_state_lookup_result", + { + "archive_relative_case_path": relative, + "status_code": None if state is None else state.status_code, + }, + ) if state is None or state.status_code != 200: - LOGGER.warning("State lookup failed for %s; deferring", relative) + summary["deferred_state_lookups"] += 1 + _log_event( + "diagnostics_scanner_state_lookup_deferred", + { + "archive_relative_case_path": relative, + "status_code": None if state is None else state.status_code, + }, + ) continue - if ( - state.status_code == 200 - and state.json() - and ( - state.json().get("settingsFilename") == candidate.settings.name - and state.json().get("fingerprint") == candidate.fingerprint - ) + state_payload = state.json() + if state_payload and ( + state_payload.get("settingsFilename") == candidate.settings.name + and state_payload.get("fingerprint") == candidate.fingerprint ): + summary["unchanged_candidates"] += 1 + _log_event( + "diagnostics_scanner_skipped_unchanged", + { + "archive_relative_case_path": relative, + "settings_filename": candidate.settings.name, + "fingerprint": candidate.fingerprint, + }, + ) continue payload = { @@ -105,8 +154,27 @@ def run() -> int: ) if response is None or response.status_code != 204: - LOGGER.warning("Diagnostics link submission failed for %s", relative) + summary["failed_link_submissions"] += 1 + _log_event( + "diagnostics_scanner_link_submission_failed", + { + "archive_relative_case_path": relative, + "status_code": None + if response is None + else response.status_code, + }, + ) + else: + summary["submitted_links"] += 1 + _log_event( + "diagnostics_scanner_link_submitted", + { + "archive_relative_case_path": relative, + "status_code": response.status_code, + }, + ) + _log_event("diagnostics_scanner_completed", summary) return 0 @@ -126,6 +194,16 @@ def _resolve_archive(machine_name: str) -> DiagnosticsArchive: return archive +def _sanitize_url(url: str) -> str: + """Return a URL safe to include in structured logs.""" + parsed = urlparse(url) + hostname = parsed.hostname or "" + netloc = hostname + if parsed.port is not None: + netloc = f"{netloc}:{parsed.port}" + return urlunparse((parsed.scheme, netloc, parsed.path, "", "", "")) + + def _discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 base = urlparse(public_base_url) candidates: list[Candidate] = [] @@ -139,7 +217,11 @@ def _discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C90 newest_by_case: dict[Path, tuple[Path, datetime]] = {} for cfg in tier_root.rglob("provenance.*.cfg"): - if cfg.is_symlink() or root not in cfg.resolve().parents: + try: + if cfg.is_symlink() or root not in cfg.resolve().parents: + continue + except OSError as exc: + _log_invalid_provenance(root, cfg, exc) continue timestamp = _timestamp(cfg) @@ -154,14 +236,14 @@ def _discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C90 for case_dir, (cfg, timestamp) in newest_by_case.items(): settings = cfg.with_suffix(".settings") - if ( - settings.is_symlink() - or root not in settings.resolve().parents - or not settings.is_file() - or not _published_output(case_dir, root) - ): - continue try: + if ( + settings.is_symlink() + or root not in settings.resolve().parents + or not settings.is_file() + or not _published_output(case_dir, root) + ): + continue settings_bytes = _read_settings_bytes(settings) values = _parse_settings_bytes(settings_bytes) url = urlparse(values["diagnostics_url"]) @@ -178,11 +260,22 @@ def _discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C90 digest = hashlib.sha256(settings_bytes).hexdigest() candidates.append(Candidate(cfg, settings, timestamp, values, digest)) except (OSError, UnicodeError, ValueError) as exc: - LOGGER.warning("Skipping invalid provenance %s: %s", cfg, exc) + _log_invalid_provenance(root, cfg, exc) return candidates +def _log_invalid_provenance(root: Path, cfg: Path, exc: Exception) -> None: + """Log a malformed or inaccessible provenance file without halting discovery.""" + _log_event( + "diagnostics_scanner_invalid_provenance", + { + "provenance_path": cfg.relative_to(root).as_posix(), + "reason": str(exc), + }, + ) + + def _request_with_retry(method, url: str, **kwargs) -> httpx.Response | None: for attempt in range(3): try: @@ -197,8 +290,25 @@ def _request_with_retry(method, url: str, **kwargs) -> httpx.Response | None: return response if attempt < 2: + _log_event( + "diagnostics_scanner_request_retry_scheduled", + { + "attempt": attempt + 1, + "max_attempts": 3, + "status_code": None if response is None else response.status_code, + "request_error": response is None, + }, + ) time.sleep(2**attempt) + _log_event( + "diagnostics_scanner_request_retry_exhausted", + { + "attempts": 3, + "status_code": None if response is None else response.status_code, + "request_error": response is None, + }, + ) return response diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index 4dbf12b1..7cd4b6d8 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -90,18 +90,73 @@ def test_discovery_rejects_external_output_directory_symlink(tmp_path: Path) -> assert _discover(tmp_path, BASE_URL) == [] +def test_discovery_continues_after_published_output_oserror( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + failing_case = _case(tmp_path, "production/type/failing") + valid_case = _case(tmp_path, "production/type/valid") + for case_dir, case_name in ((failing_case, "failing"), (valid_case, "valid")): + settings = next(case_dir.glob("*.settings")) + settings.write_text( + settings.read_text().replace("case_name = case", f"case_name = {case_name}") + ) + events: list[tuple[str, dict | None]] = [] + + def published_output(case_dir: Path, _root: Path) -> bool: + if case_dir == failing_case: + raise OSError("output unavailable") + return True + + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._published_output", + published_output, + ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._log_event", + lambda event, fields=None: events.append((event, fields)), + ) + + candidates = _discover(tmp_path, BASE_URL) + + assert [candidate.path.parent for candidate in candidates] == [valid_case] + assert ( + "diagnostics_scanner_invalid_provenance", + { + "provenance_path": "production/type/failing/" + "provenance.20260811_120000_000000.cfg", + "reason": "output unavailable", + }, + ) in events + + def test_retry_helper_retries_transient_response( monkeypatch: pytest.MonkeyPatch, ) -> None: responses = [httpx.Response(503), httpx.Response(204)] + events: list[tuple[str, dict | None]] = [] monkeypatch.setattr( "app.scripts.ingestion.diagnostics_link_scanner.time.sleep", lambda _: None ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._log_event", + lambda event, fields=None: events.append((event, fields)), + ) response = _request_with_retry( lambda *_args, **_kwargs: responses.pop(0), "https://x" ) assert response is not None assert response.status_code == 204 + retry_fields = next( + fields + for event, fields in events + if event == "diagnostics_scanner_request_retry_scheduled" + ) + assert retry_fields == { + "attempt": 1, + "max_attempts": 3, + "status_code": 503, + "request_error": False, + } class _Client: @@ -130,6 +185,7 @@ def test_run_submits_exact_payload_and_bearer_auth( ) -> None: _case(tmp_path, "production/type/case") client = _Client(httpx.Response(200, json=None)) + events: list[tuple[str, dict | None]] = [] monkeypatch.setattr( "app.scripts.ingestion.diagnostics_link_scanner._resolve_archive", lambda _machine: DiagnosticsArchive(str(tmp_path), BASE_URL), @@ -138,6 +194,10 @@ def test_run_submits_exact_payload_and_bearer_auth( "app.scripts.ingestion.diagnostics_link_scanner.httpx.Client", lambda **_kwargs: client, ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._log_event", + lambda event, fields=None: events.append((event, fields)), + ) monkeypatch.setenv("SIMBOARD_API_BASE_URL", "https://api.example.org") monkeypatch.setenv("SIMBOARD_API_TOKEN", "token") monkeypatch.setenv("DRY_RUN", "false") @@ -148,10 +208,22 @@ def test_run_submits_exact_payload_and_bearer_auth( } assert client.post_calls[0]["headers"] == {"Authorization": "Bearer token"} assert client.post_calls[0]["json"]["diagnostics"][0]["name"] == "zppy diagnostics" + startup_fields = next( + fields + for event, fields in events + if event == "diagnostics_scanner_startup_configuration" + ) + assert startup_fields is not None + assert startup_fields["has_api_token"] is True + assert "token" not in startup_fields + assert ( + "diagnostics_scanner_link_submitted", + {"archive_relative_case_path": "production/type/case", "status_code": 204}, + ) in events def test_dry_run_requires_no_api_configuration( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: _case(tmp_path, "production/type/case") monkeypatch.setattr( @@ -161,9 +233,36 @@ def test_dry_run_requires_no_api_configuration( monkeypatch.delenv("SIMBOARD_API_BASE_URL", raising=False) monkeypatch.delenv("SIMBOARD_API_TOKEN", raising=False) monkeypatch.setenv("DRY_RUN", "true") + events: list[tuple[str, dict | None]] = [] + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._log_event", + lambda event, fields=None: events.append((event, fields)), + ) assert run() == 0 + candidate_events = [ + fields + for event, fields in events + if event == "diagnostics_scanner_dry_run_candidate" + ] + assert len(candidate_events) == 1 + candidate_event = candidate_events[0] + assert candidate_event is not None + assert candidate_event["archive_relative_case_path"] == "production/type/case" assert ( - "Dry run complete: 1 diagnostics candidate(s) found." in capsys.readouterr().out + candidate_event["settings_filename"] + == "provenance.20260811_120000_000000.settings" + ) + assert isinstance(candidate_event["fingerprint"], str) + assert events[-1] == ( + "diagnostics_scanner_completed", + { + "discovered_candidates": 1, + "dry_run_candidates": 1, + "unchanged_candidates": 0, + "deferred_state_lookups": 0, + "submitted_links": 0, + "failed_link_submissions": 0, + }, ) @@ -172,6 +271,7 @@ def test_run_defers_after_exhausted_state_lookup( ) -> None: _case(tmp_path, "production/type/case") client = _Client(httpx.Response(503)) + events: list[tuple[str, dict | None]] = [] monkeypatch.setattr( "app.scripts.ingestion.diagnostics_link_scanner.time.sleep", lambda _: None ) @@ -183,8 +283,20 @@ def test_run_defers_after_exhausted_state_lookup( "app.scripts.ingestion.diagnostics_link_scanner.httpx.Client", lambda **_kwargs: client, ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._log_event", + lambda event, fields=None: events.append((event, fields)), + ) monkeypatch.setenv("SIMBOARD_API_BASE_URL", "https://api.example.org") monkeypatch.setenv("SIMBOARD_API_TOKEN", "token") monkeypatch.setenv("DRY_RUN", "false") run() assert client.post_calls == [] + assert ( + "diagnostics_scanner_request_retry_exhausted", + {"attempts": 3, "status_code": 503, "request_error": False}, + ) in events + assert ( + "diagnostics_scanner_state_lookup_deferred", + {"archive_relative_case_path": "production/type/case", "status_code": 503}, + ) in events diff --git a/docs/architecture/diagnostics-linkage.md b/docs/architecture/diagnostics-linkage.md new file mode 100644 index 00000000..f1b090d3 --- /dev/null +++ b/docs/architecture/diagnostics-linkage.md @@ -0,0 +1,32 @@ +# Diagnostics Linkage Architecture + +The diagnostics scanner is separate from performance-metadata collection. It discovers published zppy diagnostics and attaches a case-scoped diagnostic link to an already ingested SimBoard Case. + +## Terminology + +| Term | Definition | +| --- | --- | +| Diagnostics archive | A reviewed, machine-specific readable filesystem root and its corresponding public HTTP(S) base URL. It contains published diagnostics output; it is not a performance archive directory. | +| Diagnostics case | One published diagnostics case directory below a diagnostics archive. This filesystem identity is distinct from, and is resolved to, a SimBoard Case using the provenance case name, machine, and HPC username. | +| Provenance configuration | A timestamped `provenance.*.cfg` file in a diagnostics case directory. Its timestamp identifies which configuration is newest for discovery purposes. | +| Provenance settings | The non-symlink `.settings` file paired with the selected provenance configuration. It supplies the required case-resolution and diagnostics URL values and is the content used to produce the fingerprint. | +| Scanner candidate | A diagnostics case whose newest timestamped provenance configuration has valid paired settings, a published output, a diagnostics URL under the archive's public base URL, and a layout consistent with its settings. If that selected configuration or its settings are invalid or missing, the case is skipped; the scanner does not fall back to an older configuration. At most one candidate is discovered per diagnostics case directory in each archive tier. | +| Fingerprint | The SHA-256 digest of the selected provenance settings file bytes. It lets scanner state distinguish unchanged settings from changed settings without treating the provenance timestamp alone as sufficient. | +| Scanner state | The successful scanner submission record for a machine and archive-relative diagnostics case path. It records the selected settings filename, provenance timestamp, fingerprint, linked URL, submission time, and linked diagnostic link. | +| Linked candidate | A candidate for which scanner state has the same settings filename and fingerprint. It is already represented by a successful scanner submission and is not submitted again. | +| Unchanged candidate | A linked candidate: its selected settings filename and fingerprint match scanner state. “Unchanged” describes scanner submission state, not whether files in the diagnostics directory changed. | +| Deferred candidate | A candidate whose scanner-state lookup cannot complete successfully. It is left for a later scanner run rather than submitted without state. A candidate whose link submission fails is likewise not recorded as successfully linked and remains eligible later. | + +Invalid, unreadable, unsafe, or malformed provenance and settings inputs are skipped during discovery rather than becoming scanner candidates. The scanner only considers the `production` and `development` archive tiers, and selects the newest timestamped provenance configuration in each diagnostics case directory. + +## Scanner State Flow + +1. The scanner resolves the configured diagnostics archive for its machine and verifies that its root is readable and its public base URL is absolute HTTP(S). +2. It discovers scanner candidates by selecting the newest timestamped provenance configuration for each diagnostics case directory, then validating its paired settings and computing the settings fingerprint. If the selected configuration or settings are invalid or missing, the case is skipped without falling back to an older configuration. +3. For every candidate, it calls `GET /api/v1/diagnostics/scanner-state` with the configured machine and archive-relative diagnostics case path. A missing state is an unlinked candidate; matching settings filename and fingerprint make it unchanged; a failed or non-successful lookup defers it. +4. For each unlinked or changed candidate, it calls `POST /api/v1/diagnostics/scanner/link` with one diagnostic link and provenance metadata. The API resolves the target SimBoard Case from case name, machine, and HPC username, then atomically upserts the case diagnostic link and scanner state. A successful request returns no content. +5. On a later run, the persisted state makes a matching candidate unchanged. A changed filename or fingerprint is submitted again and updates the state for that machine/path identity. + +Scanner API access requires the diagnostics-scanner role. A state lookup can return no state, and it returns an error when the supplied machine is unknown. The scanner-link endpoint requires exactly one diagnostic and rejects unsafe archive-relative paths; case-resolution failures also prevent a successful state update. + +With `DRY_RUN` enabled, the scanner performs archive resolution and candidate discovery, logs the diagnostics case paths it would link, and exits without reading scanner state or submitting links. It therefore creates or updates no diagnostic links or scanner state; every discovered candidate is reported as a proposed link rather than classified as linked, unchanged, or deferred. diff --git a/docs/architecture/metadata-ingestion.md b/docs/architecture/metadata-ingestion.md index b1ea8d5c..19947953 100644 --- a/docs/architecture/metadata-ingestion.md +++ b/docs/architecture/metadata-ingestion.md @@ -34,6 +34,8 @@ that case by an execution ID derived from a CIME LID. | Staging directory | The active `PERF_ARCHIVE_DIR` tree where new performance output from E3SM runs appears before PACE moves it elsewhere. | | Archive directory | The long-term `OLD_PERF_ARCHIVE_DIR` tree managed by PACE after staging output is moved. | +Published diagnostics linking is documented in [Diagnostics Linkage](diagnostics-linkage.md). + ### Case and execution state terms Case-level state is derived from execution-level state. diff --git a/docs/developer/README.md b/docs/developer/README.md index e86c0a08..3d556e8e 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -65,6 +65,8 @@ SimBoard supports local path ingestion from NERSC / Perlmutter and remote automa See [Metadata Ingestion Architecture](../architecture/metadata-ingestion.md) for terminology, ingestion modes, submission-state flow, runner configuration, site mapping, and PACE reference scripts. +See [Diagnostics Linkage Architecture](../architecture/diagnostics-linkage.md) for published diagnostics scanner terminology and state flow. + ## Local Environment Setup Prerequisites: From fa6a64c0404a1342547c5a4a80d953990100d651 Mon Sep 17 00:00:00 2001 From: Vo Date: Thu, 13 Aug 2026 16:49:58 -0700 Subject: [PATCH 20/28] Require diagnostics scanner machine name --- backend/app/scripts/README.md | 9 ++++--- .../ingestion/diagnostics_link_scanner.py | 17 ++++++++++-- .../sites/chrysalis-diagnostics-scanner.sh | 2 +- .../sites/nersc-diagnostics-scanner.sh | 2 +- .../test_diagnostics_link_scanner.py | 26 ++++++++++++++++++- docs/architecture/diagnostics-linkage.md | 2 +- 6 files changed, 49 insertions(+), 9 deletions(-) diff --git a/backend/app/scripts/README.md b/backend/app/scripts/README.md index 4b6a761c..4cbb612e 100644 --- a/backend/app/scripts/README.md +++ b/backend/app/scripts/README.md @@ -222,13 +222,16 @@ Run through the NERSC wrapper: ```bash SIMBOARD_API_TOKEN= \ +MACHINE_NAME=perlmutter \ DRY_RUN=true \ backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh ``` -Use `sites/chrysalis-diagnostics-scanner.sh` at LCRC. Required: API base URL, -service-account token, and machine name. Roots and public URLs come only from -`diagnostics_archives.py`. +Use `sites/chrysalis-diagnostics-scanner.sh` at LCRC with +`MACHINE_NAME=chrysalis`. `MACHINE_NAME` is required for every diagnostics +scanner invocation; wrappers do not assign a machine default. A non-dry run +also requires an API base URL and service-account token. Roots and public URLs +come only from `diagnostics_archives.py`. Start with `DRY_RUN=true`; it needs no API URL or token. Inspect logs, then schedule with `DRY_RUN=false`, which requires both API URL and service token. diff --git a/backend/app/scripts/ingestion/diagnostics_link_scanner.py b/backend/app/scripts/ingestion/diagnostics_link_scanner.py index 1228781d..9c818811 100644 --- a/backend/app/scripts/ingestion/diagnostics_link_scanner.py +++ b/backend/app/scripts/ingestion/diagnostics_link_scanner.py @@ -35,10 +35,14 @@ class Candidate: def run() -> int: - machine = os.environ.get("MACHINE_NAME", "perlmutter") + machine = os.environ.get("MACHINE_NAME", "").strip() + if not machine: + raise ValueError("MACHINE_NAME is required") + archive = _resolve_archive(machine) root = Path(archive.root) dry_run = os.environ.get("DRY_RUN", "true").lower() in {"1", "true", "yes"} + summary = { "discovered_candidates": 0, "dry_run_candidates": 0, @@ -58,6 +62,7 @@ def run() -> int: "has_api_token": bool(os.environ.get("SIMBOARD_API_TOKEN")), }, ) + candidates = _discover(root, archive.public_base_url) summary["discovered_candidates"] = len(candidates) _log_event("diagnostics_scanner_discovery_completed", summary.copy()) @@ -76,6 +81,7 @@ def run() -> int: ) _log_event("diagnostics_scanner_dry_run_completed", summary.copy()) _log_event("diagnostics_scanner_completed", summary.copy()) + return 0 api_base = os.environ["SIMBOARD_API_BASE_URL"].rstrip("/") @@ -111,7 +117,7 @@ def run() -> int: ) continue - state_payload = state.json() + state_payload = state.json() if state.content else None if state_payload and ( state_payload.get("settingsFilename") == candidate.settings.name and state_payload.get("fingerprint") == candidate.fingerprint @@ -175,6 +181,7 @@ def run() -> int: ) _log_event("diagnostics_scanner_completed", summary) + return 0 @@ -199,8 +206,10 @@ def _sanitize_url(url: str) -> str: parsed = urlparse(url) hostname = parsed.hostname or "" netloc = hostname + if parsed.port is not None: netloc = f"{netloc}:{parsed.port}" + return urlunparse((parsed.scheme, netloc, parsed.path, "", "", "")) @@ -244,6 +253,7 @@ def _discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C90 or not _published_output(case_dir, root) ): continue + settings_bytes = _read_settings_bytes(settings) values = _parse_settings_bytes(settings_bytes) url = urlparse(values["diagnostics_url"]) @@ -255,6 +265,7 @@ def _discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C90 raise ValueError( "Diagnostics URL outside configured public archive" ) + _validate_layout(case_dir, root, values) digest = hashlib.sha256(settings_bytes).hexdigest() @@ -315,8 +326,10 @@ def _request_with_retry(method, url: str, **kwargs) -> httpx.Response | None: def _read_settings_bytes(path: Path) -> bytes: with path.open("rb") as settings_file: content = settings_file.read(MAX_SETTINGS_BYTES + 1) + if len(content) > MAX_SETTINGS_BYTES: raise ValueError("Provenance settings file is too large") + return content diff --git a/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh b/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh index 35d154a9..8d49feb5 100755 --- a/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh +++ b/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh @@ -9,7 +9,7 @@ PYTHON_BIN="${PYTHON_BIN:-${BACKEND_DIR}/.venv/bin/python}" [[ -x "${PYTHON_BIN}" ]] || { echo "Missing backend Python environment" >&2; exit 1; } export SIMBOARD_API_BASE_URL="${SIMBOARD_API_BASE_URL:-https://simboard-dev-api.e3sm.org}" -export MACHINE_NAME="${MACHINE_NAME:-chrysalis}" +: "${MACHINE_NAME:?MACHINE_NAME must be set}" export DRY_RUN="${DRY_RUN:-true}" if [[ "${DRY_RUN,,}" != "true" && "${DRY_RUN}" != "1" && "${DRY_RUN,,}" != "yes" ]]; then diff --git a/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh b/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh index 60f88428..8d49feb5 100755 --- a/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh +++ b/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh @@ -9,7 +9,7 @@ PYTHON_BIN="${PYTHON_BIN:-${BACKEND_DIR}/.venv/bin/python}" [[ -x "${PYTHON_BIN}" ]] || { echo "Missing backend Python environment" >&2; exit 1; } export SIMBOARD_API_BASE_URL="${SIMBOARD_API_BASE_URL:-https://simboard-dev-api.e3sm.org}" -export MACHINE_NAME="${MACHINE_NAME:-perlmutter}" +: "${MACHINE_NAME:?MACHINE_NAME must be set}" export DRY_RUN="${DRY_RUN:-true}" if [[ "${DRY_RUN,,}" != "true" && "${DRY_RUN}" != "1" && "${DRY_RUN,,}" != "yes" ]]; then diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index 7cd4b6d8..7ab9796a 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -20,13 +20,34 @@ def test_chrysalis_diagnostics_archive_settings() -> None: archive = DIAGNOSTICS_ARCHIVES_BY_MACHINE["chrysalis"] - assert archive.root == "/lcrc/group/e3sm/diagnostic_output" + assert archive.root == "/lcrc/group/e3sm/public_html/diagnostic_output" assert ( archive.public_base_url == "https://web.lcrc.anl.gov/public/e3sm/diagnostic_output" ) +@pytest.mark.parametrize("machine_name", [None, " "]) +def test_run_requires_machine_name_before_archive_resolution( + monkeypatch: pytest.MonkeyPatch, machine_name: str | None +) -> None: + def resolve_archive(_machine: str) -> DiagnosticsArchive: + pytest.fail("archive resolution must not be called without MACHINE_NAME") + raise AssertionError("unreachable") + + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._resolve_archive", + resolve_archive, + ) + if machine_name is None: + monkeypatch.delenv("MACHINE_NAME", raising=False) + else: + monkeypatch.setenv("MACHINE_NAME", machine_name) + + with pytest.raises(ValueError, match="MACHINE_NAME is required"): + run() + + def _case(root: Path, path: str, *, timestamp: str = "20260811_120000_000000") -> Path: directory = root / path directory.mkdir(parents=True) @@ -200,6 +221,7 @@ def test_run_submits_exact_payload_and_bearer_auth( ) monkeypatch.setenv("SIMBOARD_API_BASE_URL", "https://api.example.org") monkeypatch.setenv("SIMBOARD_API_TOKEN", "token") + monkeypatch.setenv("MACHINE_NAME", "perlmutter") monkeypatch.setenv("DRY_RUN", "false") assert run() == 0 assert client.get_calls[0]["params"] == { @@ -232,6 +254,7 @@ def test_dry_run_requires_no_api_configuration( ) monkeypatch.delenv("SIMBOARD_API_BASE_URL", raising=False) monkeypatch.delenv("SIMBOARD_API_TOKEN", raising=False) + monkeypatch.setenv("MACHINE_NAME", "perlmutter") monkeypatch.setenv("DRY_RUN", "true") events: list[tuple[str, dict | None]] = [] monkeypatch.setattr( @@ -289,6 +312,7 @@ def test_run_defers_after_exhausted_state_lookup( ) monkeypatch.setenv("SIMBOARD_API_BASE_URL", "https://api.example.org") monkeypatch.setenv("SIMBOARD_API_TOKEN", "token") + monkeypatch.setenv("MACHINE_NAME", "perlmutter") monkeypatch.setenv("DRY_RUN", "false") run() assert client.post_calls == [] diff --git a/docs/architecture/diagnostics-linkage.md b/docs/architecture/diagnostics-linkage.md index f1b090d3..27d4003c 100644 --- a/docs/architecture/diagnostics-linkage.md +++ b/docs/architecture/diagnostics-linkage.md @@ -21,7 +21,7 @@ Invalid, unreadable, unsafe, or malformed provenance and settings inputs are ski ## Scanner State Flow -1. The scanner resolves the configured diagnostics archive for its machine and verifies that its root is readable and its public base URL is absolute HTTP(S). +1. `MACHINE_NAME` is required and must name the machine whose diagnostics archive the scanner resolves. The scanner rejects unset or blank values before archive resolution, then verifies that the configured root is readable and its public base URL is absolute HTTP(S). 2. It discovers scanner candidates by selecting the newest timestamped provenance configuration for each diagnostics case directory, then validating its paired settings and computing the settings fingerprint. If the selected configuration or settings are invalid or missing, the case is skipped without falling back to an older configuration. 3. For every candidate, it calls `GET /api/v1/diagnostics/scanner-state` with the configured machine and archive-relative diagnostics case path. A missing state is an unlinked candidate; matching settings filename and fingerprint make it unchanged; a failed or non-successful lookup defers it. 4. For each unlinked or changed candidate, it calls `POST /api/v1/diagnostics/scanner/link` with one diagnostic link and provenance metadata. The API resolves the target SimBoard Case from case name, machine, and HPC username, then atomically upserts the case diagnostic link and scanner state. A successful request returns no content. From 365c7cd59b696f8cb7dfebdf93d7082044d94470 Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 13 Aug 2026 19:01:22 -0500 Subject: [PATCH 21/28] Fix links --- .../scripts/ingestion/diagnostics_archives.py | 20 +++++++++---------- .../ingestion/diagnostics_link_scanner.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/backend/app/scripts/ingestion/diagnostics_archives.py b/backend/app/scripts/ingestion/diagnostics_archives.py index c1124c32..33caf3c4 100644 --- a/backend/app/scripts/ingestion/diagnostics_archives.py +++ b/backend/app/scripts/ingestion/diagnostics_archives.py @@ -13,23 +13,23 @@ class DiagnosticsArchive: # Source: https://github.com/E3SM-Project/mache/tree/main/mache/machines DIAGNOSTICS_ARCHIVES_BY_MACHINE: dict[str, DiagnosticsArchive] = { "perlmutter": DiagnosticsArchive( - root="/global/cfs/cdirs/e3sm/www", - public_base_url="https://portal.nersc.gov/cfs/e3sm/", + root="/global/cfs/cdirs/e3sm/www/diagnostics_archive", + public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostics_archive", ), "pm": DiagnosticsArchive( - root="/global/cfs/cdirs/e3sm/www", - public_base_url="https://portal.nersc.gov/cfs/e3sm/", + root="/global/cfs/cdirs/e3sm/www/diagnostics_archive", + public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostics_archive", ), "pm-cpu": DiagnosticsArchive( - root="/global/cfs/cdirs/e3sm/www", - public_base_url="https://portal.nersc.gov/cfs/e3sm/", + root="/global/cfs/cdirs/e3sm/www/diagnostics_archive", + public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostics_archive", ), "pm-gpu": DiagnosticsArchive( - root="/global/cfs/cdirs/e3sm/www", - public_base_url="https://portal.nersc.gov/cfs/e3sm/", + root="/global/cfs/cdirs/e3sm/www/diagnostics_archive", + public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostics_archive", ), "chrysalis": DiagnosticsArchive( - root="/lcrc/group/e3sm/public_html/diagnostic_output", - public_base_url="https://web.lcrc.anl.gov/public/e3sm/diagnostic_output", + root="/lcrc/group/e3sm/public_html/diagnostic_output/diagnostics_archive", + public_base_url="https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/diagnostics_archive", ), } diff --git a/backend/app/scripts/ingestion/diagnostics_link_scanner.py b/backend/app/scripts/ingestion/diagnostics_link_scanner.py index 9c818811..6d355cb5 100644 --- a/backend/app/scripts/ingestion/diagnostics_link_scanner.py +++ b/backend/app/scripts/ingestion/diagnostics_link_scanner.py @@ -381,7 +381,7 @@ def _validate_layout(case_dir: Path, root: Path, values: dict[str, str]) -> None if values["case_name"] != parts[-1]: raise ValueError("Provenance case_name does not match archive layout") - expected_group = parts[-2] if len(parts) == 4 else None + expected_group = parts[-2] if len(parts) == 3 else None if values.get("case_group") != expected_group: raise ValueError("Provenance case_group does not match archive layout") From 6932049caaa24e23f540e2f35bfa0cf693b1acdd Mon Sep 17 00:00:00 2001 From: Tom Vo Date: Thu, 13 Aug 2026 19:14:58 -0500 Subject: [PATCH 22/28] Resolve diagnostics scanner conflicts --- backend/app/scripts/README.md | 3 +++ .../ingestion/test_diagnostics_link_scanner.py | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/backend/app/scripts/README.md b/backend/app/scripts/README.md index 4cbb612e..c837425d 100644 --- a/backend/app/scripts/README.md +++ b/backend/app/scripts/README.md @@ -235,6 +235,9 @@ come only from `diagnostics_archives.py`. Start with `DRY_RUN=true`; it needs no API URL or token. Inspect logs, then schedule with `DRY_RUN=false`, which requires both API URL and service token. +The scanner emits structured events for startup configuration, discovery, +candidate selection, state lookups, retry outcomes, and completion; credentials +are never logged. Dry runs also emit one candidate event per discovered link. Scanner account needs read/traverse access to `production/` and `development/`, provenance settings, and published output. Failed or not-ready candidates retry next run. Refresh registry entries from Mache `[web_portal]` cfg data only in a diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index 7ab9796a..fdfb4236 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -20,10 +20,13 @@ def test_chrysalis_diagnostics_archive_settings() -> None: archive = DIAGNOSTICS_ARCHIVES_BY_MACHINE["chrysalis"] - assert archive.root == "/lcrc/group/e3sm/public_html/diagnostic_output" + assert ( + archive.root + == "/lcrc/group/e3sm/public_html/diagnostic_output/diagnostics_archive" + ) assert ( archive.public_base_url - == "https://web.lcrc.anl.gov/public/e3sm/diagnostic_output" + == "https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/diagnostics_archive" ) @@ -53,8 +56,14 @@ def _case(root: Path, path: str, *, timestamp: str = "20260811_120000_000000") - directory.mkdir(parents=True) cfg = directory / f"provenance.{timestamp}.cfg" cfg.write_text("cfg", encoding="utf-8") + case_group = ( + f"case_group = {directory.parent.name}\n" + if len(directory.relative_to(root).parts) == 3 + else "" + ) cfg.with_suffix(".settings").write_text( - "case_name = case\nmachine = perlmutter\nhpc_username = user\n" + f"case_name = {directory.name}\nmachine = perlmutter\nhpc_username = user\n" + f"{case_group}" "diagnostics_url = https://diagnostics.example.org/archive/case\n", encoding="utf-8", ) From 2305efb31aeb3ca1a3a36937bbe18661430aabf9 Mon Sep 17 00:00:00 2001 From: Vo Date: Thu, 13 Aug 2026 17:15:32 -0700 Subject: [PATCH 23/28] Rename diagnostics bash script --- backend/app/scripts/README.md | 4 ++-- ...lis-diagnostics-scanner.sh => lcrc-diagnostics-scanner.sh} | 3 ++- backend/app/scripts/ingestion/sites/nersc.crontab.example | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) rename backend/app/scripts/ingestion/sites/{chrysalis-diagnostics-scanner.sh => lcrc-diagnostics-scanner.sh} (78%) diff --git a/backend/app/scripts/README.md b/backend/app/scripts/README.md index c837425d..711f360a 100644 --- a/backend/app/scripts/README.md +++ b/backend/app/scripts/README.md @@ -23,7 +23,7 @@ scripts/ │ ├── hpc_upload_archive_ingestor.py │ ├── nersc_archive_ingestor.py │ ├── sites/ -│ │ ├── chrysalis-diagnostics-scanner.sh +│ │ ├── lcrc-diagnostics-scanner.sh │ │ ├── nersc-diagnostics-scanner.sh │ │ └── nersc.sh │ └── v3_data/ @@ -227,7 +227,7 @@ DRY_RUN=true \ backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh ``` -Use `sites/chrysalis-diagnostics-scanner.sh` at LCRC with +Use `sites/lcrc-diagnostics-scanner.sh` at LCRC with `MACHINE_NAME=chrysalis`. `MACHINE_NAME` is required for every diagnostics scanner invocation; wrappers do not assign a machine default. A non-dry run also requires an API base URL and service-account token. Roots and public URLs diff --git a/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh b/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.sh similarity index 78% rename from backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh rename to backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.sh index 8d49feb5..ae629ab9 100755 --- a/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh +++ b/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.sh @@ -12,7 +12,8 @@ export SIMBOARD_API_BASE_URL="${SIMBOARD_API_BASE_URL:-https://simboard-dev-api. : "${MACHINE_NAME:?MACHINE_NAME must be set}" export DRY_RUN="${DRY_RUN:-true}" -if [[ "${DRY_RUN,,}" != "true" && "${DRY_RUN}" != "1" && "${DRY_RUN,,}" != "yes" ]]; then +DRY_RUN_NORMALIZED="$(printf '%s' "${DRY_RUN}" | tr '[:upper:]' '[:lower:]')" +if [[ "${DRY_RUN_NORMALIZED}" != "true" && "${DRY_RUN}" != "1" && "${DRY_RUN_NORMALIZED}" != "yes" ]]; then : "${SIMBOARD_API_TOKEN:?SIMBOARD_API_TOKEN must be set when DRY_RUN is false}" fi diff --git a/backend/app/scripts/ingestion/sites/nersc.crontab.example b/backend/app/scripts/ingestion/sites/nersc.crontab.example index ebdf18f0..974a92be 100644 --- a/backend/app/scripts/ingestion/sites/nersc.crontab.example +++ b/backend/app/scripts/ingestion/sites/nersc.crontab.example @@ -29,4 +29,4 @@ OLD_PERF_ARCHIVE_ROOT=/global/cfs/projectdirs/e3sm/OLD_PERF 20 * * * * cd ${REPO_DIR} && ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.log 2>&1 # Chrysalis diagnostics provenance scan; use its local checkout path for REPO_DIR. -25 * * * * cd ${REPO_DIR} && ${REPO_DIR}/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/chrysalis-diagnostics-scanner.log 2>&1 +25 * * * * cd ${REPO_DIR} && ${REPO_DIR}/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.log 2>&1 From 9542d797a151f6e69ea8a8143f7724d71ac1e373 Mon Sep 17 00:00:00 2001 From: Vo Date: Thu, 13 Aug 2026 17:36:13 -0700 Subject: [PATCH 24/28] Fix diagnostic provenance tests --- .../features/catalog/test_diagnostic_provenance_state.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/tests/features/catalog/test_diagnostic_provenance_state.py b/backend/tests/features/catalog/test_diagnostic_provenance_state.py index decad85d..7de3d6ce 100644 --- a/backend/tests/features/catalog/test_diagnostic_provenance_state.py +++ b/backend/tests/features/catalog/test_diagnostic_provenance_state.py @@ -1,6 +1,7 @@ from unittest.mock import patch from uuid import uuid4 +from fastapi.testclient import TestClient from sqlalchemy.orm import Session from app.api.version import API_BASE @@ -83,7 +84,7 @@ def test_scanner_link_is_idempotent_and_state_is_readable(client, db: Session) - f"{API_BASE}/diagnostics/scanner-state", params={ "machine": machine.name, - "archiveRelativeCasePath": "production/e3sm/case", + "archive_relative_case_path": "production/e3sm/case", }, headers=headers, ) @@ -110,7 +111,10 @@ def fail_only_state_insert(statement, *args, **kwargs): app.dependency_overrides[current_active_user] = lambda: service_user try: with patch.object(db, "execute", side_effect=fail_only_state_insert): - response = client.post(f"{API_BASE}/diagnostics/scanner/link", json=payload) + with TestClient(app, raise_server_exceptions=False) as error_client: + response = error_client.post( + f"{API_BASE}/diagnostics/scanner/link", json=payload + ) finally: app.dependency_overrides.pop(current_active_user, None) From de93b69255d77f4ffbcbb48146550979d4783241 Mon Sep 17 00:00:00 2001 From: Vo Date: Thu, 13 Aug 2026 17:55:53 -0700 Subject: [PATCH 25/28] Cover diagnostic scanner API branches --- .../test_diagnostic_provenance_state.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/backend/tests/features/catalog/test_diagnostic_provenance_state.py b/backend/tests/features/catalog/test_diagnostic_provenance_state.py index 7de3d6ce..ec5ee30f 100644 --- a/backend/tests/features/catalog/test_diagnostic_provenance_state.py +++ b/backend/tests/features/catalog/test_diagnostic_provenance_state.py @@ -8,6 +8,7 @@ from app.features.catalog.models import DiagnosticProvenanceState, ExternalLink from app.features.machine.models import Machine from app.features.user.manager import current_active_user +from app.features.user.models import User, UserRole from app.main import app from tests.features.catalog.test_api import ( _create_matching_execution, @@ -92,6 +93,98 @@ def test_scanner_link_is_idempotent_and_state_is_readable(client, db: Session) - assert response.json()["fingerprint"] == "a" * 64 +@use_real_auth +def test_scanner_state_returns_404_for_unknown_machine(client, db: Session) -> None: + _, _, token, _ = _matching_case(db) + + response = client.get( + f"{API_BASE}/diagnostics/scanner-state", + params={ + "machine": "unknown-machine", + "archive_relative_case_path": "production/e3sm/case", + }, + headers={"Authorization": f"Bearer {token}"}, + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "Unknown machine." + + +@use_real_auth +def test_scanner_link_rejects_multiple_diagnostics(client, db: Session) -> None: + machine, _, token, case = _matching_case(db) + payload = _payload( + case_name=case.name, machine=machine.name, path="production/e3sm/case" + ) + payload["diagnostics"].append(payload["diagnostics"][0].copy()) + + response = client.post( + f"{API_BASE}/diagnostics/scanner/link", + json=payload, + headers={"Authorization": f"Bearer {token}"}, + ) + + assert response.status_code == 422 + assert response.json()["detail"] == "Scanner payload requires one diagnostic." + + +@use_real_auth +def test_scanner_link_rejects_unsafe_archive_path(client, db: Session) -> None: + machine, _, token, case = _matching_case(db) + payload = _payload(case_name=case.name, machine=machine.name, path="../outside") + + response = client.post( + f"{API_BASE}/diagnostics/scanner/link", + json=payload, + headers={"Authorization": f"Bearer {token}"}, + ) + + assert response.status_code == 422 + assert response.json()["detail"] == "Invalid archive-relative case path." + + +@use_real_auth +def test_scanner_link_returns_404_for_unknown_machine(client, db: Session) -> None: + _, _, token, case = _matching_case(db) + payload = _payload( + case_name=case.name, machine="unknown-machine", path="production/e3sm/case" + ) + + response = client.post( + f"{API_BASE}/diagnostics/scanner/link", + json=payload, + headers={"Authorization": f"Bearer {token}"}, + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "No matching case found." + + +@use_real_auth +def test_scanner_endpoints_reject_regular_user(client, db: Session) -> None: + machine, _, _, case = _matching_case(db) + payload = _payload( + case_name=case.name, machine=machine.name, path="production/e3sm/case" + ) + regular_user = User( + id=uuid4(), + email="regular-scanner-user@example.com", + is_active=True, + is_verified=True, + role=UserRole.USER, + ) + app.dependency_overrides[current_active_user] = lambda: regular_user + try: + response = client.post(f"{API_BASE}/diagnostics/scanner/link", json=payload) + finally: + app.dependency_overrides.pop(current_active_user, None) + + assert response.status_code == 403 + assert response.json()["detail"] == ( + "Scanner access requires an administrator or service account." + ) + + @use_real_auth def test_scanner_link_rolls_back_link_when_state_write_fails( client, db: Session From bfe800157ea3f5f74ecd9a10a3cf0a0f67fd4a3f Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 18 Aug 2026 13:41:21 -0700 Subject: [PATCH 26/28] Document zppy diagnostics links --- docs/README.md | 4 ++++ docs/user/diagnostics.md | 45 ++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 3 +++ 3 files changed, 52 insertions(+) create mode 100644 docs/user/diagnostics.md diff --git a/docs/README.md b/docs/README.md index b0befac9..a9abca83 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,6 +2,10 @@ Use this directory as the documentation router by audience. +## For Users + +- Publish diagnostics links: [user/diagnostics.md](user/diagnostics.md) + ## For Contributors - Developer guide: [developer/README.md](developer/README.md) diff --git a/docs/user/diagnostics.md b/docs/user/diagnostics.md new file mode 100644 index 00000000..bd889a63 --- /dev/null +++ b/docs/user/diagnostics.md @@ -0,0 +1,45 @@ +# Configure zppy Diagnostics for SimBoard + +zppy publishes diagnostics web output; SimBoard links that output to an existing SimBoard case. The resulting link lets people open the diagnostics directly from the case. + +## Configure zppy for SimBoard + +In zppy configuration, enable SimBoard with `[simboard] enabled = True`. Set `simulation_type` to `production` or `development`; it defaults to `development`. `none` cannot be used while SimBoard is enabled. When no `www` path is provided, zppy uses the Mache web-portal configuration to infer it; you can also provide an explicit override. See the [zppy SimBoard configuration guide](https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/ac.forsyth2/zppy_docs_pr841_20260730/html/user_guide/tasks/simboard.html) for the exact configuration. + +To promote diagnostics from `development` to `production`, follow zppy's manual move/copy process. Do not treat promotion as a SimBoard link update. + +## Before you publish + +1. Confirm that the intended case is already visible in SimBoard. +2. Check that provenance `case_name`, `machine`, and `hpc_username` match that case. +3. Apply the **SimBoard archive layout rule**: + - ungrouped output must be at `simulation_type/case`; + - grouped output must be at `simulation_type/case_group/case`, using the `CASE_GROUP` parameter in E3SM run script configurations. This is not a zppy configuration option. The layout and its values must agree with the provenance, or discovery will not find the output. + +If the case is not available in SimBoard, contact the SimBoard administrator: [Tom Vo](mailto:vo13@llnl.gov). + +## Publish and find the link + +1. Run and publish the zppy diagnostics using the configured `simulation_type`. This generates the `provenance.settings` file used for SimBoard publication. +2. Confirm that the diagnostics web output is complete and opens successfully in a browser. +3. Confirm the completed public output uses the matching SimBoard archive layout. +4. Wait for the scheduled SimBoard scanner to attempt linkage. Linking is not immediate; the scanner runs periodically every 15 minutes. +5. After the link appears, open the case in SimBoard and follow its diagnostics link. + +Discovery uses the latest valid provenance for each published diagnostics case. If current provenance is incomplete or invalid, re-run and re-publish zppy diagnostics to regenerate it, then run discovery again. Do not manually edit provenance files or expect discovery to use an older provenance file. + +## URL behavior + +The initial external URL is stable for a published case path. When content is updated at that same published path, the SimBoard link continues to use that URL. + +If output is deleted or moved, restore it at the original URL to keep the link working. Otherwise, manually update or remove the link in SimBoard. SimBoard does not dynamically check or remove existing links whose external output is unavailable. + +## Troubleshooting + +**The case does not receive a diagnostics link.** Check the configured `simulation_type`, the matching grouped or ungrouped archive layout, the latest provenance and paired settings, the required case identity, and that the completed output is publicly accessible. If it is still missing, contact [Tom Vo](mailto:vo13@llnl.gov). + +**The link opens the wrong output.** Check `simulation_type`, `case_group`, and the published path. SimBoard does not semantically validate whether the selected `simulation_type` is appropriate for the output. + +**The link no longer opens.** Restore the output at its original URL, or manually update or remove the SimBoard link. + +For SimBoard scanner implementation details, see [Diagnostics Linkage Architecture](../architecture/diagnostics-linkage.md). diff --git a/mkdocs.yml b/mkdocs.yml index 98c6a1cf..336acadb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -15,6 +15,8 @@ markdown_extensions: format: !!python/name:pymdownx.superfences.fence_code_format nav: - Home: README.md + - User Guide: + - Publish Diagnostics Links: user/diagnostics.md - Developer: - Developer Guide: developer/README.md - Assistant LLM Setup: developer/assistant-llm-setup.md @@ -22,6 +24,7 @@ nav: - Frontend Guide: frontend/README.md - Architecture: - Metadata Ingestion: architecture/metadata-ingestion.md + - Diagnostics Linkage: architecture/diagnostics-linkage.md - Operations: - CI/CD: cicd/README.md - Deployment Docs Index: deploy/README.md From 2b54d63bf5842afda9b317bb1c554617f10433ab Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 25 Aug 2026 10:35:44 -0700 Subject: [PATCH 27/28] Address diagnostics scanner review feedback --- .../ingestion/diagnostics_link_scanner.py | 65 ++++++++--- .../ingestion/sites/nersc.crontab.example | 2 +- .../test_diagnostics_link_scanner.py | 110 ++++++++++++++++-- docs/architecture/diagnostics-linkage.md | 2 +- docs/user/diagnostics.md | 4 +- 5 files changed, 156 insertions(+), 27 deletions(-) diff --git a/backend/app/scripts/ingestion/diagnostics_link_scanner.py b/backend/app/scripts/ingestion/diagnostics_link_scanner.py index 6d355cb5..a63da9bc 100644 --- a/backend/app/scripts/ingestion/diagnostics_link_scanner.py +++ b/backend/app/scripts/ingestion/diagnostics_link_scanner.py @@ -4,15 +4,17 @@ import hashlib import os +import posixpath import re import time from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from urllib.parse import urlparse, urlunparse +from urllib.parse import unquote, urlparse, urlunparse import httpx +from app.features.machine.utils import canonicalize_machine_name from app.scripts.ingestion.archive_ingestor_core import _log_event from app.scripts.ingestion.diagnostics_archives import ( DIAGNOSTICS_ARCHIVES_BY_MACHINE, @@ -35,10 +37,11 @@ class Candidate: def run() -> int: - machine = os.environ.get("MACHINE_NAME", "").strip() - if not machine: + configured_machine = os.environ.get("MACHINE_NAME", "").strip() + if not configured_machine: raise ValueError("MACHINE_NAME is required") + machine = canonicalize_machine_name(configured_machine) archive = _resolve_archive(machine) root = Path(archive.root) dry_run = os.environ.get("DRY_RUN", "true").lower() in {"1", "true", "yes"} @@ -63,7 +66,7 @@ def run() -> int: }, ) - candidates = _discover(root, archive.public_base_url) + candidates = _discover(root, archive.public_base_url, machine) summary["discovered_candidates"] = len(candidates) _log_event("diagnostics_scanner_discovery_completed", summary.copy()) @@ -186,7 +189,8 @@ def run() -> int: def _resolve_archive(machine_name: str) -> DiagnosticsArchive: - archive = DIAGNOSTICS_ARCHIVES_BY_MACHINE.get(machine_name.lower()) + canonical_machine_name = canonicalize_machine_name(machine_name) + archive = DIAGNOSTICS_ARCHIVES_BY_MACHINE.get(canonical_machine_name) if archive is None: raise ValueError(f"Unsupported diagnostics scanner machine: {machine_name}") @@ -213,7 +217,9 @@ def _sanitize_url(url: str) -> str: return urlunparse((parsed.scheme, netloc, parsed.path, "", "", "")) -def _discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 +def _discover( # noqa: C901 + root: Path, public_base_url: str, machine_name: str +) -> list[Candidate]: base = urlparse(public_base_url) candidates: list[Candidate] = [] @@ -258,10 +264,13 @@ def _discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C90 values = _parse_settings_bytes(settings_bytes) url = urlparse(values["diagnostics_url"]) - if (url.scheme, url.netloc) != ( - base.scheme, - base.netloc, - ) or not url.path.startswith(base.path.rstrip("/") + "/"): + if canonicalize_machine_name(values["machine"]) != machine_name: + raise ValueError( + "Provenance machine does not match configured diagnostics archive" + ) + values["machine"] = machine_name + + if not _is_archive_url(url, base): raise ValueError( "Diagnostics URL outside configured public archive" ) @@ -375,13 +384,13 @@ def _published_output(case_dir: Path, root: Path) -> bool: def _validate_layout(case_dir: Path, root: Path, values: dict[str, str]) -> None: parts = case_dir.relative_to(root).parts - if len(parts) not in {3, 4} or parts[0] not in {"production", "development"}: + if len(parts) not in {2, 3} or parts[0] not in {"production", "development"}: raise ValueError("Invalid diagnostics archive case layout") if values["case_name"] != parts[-1]: raise ValueError("Provenance case_name does not match archive layout") - expected_group = parts[-2] if len(parts) == 3 else None + expected_group = parts[1] if len(parts) == 3 else None if values.get("case_group") != expected_group: raise ValueError("Provenance case_group does not match archive layout") @@ -393,9 +402,35 @@ def _timestamp(cfg: Path) -> datetime | None: if match is None: return None - return datetime.strptime(match.group(1), "%Y%m%d_%H%M%S_%f").replace( - tzinfo=timezone.utc - ) + try: + return datetime.strptime(match.group(1), "%Y%m%d_%H%M%S_%f").replace( + tzinfo=timezone.utc + ) + except ValueError: + return None + + +def _is_archive_url(url, archive_base) -> bool: + """Check that a decoded, normalized URL path remains within the archive.""" + if (url.scheme, url.netloc) != (archive_base.scheme, archive_base.netloc): + return False + + base_path = _normalized_url_path(archive_base.path) + url_path = _normalized_url_path(url.path) + prefix = "/" if base_path == "/" else f"{base_path}/" + return url_path.startswith(prefix) + + +def _normalized_url_path(path: str) -> str: + """Fully decode and normalize a URL path before validating its boundary.""" + decoded = path + while True: + unquoted = unquote(decoded) + if unquoted == decoded: + break + decoded = unquoted + + return posixpath.normpath(f"/{decoded.lstrip('/')}") if __name__ == "__main__": diff --git a/backend/app/scripts/ingestion/sites/nersc.crontab.example b/backend/app/scripts/ingestion/sites/nersc.crontab.example index 974a92be..4ec33111 100644 --- a/backend/app/scripts/ingestion/sites/nersc.crontab.example +++ b/backend/app/scripts/ingestion/sites/nersc.crontab.example @@ -29,4 +29,4 @@ OLD_PERF_ARCHIVE_ROOT=/global/cfs/projectdirs/e3sm/OLD_PERF 20 * * * * cd ${REPO_DIR} && ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.log 2>&1 # Chrysalis diagnostics provenance scan; use its local checkout path for REPO_DIR. -25 * * * * cd ${REPO_DIR} && ${REPO_DIR}/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.log 2>&1 +25 * * * * cd ${REPO_DIR} && MACHINE_NAME=chrysalis ${REPO_DIR}/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.log 2>&1 diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py index fdfb4236..393024df 100644 --- a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -74,16 +74,109 @@ def _case(root: Path, path: str, *, timestamp: str = "20260811_120000_000000") - def test_newest_missing_settings_defers_without_stale_fallback(tmp_path: Path) -> None: directory = _case(tmp_path, "production/type/case") (directory / "provenance.20260812_120000_000000.cfg").write_text("cfg") - assert _discover(tmp_path, BASE_URL) == [] + assert _discover(tmp_path, BASE_URL, "perlmutter") == [] -def test_discovery_rejects_case_and_group_mismatches(tmp_path: Path) -> None: - directory = _case(tmp_path, "development/type/group/case") +@pytest.mark.parametrize( + ("path", "expected_group"), + [("development/case", None), ("development/group/case", "group")], +) +def test_discovery_accepts_supported_archive_layouts( + tmp_path: Path, path: str, expected_group: str | None +) -> None: + directory = _case(tmp_path, path) + candidates = _discover(tmp_path, BASE_URL, "perlmutter") + + assert [candidate.path.parent for candidate in candidates] == [directory] + assert candidates[0].values.get("case_group") == expected_group + + +def test_discovery_requires_machine_matching_configured_archive(tmp_path: Path) -> None: + directory = _case(tmp_path, "development/case") + + matching = _discover(tmp_path, BASE_URL, "perlmutter") + mismatching = _discover(tmp_path, BASE_URL, "chrysalis") + + assert [candidate.path.parent for candidate in matching] == [directory] + assert mismatching == [] + + +@pytest.mark.parametrize("path", ["development", "development/group/extra/case"]) +def test_discovery_rejects_unsupported_archive_layouts( + tmp_path: Path, path: str +) -> None: + _case(tmp_path, path) + assert _discover(tmp_path, BASE_URL, "perlmutter") == [] + + +def test_discovery_rejects_case_name_mismatch(tmp_path: Path) -> None: + directory = _case(tmp_path, "development/group/case") settings = next(directory.glob("*.settings")) settings.write_text( settings.read_text().replace("case_name = case", "case_name = wrong") ) - assert _discover(tmp_path, BASE_URL) == [] + assert _discover(tmp_path, BASE_URL, "perlmutter") == [] + + +@pytest.mark.parametrize("replacement", ["case_group = wrong", ""]) +def test_discovery_rejects_case_group_mismatch( + tmp_path: Path, replacement: str +) -> None: + directory = _case(tmp_path, "development/group/case") + settings = next(directory.glob("*.settings")) + settings.write_text(settings.read_text().replace("case_group = group", replacement)) + assert _discover(tmp_path, BASE_URL, "perlmutter") == [] + + +def test_discovery_skips_malformed_timestamp_without_aborting(tmp_path: Path) -> None: + directory = _case(tmp_path, "production/group/valid") + (directory / "provenance.20261311_120000_000000.cfg").write_text("cfg") + + candidates = _discover(tmp_path, BASE_URL, "perlmutter") + + assert [candidate.path.parent for candidate in candidates] == [directory] + + +@pytest.mark.parametrize( + "diagnostics_path", + [ + "/archive/../outside", + "/archive/%2e%2e/outside", + "/archive%2f..%2foutside", + "/archive/%252e%252e%252foutside", + ], +) +def test_discovery_rejects_diagnostics_url_path_traversal( + tmp_path: Path, diagnostics_path: str +) -> None: + directory = _case(tmp_path, "production/group/case") + settings = next(directory.glob("*.settings")) + settings.write_text( + settings.read_text().replace( + "https://diagnostics.example.org/archive/case", + f"https://diagnostics.example.org{diagnostics_path}", + ) + ) + + assert _discover(tmp_path, BASE_URL, "perlmutter") == [] + + +def test_discovery_accepts_normalized_diagnostics_url_within_archive( + tmp_path: Path, +) -> None: + directory = _case(tmp_path, "production/group/case") + settings = next(directory.glob("*.settings")) + settings.write_text( + settings.read_text().replace( + "https://diagnostics.example.org/archive/case", + "https://diagnostics.example.org/archive/group/../case", + ) + ) + + assert [ + candidate.path.parent + for candidate in _discover(tmp_path, BASE_URL, "perlmutter") + ] == [directory] def test_parse_settings_rejects_duplicate_required_key(tmp_path: Path) -> None: @@ -107,7 +200,7 @@ def test_discovery_rejects_settings_symlink_outside_root(tmp_path: Path) -> None outside.write_text(settings.read_text(), encoding="utf-8") settings.unlink() settings.symlink_to(outside) - assert _discover(tmp_path, BASE_URL) == [] + assert _discover(tmp_path, BASE_URL, "perlmutter") == [] def test_discovery_rejects_external_output_directory_symlink(tmp_path: Path) -> None: @@ -117,7 +210,7 @@ def test_discovery_rejects_external_output_directory_symlink(tmp_path: Path) -> outside.mkdir(exist_ok=True) (outside / "index.html").write_text("ready") (directory / "output").symlink_to(outside, target_is_directory=True) - assert _discover(tmp_path, BASE_URL) == [] + assert _discover(tmp_path, BASE_URL, "perlmutter") == [] def test_discovery_continues_after_published_output_oserror( @@ -146,7 +239,7 @@ def published_output(case_dir: Path, _root: Path) -> bool: lambda event, fields=None: events.append((event, fields)), ) - candidates = _discover(tmp_path, BASE_URL) + candidates = _discover(tmp_path, BASE_URL, "perlmutter") assert [candidate.path.parent for candidate in candidates] == [valid_case] assert ( @@ -230,7 +323,7 @@ def test_run_submits_exact_payload_and_bearer_auth( ) monkeypatch.setenv("SIMBOARD_API_BASE_URL", "https://api.example.org") monkeypatch.setenv("SIMBOARD_API_TOKEN", "token") - monkeypatch.setenv("MACHINE_NAME", "perlmutter") + monkeypatch.setenv("MACHINE_NAME", "pm-gpu") monkeypatch.setenv("DRY_RUN", "false") assert run() == 0 assert client.get_calls[0]["params"] == { @@ -238,6 +331,7 @@ def test_run_submits_exact_payload_and_bearer_auth( "archive_relative_case_path": "production/type/case", } assert client.post_calls[0]["headers"] == {"Authorization": "Bearer token"} + assert client.post_calls[0]["json"]["machine"] == "perlmutter" assert client.post_calls[0]["json"]["diagnostics"][0]["name"] == "zppy diagnostics" startup_fields = next( fields diff --git a/docs/architecture/diagnostics-linkage.md b/docs/architecture/diagnostics-linkage.md index 27d4003c..7ee95fed 100644 --- a/docs/architecture/diagnostics-linkage.md +++ b/docs/architecture/diagnostics-linkage.md @@ -27,6 +27,6 @@ Invalid, unreadable, unsafe, or malformed provenance and settings inputs are ski 4. For each unlinked or changed candidate, it calls `POST /api/v1/diagnostics/scanner/link` with one diagnostic link and provenance metadata. The API resolves the target SimBoard Case from case name, machine, and HPC username, then atomically upserts the case diagnostic link and scanner state. A successful request returns no content. 5. On a later run, the persisted state makes a matching candidate unchanged. A changed filename or fingerprint is submitted again and updates the state for that machine/path identity. -Scanner API access requires the diagnostics-scanner role. A state lookup can return no state, and it returns an error when the supplied machine is unknown. The scanner-link endpoint requires exactly one diagnostic and rejects unsafe archive-relative paths; case-resolution failures also prevent a successful state update. +Scanner API access is permitted only for `ADMIN` and `SERVICE_ACCOUNT` roles. A state lookup can return no state, and it returns an error when the supplied machine is unknown. The scanner-link endpoint requires exactly one diagnostic and rejects unsafe archive-relative paths; case-resolution failures also prevent a successful state update. With `DRY_RUN` enabled, the scanner performs archive resolution and candidate discovery, logs the diagnostics case paths it would link, and exits without reading scanner state or submitting links. It therefore creates or updates no diagnostic links or scanner state; every discovered candidate is reported as a proposed link rather than classified as linked, unchanged, or deferred. diff --git a/docs/user/diagnostics.md b/docs/user/diagnostics.md index bd889a63..0f10ccfc 100644 --- a/docs/user/diagnostics.md +++ b/docs/user/diagnostics.md @@ -20,10 +20,10 @@ If the case is not available in SimBoard, contact the SimBoard administrator: [T ## Publish and find the link -1. Run and publish the zppy diagnostics using the configured `simulation_type`. This generates the `provenance.settings` file used for SimBoard publication. +1. Run and publish the zppy diagnostics using the configured `simulation_type`. This generates timestamped paired provenance files: `provenance.*.cfg` and its corresponding `provenance.*.settings`, which SimBoard uses for publication. 2. Confirm that the diagnostics web output is complete and opens successfully in a browser. 3. Confirm the completed public output uses the matching SimBoard archive layout. -4. Wait for the scheduled SimBoard scanner to attempt linkage. Linking is not immediate; the scanner runs periodically every 15 minutes. +4. Wait for the scheduled SimBoard scanner to attempt linkage. Linking is not immediate; the scanner runs periodically. 5. After the link appears, open the case in SimBoard and follow its diagnostics link. Discovery uses the latest valid provenance for each published diagnostics case. If current provenance is incomplete or invalid, re-run and re-publish zppy diagnostics to regenerate it, then run discovery again. Do not manually edit provenance files or expect discovery to use an older provenance file. From 3eaca05b639830c39df764997964b1f944ca6b80 Mon Sep 17 00:00:00 2001 From: Vo Date: Tue, 25 Aug 2026 10:52:12 -0700 Subject: [PATCH 28/28] Add full test coverage --- backend/tests/features/catalog/test_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/tests/features/catalog/test_api.py b/backend/tests/features/catalog/test_api.py index 95c0f3e9..e8aa19fb 100644 --- a/backend/tests/features/catalog/test_api.py +++ b/backend/tests/features/catalog/test_api.py @@ -803,13 +803,15 @@ def test_filter_options_cascade_case_and_execution_facets( f"{API_BASE}/cases/filter-options", params={ "machine_id": str(first_machine.id), + "search": "cascade-first", + "execution_id": "cascade-first-execution", "campaign": "campaign-a", }, ).json() assert execution_facets["hpcUsernames"] == ["alpha"] assert execution_facets["statuses"] == [ExecutionStatus.CREATED.value] - # Campaign remains selectable because its own active value is excluded. - assert execution_facets["campaigns"] == ["campaign-a", "campaign-b"] + # Search and execution filters keep only the matching campaign. + assert execution_facets["campaigns"] == ["campaign-a"] def test_overview_is_fixed_size_and_recent_first(self, client, db: Session): cases = [_create_case(db, f"overview-{index}") for index in range(7)]