diff --git a/CHANGELOG.md b/CHANGELOG.md index d8d0c423a..e7126bde6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ version in lockstep (enforced by `scripts/release/version_sync_check.py`). ## [Unreleased] +### Added + +- **Gold factor model, computed by argon.** A port of the signal-lab gold study: a rolling 252-day fit of gold's daily move (XAUUSD) on the 10-year real yield (DFII10), a synthetic ICE dollar index and a safe-haven spread, with coefficients lagged two rows. It produces the 95% intervals, the 63-day explained share and its label, and each day's move split into real-yield, dollar, safe-haven, last-year's-pace and unexplained parts. It is a historical description, not a forecast: the study's direction test failed. Engine `macro/gold_factor.py`. Results go to `gold_factor_daily` (migration 162). The new `gold_factor_compute` job (17:40 ET Mon–Fri, massive-0) fetches the last month of IB 1-day MIDPOINT closes through xenon `POST /historical/bars`, stores them in `macro_series_daily` (source `IB`), then recomputes and upserts the whole history. The IB fetch runs only when `XENON_QUERY_API_KEY` is set, and needs xenon PR #174 deployed. `GET /api/gold/factor` serves daily points and yearly sums. The Gold tab opens with a "What moved gold" band of six charts. `scripts/backfill/gold_factor_seed.py` seeds the history: IB closes through xenon in overlapping 6-year windows (one call of 17 years or more outruns IB's 60 s timeout), plus FRED DFII10/DGS10 from 2010 run through the real FRED job. `scripts/research/gold_factor_parity.py` checks the port: from the study's own IB closes, argon matches the study exactly on inputs, coefficients, intervals, parts and labels; with `--xenon`, the same run on closes pulled through xenon matches it exactly too, on all 4079 rows (`docs/research/2026-10-10-gold-factor-parity.json`). `statsmodels` (already installed via `arch`) is now declared. + ## [0.13.29] — 2026-10-05 diff --git a/docs/research/2026-10-10-gold-factor-parity.json b/docs/research/2026-10-10-gold-factor-parity.json new file mode 100644 index 000000000..adfc6e830 --- /dev/null +++ b/docs/research/2026-10-10-gold-factor-parity.json @@ -0,0 +1,119 @@ +{ + "study": "/Users/chenxi/projects/signal-lab/.worktrees/research-gold-yield-usd-triad", + "study_commit": "2408775", + "frozen_end": "2026-10-07", + "inputs": { + "rows_argon": 4079, + "rows_study": 4079, + "same_dates": true, + "max_abs_y": 0.0, + "max_abs_r": 0.0, + "max_abs_d": 0.0, + "max_abs_s": 0.0 + }, + "estimator": { + "01.a": 0.0, + "01.b_r": 0.0, + "01.b_d": 0.0, + "01.b_s": 0.0, + "01.b_r_lo95": 0.0, + "01.b_r_hi95": 0.0, + "01.b_d_lo95": 0.0, + "01.b_d_hi95": 0.0, + "01.b_s_lo95": 0.0, + "01.b_s_hi95": 0.0, + "01.e": 0.0, + "03b.gold_change": 0.0, + "03b.own_trailing_drift": 0.0, + "03b.real_yield_part": 0.0, + "03b.dollar_part": 0.0, + "03b.haven_part": 0.0, + "03b.U_1": 4.440892098500626e-15, + "03b.U_5": 0.0, + "03b.U_21": 0.0, + "01.published_name_mismatches": 0, + "01.raw_name_mismatches": 0, + "thresholds_recomputed": [ + 0.21839387732373133, + 0.4020650422009684 + ], + "thresholds_constant": [ + 0.21839387732373133, + 0.4020650422009684 + ] + }, + "xenon": { + "dir": "/tmp/xenon-argon-verify", + "legs": { + "XAUUSD": { + "first": "2010-06-09", + "overlap": 4218, + "mismatch_dates": [], + "study_only_dates": [] + }, + "EURUSD": { + "first": "2005-03-09", + "overlap": 5597, + "mismatch_dates": [ + "2005-12-27", + "2006-01-03", + "2006-05-29" + ], + "study_only_dates": [] + }, + "USDJPY": { + "first": "2005-03-09", + "overlap": 5597, + "mismatch_dates": [ + "2005-12-27", + "2006-01-03", + "2006-05-29" + ], + "study_only_dates": [] + }, + "GBPUSD": { + "first": "2005-03-09", + "overlap": 5597, + "mismatch_dates": [ + "2005-12-27", + "2006-01-03", + "2006-05-29" + ], + "study_only_dates": [] + }, + "USDCAD": { + "first": "2005-03-09", + "overlap": 5597, + "mismatch_dates": [ + "2005-12-27", + "2006-01-03", + "2006-05-29" + ], + "study_only_dates": [] + }, + "USDSEK": { + "first": "2006-09-01", + "overlap": 5212, + "mismatch_dates": [], + "study_only_dates": [] + }, + "USDCHF": { + "first": "2005-03-09", + "overlap": 5597, + "mismatch_dates": [ + "2005-12-27", + "2006-01-03", + "2006-05-29" + ], + "study_only_dates": [] + } + }, + "rows": [ + 4079, + 4079 + ], + "same_dates": true, + "max_abs_diff": 0.0, + "label_mismatches": 0 + } +} diff --git a/docs/runbooks/data-gap-dataset-policy.md b/docs/runbooks/data-gap-dataset-policy.md index ea3cc528e..ddf0d2486 100644 --- a/docs/runbooks/data-gap-dataset-policy.md +++ b/docs/runbooks/data-gap-dataset-policy.md @@ -6,7 +6,7 @@ Generated from `REGISTRY` in `src/uw_scan/reports/data_gap_registry/` (one sourc uv run python -c "from uw_scan.reports.data_gap_healer import render_dataset_policy_markdown as r; open('docs/runbooks/data-gap-dataset-policy.md','w').write(r())" ``` -**186 datasets** across 11 groups. +**187 datasets** across 11 groups. ## core_watchlist @@ -73,6 +73,7 @@ uv run python -c "from uw_scan.reports.data_gap_healer import render_dataset_pol | etf_flows_daily | freshness_only | none | none | | equity_session | EXTERNAL-PROVIDER BLOCK, not a healer gap: the source requires an interactive auth cookie and exposes no historical API, so there is nothing for an adapter to call. Re-probe if a credential is ever provisioned. | 2026-08-16 | | etf_holdings_daily | freshness_only | none | none | | equity_session | EXTERNAL-PROVIDER BLOCK, not a healer gap: the source requires an interactive auth cookie and exposes no historical API, so there is nothing for an adapter to call. Re-probe if a credential is ever provisioned. | 2026-08-16 | | exchange_inventory_daily | freshness_only | external | run_once | gold_lbma | monthly | | | +| gold_factor_daily | freshness_only | db | run_once | gold_factor | equity_session | | | | gold_posture_daily | freshness_only | db | run_once | gold_posture | equity_session | | | | macro_series_daily | freshness_only | external | run_once_lookback | macro_fred | daily | | | | macro_series_monthly | freshness_only | external | run_once_lookback | macro_fred | monthly | | | diff --git a/docs/superpowers/plans/2026-08-16-healer-coverage-hardening.md b/docs/superpowers/plans/2026-08-16-healer-coverage-hardening.md index 370a48090..3cd5b2e7a 100644 --- a/docs/superpowers/plans/2026-08-16-healer-coverage-hardening.md +++ b/docs/superpowers/plans/2026-08-16-healer-coverage-hardening.md @@ -161,6 +161,12 @@ forbidden, and `select_sessions` is what forbids it. The trigger was `2026-08-14` going missing during the 08-11..14 outage with no mechanism able to reach it. +**Post-plan amendment, 2026-10-10 — scope 59 → 60.** `gold_factor_daily` +(migration 162, the gold factor model) entered the daily scope wired: its +`gold_factor` adapter reruns `gold_factor_compute_job`, which recomputes the whole +history from `macro_series_daily` with no external spend. **33 wired + 27 dated +refusals.** + | task | datasets it makes healable | |---|---| | already wired | `option_surface_grid_daily`, `greek_exposure_daily`, `uw_gex_levels_daily`, `uw_volatility_signal_daily`, `uw_short_pressure_daily`, `daily_ohlc`, `vrp_daily`, `stock_analytics_daily`, `realized_volatility_history`, `volatility_stats_history`, `market_tide_sentiment_daily`, `macro_series_daily`, `rates_*`, `gold_posture_daily`, `uw_gold_options_daily` | diff --git a/pyproject.toml b/pyproject.toml index 5ccd643fb..68df01c7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "websockets>=12.0", "scipy>=1.17.1", "arch==8.0.0", + "statsmodels>=0.14", ] [project.scripts] diff --git a/scripts/backfill/gold_factor_seed.py b/scripts/backfill/gold_factor_seed.py new file mode 100644 index 000000000..7a67574c3 --- /dev/null +++ b/scripts/backfill/gold_factor_seed.py @@ -0,0 +1,65 @@ +"""One-time seed of the gold factor model's history, then one compute. + + uv run python scripts/backfill/gold_factor_seed.py + +1. IB 1-day MIDPOINT closes (XAUUSD + the six dollar-index pairs) through xenon + POST /historical/bars -> macro_series_daily, source 'IB'. One call of 17 Y or more outruns + IB's 60 s timeout, and IB's "6 Y" is not an exact calendar span, so the history comes in + overlapping 6 Y windows plus a final 2 Y to now; repeated dates are identical and the + latest vintage wins on read. XAUUSD starts 2010-06-09 at IB, which bounds the model. +2. FRED DFII10 / DGS10 from 2010 through the real gold_fred_ingest_job (argon's stored FRED + history starts 2021-05). +3. gold_factor_compute_job. + +Needs XENON_QUERY_API_URL / XENON_QUERY_API_KEY. Idempotent: inserts are ON CONFLICT DO +NOTHING on (series_id, obs_date, as_of) and the compute upserts. Parity of these xenon closes +with the study: the "xenon" block of docs/research/2026-10-10-gold-factor-parity.json. +""" + +from __future__ import annotations + +from datetime import date + +import psycopg + +from uw_scan.config import Settings +from uw_scan.worker.jobs.gold_factor import fetch_ib_legs, gold_factor_compute_job +from uw_scan.worker.jobs.gold_jobs import gold_fred_ingest_job + +# (duration, end) per window, oldest first; each 6 Y window overlaps the next by a year. +WINDOWS = [ + ("6 Y", "20150201-00:00:00"), + ("6 Y", "20200201-00:00:00"), + ("6 Y", "20250201-00:00:00"), + ("2 Y", ""), +] +FRED_START = date(2010, 1, 1) + + +def main() -> None: + s = Settings.from_env() + if not (s.xenon_query_api_url and s.xenon_query_api_key): + raise SystemExit("XENON_QUERY_API_URL and XENON_QUERY_API_KEY must be set") + dsn = s.db_dsn() + with psycopg.connect(dsn) as conn: + for duration, end in WINDOWS: + failures = fetch_ib_legs( + conn, + base_url=s.xenon_query_api_url, + api_key=s.xenon_query_api_key.get_secret_value(), + duration=duration, + end_date_time=end, + ) + print(f"{duration} to {end or 'now'}: {'ok' if not failures else failures}") + if failures: + raise SystemExit("xenon fetch failed; rerun (inserts are idempotent)") + gold_fred_ingest_job( + dsn=dsn, + series_ids=["DFII10", "DGS10"], + lookback_days=(date.today() - FRED_START).days, + ) + print(f"gold_factor_daily: {gold_factor_compute_job(dsn=dsn)} rows") + + +if __name__ == "__main__": + main() diff --git a/scripts/research/gold_factor_parity.py b/scripts/research/gold_factor_parity.py new file mode 100644 index 000000000..acfe0d1a0 --- /dev/null +++ b/scripts/research/gold_factor_parity.py @@ -0,0 +1,142 @@ +"""Parity of argon's gold factor engine (uw_scan.macro.gold_factor) with the signal-lab study. + + uv run python scripts/research/gold_factor_parity.py \ + --study ~/projects/signal-lab/.worktrees/research-gold-yield-usd-triad \ + [--xenon /tmp/xenon-argon-verify] + +Two checks, written to docs/research/2026-10-10-gold-factor-parity.json: + inputs: the study's raw IB 1-day midpoint closes + FRED files -> argon `changes` vs the + study's grid_ext y/r/d/s (rows to the frozen end 2026-10-07) + estimator: grid_ext y/r/d/s -> argon `compute` vs the study's coefficient and daily-parts + tables (01_coef_A_daily, 03b_unexplained_daily) + xenon: (with --xenon DIR) the same estimator run on closes pulled through xenon + POST /historical/bars, one .json bar list per price leg, in place of the + study's IB files; outputs compared column by column to the frozen end +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path + +import numpy as np +import pandas as pd + +from uw_scan.macro import gold_factor as gf + +RUN = "research/runs/2026-10-09-gold-regime-direction" +SHEET = "research/forward/gold-regime-direction/statesheet" +FROZEN_END = pd.Timestamp("2026-10-07") +OUT = Path("docs/research/2026-10-10-gold-factor-parity.json") + + +def maxdiff(a: pd.Series, b: pd.Series) -> float: + a, b = a.align(b, join="inner") + m = np.isfinite(a.to_numpy(float)) | np.isfinite(b.to_numpy(float)) + return float(np.nanmax(np.abs(a.to_numpy(float)[m] - b.to_numpy(float)[m]))) + + +def load_levels(study: Path) -> pd.DataFrame: + ib, fred = study / RUN / "data/d1/data/ib", study / RUN / "data/lead/fred" + cols = { + leg: pd.read_csv(ib / f"ib_{leg.lower()}_1d_midpoint.csv", parse_dates=["date"]).set_index("date")["close"] + for leg in gf.PRICE_LEGS + } + for sid in ("DFII10", "DGS10"): + cols[sid] = pd.read_csv(fred / f"{sid}.csv", parse_dates=["date"]).set_index("date")["value"].astype(float) + return pd.DataFrame(cols) + + +def xenon_check(study: Path, xdir: Path) -> dict: + """Swap the study's IB closes for xenon's, rerun everything, diff the two outputs.""" + lv = load_levels(study) + xen, legs = lv.copy(), {} + for leg in gf.PRICE_LEGS: + bars = pd.DataFrame(json.loads((xdir / f"{leg}.json").read_text())) + s = bars.assign(date=pd.to_datetime(bars.date)).drop_duplicates("date", keep="last").set_index("date")["close"] + a, b = lv[leg].loc[:FROZEN_END].dropna().align(s, join="inner") + legs[leg] = { + "first": str(s.index[0].date()), + "overlap": len(a), + "mismatch_dates": [str(d.date()) for d in a.index[(b / a - 1).abs() > 1e-6]], + "study_only_dates": [str(d.date()) for d in lv[leg].loc[:FROZEN_END].dropna().index.difference(s.index)], + } + xen[leg] = s + + def run(levels: pd.DataFrame) -> pd.DataFrame: + chg = gf.changes(levels.loc[levels.dropna().index[0] : FROZEN_END]) + return gf.compute(chg[["y", "r", "d", "s"]]) + + mine, theirs = run(lv), run(xen) + num = [c for c in mine.columns if mine[c].dtype.kind == "f"] + return { + "legs": legs, + "rows": [len(mine), len(theirs)], + "same_dates": bool(mine.index.equals(theirs.index)), + "max_abs_diff": max(maxdiff(mine[c], theirs[c]) for c in num), + "label_mismatches": int(sum((mine[c].fillna("U") != theirs[c].fillna("U")).sum() for c in ("raw_label", "published_label"))), + } + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--study", type=Path, required=True) + ap.add_argument("--xenon", type=Path, help="dir of .json xenon bar lists") + args = ap.parse_args() + study = args.study.expanduser() + ext = pd.read_parquet(study / RUN / "panel/data/grid_ext.parquet") + c01 = pd.read_parquet(study / SHEET / "01_coef_A_daily.parquet") + c3b = pd.read_parquet(study / SHEET / "03b_unexplained_daily.parquet") + + # inputs: the study builds its window from the first complete date to the frozen end + lv = load_levels(study) + first = lv.dropna().index[0] + mine = gf.changes(lv.loc[first:FROZEN_END]) + inputs = { + "rows_argon": len(mine), + "rows_study": len(ext), + "same_dates": bool(mine.index.equals(ext.index)), + **{f"max_abs_{c}": maxdiff(mine[c], ext[c]) for c in ("y", "r", "d", "s")}, + } + + out = gf.compute(ext[["y", "r", "d", "s"]]) + pairs = {c: c for c in ("a", "b_r", "b_d", "b_s", "b_r_lo95", "b_r_hi95", "b_d_lo95", "b_d_hi95", "b_s_lo95", "b_s_hi95")} + pairs["e"] = "explained_share" + est = {f"01.{k}": maxdiff(out[v], c01[k]) for k, v in pairs.items()} + for k, v in { + "gold_change": "gold_change", + "own_trailing_drift": "own_trailing_drift", + "real_yield_part": "real_yield_part", + "dollar_part": "dollar_part", + "haven_part": "haven_part", + "U_1": "unexplained_1", + "U_5": "unexplained_5", + "U_21": "unexplained_21", + }.items(): + est[f"03b.{k}"] = maxdiff(out[v], c3b[k]) + pub = out["published_label"].map(gf.STUDY_NAME).reindex(c01.index) + raw = out["raw_label"].map(gf.STUDY_NAME).reindex(c01.index) + est["01.published_name_mismatches"] = int((pub.fillna("UNDEF") != c01["published_name"]).sum()) + est["01.raw_name_mismatches"] = int((raw.fillna("UNDEF") != c01["raw_name"]).sum()) + e = out.loc[:"2021-12-31", "explained_share"].dropna() + est["thresholds_recomputed"] = np.percentile(e, [33, 67]).tolist() + est["thresholds_constant"] = [gf.E_LO, gf.E_HI] + + commit = subprocess.run( + ["git", "-C", str(study), "rev-parse", "--short", "HEAD"], capture_output=True, text=True + ).stdout.strip() + res = {"study": str(study), "study_commit": commit, "frozen_end": str(FROZEN_END.date()), "inputs": inputs, "estimator": est} + if args.xenon: + res["xenon"] = {"dir": str(args.xenon), **xenon_check(study, args.xenon)} + elif OUT.exists(): # keep a previous xenon block when rerunning without the bars + prev = json.loads(OUT.read_text()).get("xenon") + if prev: + res["xenon"] = prev + OUT.write_text(json.dumps(res, indent=1) + "\n") + print(json.dumps(res, indent=1)) + + +if __name__ == "__main__": + main() diff --git a/src/uw_scan/api/routers/gold_factor.py b/src/uw_scan/api/routers/gold_factor.py new file mode 100644 index 000000000..a187bd795 --- /dev/null +++ b/src/uw_scan/api/routers/gold_factor.py @@ -0,0 +1,67 @@ +"""GET /api/gold/factor — the gold factor model (real yield / dollar / haven), read-only. + +Daily points from `from` (default five years back) plus per-year sums over the whole history. +Historical description, not a forecast. +""" + +from __future__ import annotations + +from datetime import date, timedelta +from itertools import groupby + +from fastapi import APIRouter, Depends, HTTPException, Query + +from uw_scan.api.deps import get_repo +from uw_scan.macro.gold_factor import E_HI, E_LO +from uw_scan.models import GoldFactorPoint, GoldFactorResponse, GoldFactorYear +from uw_scan.storage.gold_factor import fetch_rows +from uw_scan.storage.repository import Repository + +router = APIRouter(prefix="/gold", tags=["gold"]) +_PARTS = ("own_trailing_drift", "real_yield_part", "dollar_part", "haven_part", "unexplained_1") + + +def _years(rows: list[dict]) -> list[GoldFactorYear]: + last_year = rows[-1]["obs_date"].year + out = [] + for year, grp in groupby(rows, key=lambda r: r["obs_date"].year): + grp = list(grp) + ok = [r for r in grp if all(r[p] is not None for p in _PARTS)] + if not ok: + continue + s = {p: sum(r[p] for r in ok) for p in (*_PARTS, "gold_change")} + out.append( + GoldFactorYear( + year=year, + rows=len(ok), + partial_year=len(ok) < len(grp) or year == last_year, + gold_change=s["gold_change"], + own_trailing_drift=s["own_trailing_drift"], + real_yield_part=s["real_yield_part"], + dollar_part=s["dollar_part"], + haven_part=s["haven_part"], + unexplained=s["unexplained_1"], + explained_share=1 + - sum(r["unexplained_1"] ** 2 for r in ok) / sum(r["gold_change"] ** 2 for r in ok), + ) + ) + return out + + +@router.get("/factor", response_model=GoldFactorResponse) +def get_gold_factor( + from_date: date | None = Query(None, alias="from"), + repo: Repository = Depends(get_repo), +) -> GoldFactorResponse: + rows = fetch_rows(repo.conn) + if not rows: + raise HTTPException(404, "gold factor model not computed yet") + start = from_date or rows[-1]["obs_date"] - timedelta(days=5 * 365) + return GoldFactorResponse( + computed_at=max(r["computed_at"] for r in rows), + last_date=rows[-1]["obs_date"], + explained_low=E_LO, + explained_high=E_HI, + points=[GoldFactorPoint.model_validate(r) for r in rows if r["obs_date"] >= start], + years=_years(rows), + ) diff --git a/src/uw_scan/api/server.py b/src/uw_scan/api/server.py index 7a16d9d13..f38b03125 100644 --- a/src/uw_scan/api/server.py +++ b/src/uw_scan/api/server.py @@ -16,6 +16,7 @@ cockpit, fundamentals_desk, gold, + gold_factor, health, jobs, macro, @@ -94,6 +95,7 @@ def create_app() -> FastAPI: app.include_router(regime.router, prefix="/api", tags=["regime"]) app.include_router(regime_validation.router, prefix="/api", tags=["regime"]) app.include_router(gold.router, prefix="/api", tags=["gold"]) + app.include_router(gold_factor.router, prefix="/api", tags=["gold"]) app.include_router(rates.router, prefix="/api", tags=["rates"]) app.include_router(macro.router, prefix="/api", tags=["macro"]) app.include_router(scanner.router, prefix="/api", tags=["scanner"]) diff --git a/src/uw_scan/macro/CLAUDE.md b/src/uw_scan/macro/CLAUDE.md index 238b5d439..798ae3dfb 100644 --- a/src/uw_scan/macro/CLAUDE.md +++ b/src/uw_scan/macro/CLAUDE.md @@ -29,6 +29,7 @@ and the API in `api/routers/macro.py`. The web surface is `web/components/macro/ | `usd.py` | Point-in-time USD transmission state | | `gold.py` | Gold's declared inputs, and the manifest that proves what was read | | `gold_state.py` | The gold domain state: whether the relationship Lens 2 rests on is in force | +| `gold_factor.py` | Gold factor model: rolling fit of gold on real yield / dollar / haven, daily parts (signal-lab port; parity `scripts/research/gold_factor_parity.py`) | | `gold_ingest.py` | Gold's own two series, promoted from the warm store into the evidence store | | `policy.py` | Pure assembly of independent policy paths | | `policy_report.py` | Point-in-time policy comparison assembly from immutable observations | diff --git a/src/uw_scan/macro/gold_factor.py b/src/uw_scan/macro/gold_factor.py new file mode 100644 index 000000000..349d9f80f --- /dev/null +++ b/src/uw_scan/macro/gold_factor.py @@ -0,0 +1,174 @@ +"""Gold factor model: how much of gold's daily move the 10Y real yield, the dollar and the +safe-haven spread explain. A port of the signal-lab gold-regime study (estimator A, regime +label, daily parts), recomputed by argon from its own stored inputs. + +Historical description only. The direction test (Phase 3) failed, so nothing here is a +forecast: a slope is the fit over the last 252 rows, not a forward sensitivity. + +Source of truth (signal-lab, research/gold-regime-phase3): + runs/2026-10-09-gold-regime-direction/scripts/{panel,estimators,layers}.py + forward/gold-regime-direction/statesheet/build_sheet.py (items 1, 3, 3b) + +Inputs per date (levels): XAUUSD and the six ICE dollar-index pairs as IB 1-day MIDPOINT +closes (the bar ending 17:00 New York, keyed by New York date), DFII10 and DGS10 from FRED. +A row exists only on dates with every input present; each row changes from the previous such +date (no forward fill). DGS10 only gates the calendar, as in the study. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import statsmodels.api as sm + +DXY_CONSTANT = 50.14348112 +DXY_EXPONENT = { + "EURUSD": -0.576, + "USDJPY": 0.136, + "GBPUSD": -0.119, + "USDCAD": 0.091, + "USDSEK": 0.042, + "USDCHF": 0.036, +} +PAIRS = tuple(DXY_EXPONENT) +PRICE_LEGS = ("XAUUSD", *PAIRS) +INPUTS = (*PRICE_LEGS, "DFII10", "DGS10") + +W = 252 # rolling rows +E_WINDOW = 63 # explained-share rows +SWITCH_ROWS = 5 # a new label publishes after this many consecutive rows +# 33rd/67th percentiles of the explained share on rows to 2021-12-31, fixed by the study. +# ponytail: frozen constants, not refit; re-derive only if the study re-freezes its thresholds. +E_LO = 0.21839387732373133 +E_HI = 0.4020650422009684 + +MUCH, PART, LITTLE, UNDEF = "much", "part", "little", None +# the study's names, kept in the parity check +STUDY_NAME = {MUCH: "DRIVEN", PART: "PARTIAL", LITTLE: "OWN-DRIVER"} + + +def changes(levels: pd.DataFrame) -> pd.DataFrame: + """Daily variables from a date-indexed frame of input levels. + + y, d, h, s in percent (100 x log change); r in bp. Rows: dates with every input present, + each changing from the previous complete date; the first complete date is the base only. + """ + g = levels.loc[:, list(INPUTS)].sort_index().dropna() + lg = 100 * np.log(g[list(PRICE_LEGS)]).diff() + dxy = DXY_CONSTANT * np.exp(sum(w * np.log(g[p]) for p, w in DXY_EXPONENT.items())) + out = pd.DataFrame(index=g.index) + out["y"] = lg["XAUUSD"] + out["r"] = 100 * g["DFII10"].diff() + out["d"] = 100 * np.log(dxy).diff() + out["s"] = 0.5 * (lg["USDCHF"] + lg["USDJPY"]) - out["d"] + return out.iloc[1:] + + +def rolling_ols(y: np.ndarray, X: np.ndarray, w: int = W) -> np.ndarray: + """b_t from rows t-w+1..t; NaN before row w-1.""" + n, k = X.shape + zz = np.concatenate( + [np.zeros((1, k, k)), np.cumsum(X[:, :, None] * X[:, None, :], axis=0)] + ) + zy = np.concatenate([np.zeros((1, k)), np.cumsum(X * y[:, None], axis=0)]) + out = np.full((n, k), np.nan) + out[w - 1 :] = np.linalg.solve(zz[w:] - zz[:-w], (zy[w:] - zy[:-w])[..., None])[ + ..., 0 + ] + return out + + +def lag2(a: np.ndarray) -> np.ndarray: + out = np.full_like(a, np.nan) + out[2:] = a[:-2] + return out + + +def rsum(a: np.ndarray, h: int) -> np.ndarray: + """Sum over rows t-h+1..t; NaN unless all h rows are finite.""" + ok = np.isfinite(a) + c = np.concatenate([[0.0], np.cumsum(np.where(ok, a, 0.0))]) + k = np.concatenate([[0.0], np.cumsum(ok)]) + out = np.full(len(a), np.nan) + out[h - 1 :] = np.where(k[h:] - k[:-h] == h, c[h:] - c[:-h], np.nan) + return out + + +def publish(raw: list) -> list: + """The first defined raw label publishes at once; later a new label publishes at the row + where the raw label has equalled it on SWITCH_ROWS consecutive rows.""" + pub: list = [UNDEF] * len(raw) + first = next((i for i, v in enumerate(raw) if v is not UNDEF), None) + if first is None: + return pub + cur, run_lab, run = raw[first], UNDEF, 0 + for t in range(first, len(raw)): + run = run + 1 if raw[t] == run_lab else 1 + run_lab = raw[t] + if run_lab != cur and run_lab is not UNDEF and run >= SWITCH_ROWS: + cur = run_lab + pub[t] = cur + return pub + + +def nw_lag(n: int) -> int: + """Newey-West (1994) automatic lag, Bartlett kernel.""" + return int(np.floor(4 * (n / 100) ** (2 / 9))) + + +def compute(chg: pd.DataFrame, *, intervals: bool = True) -> pd.DataFrame: + """All per-row outputs from the output of `changes`. Rows before the first full window + carry NaN coefficients; parts and U need two more rows.""" + y = chg["y"].to_numpy(float) + F = chg[["r", "d", "s"]].to_numpy(float) + X = np.c_[np.ones(len(y)), F] + B = rolling_ols(y, X) + BL = lag2(B) + U = y - np.einsum("ij,ij->i", X, BL) + e = 1 - rsum(U**2, E_WINDOW) / rsum(y**2, E_WINDOW) + ok = np.isfinite(e) & np.isfinite(B[:, 1]) & np.isfinite(B[:, 2]) + raw = [ + UNDEF + if not ok[t] + else MUCH + if e[t] >= E_HI and B[t, 1] < 0 and B[t, 2] < 0 + else LITTLE + if e[t] <= E_LO or (B[t, 1] >= 0 and B[t, 2] >= 0) + else PART + for t in range(len(y)) + ] + out = pd.DataFrame( + { + "a": B[:, 0], + "b_r": B[:, 1], + "b_d": B[:, 2], + "b_s": B[:, 3], + "explained_share": e, + "raw_label": raw, + "published_label": publish(raw), + "gold_change": y, + "own_trailing_drift": BL[:, 0], + "real_yield_part": BL[:, 1] * F[:, 0], + "dollar_part": BL[:, 2] * F[:, 1], + "haven_part": BL[:, 3] * F[:, 2], + "unexplained_1": U, + "unexplained_5": rsum(U, 5), + "unexplained_21": rsum(U, 21), + "real_yield_change_bp": F[:, 0], + "dollar_change_pct": F[:, 1], + "haven_change_pct": F[:, 2], + }, + index=chg.index, + ) + if intervals: + ci = np.full((len(y), 6), np.nan) + lag = nw_lag(W) + for t in range(W - 1, len(y)): + w = slice(t - W + 1, t + 1) + res = sm.OLS(y[w], X[w]).fit(cov_type="HAC", cov_kwds={"maxlags": lag}) + ci[t] = np.asarray(res.conf_int(0.05))[1:].ravel() + for j, nm in enumerate( + ("b_r_lo95", "b_r_hi95", "b_d_lo95", "b_d_hi95", "b_s_lo95", "b_s_hi95") + ): + out[nm] = ci[:, j] + return out diff --git a/src/uw_scan/models/__init__.py b/src/uw_scan/models/__init__.py index 9cef37573..ece1bc292 100644 --- a/src/uw_scan/models/__init__.py +++ b/src/uw_scan/models/__init__.py @@ -82,6 +82,7 @@ ProfitPoolLayer, ScopeGroup, ) +from .gold_factor import GoldFactorPoint, GoldFactorResponse, GoldFactorYear from .gold import ( GoldCbCountryHistory, GoldCorrelationBand, @@ -417,6 +418,9 @@ __all__ = [ "_UwBase", + "GoldFactorPoint", + "GoldFactorResponse", + "GoldFactorYear", "AgentRunIndexRow", "AgentRunIngest", "AgentRunIngestResult", diff --git a/src/uw_scan/models/gold_factor.py b/src/uw_scan/models/gold_factor.py new file mode 100644 index 000000000..a5d86a850 --- /dev/null +++ b/src/uw_scan/models/gold_factor.py @@ -0,0 +1,62 @@ +"""Gold factor model API contracts (GET /api/gold/factor). Historical description, not a +forecast: the study's direction test failed.""" + +from __future__ import annotations + +from datetime import date, datetime +from typing import Literal + +from pydantic import BaseModel + +from ._base import _preserve_public_module + +GoldFactorLabel = Literal["much", "part", "little"] + + +class GoldFactorPoint(BaseModel): + obs_date: date + b_r: float | None + b_r_lo95: float | None + b_r_hi95: float | None + b_d: float | None + b_d_lo95: float | None + b_d_hi95: float | None + b_s: float | None + b_s_lo95: float | None + b_s_hi95: float | None + explained_share: float | None + published_label: GoldFactorLabel | None + gold_change: float + own_trailing_drift: float | None + real_yield_part: float | None + dollar_part: float | None + haven_part: float | None + unexplained_1: float | None + unexplained_21: float | None + + +class GoldFactorYear(BaseModel): + """Sums over the rows of a calendar year that have every part (percent).""" + + year: int + rows: int + partial_year: bool + gold_change: float + own_trailing_drift: float + real_yield_part: float + dollar_part: float + haven_part: float + unexplained: float + explained_share: float # 1 - sum U^2 / sum y^2 in the year; can be negative + + +class GoldFactorResponse(BaseModel): + computed_at: datetime + last_date: date + explained_low: float # at or below: "little" + explained_high: float # at or above (with both slopes negative): "much" + points: list[GoldFactorPoint] + years: list[GoldFactorYear] + + +_preserve_public_module(GoldFactorPoint, GoldFactorYear, GoldFactorResponse) diff --git a/src/uw_scan/reports/data_gap_registry/gold_rates_macro.py b/src/uw_scan/reports/data_gap_registry/gold_rates_macro.py index eccd09d81..ce750786e 100644 --- a/src/uw_scan/reports/data_gap_registry/gold_rates_macro.py +++ b/src/uw_scan/reports/data_gap_registry/gold_rates_macro.py @@ -90,6 +90,17 @@ healer_adapter="gold_posture", source_system="derived", ), + DatasetRegistryEntry( + "gold_factor_daily", + "gold_rates_macro", + "freshness_only", + date_col="obs_date", + ticker_col=None, + provider="db", + granularity="run_once", + healer_adapter="gold_factor", + source_system="derived", + ), DatasetRegistryEntry( "uw_gold_options_daily", "gold_rates_macro", diff --git a/src/uw_scan/sources/xenon_query.py b/src/uw_scan/sources/xenon_query.py index f6ffa0095..922025430 100644 --- a/src/uw_scan/sources/xenon_query.py +++ b/src/uw_scan/sources/xenon_query.py @@ -144,3 +144,48 @@ def fetch_ib_option_quote( "vega": greeks.get("vega"), "theta": greeks.get("theta"), } + + +def fetch_ib_daily_midpoints( + *, + base_url: str, + api_key: str | None, + sec_type: str, + symbol: str, + currency: str = "USD", + duration: str = "1 M", + end_date_time: str = "", + timeout_s: float = 60.0, + client: httpx.Client | None = None, +) -> list[tuple[str, float]]: + """IB 1-day MIDPOINT closes via POST /historical/bars, as (YYYY-MM-DD, close), oldest first. + + For FX (sec_type CASH, xenon resolves IDEALPRO) and CMDTY (XAUUSD). The bar dated D is + 17:15 New York on D-1 to 17:00 New York on D. ``duration`` is IB's ("1 M", "2 Y"); ``end_date_time`` + ("YYYYMMDD-HH:MM:SS" UTC, "" = now) ends the window. One call of 17 Y or more outruns IB's + 60 s timeout, so long history is fetched in overlapping windows. + Raises SourceUnavailable when xenon could not answer (same contract as the greeks calls). + """ + headers = {"X-API-Key": api_key} if api_key else {} + payload = { + "contract": {"sec_type": sec_type, "symbol": symbol, "currency": currency}, + "duration": duration, + "end_date_time": end_date_time, + "bar_size": "1 day", + "what_to_show": "MIDPOINT", + "use_rth": False, + } + what = f"xenon daily midpoints {symbol}.{currency} {duration} to {end_date_time or 'now'}" + own = client is None + c = client or httpx.Client(timeout=timeout_s, trust_env=False) + try: + try: + resp = c.post(f"{base_url}/historical/bars", json=payload, headers=headers) + resp.raise_for_status() + bars = resp.json()["bars"] + return [(str(b["date"])[:10], float(b["close"])) for b in bars] + except (httpx.HTTPError, ValueError, KeyError, TypeError) as exc: + raise SourceUnavailable("xenon_query", f"{what}: {exc!r}") from exc + finally: + if own: + c.close() diff --git a/src/uw_scan/storage/gold_factor.py b/src/uw_scan/storage/gold_factor.py new file mode 100644 index 000000000..94367ccec --- /dev/null +++ b/src/uw_scan/storage/gold_factor.py @@ -0,0 +1,81 @@ +"""gold_factor_daily persistence (migration 162) and its input read. + +Standalone functions over one connection, not a `Repository` mixin: repository.py is +closed to new query methods (root CLAUDE.md module-size rule). +""" + +from __future__ import annotations + +import math +from datetime import date, datetime +from typing import Any + +import pandas as pd +import psycopg + +from uw_scan.macro.gold_factor import INPUTS + +COLS = ( + "obs_date", "a", + "b_r", "b_r_lo95", "b_r_hi95", + "b_d", "b_d_lo95", "b_d_hi95", + "b_s", "b_s_lo95", "b_s_hi95", + "explained_share", "raw_label", "published_label", + "gold_change", "own_trailing_drift", "real_yield_part", "dollar_part", "haven_part", + "unexplained_1", "unexplained_5", "unexplained_21", + "real_yield_change_bp", "dollar_change_pct", "haven_change_pct", +) # fmt: skip + + +def fetch_inputs(conn: psycopg.Connection) -> pd.DataFrame: + """Latest vintage of every model input, one column per series, indexed by date.""" + rows = conn.execute( + """ + SELECT DISTINCT ON (series_id, obs_date) series_id, obs_date, value::float8 + FROM uw_scan.macro_series_daily + WHERE series_id = ANY(%s) + ORDER BY series_id, obs_date, as_of DESC + """, + (list(INPUTS),), + ).fetchall() + df = pd.DataFrame(rows, columns=["series_id", "obs_date", "value"]) + wide = df.pivot(index="obs_date", columns="series_id", values="value") + wide.index = pd.DatetimeIndex(wide.index) + return wide.reindex(columns=list(INPUTS)) + + +def _clean(v: Any) -> Any: + return None if isinstance(v, float) and math.isnan(v) else v + + +def upsert_rows(conn: psycopg.Connection, out: pd.DataFrame, computed_at: datetime) -> int: + """Upsert every row of `gold_factor.compute` output; returns the row count.""" + body = COLS[1:] + params = [ + (ts.date(), *(_clean(r[c]) for c in body), computed_at) + for ts, r in out.iterrows() + ] + sets = ", ".join(f"{c} = EXCLUDED.{c}" for c in (*body, "computed_at")) + with conn.cursor() as cur: + cur.executemany( + f""" + INSERT INTO uw_scan.gold_factor_daily ({", ".join(COLS)}, computed_at) + VALUES ({", ".join(["%s"] * (len(COLS) + 1))}) + ON CONFLICT (obs_date) DO UPDATE SET {sets} + """, + params, + ) + return len(params) + + +def fetch_rows(conn: psycopg.Connection, *, from_date: date | None = None) -> list[dict[str, Any]]: + cur = conn.execute( + f""" + SELECT {", ".join(COLS)}, computed_at FROM uw_scan.gold_factor_daily + WHERE %(f)s::date IS NULL OR obs_date >= %(f)s::date + ORDER BY obs_date + """, + {"f": from_date}, + ) + names = [c.name for c in cur.description] + return [dict(zip(names, r, strict=True)) for r in cur.fetchall()] diff --git a/src/uw_scan/storage/migrations/162_gold_factor_daily.sql b/src/uw_scan/storage/migrations/162_gold_factor_daily.sql new file mode 100644 index 000000000..bc6c3df50 --- /dev/null +++ b/src/uw_scan/storage/migrations/162_gold_factor_daily.sql @@ -0,0 +1,51 @@ +-- 162_gold_factor_daily.sql — gold factor model, one row per grid date. +-- +-- How much of gold's daily move the 10Y real yield, the dollar (synthetic ICE +-- index) and the safe-haven spread explain: a 252-row rolling OLS with +-- coefficients lagged 2 rows, the 63-row explained share, its label, and each +-- day's move split into parts. Historical description only; the study's +-- direction test failed, so nothing here is a forecast. +-- Engine: src/uw_scan/macro/gold_factor.py (a port of the signal-lab study, +-- parity in docs/research/2026-10-10-gold-factor-parity.json). +-- +-- Units: *_part, gold_change, unexplained_* and own_trailing_drift are percent +-- (100 x log change); b_r is % per bp of DFII10; b_d, b_s are % per %. +-- gold_change = own_trailing_drift + real_yield_part + dollar_part +-- + haven_part + unexplained_1 (per row, exactly). +-- +-- Writer: worker/jobs/gold_factor.py (nightly, massive-0) recomputes the whole +-- history from macro_series_daily and upserts every row, so a revised input +-- converges on the next run. +-- +-- Idempotent: IF NOT EXISTS. + +SET search_path TO uw_scan, public; + +CREATE TABLE IF NOT EXISTS uw_scan.gold_factor_daily ( + obs_date DATE PRIMARY KEY, + a DOUBLE PRECISION, + b_r DOUBLE PRECISION, + b_r_lo95 DOUBLE PRECISION, + b_r_hi95 DOUBLE PRECISION, + b_d DOUBLE PRECISION, + b_d_lo95 DOUBLE PRECISION, + b_d_hi95 DOUBLE PRECISION, + b_s DOUBLE PRECISION, + b_s_lo95 DOUBLE PRECISION, + b_s_hi95 DOUBLE PRECISION, + explained_share DOUBLE PRECISION, + raw_label TEXT CHECK (raw_label IN ('much','part','little')), + published_label TEXT CHECK (published_label IN ('much','part','little')), + gold_change DOUBLE PRECISION NOT NULL, + own_trailing_drift DOUBLE PRECISION, + real_yield_part DOUBLE PRECISION, + dollar_part DOUBLE PRECISION, + haven_part DOUBLE PRECISION, + unexplained_1 DOUBLE PRECISION, + unexplained_5 DOUBLE PRECISION, + unexplained_21 DOUBLE PRECISION, + real_yield_change_bp DOUBLE PRECISION NOT NULL, + dollar_change_pct DOUBLE PRECISION NOT NULL, + haven_change_pct DOUBLE PRECISION NOT NULL, + computed_at TIMESTAMPTZ NOT NULL +); diff --git a/src/uw_scan/worker/jobs/data_gap_adapters.py b/src/uw_scan/worker/jobs/data_gap_adapters.py index b164fe312..13ed56983 100644 --- a/src/uw_scan/worker/jobs/data_gap_adapters.py +++ b/src/uw_scan/worker/jobs/data_gap_adapters.py @@ -32,6 +32,7 @@ _run_gex_levels, _run_gold_cot, _run_gold_lbma, + _run_gold_factor, _run_gold_posture, _run_gold_uw_options, _run_greek_exposure, @@ -229,6 +230,9 @@ "rates_fred": HealSpec( "rates_fred", "external", "run_once_lookback", _run_rates_fred, est_per_item=0 ), + "gold_factor": HealSpec( + "gold_factor", "db", "run_once", _run_gold_factor, est_per_item=0 + ), "gold_posture": HealSpec( "gold_posture", "db", "run_once", _run_gold_posture, est_per_item=0 ), diff --git a/src/uw_scan/worker/jobs/data_gap_heal_runners.py b/src/uw_scan/worker/jobs/data_gap_heal_runners.py index b7d3084ec..6ece36f6d 100644 --- a/src/uw_scan/worker/jobs/data_gap_heal_runners.py +++ b/src/uw_scan/worker/jobs/data_gap_heal_runners.py @@ -214,6 +214,14 @@ def _run_rates_fred(ctx: HealContext, lookback_days: int) -> int: return 0 +def _run_gold_factor(ctx: HealContext) -> int: + from uw_scan.worker.jobs.gold_factor import gold_factor_compute_job + + # Recomputes and upserts the whole history from stored inputs; no external spend. + gold_factor_compute_job(dsn=ctx.settings.db_dsn()) + return 0 + + def _run_gold_posture(ctx: HealContext) -> int: from uw_scan.worker.jobs.gold_jobs import ( _latest_gold_market_date, diff --git a/src/uw_scan/worker/jobs/gold_factor.py b/src/uw_scan/worker/jobs/gold_factor.py new file mode 100644 index 000000000..561b1cbbf --- /dev/null +++ b/src/uw_scan/worker/jobs/gold_factor.py @@ -0,0 +1,81 @@ +"""Nightly gold factor: IB prices via xenon -> macro_series_daily -> gold_factor_daily. + +1. With a xenon key, fetch the last month of IB 1-day MIDPOINT closes for XAUUSD (CMDTY) and the + six dollar-index pairs (CASH, IDEALPRO) and store them in macro_series_daily, source 'IB'. + A month, not a day, so a missed night or a revised bar heals on the next run. +2. Recompute the whole history from macro_series_daily (with FRED DFII10/DGS10) and upsert it: + the rolling fit is cheap and a revised input converges at once. +A leg xenon could not fetch still lets the compute run on what is stored; the job then raises +naming the legs, so the failure reaches job_failures. +""" + +from __future__ import annotations + +import logging +from datetime import UTC, date, datetime + +import psycopg + +from uw_scan.macro import gold_factor as gf +from uw_scan.sources.xenon_query import fetch_ib_daily_midpoints +from uw_scan.storage import gold_factor as store +from uw_scan.storage.repository import Repository + +logger = logging.getLogger(__name__) + + +def _ib_contract(leg: str) -> tuple[str, str, str]: + return ("CMDTY", leg, "USD") if leg == "XAUUSD" else ("CASH", leg[:3], leg[3:]) + + +def fetch_ib_legs( + conn: psycopg.Connection, + *, + base_url: str, + api_key: str, + duration: str = "1 M", + end_date_time: str = "", +) -> list[str]: + """Store the IB closes of every price leg; returns the failures as 'LEG: error'.""" + repo = Repository(conn, schema="uw_scan") + now = datetime.now(UTC) + failures = [] + for leg in gf.PRICE_LEGS: + sec_type, symbol, currency = _ib_contract(leg) + try: + bars = fetch_ib_daily_midpoints( + base_url=base_url, api_key=api_key, sec_type=sec_type, symbol=symbol, + currency=currency, duration=duration, end_date_time=end_date_time, + ) # fmt: skip + except Exception as exc: + logger.exception("gold_factor: xenon fetch %s failed: %r", leg, exc) + failures.append(f"{leg}: {repr(exc)[:200]}") + continue + repo.insert_macro_series_daily_rows( + ({"series_id": leg, "obs_date": date.fromisoformat(d), "value": v} for d, v in bars), + as_of=now, + source="IB", + ) + conn.commit() + return failures + + +def gold_factor_compute_job( + *, dsn: str, xenon_url: str | None = None, xenon_key: str | None = None +) -> int: + failures: list[str] = [] + with psycopg.connect(dsn) as conn: + if xenon_key and xenon_url: + failures = fetch_ib_legs(conn, base_url=xenon_url, api_key=xenon_key) + levels = store.fetch_inputs(conn) + chg = gf.changes(levels) + if len(chg) < gf.W + 2: + raise RuntimeError( + f"gold_factor: {len(chg)} complete rows, need {gf.W + 2}; " + "seed with scripts/backfill/gold_factor_seed.py" + ) + n = store.upsert_rows(conn, gf.compute(chg), datetime.now(UTC)) + logger.info("gold_factor: upserted %d rows through %s", n, chg.index[-1].date()) + if failures: + raise RuntimeError(f"gold_factor: computed, but xenon fetch failed for {failures}") + return n diff --git a/src/uw_scan/worker/schedule/gold.py b/src/uw_scan/worker/schedule/gold.py index 1b13b69a8..d19d86f76 100644 --- a/src/uw_scan/worker/schedule/gold.py +++ b/src/uw_scan/worker/schedule/gold.py @@ -24,6 +24,7 @@ gold_uw_options_ingest_job, gold_wgc_cb_ingest_job, ) +from uw_scan.worker.jobs.gold_factor import gold_factor_compute_job logger = logging.getLogger(__name__) @@ -105,6 +106,14 @@ def _gold_wgc_cb_ingest() -> None: record_request=lambda _provider, event: recorder.record(event), ) + def _gold_factor_compute() -> None: + key = settings.xenon_query_api_key + gold_factor_compute_job( + dsn=settings.db_dsn(), + xenon_url=settings.xenon_query_api_url, + xenon_key=key.get_secret_value() if key is not None else None, + ) + def _gold_posture_compute() -> None: gold_posture_compute_job(dsn=settings.db_dsn()) @@ -118,6 +127,14 @@ def _gold_posture_compute() -> None: id="gold_fred_ingest", name="Gold: FRED daily refresh", ) + # 17:40, after the 17:00 FRED refresh: FRED posts DFII10 for day t on t+1, so this run + # completes row t; row t+1 waits for tomorrow's yield (no forward fill). + sched.add_job( + _gold_factor_compute, + CronTrigger.from_crontab("40 17 * * 0-4", timezone=settings.rth_tz), + id="gold_factor_compute", + name="Gold: factor model (real yield / dollar / haven)", + ) sched.add_job( _gold_spot_ingest, CronTrigger.from_crontab("5 17 * * 0-4", timezone=settings.rth_tz), diff --git a/tests/integration/api/openapi.snapshot.json b/tests/integration/api/openapi.snapshot.json index 26a830129..46e4b9996 100644 --- a/tests/integration/api/openapi.snapshot.json +++ b/tests/integration/api/openapi.snapshot.json @@ -8497,6 +8497,339 @@ "title": "GoldDecompositionRow", "type": "object" }, + "GoldFactorPoint": { + "properties": { + "b_d": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "B D" + }, + "b_d_hi95": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "B D Hi95" + }, + "b_d_lo95": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "B D Lo95" + }, + "b_r": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "B R" + }, + "b_r_hi95": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "B R Hi95" + }, + "b_r_lo95": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "B R Lo95" + }, + "b_s": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "B S" + }, + "b_s_hi95": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "B S Hi95" + }, + "b_s_lo95": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "B S Lo95" + }, + "dollar_part": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Dollar Part" + }, + "explained_share": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Explained Share" + }, + "gold_change": { + "title": "Gold Change", + "type": "number" + }, + "haven_part": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Haven Part" + }, + "obs_date": { + "format": "date", + "title": "Obs Date", + "type": "string" + }, + "own_trailing_drift": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Own Trailing Drift" + }, + "published_label": { + "anyOf": [ + { + "enum": [ + "much", + "part", + "little" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published Label" + }, + "real_yield_part": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Real Yield Part" + }, + "unexplained_1": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Unexplained 1" + }, + "unexplained_21": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Unexplained 21" + } + }, + "required": [ + "obs_date", + "b_r", + "b_r_lo95", + "b_r_hi95", + "b_d", + "b_d_lo95", + "b_d_hi95", + "b_s", + "b_s_lo95", + "b_s_hi95", + "explained_share", + "published_label", + "gold_change", + "own_trailing_drift", + "real_yield_part", + "dollar_part", + "haven_part", + "unexplained_1", + "unexplained_21" + ], + "title": "GoldFactorPoint", + "type": "object" + }, + "GoldFactorResponse": { + "properties": { + "computed_at": { + "format": "date-time", + "title": "Computed At", + "type": "string" + }, + "explained_high": { + "title": "Explained High", + "type": "number" + }, + "explained_low": { + "title": "Explained Low", + "type": "number" + }, + "last_date": { + "format": "date", + "title": "Last Date", + "type": "string" + }, + "points": { + "items": { + "$ref": "#/components/schemas/GoldFactorPoint" + }, + "title": "Points", + "type": "array" + }, + "years": { + "items": { + "$ref": "#/components/schemas/GoldFactorYear" + }, + "title": "Years", + "type": "array" + } + }, + "required": [ + "computed_at", + "last_date", + "explained_low", + "explained_high", + "points", + "years" + ], + "title": "GoldFactorResponse", + "type": "object" + }, + "GoldFactorYear": { + "description": "Sums over the rows of a calendar year that have every part (percent).", + "properties": { + "dollar_part": { + "title": "Dollar Part", + "type": "number" + }, + "explained_share": { + "title": "Explained Share", + "type": "number" + }, + "gold_change": { + "title": "Gold Change", + "type": "number" + }, + "haven_part": { + "title": "Haven Part", + "type": "number" + }, + "own_trailing_drift": { + "title": "Own Trailing Drift", + "type": "number" + }, + "partial_year": { + "title": "Partial Year", + "type": "boolean" + }, + "real_yield_part": { + "title": "Real Yield Part", + "type": "number" + }, + "rows": { + "title": "Rows", + "type": "integer" + }, + "unexplained": { + "title": "Unexplained", + "type": "number" + }, + "year": { + "title": "Year", + "type": "integer" + } + }, + "required": [ + "year", + "rows", + "partial_year", + "gold_change", + "own_trailing_drift", + "real_yield_part", + "dollar_part", + "haven_part", + "unexplained", + "explained_share" + ], + "title": "GoldFactorYear", + "type": "object" + }, "GoldGauge60dTimeSeriesPoint": { "properties": { "corr_60d": { @@ -31576,6 +31909,57 @@ ] } }, + "/api/gold/factor": { + "get": { + "tags": [ + "gold", + "gold" + ], + "summary": "Get Gold Factor", + "operationId": "get_gold_factor_api_gold_factor_get", + "parameters": [ + { + "name": "from", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string", + "format": "date" + }, + { + "type": "null" + } + ], + "title": "From" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GoldFactorResponse" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/api/gold/state": { "get": { "operationId": "get_state_api_gold_state_get", diff --git a/tests/integration/worker/test_gold_factor_compute_job.py b/tests/integration/worker/test_gold_factor_compute_job.py new file mode 100644 index 000000000..ccaa11280 --- /dev/null +++ b/tests/integration/worker/test_gold_factor_compute_job.py @@ -0,0 +1,88 @@ +"""Worker job — gold_factor_compute_job: macro_series_daily inputs -> gold_factor_daily.""" + +from __future__ import annotations + +import os +from datetime import UTC, datetime + +import numpy as np +import psycopg +import pytest + +from uw_scan.config import Settings +from uw_scan.storage.gold_factor import fetch_rows +from uw_scan.storage.repository import Repository +from uw_scan.worker.jobs.gold_factor import gold_factor_compute_job + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def fresh_db(seeded_db_empty_cards) -> Settings: + _ = seeded_db_empty_cards + test_db = os.environ.get("UW_SCAN_TEST_DB_NAME") + if not test_db: + pytest.fail("UW_SCAN_TEST_DB_NAME is not set.", pytrace=False) + os.environ.setdefault("UW_SCAN_API_KEY", "test-dummy-not-used-by-db-tests") + return Settings.from_env().model_copy(update={"db_name": test_db}) + + +def _seed(dsn: str, n: int) -> None: + from tests.unit.macro.test_gold_factor import _levels + + lv = _levels(n) + with psycopg.connect(dsn) as conn: + repo = Repository(conn, schema="uw_scan") + for sid in lv.columns: + repo.insert_macro_series_daily_rows( + ({"series_id": sid, "obs_date": d.date(), "value": float(v)} for d, v in lv[sid].items()), + as_of=datetime(2026, 10, 10, tzinfo=UTC), + source="IB" if sid not in ("DFII10", "DGS10") else "FRED", + ) + + +def test_job_writes_every_grid_row_and_is_idempotent(fresh_db: Settings) -> None: + dsn = fresh_db.db_dsn() + _seed(dsn, 300) + assert gold_factor_compute_job(dsn=dsn) == 299 + assert gold_factor_compute_job(dsn=dsn) == 299 + with psycopg.connect(dsn) as conn: + rows = fetch_rows(conn) + assert len(rows) == 299 + last = rows[-1] + parts = ("own_trailing_drift", "real_yield_part", "dollar_part", "haven_part", "unexplained_1") + assert np.isclose(sum(last[p] for p in parts), last["gold_change"]) + assert last["b_r_lo95"] <= last["b_r"] <= last["b_r_hi95"] + assert rows[0]["b_r"] is None and rows[0]["computed_at"] is not None + + +def test_job_refuses_too_short_a_history(fresh_db: Settings) -> None: + _seed(fresh_db.db_dsn(), 100) + with pytest.raises(RuntimeError, match="gold_factor_seed"): + gold_factor_compute_job(dsn=fresh_db.db_dsn()) + + +def test_api_serves_points_and_years(fresh_db: Settings) -> None: + from fastapi.testclient import TestClient + + from uw_scan.api.deps import get_repo, get_settings + from uw_scan.api.server import create_app + + dsn = fresh_db.db_dsn() + _seed(dsn, 300) + gold_factor_compute_job(dsn=dsn) + + def _repo(): + with psycopg.connect(dsn) as conn: + yield Repository(conn, schema="uw_scan") + + app = create_app() + app.dependency_overrides[get_settings] = lambda: fresh_db + app.dependency_overrides[get_repo] = _repo + with TestClient(app) as client: + assert client.get("/api/gold/factor").status_code == 200 + body = client.get("/api/gold/factor", params={"from": "2020-01-01"}).json() + assert len(body["points"]) == 299 + yr = body["years"][-1] + parts = ("own_trailing_drift", "real_yield_part", "dollar_part", "haven_part", "unexplained") + assert np.isclose(sum(yr[p] for p in parts), yr["gold_change"]) and yr["partial_year"] diff --git a/tests/unit/macro/test_gold_factor.py b/tests/unit/macro/test_gold_factor.py new file mode 100644 index 000000000..dc1970b3f --- /dev/null +++ b/tests/unit/macro/test_gold_factor.py @@ -0,0 +1,53 @@ +"""uw_scan.macro.gold_factor on labelled synthetic data. Parity with the signal-lab study on +real data is scripts/research/gold_factor_parity.py (result in docs/research/).""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import statsmodels.api as sm + +from uw_scan.macro import gold_factor as gf + + +def _levels(n: int = 400, seed: int = 3) -> pd.DataFrame: + rng = np.random.default_rng(seed) + idx = pd.bdate_range("2020-01-01", periods=n) + walk = lambda s0, vol: s0 * np.exp(np.cumsum(rng.normal(0, vol, n))) # noqa: E731 + lv = {leg: walk(1.0 + i, 0.005) for i, leg in enumerate(gf.PAIRS)} + lv["XAUUSD"] = walk(1800.0, 0.01) + lv["DFII10"] = 1.0 + np.cumsum(rng.normal(0, 0.03, n)) + lv["DGS10"] = 3.0 + np.cumsum(rng.normal(0, 0.03, n)) + return pd.DataFrame(lv, index=idx) + + +def test_changes_skip_incomplete_dates_without_forward_fill() -> None: + lv = _levels(10) + lv.iloc[4, lv.columns.get_loc("DFII10")] = np.nan + chg = gf.changes(lv) + assert lv.index[4] not in chg.index and len(chg) == 8 # base row + the gap dropped + # the row after the gap changes from the last complete date, two dates back + assert np.isclose(chg.loc[lv.index[5], "r"], 100 * (lv["DFII10"].iloc[5] - lv["DFII10"].iloc[3])) + + +def test_rolling_fit_matches_statsmodels_and_parts_sum_to_gold() -> None: + chg = gf.changes(_levels()) + out = gf.compute(chg) + X = np.c_[np.ones(len(chg)), chg[["r", "d", "s"]].to_numpy()] + t = len(chg) - 1 + res = sm.OLS(chg["y"].to_numpy()[t - 251 : t + 1], X[t - 251 : t + 1]).fit( + cov_type="HAC", cov_kwds={"maxlags": 4} + ) + assert np.allclose(out.iloc[t][["a", "b_r", "b_d", "b_s"]].to_numpy(float), res.params) + assert np.isclose(out.iloc[t]["b_d_lo95"], res.conf_int(0.05)[2, 0]) + assert out["b_r"].iloc[: gf.W - 1].isna().all() and np.isfinite(out["b_r"].iloc[gf.W - 1]) + parts = out[["own_trailing_drift", "real_yield_part", "dollar_part", "haven_part", "unexplained_1"]] + ok = parts.notna().all(axis=1) + assert ok.sum() == len(chg) - gf.W - 1 + assert np.allclose(parts[ok].sum(axis=1), out.loc[ok, "gold_change"], atol=1e-12) + + +def test_publish_needs_five_consecutive_rows() -> None: + U, M, P, L = gf.UNDEF, gf.MUCH, gf.PART, gf.LITTLE + raw = [U, M, P, P, P, P, M, P, P, P, P, P, L, L] + assert gf.publish(raw) == [U, M, M, M, M, M, M, M, M, M, M, P, P, P] diff --git a/tests/unit/reports/test_full_coverage.py b/tests/unit/reports/test_full_coverage.py index 5a2360b75..d53209c14 100644 --- a/tests/unit/reports/test_full_coverage.py +++ b/tests/unit/reports/test_full_coverage.py @@ -36,7 +36,7 @@ def test_the_coverage_ledger_numbers_are_still_true() -> None: if e.audit_mode not in BY_DESIGN_AUDIT_MODES and e.expected_frequency in ("equity_session", "daily") ] - assert len(scoped) == 59, ( + assert len(scoped) == 60, ( f"the daily/equity_session scope moved to {len(scoped)}; update the " "Coverage Ledger in docs/superpowers/plans/" "2026-08-16-healer-coverage-hardening.md" diff --git a/tests/unit/test_xenon_query.py b/tests/unit/test_xenon_query.py index d2a5155a3..72fa7d66c 100644 --- a/tests/unit/test_xenon_query.py +++ b/tests/unit/test_xenon_query.py @@ -6,7 +6,11 @@ import pytest from uw_scan.sources.source_errors import SourceUnavailable -from uw_scan.sources.xenon_query import fetch_ib_option_iv, fetch_ib_option_quote +from uw_scan.sources.xenon_query import ( + fetch_ib_daily_midpoints, + fetch_ib_option_iv, + fetch_ib_option_quote, +) def _client(handler) -> httpx.Client: @@ -169,3 +173,41 @@ def handler(request): right="C", client=_client(handler), ) + + +def test_daily_midpoints_sends_the_research_contract_and_returns_closes(): + import json + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/historical/bars" + assert request.headers["X-API-Key"] == "k" + body = json.loads(request.content) + assert body["contract"] == {"sec_type": "CASH", "symbol": "EUR", "currency": "USD"} + assert (body["what_to_show"], body["use_rth"], body["bar_size"]) == ("MIDPOINT", False, "1 day") + assert (body["duration"], body["end_date_time"]) == ("6 Y", "20200201-00:00:00") + # xenon's live sample row, 2026-10-10 (volume -1 is IB's MIDPOINT sentinel) + bar = {"date": "2026-10-09", "open": 1.1206, "high": 1.1243, "low": 1.11875, "close": 1.12, "volume": -1} + return httpx.Response(200, json={"bars": [bar]}) + + out = fetch_ib_daily_midpoints( + base_url="http://x", api_key="k", sec_type="CASH", symbol="EUR", + duration="6 Y", end_date_time="20200201-00:00:00", client=_client(handler), + ) # fmt: skip + assert out == [("2026-10-09", 1.12)] + + +@pytest.mark.parametrize( + "resp", + [ + httpx.Response(401), + # xenon #174: a contract IB cannot qualify is a 404, never 200 {"bars": []} + httpx.Response(404, json={"detail": "IB could not qualify contract(s): CASH XXX USD@IDEALPRO"}), + httpx.Response(200, json={"nope": []}), + ], +) +def test_daily_midpoints_raise_source_unavailable(resp): + with pytest.raises(SourceUnavailable): + fetch_ib_daily_midpoints( + base_url="http://x", api_key=None, sec_type="CMDTY", symbol="XAUUSD", + client=_client(lambda _r: resp), + ) diff --git a/tests/unit/worker/fixtures/scheduler_jobs_golden.json b/tests/unit/worker/fixtures/scheduler_jobs_golden.json index ec1280086..6c89ba576 100644 --- a/tests/unit/worker/fixtures/scheduler_jobs_golden.json +++ b/tests/unit/worker/fixtures/scheduler_jobs_golden.json @@ -334,6 +334,14 @@ }, "trigger": "cron[month='*', day='*', day_of_week='0-4', hour='18', minute='30'] tz=America/New_York" }, + { + "func": "_gold_factor_compute", + "id": "gold_factor_compute", + "options": { + "name": "'Gold: factor model (real yield / dollar / haven)'" + }, + "trigger": "cron[month='*', day='*', day_of_week='0-4', hour='17', minute='40'] tz=America/New_York" + }, { "func": "_gold_fred_ingest", "id": "gold_fred_ingest", @@ -1141,6 +1149,14 @@ }, "trigger": "cron[month='*', day='*', day_of_week='0-4', hour='18', minute='30'] tz=America/New_York" }, + { + "func": "_gold_factor_compute", + "id": "gold_factor_compute", + "options": { + "name": "'Gold: factor model (real yield / dollar / haven)'" + }, + "trigger": "cron[month='*', day='*', day_of_week='0-4', hour='17', minute='40'] tz=America/New_York" + }, { "func": "_gold_fred_ingest", "id": "gold_fred_ingest", @@ -2436,6 +2452,14 @@ }, "trigger": "cron[month='*', day='*', day_of_week='0-4', hour='18', minute='30'] tz=America/New_York" }, + { + "func": "_gold_factor_compute", + "id": "gold_factor_compute", + "options": { + "name": "'Gold: factor model (real yield / dollar / haven)'" + }, + "trigger": "cron[month='*', day='*', day_of_week='0-4', hour='17', minute='40'] tz=America/New_York" + }, { "func": "_gold_fred_ingest", "id": "gold_fred_ingest", @@ -3073,6 +3097,14 @@ }, "trigger": "cron[month='*', day='*', day_of_week='0-4', hour='18', minute='30'] tz=America/New_York" }, + { + "func": "_gold_factor_compute", + "id": "gold_factor_compute", + "options": { + "name": "'Gold: factor model (real yield / dollar / haven)'" + }, + "trigger": "cron[month='*', day='*', day_of_week='0-4', hour='17', minute='40'] tz=America/New_York" + }, { "func": "_gold_fred_ingest", "id": "gold_fred_ingest", diff --git a/uv.lock b/uv.lock index ac13b5db6..dd3d04f6a 100644 --- a/uv.lock +++ b/uv.lock @@ -2739,6 +2739,7 @@ dependencies = [ { name = "pyyaml" }, { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "statsmodels" }, { name = "tzdata" }, { name = "uvicorn", extra = ["standard"] }, { name = "websockets" }, @@ -2782,6 +2783,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.7" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "scipy", specifier = ">=1.17.1" }, + { name = "statsmodels", specifier = ">=0.14" }, { name = "tzdata", specifier = ">=2024.2" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.32,<1.0" }, { name = "websockets", specifier = ">=12.0" }, diff --git a/web/app/macro/[tab]/goldTab.tsx b/web/app/macro/[tab]/goldTab.tsx index a261c12b0..84abbc034 100644 --- a/web/app/macro/[tab]/goldTab.tsx +++ b/web/app/macro/[tab]/goldTab.tsx @@ -1,4 +1,5 @@ import { GoldCompassLayout } from "@/components/gold/GoldCompassLayout"; +import { GoldFactorBand } from "@/components/gold/factor/GoldFactorBand"; import { GoldMetaChips } from "@/components/gold/GoldMetaChips"; import { GoldPostureNotice } from "@/components/gold/GoldPostureNotice"; import { @@ -131,7 +132,7 @@ export async function GoldTab({ replay }: MacroTabProps) { // tab's date to that endpoint would put two different questions under one pill and let // the answer to the second pass for the answer to the first. Live, both mean "newest", // so they agree and the pill is honest. - const [posture, domain, gauge] = await Promise.all([ + const [posture, domain, gauge, factor] = await Promise.all([ settle( () => (asOf === null ? api.goldState() : api.goldReplay(asOf)), "gold posture API", @@ -147,7 +148,22 @@ export async function GoldTab({ replay }: MacroTabProps) { asOf === null ? settle(() => api.goldGauge(), "gold gauge API") : Promise.resolve({ value: null, error: undefined }), + // The factor model, live-only for the same clock reason: `/api/gold/factor` takes no + // date. Settled on its own so an outage costs this band, not the tab. + asOf === null + ? settle(() => api.goldFactor(), "gold factor API") + : Promise.resolve({ value: null, error: undefined }), ]); + const factorBand = factor.value ? ( + + ) : factor.error ? ( + + ) : null; const verdict = replayVerdictForObsDate(replay, { obsDate: posture.value?.obs_date, @@ -196,6 +212,7 @@ export async function GoldTab({ replay }: MacroTabProps) { <> {status}
+ {factorBand} ) { + return f.x([0, Math.max(1, n - 1)]); +} + +function yearTicks(points: FactorPoint[]): { i: number; label: string }[] { + const out: { i: number; label: string }[] = []; + points.forEach((p, i) => { + const y = p.obs_date.slice(0, 4); + // a window that starts late in a year would print that label on top of the next one + const first = i === 0 && p.obs_date.slice(5, 7) <= "03"; + if (first || (i > 0 && points[i - 1].obs_date.slice(0, 4) !== y)) + out.push({ i, label: y }); + }); + return out; +} + +/** Each slope on its own axis: the units differ (% per bp against % per %), so one shared + * axis would flatten the yield slope into its zero line. */ +function SensitivityPanel({ last }: { last: FactorPoint }) { + const rows = [ + { + label: "Real yield (% per bp)", + b: last.b_r, + lo: last.b_r_lo95, + hi: last.b_r_hi95, + color: PARTS[0].color, + }, + { + label: "Dollar (% per %)", + b: last.b_d, + lo: last.b_d_lo95, + hi: last.b_d_hi95, + color: PARTS[1].color, + }, + { + label: "Safe haven (% per %)", + b: last.b_s, + lo: last.b_s_lo95, + hi: last.b_s_hi95, + color: PARTS[2].color, + }, + ]; + const W = 620; + const RH = 74; + return ( + +
+ + {rows.map((r, k) => { + if (r.b == null || r.lo == null || r.hi == null) return null; + const m = Math.max(Math.abs(r.lo), Math.abs(r.hi)) * 1.15; + const x = (v: number) => 150 + ((v + m) / (2 * m)) * (W - 160); + const y = k * RH + 26; + const crosses = r.lo < 0 && r.hi > 0; + return ( + + + {r.label} + + + {fmt(r.b, 3)} + + + + + + {r.lo.toFixed(2)} + + + {r.hi.toFixed(2)} + + {crosses ? ( + + interval crosses zero + + ) : null} + + ); + })} + +
+

+ Dot = fitted slope, bar = 95% interval · as of {last.obs_date} ·{" "} + {RESEARCH_LABEL} +

+ + ); +} + +function ExplainedSharePanel({ data }: { data: GoldFactor }) { + const pts = data.points.filter((p) => p.explained_share != null); + const f = chartFrame(620, 260, { top: 10, right: 12, bottom: 22, left: 34 }); + const x = xByIndex(pts.length, f); + const y = f.y([-0.4, 1]); + const last = pts.at(-1); + return ( + +
+ + {[0, 0.5, 1].map((t) => ( + {`${t * 100}%`} + ))} + {[data.explained_low, data.explained_high].map((t) => ( + + ))} + [x(i), y(p.explained_share as number)]), + )} + fill="none" + stroke="var(--text-primary)" + strokeWidth={1.4} + /> + {yearTicks(pts).map((t) => ( + + {t.label} + + ))} + +
+

+ Share of the last 63 days' variation the fit explains. Below{" "} + {Math.round(data.explained_low * 100)}%: explains little; above{" "} + {Math.round(data.explained_high * 100)}% with both slopes negative: + explains much. + {last ? ( + <> + {" "} + Now {Math.round((last.explained_share as number) * 100)}% —{" "} + {LABEL_TEXT[last.published_label ?? ""] ?? "not yet labelled"}. + + ) : null} +

+ + ); +} + +function YearToDatePanel({ data }: { data: GoldFactor }) { + const year = data.last_date.slice(0, 4); + const rows = complete(data.points).filter((p) => p.obs_date.startsWith(year)); + const series = [ + { + key: "gold_change", + label: "Gold", + color: "var(--text-primary)", + } as const, + ...PARTS, + ]; + const cum = series.map((s) => { + let acc = 0; + return rows.map((r) => (acc += (r[s.key] as number) ?? 0)); + }); + const all = cum.flat(); + const f = chartFrame(620, 260, { top: 10, right: 12, bottom: 22, left: 34 }); + const lo = Math.min(0, ...all); + const hi = Math.max(0, ...all); + const x = xByIndex(rows.length, f); + const y = f.y([lo, hi]); + return ( + +
+ {series.map((s) => ( + + + {s.label} + + ))} +
+
+ + {niceTicks(lo, hi, 5) + .filter((t) => t >= lo && t <= hi) + .map((t) => ( + + + + {t} + + + ))} + {series.map((s, k) => ( + [x(i), y(v)]))} + fill="none" + stroke={s.color} + strokeWidth={s.key === "gold_change" ? 2 : 1.3} + strokeDasharray={ + s.key === "unexplained_1" || s.key === "own_trailing_drift" + ? "5 4" + : undefined + } + /> + ))} + +
+

+ Percent (100 × log change), {rows.length} days · {RESEARCH_LABEL} +

+ + ); +} + +function SlopeHistory({ + pts, + k, + label, + color, +}: { + pts: FactorPoint[]; + k: "b_d" | "b_r"; + label: string; + color: string; +}) { + const rows = pts.filter((p) => p[k] != null); + const lo = rows.map((p) => p[`${k}_lo95`] as number); + const hi = rows.map((p) => p[`${k}_hi95`] as number); + const f = chartFrame(620, 150, { top: 14, right: 12, bottom: 18, left: 40 }); + const x = xByIndex(rows.length, f); + const y = f.y([Math.min(0, ...lo), Math.max(0, ...hi)]); + return ( + + + {label} + + [x(i), y(v)]), + lo.map((v, i) => [x(i), y(v)]), + )} + fill={color} + opacity={0.18} + /> + + + 0 + + [x(i), y(p[k] as number)]))} + fill="none" + stroke={color} + strokeWidth={1.4} + /> + {yearTicks(rows).map((t) => ( + + {t.label} + + ))} + + ); +} + +function SensitivityHistoryPanel({ data }: { data: GoldFactor }) { + return ( + +
+ + +
+

+ Shade = 95% interval of each 252-day fit · {RESEARCH_LABEL} +

+ + ); +} + +/** + * The gold factor model: how much of gold's daily move the 10-year real yield, the dollar + * and the safe-haven spread explain, and each day's move split into those parts. Live only: + * the endpoint takes no date, so under a replayed observation date it would answer about + * today. + */ +export function GoldFactorBand({ data }: { data: GoldFactor }) { + const last = data.points.at(-1); + return ( +
+
+

What moved gold

+
+

+ Gold's daily move split into the 10-year real yield, the dollar and + the safe-haven spread, from a rolling 252-day fit. {RESEARCH_LABEL}. +

+
+ {last ? : null} + +
+
+ + +
+
+ + +
+
+ ); +} diff --git a/web/components/gold/factor/GoldFactorToggles.tsx b/web/components/gold/factor/GoldFactorToggles.tsx new file mode 100644 index 000000000..02d5bcf8d --- /dev/null +++ b/web/components/gold/factor/GoldFactorToggles.tsx @@ -0,0 +1,320 @@ +"use client"; + +import { useState } from "react"; + +import { BoardPanel } from "@/components/macro/domain/BoardPanel"; +import { chartFrame, niceTicks } from "@/lib/svgChart"; + +import { + PARTS, + RESEARCH_LABEL, + complete, + fmt, + sumParts, + type FactorYear, + type GoldFactor, +} from "./shared"; + +const AXIS = { fontSize: 10, fill: "var(--text-muted)" } as const; + +function Toggle({ + options, + value, + onChange, + label, +}: { + options: readonly { id: T; text: string }[]; + value: T; + onChange: (v: T) => void; + label: string; +}) { + return ( + + {options.map((o) => ( + + ))} + + ); +} + +const WINDOWS = [ + { id: "week", text: "5 days", rows: 5 }, + { id: "month", text: "21 days", rows: 21 }, + { id: "ytd", text: "This year", rows: 0 }, +] as const; +type WindowId = (typeof WINDOWS)[number]["id"]; + +/** Waterfall: each part steps from where the previous one ended, and the last bar is gold's + * own change, which the parts add up to exactly. */ +export function MoveBreakdownPanel({ data }: { data: GoldFactor }) { + const [win, setWin] = useState("month"); + const rows = complete(data.points); + const w = WINDOWS.find((o) => o.id === win)!; + const year = data.last_date.slice(0, 4); + const picked = w.rows + ? rows.slice(-w.rows) + : rows.filter((r) => r.obs_date.startsWith(year)); + const s = sumParts(picked); + const steps = PARTS.map((p) => s[p.key]); + const bars: { + label: string; + color: string; + from: number; + to: number; + v: number; + }[] = PARTS.map((p, i) => { + const from = steps.slice(0, i).reduce((a, b) => a + b, 0); + return { + label: p.label, + color: p.color, + from, + to: from + steps[i], + v: steps[i], + }; + }); + bars.push({ + label: "Gold", + color: "var(--text-primary)", + from: 0, + to: s.gold_change, + v: s.gold_change, + }); + const ends = bars.flatMap((b) => [b.from, b.to]); + const lo = Math.min(0, ...ends); + const hi = Math.max(0, ...ends); + const f = chartFrame(620, 270, { top: 16, right: 8, bottom: 30, left: 34 }); + const y = f.y([lo, hi]); + const bw = (f.W - f.M.left - f.M.right) / bars.length; + return ( + + +
+ + {niceTicks(lo, hi, 5) + .filter((t) => t >= lo && t <= hi) + .map((t) => ( + + + + {t} + + + ))} + {bars.map((b, i) => { + const x0 = f.M.left + i * bw + bw * 0.15; + const top = y(Math.max(b.from, b.to)); + return ( + + + + {fmt(b.v, 1)} + + + {b.label} + + + ); + })} + +
+

+ Percent (100 × log change), {picked.length} days to {data.last_date} ·{" "} + {RESEARCH_LABEL} +

+ + ); +} + +const VIEWS = [ + { id: "grouped", text: "Three factors vs the rest" }, + { id: "all", text: "All five parts" }, +] as const; +type ViewId = (typeof VIEWS)[number]["id"]; + +function segments(y: FactorYear, view: ViewId) { + const factors = y.real_yield_part + y.dollar_part + y.haven_part; + const rest = y.own_trailing_drift + y.unexplained; + return view === "grouped" + ? [ + { label: "Three factors", color: PARTS[0].color, v: factors }, + { label: "The rest", color: PARTS[4].color, v: rest }, + ] + : [ + { label: PARTS[0].label, color: PARTS[0].color, v: y.real_yield_part }, + { label: PARTS[1].label, color: PARTS[1].color, v: y.dollar_part }, + { label: PARTS[2].label, color: PARTS[2].color, v: y.haven_part }, + { + label: PARTS[3].label, + color: PARTS[3].color, + v: y.own_trailing_drift, + }, + { label: PARTS[4].label, color: PARTS[4].color, v: y.unexplained }, + ]; +} + +/** Stacked by sign: positive parts build up from zero, negative ones down, and the dot is + * gold's own change for the year. "The rest" = last year's pace + unexplained — a regrouping, + * not a new fit. */ +export function YearlyBreakdownPanel({ years }: { years: FactorYear[] }) { + const [view, setView] = useState("grouped"); + const stacks = years.map((yr) => { + let up = 0; + let down = 0; + return segments(yr, view).map((s) => { + const from = s.v >= 0 ? up : down; + if (s.v >= 0) up += s.v; + else down += s.v; + return { ...s, from, to: from + s.v }; + }); + }); + const ends = stacks + .flat() + .flatMap((s) => [s.from, s.to]) + .concat(years.map((y) => y.gold_change)); + const lo = Math.min(0, ...ends); + const hi = Math.max(0, ...ends); + const f = chartFrame(620, 270, { top: 10, right: 8, bottom: 22, left: 34 }); + const y = f.y([lo, hi]); + const bw = (f.W - f.M.left - f.M.right) / Math.max(1, years.length); + const legend = segments(years[0] ?? ({} as FactorYear), view); + return ( + + +
+ {legend.map((s) => ( + + + {s.label} + + ))} + + + Gold + +
+
+ + {niceTicks(lo, hi, 5) + .filter((t) => t >= lo && t <= hi) + .map((t) => ( + + + + {t} + + + ))} + {years.map((yr, i) => { + const x0 = f.M.left + i * bw + bw * 0.15; + return ( + + {stacks[i].map((s) => ( + + ))} + + {i % 2 === 0 ? ( + {`'${String(yr.year).slice(2)}`} + ) : null} + + ); + })} + +
+

+ Percent (100 × log change); faded = partial year · {RESEARCH_LABEL} +

+ + ); +} diff --git a/web/components/gold/factor/shared.tsx b/web/components/gold/factor/shared.tsx new file mode 100644 index 000000000..974071257 --- /dev/null +++ b/web/components/gold/factor/shared.tsx @@ -0,0 +1,61 @@ +import type { components } from "@/lib/types"; + +export type GoldFactor = components["schemas"]["GoldFactorResponse"]; +export type FactorPoint = components["schemas"]["GoldFactorPoint"]; +export type FactorYear = components["schemas"]["GoldFactorYear"]; + +/** One line on every research panel. The study described the past and its direction test + * failed; the label says so where the numbers are, not once at the top of the page. */ +export const RESEARCH_LABEL = + "Historical description, not a forecast · the direction test (Phase 3) failed"; + +/** The four parts and the remainder, in the order the breakdown charts stack them. */ +export const PARTS = [ + { + key: "real_yield_part", + label: "Real yield", + color: "var(--positive, #05ad98)", + }, + { key: "dollar_part", label: "Dollar", color: "var(--warning, #f5a623)" }, + { key: "haven_part", label: "Safe haven", color: "var(--info, #8b5cf6)" }, + { + key: "own_trailing_drift", + label: "Last year's pace", + color: "var(--text-muted, #64748b)", + }, + { + key: "unexplained_1", + label: "Unexplained", + color: "var(--text-secondary, #94a3b8)", + }, +] as const; + +export type PartKey = (typeof PARTS)[number]["key"]; + +export const LABEL_TEXT: Record = { + much: "the three factors explain much", + part: "the three factors explain part", + little: "the three factors explain little", +}; + +export function fmt(v: number, digits = 2): string { + const s = v.toFixed(digits); + return v > 0 ? `+${s}` : s; +} + +/** Rows with every part, so a sum over them adds up to gold's change exactly. */ +export function complete(points: FactorPoint[]): FactorPoint[] { + return points.filter((p) => PARTS.every((s) => p[s.key] != null)); +} + +export function sumParts( + rows: FactorPoint[], +): Record { + const out = { gold_change: 0 } as Record; + for (const s of PARTS) out[s.key] = 0; + for (const r of rows) { + out.gold_change += r.gold_change; + for (const s of PARTS) out[s.key] += r[s.key] ?? 0; + } + return out; +} diff --git a/web/lib/api.ts b/web/lib/api.ts index 07ad7a3ff..b9e0d863c 100644 --- a/web/lib/api.ts +++ b/web/lib/api.ts @@ -149,6 +149,7 @@ type MacroContextSnapshotResponse = Json<"/api/macro/snapshot", "get">; type GoldStateResponse = Json<"/api/gold/state", "get">; type GoldReplayResponse = Json<"/api/gold/replay", "get">; type GoldGaugeResponse = Json<"/api/gold/gauge", "get">; +type GoldFactorResponse = Json<"/api/gold/factor", "get">; type GoldLensResponse = Json<"/api/gold/lenses/{lens_id}", "get">; type GoldInputSeriesResponse = Json<"/api/gold/inputs/{series_id}", "get">; type PositioningSnapshot = Json<"/api/positioning/{ticker}", "get">; @@ -422,6 +423,12 @@ export const api = { allow404: true, }, ), + // The gold factor model (real yield / dollar / haven). No date parameter: live only. + // 404 = not computed yet, which is a fact, not a failure. + goldFactor: (): Promise => + _fetch(`/api/gold/factor`, undefined, { + allow404: true, + }), goldLens: ( lensId: "structural" | "cyclical" | "valuation", ): Promise => diff --git a/web/lib/types.ts b/web/lib/types.ts index 941c5e523..bec753b83 100644 --- a/web/lib/types.ts +++ b/web/lib/types.ts @@ -1709,6 +1709,23 @@ export interface paths { patch?: never; trace?: never; }; + "/api/gold/factor": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Gold Factor */ + get: operations["get_gold_factor_api_gold_factor_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/rates/snapshot": { parameters: { query?: never; @@ -5570,6 +5587,97 @@ export interface components { /** Contribution */ contribution: string; }; + /** GoldFactorPoint */ + GoldFactorPoint: { + /** + * Obs Date + * Format: date + */ + obs_date: string; + /** B R */ + b_r: number | null; + /** B R Lo95 */ + b_r_lo95: number | null; + /** B R Hi95 */ + b_r_hi95: number | null; + /** B D */ + b_d: number | null; + /** B D Lo95 */ + b_d_lo95: number | null; + /** B D Hi95 */ + b_d_hi95: number | null; + /** B S */ + b_s: number | null; + /** B S Lo95 */ + b_s_lo95: number | null; + /** B S Hi95 */ + b_s_hi95: number | null; + /** Explained Share */ + explained_share: number | null; + /** Published Label */ + published_label: ("much" | "part" | "little") | null; + /** Gold Change */ + gold_change: number; + /** Own Trailing Drift */ + own_trailing_drift: number | null; + /** Real Yield Part */ + real_yield_part: number | null; + /** Dollar Part */ + dollar_part: number | null; + /** Haven Part */ + haven_part: number | null; + /** Unexplained 1 */ + unexplained_1: number | null; + /** Unexplained 21 */ + unexplained_21: number | null; + }; + /** GoldFactorResponse */ + GoldFactorResponse: { + /** + * Computed At + * Format: date-time + */ + computed_at: string; + /** + * Last Date + * Format: date + */ + last_date: string; + /** Explained Low */ + explained_low: number; + /** Explained High */ + explained_high: number; + /** Points */ + points: components["schemas"]["GoldFactorPoint"][]; + /** Years */ + years: components["schemas"]["GoldFactorYear"][]; + }; + /** + * GoldFactorYear + * @description Sums over the rows of a calendar year that have every part (percent). + */ + GoldFactorYear: { + /** Year */ + year: number; + /** Rows */ + rows: number; + /** Partial Year */ + partial_year: boolean; + /** Gold Change */ + gold_change: number; + /** Own Trailing Drift */ + own_trailing_drift: number; + /** Real Yield Part */ + real_yield_part: number; + /** Dollar Part */ + dollar_part: number; + /** Haven Part */ + haven_part: number; + /** Unexplained */ + unexplained: number; + /** Explained Share */ + explained_share: number; + }; /** GoldGauge60dTimeSeriesPoint */ GoldGauge60dTimeSeriesPoint: { /** @@ -15851,6 +15959,37 @@ export interface operations { }; }; }; + get_gold_factor_api_gold_factor_get: { + parameters: { + query?: { + from?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GoldFactorResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; rates_snapshot_api_rates_snapshot_get: { parameters: { query?: { diff --git a/web/tests/unit/goldFactorBand.test.tsx b/web/tests/unit/goldFactorBand.test.tsx new file mode 100644 index 000000000..6369516e5 --- /dev/null +++ b/web/tests/unit/goldFactorBand.test.tsx @@ -0,0 +1,55 @@ +import { fireEvent, render, screen, within } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { GoldFactorBand } from "@/components/gold/factor/GoldFactorBand"; +import type { components } from "@/lib/types"; + +type Factor = components["schemas"]["GoldFactorResponse"]; + +// Labelled synthetic model output (not market data): 30 rows whose parts add up exactly. +const points = Array.from({ length: 30 }, (_, i) => ({ + obs_date: `2026-09-${String(i + 1).padStart(2, "0")}`, + b_r: -0.05, b_r_lo95: -0.12, b_r_hi95: 0.01, + b_d: -2.3, b_d_lo95: -3.4, b_d_hi95: -1.3, + b_s: 0.7, b_s_lo95: -0.4, b_s_hi95: 1.7, + explained_share: 0.26, + published_label: "part" as const, + real_yield_part: -0.1, dollar_part: 0.2, haven_part: 0.05, + own_trailing_drift: 0.1, unexplained_1: 0.25, + gold_change: 0.5, + unexplained_21: null, +})); +const FIXTURE: Factor = { + computed_at: "2026-10-10T21:40:00Z", + last_date: "2026-09-30", + explained_low: 0.218, + explained_high: 0.402, + points, + years: [ + { year: 2025, rows: 250, partial_year: false, gold_change: 40, own_trailing_drift: 20, + real_yield_part: 5, dollar_part: 8, haven_part: 1, unexplained: 6, explained_share: 0.3 }, + { year: 2026, rows: 30, partial_year: true, gold_change: 15, own_trailing_drift: 3, + real_yield_part: -3, dollar_part: 6, haven_part: 1.5, unexplained: 7.5, explained_share: 0.2 }, + ], +}; // prettier-ignore + +describe("GoldFactorBand", () => { + it("renders the six panels, each labelled as historical description", () => { + render(); + const band = screen.getByTestId("gold-factor-band"); + for (const id of ["sensitivity", "explained", "breakdown", "ytd", "yearly", "slope-history"]) { + expect(band.querySelector(`#gold-factor-${id}`)).not.toBeNull(); + } + expect(within(band).getAllByText(/Historical description, not a forecast/).length).toBeGreaterThanOrEqual(6); + expect(within(band).getByText(/explain part/)).toBeTruthy(); + }); + + it("waterfall sums the window and ends at gold's own change", () => { + render(); + const panel = document.getElementById("gold-factor-breakdown")!; + // 21 rows x 0.5 = +10.5 on the default window; 5 rows x 0.5 = +2.5 after the toggle + expect(within(panel).getByText("+10.5")).toBeTruthy(); + fireEvent.click(within(panel).getByRole("button", { name: "5 days" })); + expect(within(panel).getByText("+2.5")).toBeTruthy(); + }); +});