From c049daf3148fcbc5d34512fbaa594768d82a217c Mon Sep 17 00:00:00 2001 From: Seth Stoudenmier Date: Tue, 3 Nov 2026 15:15:15 -0500 Subject: [PATCH 1/3] [DEV-16758] Make date fields dynamic in eval framework --- .../llm/assistants/filter_search.py | 11 +- .../llm/evals/assistants/filter_search.py | 1 + usaspending_api/llm/evals/base.py | 12 +- usaspending_api/llm/evals/data/config.json | 186 ++++-------------- usaspending_api/llm/evals/loader.py | 22 ++- .../llm/evals/managing_llm_eval_framework.md | 23 +++ usaspending_api/llm/evals/models.py | 2 + usaspending_api/llm/evals/temporal.py | 130 ++++++++++++ .../llm/management/commands/run_llm_eval.py | 10 + 9 files changed, 235 insertions(+), 162 deletions(-) create mode 100644 usaspending_api/llm/evals/temporal.py diff --git a/usaspending_api/llm/assistants/filter_search.py b/usaspending_api/llm/assistants/filter_search.py index 932ec4a50c..0fb10a3e2c 100644 --- a/usaspending_api/llm/assistants/filter_search.py +++ b/usaspending_api/llm/assistants/filter_search.py @@ -22,8 +22,9 @@ class FilterSearchAssistant: "You are USAspending search assistant. Help the user select filters to search for federal spending" ) - def __init__(self, assistant: Assistant, tools: list[AITool], session: Session) -> None: + def __init__(self, assistant: Assistant, tools: list[AITool], session: Session, as_of: date | None = None) -> None: self.assistant = assistant + self.as_of = as_of self.tools = tools self.tools_by_name = {tool.description.name: tool for tool in tools} self.session = session @@ -126,9 +127,9 @@ async def aget_tool_config(self) -> dict[str, list[dict]]: return self._tool_config @staticmethod - def _fiscal_year_date_context() -> str: - """Build the current-date/fiscal-year string appended to the system prompt.""" - today = date.today() + def _fiscal_year_date_context(today: date | None = None) -> str: + """Build the current-date/fiscal-year string appended to the system prompt (``today`` defaults to now).""" + today = today or date.today() current_fy = today.year + 1 if today.month >= 10 else today.year return ( f"\nThe current date is {today.strftime('%m/%d/%Y')}. The current federal fiscal year is " @@ -143,7 +144,7 @@ def _fiscal_year_date_context() -> str: def system_message(self) -> str: """Return the active Assistant's system prompt or the default prompt.""" if self.assistant.system_prompt: - return self.assistant.system_prompt.text + self._fiscal_year_date_context() + return self.assistant.system_prompt.text + self._fiscal_year_date_context(self.as_of) return self.DEFAULT_SYSTEM_MESSAGE @cached_property diff --git a/usaspending_api/llm/evals/assistants/filter_search.py b/usaspending_api/llm/evals/assistants/filter_search.py index 043b4f9eeb..36196aa59e 100644 --- a/usaspending_api/llm/evals/assistants/filter_search.py +++ b/usaspending_api/llm/evals/assistants/filter_search.py @@ -111,6 +111,7 @@ async def run_eval_case(case: EvalCase) -> EvalObservation: assistant=assistant_config, tools=filter_search_tools, session=session, + as_of=case.as_of, ) events: list[dict[str, Any]] = [] diff --git a/usaspending_api/llm/evals/base.py b/usaspending_api/llm/evals/base.py index 2b5c330ac1..95a3cb9e06 100644 --- a/usaspending_api/llm/evals/base.py +++ b/usaspending_api/llm/evals/base.py @@ -1,5 +1,6 @@ import logging from abc import ABC, abstractmethod +from datetime import date from pathlib import Path from statistics import fmean @@ -24,7 +25,7 @@ class BaseEval(ABC): assistant_name: str default_dataset_name: str - def __init__( + def __init__( # noqa: PLR0913 self, *, dataset_name: str | None = None, @@ -33,6 +34,7 @@ def __init__( tags: set[str] | None = None, fail_under: float | None = None, incremental_output: Path | None = None, + as_of: date | None = None, ) -> None: if not self.assistant_name: raise ValueError("assistant_name must be defined.") @@ -47,6 +49,8 @@ def __init__( self.tags = tags self.fail_under = fail_under self.incremental_output = incremental_output + # Fixed once per run so relative dataset values (e.g. "last quarter") and the assistant agree on today. + self.as_of = as_of or date.today() def load_cases(self) -> list[EvalCase]: """ @@ -63,6 +67,7 @@ def load_cases(self) -> list[EvalCase]: selected_case_names=self.selected_case_names, include_unapproved=self.include_unapproved, tags=self.tags, + as_of=self.as_of, ) @abstractmethod @@ -140,13 +145,13 @@ def run(self) -> EvalSummary: Progress is logged incrementally so results are not lost if the run fails. """ - all_cases = load_all_cases(self.dataset_name) + all_cases = load_all_cases(self.dataset_name, self.as_of) cases = self.load_cases() if not cases: raise ValueError(f"Dataset `{self.dataset_name}` does not contain evaluation cases.") - logger.info(f"Starting evaluation: {len(cases)} case(s) to run") + logger.info(f"Starting evaluation: {len(cases)} case(s) to run (as of {self.as_of.isoformat()})") # Prepare incremental output file if specified if self.incremental_output: @@ -155,6 +160,7 @@ def run(self) -> EvalSummary: with self.incremental_output.open("w", encoding="utf-8") as f: f.write(f"# Incremental evaluation results for {self.assistant_name}\n") f.write(f"# Dataset: {self.dataset_name}\n") + f.write(f"# As of: {self.as_of.isoformat()}\n") f.write(f"# Total cases: {len(cases)}\n\n") # Execute cases one by one with progress logging diff --git a/usaspending_api/llm/evals/data/config.json b/usaspending_api/llm/evals/data/config.json index 73437ebd73..b0eb8ec275 100644 --- a/usaspending_api/llm/evals/data/config.json +++ b/usaspending_api/llm/evals/data/config.json @@ -145,8 +145,8 @@ "04", "F001" ], - "timePeriodFY": ["2026"] - }, + "$time": {"unit": "fiscal_year", "offset": -1} + }, "expected_tools": [ "lookup_award", "execute_filter" @@ -278,9 +278,8 @@ } } }, - "timePeriodType": "fy", - "timePeriodFY": ["2026"] - }, + "$time": {"unit": "fiscal_year", "offset": -1} + }, "expected_tools": [ "lookup_recipient", "lookup_agencies", @@ -311,9 +310,8 @@ } } }, - "timePeriodType": "dr", - "time_period": [{"start_date": "2026-09-01", "end_date": "2026-09-30"}] - }, + "$time": {"unit": "month", "offset": -1} + }, "expected_tools": [ "lookup_agencies", "execute_filter" @@ -484,8 +482,8 @@ "selectedRecipients": [ "SPACE EXPLORATION TECHNOLOGIES CORP." ], - "timePeriodFY": ["2023", "2023", "2024", "2025", "2026"] - }, + "$time": {"unit": "fiscal_year", "offset": -1, "span": 5} + }, "expected_tools": [ "lookup_recipient", "execute_filter" @@ -539,13 +537,7 @@ "id": 26, "query": "Total funding given to companies in Virginia last quarter?", "expected_output": { - "timePeriodType": "dr", - "time_period": [ - { - "start_date": "2026-07-01", - "end_date": "2026-09-30" - } - ], + "$time": {"unit": "fiscal_quarter", "offset": -1}, "recipientType": [ "business", "minority_owned_business", @@ -596,13 +588,7 @@ "id": 27, "query": "Total funding given to recipients in Virginia last quarter?", "expected_output": { - "timePeriodType": "dr", - "time_period": [ - { - "start_date": "2026-07-01", - "end_date": "2026-09-30" - } - ], + "$time": {"unit": "fiscal_quarter", "offset": -1}, "selectedRecipientLocations": { "USA_VA": { "identifier": "USA_VA", @@ -726,10 +712,7 @@ "id": 30, "query": "How much funding did USAID give to overseas companies last fiscal year?", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "selectedFundingAgencies": { "801_toptier": { "id": 801, @@ -912,13 +895,7 @@ "id": 36, "query": "How much was contracted to Booz Allen for AI last month?", "expected_output": { - "timePeriodType": "dr", - "time_period": [ - { - "start_date": "2026-09-01", - "end_date": "2026-09-30" - } - ], + "$time": {"unit": "month", "offset": -1}, "selectedRecipients": [ "Booz Allen" ], @@ -975,10 +952,7 @@ "id": 38, "query": "How many loans were given to small businesses by the SBA last year?", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "awardType": [ "07", "08", @@ -1078,10 +1052,7 @@ "id": 42, "query": "How many contract idvs were given to companies in the DMV last year?", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "awardType": [ "IDV_A", "IDV_B", @@ -1150,10 +1121,7 @@ "id": 43, "query": "How much funding did CBP receive for this fiscal year?", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2027" - ], + "$time": {"unit": "fiscal_year", "offset": 0}, "selectedFundingAgencies": { "778_subtier": { "id": 778, @@ -1188,10 +1156,7 @@ "id": 44, "query": "Which state received the most funding for medicare last FY?", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "selectedCFDA": { "93.071": {"identifier": "93.071", "program_number": "93.071", "program_title": "Medicare Enrollment Assistance Program"}, "93.518": {"identifier": "93.518", "program_number": "93.518", "program_title": "Affordable Care Act - Medicare Improvements for Patients and Providers"}, @@ -1224,13 +1189,7 @@ "id": 45, "query": "Which agencies financed construction contracts in Texas last month?", "expected_output": { - "timePeriodType": "dr", - "time_period": [ - { - "start_date": "2026-09-01", - "end_date": "2026-09-30" - } - ], + "$time": {"unit": "month", "offset": -1}, "awardType": [ "A", "B", @@ -1318,10 +1277,7 @@ "id": 47, "query": "What was the total amount of funding that went to black owned businesses last year?", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "recipientType": [ "black_american_owned_business" ] @@ -1452,10 +1408,7 @@ "id": 56, "query": "All contracts over $100000 awarded to small businesses, minority owned businesses and veteran owned businesses in Tennessee last year", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "recipientType": [ "small_business", "minority_owned_business", @@ -1664,10 +1617,7 @@ "id": 64, "query": "Find all grants between $100000 and $200000 last year", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "awardType": [ "02", "03", @@ -1758,10 +1708,7 @@ "IDV_D", "IDV_E" ], - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "selectedAwardingAgencies": { "766_toptier": { "id": 766, @@ -2409,10 +2356,7 @@ "id": 94, "query": "Awards with NAICS 111910 for last year", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "naicsCodes": { "require": [ "111910" @@ -2616,10 +2560,7 @@ "id": 101, "query": "Contracts for chemical weapons last year", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "pscCodes": { "require": [ ["Product", "1040"] @@ -3047,13 +2988,7 @@ "id": 117, "query": "List all fixed price and time and materials contracts awarded last month", "expected_output": { - "timePeriodType": "dr", - "time_period": [ - { - "start_date": "2026-09-01", - "end_date": "2026-09-30" - } - ], + "$time": {"unit": "month", "offset": -1}, "pricingType": [ "A", "B", @@ -3086,10 +3021,7 @@ "id": 118, "query": "Show me cost reimbursement contracts awarded to small businesses in Virginia last year for technology services", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "pricingType": [ "R", "S", @@ -3154,10 +3086,7 @@ "id": 119, "query": "Find all cost reimbursement contracts set aside for small businesses in Virginia last year for technology services", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "pricingType": [ "R", "S", @@ -3349,10 +3278,7 @@ "id": 125, "query": "How many fixed priced contracts set aside for veteran owned business last year for consulting services were there?", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "pricingType": [ "A", "B", @@ -3487,10 +3413,7 @@ "id": 129, "query": "All no bid contracts awarded last year for mining services", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "extentCompeted": [ "B" ], @@ -3794,10 +3717,7 @@ "id": 139, "query": "All spending on the updates to the White house ball room this year", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2027" - ], + "$time": {"unit": "fiscal_year", "offset": 0}, "keyword": [ "updates to the White house ball room" ] @@ -4032,10 +3952,7 @@ "id": 147, "query": "All contracts with 123 Main St. in the award description last year", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "awardDescription": "123 Main St.", "awardType": [ "A", @@ -4125,10 +4042,7 @@ "id": 151, "query": "All awards over 1 trillion last fiscal year", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "awardAmounts": { "specific": [ 1000000000000, @@ -4199,10 +4113,7 @@ "id": 153, "query": "Show me all awards for assistance listing 16.014 last year", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "selectedCFDA": { "16.014": {"identifier": "16.014", "program_number": "16.014", "program_title": "Engaging Men and Youth in Preventing Domestic Violence, Dating Violence, Sexual Assault, and Stalking"} } @@ -4258,10 +4169,7 @@ "id": 155, "query": "List all spending under Treasury Account Symbol 310-2012/2012-3200-000 last fiscal year", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "tasCodes": { "require": [ ["310", "310-3200", "310-2012/2012-3200-000"] @@ -4390,13 +4298,7 @@ "id": 160, "query": "All contracts that available for competition in Utah so far this year", "expected_output": { - "timePeriodType": "dr", - "time_period": [ - { - "start_date": "2026-10-01", - "end_date": "2026-10-05" - } - ], + "$time": {"unit": "fiscal_year", "offset": 0, "to_date": true, "as": "dr"}, "extentCompeted": [ "A" ], @@ -4617,10 +4519,7 @@ "id": 170, "query": "Show me all awards given to Nonprofits this year", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2027" - ], + "$time": {"unit": "fiscal_year", "offset": 0}, "recipientType": [ "nonprofit" ] @@ -4667,10 +4566,7 @@ "id": 175, "query": "How much taxpayer dollars went to defense contractors this year?", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2027" - ], + "$time": {"unit": "fiscal_year", "offset": 0}, "keyword": [ "defense" ] @@ -4788,10 +4684,7 @@ "id": 181, "query": "How much did we give to foreign aid last year?", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "locationDomesticForeign": "foreign" }, "expected_tools": [ @@ -4836,10 +4729,7 @@ "id": 183, "query": "8(a) contracts in the last year", "expected_output": { - "timePeriodType": "fy", - "timePeriodFY": [ - "2026" - ], + "$time": {"unit": "fiscal_year", "offset": -1}, "setAside": [ "8A", "8AN", diff --git a/usaspending_api/llm/evals/loader.py b/usaspending_api/llm/evals/loader.py index 7239ec9fa4..85b66c7984 100644 --- a/usaspending_api/llm/evals/loader.py +++ b/usaspending_api/llm/evals/loader.py @@ -6,6 +6,7 @@ """ import json +from datetime import date from pathlib import Path from typing import Any @@ -13,6 +14,7 @@ from usaspending_api.llm.evals.exceptions import DatasetError from usaspending_api.llm.evals.models import EvalCase +from usaspending_api.llm.evals.temporal import TIME_KEY, resolve_time_spec from usaspending_api.llm.models.py_models import Filters REQUIRED_FIELDS = { @@ -72,14 +74,17 @@ def parse_approved(value: Any, case_id: str) -> bool: raise DatasetError(f"Case '{case_id}' has invalid approved value '{value}'. Use a JSON boolean.") -def parse_expected_output(value: Any, case_id: str) -> dict[str, Any]: - """Validates and returns the expected output object from a JSON case.""" +def parse_expected_output(value: Any, case_id: str, as_of: date) -> dict[str, Any]: + """Validates and returns the expected output object from a JSON case, resolving any relative `$time` spec.""" if not isinstance(value, dict) or not value: raise DatasetError(f"Case '{case_id}' must define a non-empty expected_output object.") if not all(isinstance(key, str) and key for key in value): raise DatasetError(f"Case '{case_id}' expected_output keys must be non-empty strings.") + if TIME_KEY in value: + value = {k: v for k, v in value.items() if k != TIME_KEY} | resolve_time_spec(value[TIME_KEY], as_of, case_id) + # Pass through Filters model to normalize and include all fields with defaults. result = Filters(**value).model_dump(exclude_none=True) @@ -145,7 +150,7 @@ def _parse_query(value: Any, case_id: str) -> str: return query -def _parse_case(row: Any, case_number: int, seen_case_ids: set[str]) -> EvalCase: +def _parse_case(row: Any, case_number: int, seen_case_ids: set[str], as_of: date) -> EvalCase: if not isinstance(row, dict): raise DatasetError(f"Dataset case {case_number} must be a JSON object.") @@ -167,11 +172,12 @@ def _parse_case(row: Any, case_number: int, seen_case_ids: set[str]) -> EvalCase name=case_id, input={"query": _parse_query(row["query"], case_id)}, expected_tool_calls=parse_expected_tools(row["expected_tools"], case_id), - expected_output=parse_expected_output(row["expected_output"], case_id), + expected_output=parse_expected_output(row["expected_output"], case_id, as_of), metadata={ "approved": approved, "tags": case_tags, }, + as_of=as_of, ) @@ -184,6 +190,7 @@ def _case_is_selected(case: EvalCase, include_unapproved: bool, tags: set[str] | def parse_json_cases( dataset_path: Path, *, + as_of: date, include_unapproved: bool = False, tags: set[str] | None = None, ) -> list[EvalCase]: @@ -192,17 +199,18 @@ def parse_json_cases( seen_case_ids: set[str] = set() for case_number, row in enumerate(_load_dataset(dataset_path), start=1): - case = _parse_case(row, case_number, seen_case_ids) + case = _parse_case(row, case_number, seen_case_ids, as_of) if _case_is_selected(case, include_unapproved, tags): cases.append(case) return cases -def load_all_cases(dataset_name: str) -> list[EvalCase]: +def load_all_cases(dataset_name: str, as_of: date | None = None) -> list[EvalCase]: """Loads every validated case without approval, tag, or case filtering.""" return parse_json_cases( resolve_dataset_path(dataset_name), + as_of=as_of or date.today(), include_unapproved=True, ) @@ -213,10 +221,12 @@ def load_cases( selected_case_names: set[str] | None = None, include_unapproved: bool = False, tags: set[str] | None = None, + as_of: date | None = None, ) -> list[EvalCase]: """Loads all eligible cases, then optionally narrows them by case ID.""" cases = parse_json_cases( resolve_dataset_path(dataset_name), + as_of=as_of or date.today(), include_unapproved=include_unapproved, tags=tags, ) diff --git a/usaspending_api/llm/evals/managing_llm_eval_framework.md b/usaspending_api/llm/evals/managing_llm_eval_framework.md index a2f160e08e..891d1f9fd0 100644 --- a/usaspending_api/llm/evals/managing_llm_eval_framework.md +++ b/usaspending_api/llm/evals/managing_llm_eval_framework.md @@ -458,6 +458,29 @@ The report is the stakeholder-facing output. It should not be edited and fed back into `config.json` automatically. Any ground-truth change should be made intentionally in the JSON source. +## Relative Time Periods (`$time`) + +Queries such as "last quarter" or "this fiscal year" should not hardcode dates in `expected_output`. Use a `$time` object instead; the loader replaces it with `timePeriodType` plus `timePeriodFY` or `time_period` when the dataset is loaded (see `llm/evals/temporal.py`). + +```json +"expected_output": { + "selectedRecipients": ["Booz Allen"], + "$time": {"unit": "month", "offset": -1} +} +``` + +| Key | Description | +| --- | --- | +| `unit` | Required. `fiscal_year`, `fiscal_quarter`, `calendar_year` or `month`. | +| `offset` | Units relative to the current one; `0` is current, `-1` is previous. Default `0`. | +| `span` | Number of consecutive units ending at `offset` (e.g. "last 5 years" is `offset: -1, span: 5`). Default `1`. | +| `to_date` | Cap the end of a date range at the as-of date ("so far this year"). Default `false`. | +| `as` | Force `fy` (fiscal years only) or `dr` (date range). Defaults to `fy` for `fiscal_year` and `dr` otherwise. | + +Fixed periods that never change (e.g. "FY25", "Q2 2024") stay as literal `timePeriodType`/`timePeriodFY`/`time_period` values. + +Use `--as-of YYYY-MM-DD` on `run_llm_eval` to pin "today" for a run. The same date is used to resolve `$time` and is given to the assistant as the current date, so a run can be reproduced later. It defaults to the current date. + ## Ground Truth Values and Normalization The framework compares expected filter values to the canonical filter payload passed to `execute_filter`. diff --git a/usaspending_api/llm/evals/models.py b/usaspending_api/llm/evals/models.py index 5e78ba5723..ecbcbfcd7c 100644 --- a/usaspending_api/llm/evals/models.py +++ b/usaspending_api/llm/evals/models.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field +from datetime import date from typing import Any @@ -46,6 +47,7 @@ class EvalCase: expected_tool_calls: tuple[str, ...] = () expected_output: dict[str, Any] = field(default_factory=dict) metadata: dict[str, Any] = field(default_factory=dict) + as_of: date | None = None @dataclass(frozen=True) diff --git a/usaspending_api/llm/evals/temporal.py b/usaspending_api/llm/evals/temporal.py new file mode 100644 index 0000000000..3cf237d84c --- /dev/null +++ b/usaspending_api/llm/evals/temporal.py @@ -0,0 +1,130 @@ +""" +Resolves relative time period specs from the ground truth dataset into concrete filter values. + +A case can replace hardcoded time period fields in ``expected_output`` with a ``$time`` object so that queries like +"last quarter" or "this fiscal year" stay correct no matter when the evaluation runs: + + "$time": {"unit": "fiscal_quarter", "offset": -1} + +Spec keys: + unit: One of ``fiscal_year``, ``fiscal_quarter``, ``calendar_year`` or ``month``. (required) + offset: Whole units relative to the unit containing the as-of date; 0 is current, -1 is previous. (default 0) + span: Number of consecutive units, ending at ``offset``. (default 1) + to_date: Cap the end of a date range at the as-of date, e.g. "so far this year". (default false) + as: ``fy`` or ``dr`` to force the output form. Defaults to ``fy`` for ``fiscal_year`` and ``dr`` otherwise. + +Federal fiscal years are named for the calendar year they end in and run October 1 through September 30. +""" + +import calendar +from datetime import date +from typing import Any + +from usaspending_api.llm.evals.exceptions import DatasetError + +TIME_KEY = "$time" + +UNITS = {"fiscal_year", "fiscal_quarter", "calendar_year", "month"} +OUTPUT_FORMS = {"fy", "dr"} +SPEC_KEYS = {"unit", "offset", "span", "to_date", "as"} + +# First month of each fiscal quarter, Q1 through Q4. +FISCAL_QUARTER_START_MONTHS = (10, 1, 4, 7) + + +def fiscal_year_of(day: date) -> int: + return day.year + 1 if day.month >= 10 else day.year + + +def _fiscal_quarter_index(day: date) -> int: + """Return a continuous index (fiscal year * 4 + zero-based quarter) for the quarter containing ``day``.""" + return fiscal_year_of(day) * 4 + ((day.month - 10) % 12) // 3 + + +def _fiscal_quarter_start(index: int) -> date: + fiscal_year, quarter = divmod(index, 4) + month = FISCAL_QUARTER_START_MONTHS[quarter] + return date(fiscal_year - 1 if quarter == 0 else fiscal_year, month, 1) + + +def _month_start(index: int) -> date: + """``index`` is year * 12 + zero-based month.""" + year, month = divmod(index, 12) + return date(year, month + 1, 1) + + +def _month_end(index: int) -> date: + year, month = divmod(index, 12) + return date(year, month + 1, calendar.monthrange(year, month + 1)[1]) + + +def _date_range(unit: str, first: int, last: int) -> tuple[date, date]: + """Return the start and end dates covering units ``first`` through ``last`` (inclusive, unit-specific index).""" + result = None + if unit == "fiscal_year": + result = date(first - 1, 10, 1), date(last, 9, 30) + if unit == "calendar_year": + result = date(first, 1, 1), date(last, 12, 31) + if unit == "fiscal_quarter": + result = _fiscal_quarter_start(first), _fiscal_quarter_start(last + 1) - date.resolution + return result or (_month_start(first), _month_end(last)) + + +def _current_index(unit: str, today: date) -> int: + result = None + if unit == "fiscal_year": + result = fiscal_year_of(today) + if unit == "calendar_year": + result = today.year + if unit == "fiscal_quarter": + result = _fiscal_quarter_index(today) + return result or (today.year * 12 + today.month - 1) + + +def _validate(spec: Any, case_id: str) -> dict[str, Any]: + if not isinstance(spec, dict): + raise DatasetError(f"Case '{case_id}' {TIME_KEY} must be an object.") + + unknown_keys = set(spec) - SPEC_KEYS + if unknown_keys: + raise DatasetError(f"Case '{case_id}' {TIME_KEY} has unknown key(s): {', '.join(sorted(unknown_keys))}.") + + if spec.get("unit") not in UNITS: + raise DatasetError(f"Case '{case_id}' {TIME_KEY} unit must be one of: {', '.join(sorted(UNITS))}.") + + for key in ("offset", "span"): + value = spec.get(key, 0 if key == "offset" else 1) + if isinstance(value, bool) or not isinstance(value, int): + raise DatasetError(f"Case '{case_id}' {TIME_KEY} {key} must be an integer.") + + if spec.get("span", 1) < 1: + raise DatasetError(f"Case '{case_id}' {TIME_KEY} span must be at least 1.") + + if "as" in spec and spec["as"] not in OUTPUT_FORMS: + raise DatasetError(f"Case '{case_id}' {TIME_KEY} as must be one of: {', '.join(sorted(OUTPUT_FORMS))}.") + + if spec.get("as") == "fy" and spec["unit"] != "fiscal_year": + raise DatasetError(f"Case '{case_id}' {TIME_KEY} can only output 'fy' for the fiscal_year unit.") + + return spec + + +def resolve_time_spec(spec: Any, today: date, case_id: str = "?") -> dict[str, Any]: + """Convert a ``$time`` spec into ``timePeriodType`` plus ``timePeriodFY`` or ``time_period`` filter values.""" + spec = _validate(spec, case_id) + unit = spec["unit"] + last = _current_index(unit, today) + spec.get("offset", 0) + first = last - spec.get("span", 1) + 1 + output_form = spec.get("as", "fy" if unit == "fiscal_year" else "dr") + + if output_form == "fy": + return {"timePeriodType": "fy", "timePeriodFY": [str(year) for year in range(first, last + 1)]} + + start_date, end_date = _date_range(unit, first, last) + if spec.get("to_date"): + end_date = min(end_date, today) + + return { + "timePeriodType": "dr", + "time_period": [{"start_date": start_date.isoformat(), "end_date": end_date.isoformat()}], + } diff --git a/usaspending_api/llm/management/commands/run_llm_eval.py b/usaspending_api/llm/management/commands/run_llm_eval.py index eb543ecc65..88a2e09474 100644 --- a/usaspending_api/llm/management/commands/run_llm_eval.py +++ b/usaspending_api/llm/management/commands/run_llm_eval.py @@ -1,4 +1,5 @@ import logging +from datetime import date from pathlib import Path from django.core.management.base import BaseCommand, CommandError, CommandParser @@ -50,6 +51,14 @@ def add_arguments(self, parser: CommandParser) -> None: parser.add_argument( "--include-unapproved", action="store_true", help="Include draft cases whose approved value is false." ) + parser.add_argument( + "--as-of", + type=date.fromisoformat, + help=( + "Date (YYYY-MM-DD) treated as today when resolving relative dataset values such as 'last quarter' " + "and when telling the assistant the current date. (default: today)" + ), + ) parser.add_argument( "--fail-under", type=float, @@ -99,6 +108,7 @@ def _run_evaluation(self, options: dict) -> EvalSummary: tags=set(options["tags"] or []), fail_under=options["fail_under"], incremental_output=options.get("incremental_output"), + as_of=options.get("as_of"), ) return evaluator.run() except (EvalError, ValueError) as exc: From c5d2d90698edb01c45dbb4fa0e1e87072ce8047e Mon Sep 17 00:00:00 2001 From: Seth Stoudenmier Date: Wed, 4 Nov 2026 09:13:53 -0500 Subject: [PATCH 2/3] [DEV-16758] Fix failing tests --- usaspending_api/llm/evals/loader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/usaspending_api/llm/evals/loader.py b/usaspending_api/llm/evals/loader.py index 85b66c7984..a7d032672c 100644 --- a/usaspending_api/llm/evals/loader.py +++ b/usaspending_api/llm/evals/loader.py @@ -190,11 +190,12 @@ def _case_is_selected(case: EvalCase, include_unapproved: bool, tags: set[str] | def parse_json_cases( dataset_path: Path, *, - as_of: date, + as_of: date | None = None, include_unapproved: bool = False, tags: set[str] | None = None, ) -> list[EvalCase]: - """Load a JSON file and return the selected, validated cases.""" + """Load a JSON file and return the selected, validated cases. ``as_of`` defaults to today.""" + as_of = as_of or date.today() cases = [] seen_case_ids: set[str] = set() From bdadec37e5b18d305b4a1e17057e60b1a100c5da Mon Sep 17 00:00:00 2001 From: Seth Stoudenmier Date: Wed, 4 Nov 2026 16:55:57 -0500 Subject: [PATCH 3/3] [DEV-16758] Remove cached_property for system message --- usaspending_api/llm/assistants/filter_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usaspending_api/llm/assistants/filter_search.py b/usaspending_api/llm/assistants/filter_search.py index 0fb10a3e2c..07aaa6a60d 100644 --- a/usaspending_api/llm/assistants/filter_search.py +++ b/usaspending_api/llm/assistants/filter_search.py @@ -140,7 +140,7 @@ def _fiscal_year_date_context(today: date | None = None) -> str: f"timePeriodFY, use the bare four-digit year (e.g. '{current_fy}'), never the 'FY' prefix." ) - @cached_property + @property def system_message(self) -> str: """Return the active Assistant's system prompt or the default prompt.""" if self.assistant.system_prompt: