[DEV-16758] Dynamic time period values in eval config

usaspending_api/llm/assistants/filter_search.py

@@ -22,8 +22,9 @@ class FilterSearchAssistant:
         "You are USAspending search assistant. Help the user select filters to search for federal spending"
     )
 
-    def __init__(self, assistant: Assistant, tools: list[AITool], session: Session) -> None:
+    def __init__(self, assistant: Assistant, tools: list[AITool], session: Session, as_of: date | None = None) -> None:
         self.assistant = assistant
+        self.as_of = as_of
         self.tools = tools
         self.tools_by_name = {tool.description.name: tool for tool in tools}
         self.session = session
@@ -127,9 +128,9 @@ async def aget_tool_config(self) -> dict[str, Any]:
         return {**self._tool_config, "toolChoice": {"auto": {}}}
 
     @staticmethod
-    def _fiscal_year_date_context() -> str:
-        """Build the current-date/fiscal-year string appended to the system prompt."""
-        today = date.today()
+    def _fiscal_year_date_context(today: date | None = None) -> str:
+        """Build the current-date/fiscal-year string appended to the system prompt (``today`` defaults to now)."""
+        today = today or date.today()
         current_fy = today.year + 1 if today.month >= 10 else today.year
         return (
             f"\nThe current date is {today.strftime('%m/%d/%Y')}. The current federal fiscal year is "
@@ -140,11 +141,11 @@ def _fiscal_year_date_context() -> str:
             f"timePeriodFY, use the bare four-digit year (e.g. '{current_fy}'), never the 'FY' prefix."
         )
 
-    @cached_property
+    @property
     def system_message(self) -> str:
         """Return the active Assistant's system prompt or the default prompt."""
         if self.assistant.system_prompt:
-            return self.assistant.system_prompt.text + self._fiscal_year_date_context()
+            return self.assistant.system_prompt.text + self._fiscal_year_date_context(self.as_of)
         return self.DEFAULT_SYSTEM_MESSAGE
 
     @cached_property

usaspending_api/llm/evals/assistants/filter_search.py

@@ -111,6 +111,7 @@ async def run_eval_case(case: EvalCase) -> EvalObservation:
         assistant=assistant_config,
         tools=filter_search_tools,
         session=session,
+        as_of=case.as_of,
     )
 
     events: list[dict[str, Any]] = []

usaspending_api/llm/evals/base.py

@@ -1,5 +1,6 @@
 import logging
 from abc import ABC, abstractmethod
+from datetime import date
 from pathlib import Path
 from statistics import fmean
 
@@ -24,7 +25,7 @@ class BaseEval(ABC):
     assistant_name: str
     default_dataset_name: str
 
-    def __init__(
+    def __init__(  # noqa: PLR0913
         self,
         *,
         dataset_name: str | None = None,
@@ -33,6 +34,7 @@ def __init__(
         tags: set[str] | None = None,
         fail_under: float | None = None,
         incremental_output: Path | None = None,
+        as_of: date | None = None,
     ) -> None:
         if not self.assistant_name:
             raise ValueError("assistant_name must be defined.")
@@ -47,6 +49,8 @@ def __init__(
         self.tags = tags
         self.fail_under = fail_under
         self.incremental_output = incremental_output
+        # Fixed once per run so relative dataset values (e.g. "last quarter") and the assistant agree on today.
+        self.as_of = as_of or date.today()
 
     def load_cases(self) -> list[EvalCase]:
         """
@@ -63,6 +67,7 @@ def load_cases(self) -> list[EvalCase]:
             selected_case_names=self.selected_case_names,
             include_unapproved=self.include_unapproved,
             tags=self.tags,
+            as_of=self.as_of,
         )
 
     @abstractmethod
@@ -140,13 +145,13 @@ def run(self) -> EvalSummary:
 
         Progress is logged incrementally so results are not lost if the run fails.
         """
-        all_cases = load_all_cases(self.dataset_name)
+        all_cases = load_all_cases(self.dataset_name, self.as_of)
         cases = self.load_cases()
 
         if not cases:
             raise ValueError(f"Dataset `{self.dataset_name}` does not contain evaluation cases.")
 
-        logger.info(f"Starting evaluation: {len(cases)} case(s) to run")
+        logger.info(f"Starting evaluation: {len(cases)} case(s) to run (as of {self.as_of.isoformat()})")
 
         # Prepare incremental output file if specified
         if self.incremental_output:
@@ -155,6 +160,7 @@ def run(self) -> EvalSummary:
             with self.incremental_output.open("w", encoding="utf-8") as f:
                 f.write(f"# Incremental evaluation results for {self.assistant_name}\n")
                 f.write(f"# Dataset: {self.dataset_name}\n")
+                f.write(f"# As of: {self.as_of.isoformat()}\n")
                 f.write(f"# Total cases: {len(cases)}\n\n")
 
         # Execute cases one by one with progress logging