docs(examples): add product fact provenance workflow

docs/examples/product_fact_provenance.md

@@ -0,0 +1,117 @@
+---
+title: Product Fact Extraction with Provenance
+description: Keep source IDs and exact quotes for product facts, then mark unsupported AI output for human review.
+---
+
+# Product Fact Extraction with Provenance
+
+Structured output proves that an LLM returned the right data shape. It does not
+prove that a material, capacity, certification, or product claim is true.
+
+This example adds a conservative check after generation. Each product fact keeps
+a stable source ID and an exact quote. Local validation then checks that:
+
+1. The source ID exists.
+2. The quote appears exactly in that source.
+3. The extracted value appears in the quote after case and whitespace normalization.
+
+Unsupported facts stay in the result with an `unverified` status. This makes them
+easy to send to a human reviewer instead of silently dropping them.
+
+## Validate facts without another LLM call
+
+The example models compute `verified`, `status`, and `confidence` locally. These
+fields are not requested from the model, so the model cannot mark its own claim
+as verified. Here, `confidence` means the fraction of evidence items that passed
+the deterministic checks. It is not a probability supplied by the LLM.
+
+```python
+from examples.product_fact_provenance import ProductExtraction
+
+sources = {
+    "supplier-sheet:row-7": "Material: 304 stainless steel.",
+    "package-photo:ocr": "Capacity: 750 ml.",
+}
+
+extraction = ProductExtraction.model_validate(
+    {
+        "facts": [
+            {
+                "field_name": "material",
+                "value": "304 stainless steel",
+                "evidence": [
+                    {
+                        "source_id": "supplier-sheet:row-7",
+                        "quote": "Material: 304 stainless steel.",
+                    }
+                ],
+            },
+            {
+                "field_name": "waterproof_rating",
+                "value": "IPX7",
+                "evidence": [],
+            },
+        ]
+    },
+    context={"sources": sources},
+)
+
+assert extraction.facts[0].status == "verified"
+assert extraction.facts[0].confidence == 1.0
+assert extraction.facts[1].status == "unverified"
+assert extraction.facts[1].confidence == 0.0
+```
+
+## Use the models with Instructor
+
+Pass the same source mapping to the prompt and to Instructor's validation
+`context`. Ask the model to copy exact quotes and use `null` when the sources do
+not support a value.
+
+```python
+import json
+from collections.abc import Mapping
+
+import instructor
+
+from examples.product_fact_provenance import ProductExtraction
+
+
+def extract_product_facts(sources: Mapping[str, str]) -> ProductExtraction:
+    client = instructor.from_provider("openai/gpt-5-nano")
+    return client.create(
+        response_model=ProductExtraction,
+        messages=[
+            {
+                "role": "system",
+                "content": (
+                    "Extract product facts only from the supplied sources. For every "
+                    "value, copy an exact supporting quote and its source ID. Use null "
+                    "and an empty evidence list when a value is not supported."
+                ),
+            },
+            {
+                "role": "user",
+                "content": json.dumps(sources, ensure_ascii=False, indent=2),
+            },
+        ],
+        context={"sources": sources},
+    )
+```
+
+The complete runnable example is in
+[`examples/product_fact_provenance`](https://github.com/567-labs/instructor/tree/main/examples/product_fact_provenance).
+
+## Limits
+
+This check is intentionally strict. It catches missing sources, invented quotes,
+and values that do not appear in the cited text. It does not prove that a source
+document is correct, resolve conflicting supplier documents, or verify a
+paraphrase. Those cases should stay unverified until a human or a separate
+domain-specific check reviews them.
+
+## See Also
+
+- [Exact Citations for RAG](./exact_citations.md)
+- [Validation](../concepts/validation.md)
+- [CitationMixin](../concepts/citation.md)

examples/product_fact_provenance/__init__.py

@@ -0,0 +1,5 @@
+"""Models for the product fact provenance example."""
+
+from .models import Evidence, FactStatus, ProductExtraction, ProductFact
+
+__all__ = ["Evidence", "FactStatus", "ProductExtraction", "ProductFact"]

examples/product_fact_provenance/models.py

@@ -0,0 +1,104 @@
+"""Conservative, source-backed validation for extracted product facts."""
+
+from collections.abc import Mapping
+from enum import Enum
+from typing import Optional
+
+from pydantic import (
+    BaseModel,
+    Field,
+    PrivateAttr,
+    ValidationInfo,
+    computed_field,
+    model_validator,
+)
+
+
+class FactStatus(str, Enum):
+    """Whether every evidence item for a product fact was verified locally."""
+
+    VERIFIED = "verified"
+    UNVERIFIED = "unverified"
+
+
+class Evidence(BaseModel):
+    """An exact quote and the source document that contains it."""
+
+    source_id: str = Field(description="Stable ID of the source document")
+    quote: str = Field(description="Exact quote copied from the source document")
+    _verified: bool = PrivateAttr(default=False)
+
+    @computed_field
+    @property
+    def verified(self) -> bool:
+        """Return the result of local source verification."""
+
+        return self._verified
+
+
+class ProductFact(BaseModel):
+    """A product field whose evidence is checked after model generation."""
+
+    field_name: str = Field(description="Product field, such as material or capacity")
+    value: Optional[str] = Field(
+        default=None,
+        description="Extracted value, or null when the source does not support a value",
+    )
+    evidence: list[Evidence] = Field(
+        default_factory=list,
+        description="Source IDs and exact quotes that support this value",
+    )
+    _status: FactStatus = PrivateAttr(default=FactStatus.UNVERIFIED)
+    _confidence: float = PrivateAttr(default=0.0)
+
+    @computed_field
+    @property
+    def status(self) -> FactStatus:
+        """Return verified only when every evidence item passes local checks."""
+
+        return self._status
+
+    @computed_field
+    @property
+    def confidence(self) -> float:
+        """Return the fraction of evidence items that passed local checks."""
+
+        return self._confidence
+
+    @model_validator(mode="after")
+    def verify_evidence(self, info: ValidationInfo) -> "ProductFact":
+        """Check source IDs, exact quotes, and value support without another LLM call."""
+
+        sources = (info.context or {}).get("sources")
+        if not isinstance(sources, Mapping) or not self.value or not self.evidence:
+            return self
+
+        normalized_value = _normalize(self.value)
+        verified_count = 0
+
+        for evidence in self.evidence:
+            source_text = sources.get(evidence.source_id)
+            evidence._verified = (
+                isinstance(source_text, str)
+                and evidence.quote in source_text
+                and normalized_value in _normalize(evidence.quote)
+            )
+            verified_count += int(evidence._verified)
+
+        self._confidence = verified_count / len(self.evidence)
+        if verified_count == len(self.evidence):
+            self._status = FactStatus.VERIFIED
+
+        return self
+
+
+class ProductExtraction(BaseModel):
+    """Product facts extracted from one or more source documents."""
+
+    facts: list[ProductFact]
+
+
+def _normalize(text: str) -> str:
+    """Normalize case and whitespace for a conservative value-in-quote check."""
+
+    return " ".join(text.casefold().split())

examples/product_fact_provenance/run.py

@@ -0,0 +1,44 @@
+"""Extract product facts and keep unsupported claims visible for review."""
+
+import json
+from collections.abc import Mapping
+
+import instructor
+
+from examples.product_fact_provenance import ProductExtraction
+
+
+SOURCE_DOCUMENTS = {
+    "supplier-sheet:row-7": "Material: 304 stainless steel.",
+    "package-photo:ocr": "Capacity: 750 ml.",
+    "supplier-copy:paragraph-2": "Designed for everyday travel.",
+}
+
+
+def extract_product_facts(sources: Mapping[str, str]) -> ProductExtraction:
+    """Extract facts and validate their evidence against the supplied documents."""
+
+    client = instructor.from_provider("openai/gpt-5-nano")
+    return client.create(
+        response_model=ProductExtraction,
+        messages=[
+            {
+                "role": "system",
+                "content": (
+                    "Extract product facts only from the supplied sources. For every "
+                    "value, copy an exact supporting quote and its source ID. Use null "
+                    "and an empty evidence list when a value is not supported."
+                ),
+            },
+            {
+                "role": "user",
+                "content": json.dumps(sources, ensure_ascii=False, indent=2),
+            },
+        ],
+        context={"sources": sources},
+    )
+
+
+if __name__ == "__main__":
+    result = extract_product_facts(SOURCE_DOCUMENTS)
+    print(result.model_dump_json(indent=2))

mkdocs.yml

@@ -205,6 +205,7 @@ nav:
     - "Structured Outputs with Ollama": 'examples/ollama.md'
     - "Multi-Modal Data with Gemini": 'examples/multi_modal_gemini.md'
     - "Exact Citations for RAG": 'examples/exact_citations.md'
+    - "Product Fact Extraction with Provenance": 'examples/product_fact_provenance.md'
     - "Extracting Knowledge Graphs": 'examples/knowledge_graph.md'
     - "Table Extraction with GPT-4 Vision": 'examples/extracting_tables.md'
     - "User-Defined Bulk Classification": 'examples/bulk_classification.md'

tests/test_product_fact_provenance_example.py

@@ -0,0 +1,113 @@
+from examples.product_fact_provenance import FactStatus, ProductExtraction
+
+
+SOURCES = {
+    "supplier-sheet:row-7": "Material: 304 stainless steel.",
+    "package-photo:ocr": "Capacity: 750 ml.",
+}
+
+
+def test_marks_source_backed_fact_as_verified() -> None:
+    extraction = ProductExtraction.model_validate(
+        {
+            "facts": [
+                {
+                    "field_name": "material",
+                    "value": "304 stainless steel",
+                    "evidence": [
+                        {
+                            "source_id": "supplier-sheet:row-7",
+                            "quote": "Material: 304 stainless steel.",
+                        }
+                    ],
+                }
+            ]
+        },
+        context={"sources": SOURCES},
+    )
+
+    fact = extraction.facts[0]
+    assert fact.status is FactStatus.VERIFIED
+    assert fact.confidence == 1.0
+    assert fact.evidence[0].verified is True
+
+
+def test_keeps_unsupported_fact_visible_as_unverified() -> None:
+    extraction = ProductExtraction.model_validate(
+        {
+            "facts": [
+                {
+                    "field_name": "waterproof_rating",
+                    "value": "IPX7",
+                    "evidence": [
+                        {
+                            "source_id": "supplier-sheet:row-7",
+                            "quote": "Waterproof rating: IPX7.",
+                        }
+                    ],
+                }
+            ]
+        },
+        context={"sources": SOURCES},
+    )
+
+    fact = extraction.facts[0]
+    assert fact.status is FactStatus.UNVERIFIED
+    assert fact.confidence == 0.0
+    assert fact.evidence[0].verified is False
+
+
+def test_reports_partial_evidence_confidence() -> None:
+    extraction = ProductExtraction.model_validate(
+        {
+            "facts": [
+                {
+                    "field_name": "capacity",
+                    "value": "750 ml",
+                    "evidence": [
+                        {
+                            "source_id": "package-photo:ocr",
+                            "quote": "Capacity: 750 ml.",
+                        },
+                        {
+                            "source_id": "missing-source",
+                            "quote": "Capacity: 750 ml.",
+                        },
+                    ],
+                }
+            ]
+        },
+        context={"sources": SOURCES},
+    )
+
+    fact = extraction.facts[0]
+    assert fact.status is FactStatus.UNVERIFIED
+    assert fact.confidence == 0.5
+    assert [item.verified for item in fact.evidence] == [True, False]
+
+
+def test_missing_validation_context_cannot_mark_fact_verified() -> None:
+    extraction = ProductExtraction.model_validate(
+        {
+            "facts": [
+                {
+                    "field_name": "capacity",
+                    "value": "750 ml",
+                    "evidence": [
+                        {
+                            "source_id": "package-photo:ocr",
+                            "quote": "Capacity: 750 ml.",
+                            "verified": True,
+                        }
+                    ],
+                    "status": "verified",
+                    "confidence": 1.0,
+                }
+            ]
+        }
+    )
+
+    fact = extraction.facts[0]
+    assert fact.status is FactStatus.UNVERIFIED
+    assert fact.confidence == 0.0
+    assert fact.evidence[0].verified is False