feat(addie): add attributable shadow replay judgments

.changeset/catalog-creative-vector-sets.md

@@ -0,0 +1,4 @@
+---
+---
+
+Catalog the two versioned creative vector sets already published with the creative revision work; no package or protocol behavior changes.

.env.local.example

@@ -153,6 +153,9 @@ WORKOS_REDIRECT_URI=http://localhost:3000/auth/callback
 # SHADOW_EVAL_OFFICIAL_DOCS_REPLAY_ENABLED=false
 # SHADOW_EVAL_OFFICIAL_DOCS_REPLAY_CHANNEL_IDS=C0123456789
 # SHADOW_EVAL_OFFICIAL_DOCS_REPLAY_DAILY_LIMIT=0
+# Independent judgment can only narrow the enabled replay cohort and shares its quota.
+# SHADOW_EVAL_OFFICIAL_DOCS_JUDGE_ENABLED=false
+# SHADOW_EVAL_OFFICIAL_DOCS_JUDGE_CHANNEL_IDS=C0123456789
 
 # Required Bot Token Scopes:
 # - chat:write (send DMs)

docs/reference/test-vectors/index.mdx

@@ -6,7 +6,7 @@ description: "Machine-readable fixtures SDKs and implementations diff against to
 ---
 
 **Status**: Request for Comments
-**Last Updated**: August 24, 2026
+**Last Updated**: August 25, 2026
 
 ## What these are
 
@@ -35,6 +35,8 @@ SDKs SHOULD fetch versioned paths where available and record the version under t
 | [`webhook-receiver-envelope`](https://github.com/adcontextprotocol/adcp/blob/main/static/compliance/source/test-vectors/webhook-receiver-envelope.json) | Receiver-side replay vectors for full MCP webhook POST envelopes: canonical delivery-report envelope acceptance, retry idempotency preservation, and rejection of bare result payloads or malformed envelopes | `static/compliance/source/test-vectors/webhook-receiver-envelope.json` | `/compliance/latest/test-vectors/webhook-receiver-envelope.json` |
 | [`catalog-macro-substitution`](https://github.com/adcontextprotocol/adcp/blob/main/static/compliance/source/test-vectors/catalog-macro-substitution.json) | Catalog-item macro substitution safety: NFC normalization, RFC 3986 percent-encoding, nested-expansion preservation, CRLF neutralization, bidi override neutralization, and URL-scheme injection neutralization | `static/compliance/source/test-vectors/catalog-macro-substitution.json` | `/compliance/latest/test-vectors/catalog-macro-substitution.json` |
 | [`universal-macro-translation`](https://github.com/adcontextprotocol/adcp/blob/main/static/compliance/source/test-vectors/universal-macro-translation.json) | Producer-side universal-macro translation: RFC 3986 unreserved-whitelist value encoding, verbatim native insertion, unmapped-macro param dropping, privacy/consent macro advisory, suspect native-value detection, single-pass substitution, and control-character rejection in native mappings | `static/compliance/source/test-vectors/universal-macro-translation.json` | `/compliance/latest/test-vectors/universal-macro-translation.json` |
+| [`creative-representation-resolution`](https://github.com/adcontextprotocol/adcp/blob/main/static/compliance/source/test-vectors/creative-representation-resolution.json) | Deterministic creative-representation selection across product and seller capability intersections: paired redirects, highest-compatible VAST selection, representation-order tie breaking, retained alternatives, and exhaustive rejection output when no representation is compatible | `static/compliance/source/test-vectors/creative-representation-resolution.json` | `/compliance/latest/test-vectors/creative-representation-resolution.json` |
+| [`macro-processing`](https://github.com/adcontextprotocol/adcp/blob/main/static/compliance/source/test-vectors/macro-processing.json) | Creative macro-processing capability matching: byte-preserving unknown tokens, exact dialect/namespace/registry-revision/operation/context/owner/depth checks, seller-side universal-token translation, and the required two RFC 3986 encoding passes for nested click values | `static/compliance/source/test-vectors/macro-processing.json` | `/compliance/latest/test-vectors/macro-processing.json` |
 | [`attestations`](https://github.com/adcontextprotocol/adcp/tree/main/static/compliance/source/test-vectors/attestations) | Portable attestation trust and resolution: allowlist checks, delivery selection, validity windows, revocation, subject and digest comparison, stable outcomes, and the no-network rule for off-policy inputs. Uses a fictional proof format; cryptographic verification bytes are format-specific and tested by each format's own suite | `static/compliance/source/test-vectors/attestations/` | `/compliance/latest/test-vectors/attestations/` |
 | [`rights-attestations`](https://github.com/adcontextprotocol/adcp/tree/main/static/compliance/source/test-vectors/rights-attestations) | Rights-grant profile composed with portable attestations: holder/claim/resolver policy, rights-subject checks, JWKS purpose binding, issuer-bound status evidence, constraint-digest comparison, and fail-closed outcomes for paused, revoked, stale, or untrusted grants | `static/compliance/source/test-vectors/rights-attestations/` | `/compliance/latest/test-vectors/rights-attestations/` |
 | [`governance-runtime-attestations`](https://github.com/adcontextprotocol/adcp/tree/main/static/compliance/source/test-vectors/governance-runtime-attestations) | `check_governance` signal-activation with portable attestations: normalized outcome and governance verdict, binding-digest computation, deactivation negative-policy, and cache-reuse validity constraints | `static/compliance/source/test-vectors/governance-runtime-attestations/` | `/compliance/latest/test-vectors/governance-runtime-attestations/` |

server/src/addie/bolt-app.ts

@@ -507,33 +507,148 @@ async function checkAddieThreadParticipation(
  *  Gives humans time to reply first. After the delay, re-checks the thread
  *  and skips if a human has already responded. */
 const THREAD_RESPONSE_DELAY_MS = 45_000; // 45 seconds
+const SUBSTANTIVE_HUMAN_REPLY_MIN_BYTES = 20;
+const MAX_SHADOW_HUMAN_EVIDENCE_BYTES = 1500;
+const MAX_SHADOW_HUMAN_EVIDENCE_ID_CHARS = 64;
+export const HUMAN_EVIDENCE_UNATTRIBUTABLE_REASON = 'human_evidence_unattributable' as const;
+
+interface DelayedHumanReplyMessage {
+  user?: string;
+  text?: string;
+  ts: string;
+  bot_id?: string;
+  subtype?: string;
+}
+
+export interface AttributableHumanReply {
+  slackMessageTs: string;
+  userId: string;
+  content: string;
+}
+
+export interface DelayedResponseDecision {
+  shouldRespond: boolean;
+  humanEvidence: AttributableHumanReply | null;
+  humanEvidenceUnavailableReason:
+    | 'human_evidence_invalid'
+    | 'human_evidence_not_substantive'
+    | 'human_evidence_too_large'
+    | null;
+}
+
+/** Select exactly the first substantive human turn after the signed question. */
+export function findEarliestSubstantiveHumanReplyAfter(
+  messages: DelayedHumanReplyMessage[],
+  questionTs: string,
+  botUserId: string,
+): AttributableHumanReply | null {
+  const reply = [...messages]
+    // Slack timestamps are fixed-width epoch.sequence values whose lexical
+    // order is chronological; avoid locale-dependent collation here.
+    .sort((left, right) => left.ts < right.ts ? -1 : left.ts > right.ts ? 1 : 0)
+    .find((message) => Boolean(
+      message.ts > questionTs
+      && message.user
+      && message.user !== botUserId
+      && !message.bot_id
+      && !message.subtype
+      && message.text
+      && Buffer.byteLength(message.text.trim(), 'utf8') >= SUBSTANTIVE_HUMAN_REPLY_MIN_BYTES
+    ));
+  if (!reply?.user || !reply.text) return null;
+  return {
+    slackMessageTs: reply.ts,
+    userId: reply.user,
+    content: reply.text,
+  };
+}
+
+export function selectDelayedResponseDecision(
+  messages: DelayedHumanReplyMessage[],
+  questionTs: string,
+  botUserId: string,
+): DelayedResponseDecision {
+  const hasNewerHumanReply = messages.some((message) => Boolean(
+    message.ts > questionTs
+    && message.user
+    && message.user !== botUserId
+    && !message.bot_id
+    && (!message.subtype || message.subtype === 'thread_broadcast')
+  ));
+  if (!hasNewerHumanReply) {
+    return {
+      shouldRespond: true,
+      humanEvidence: null,
+      humanEvidenceUnavailableReason: null,
+    };
+  }
+  const humanEvidence = findEarliestSubstantiveHumanReplyAfter(
+    messages,
+    questionTs,
+    botUserId,
+  );
+  if (!humanEvidence) {
+    return {
+      shouldRespond: false,
+      humanEvidence: null,
+      humanEvidenceUnavailableReason: 'human_evidence_not_substantive',
+    };
+  }
+  if (
+    humanEvidence.slackMessageTs.length > MAX_SHADOW_HUMAN_EVIDENCE_ID_CHARS
+    || /\s/.test(humanEvidence.slackMessageTs)
+    || humanEvidence.userId.length > MAX_SHADOW_HUMAN_EVIDENCE_ID_CHARS
+    || /\s/.test(humanEvidence.userId)
+  ) {
+    return {
+      shouldRespond: false,
+      humanEvidence: null,
+      humanEvidenceUnavailableReason: 'human_evidence_invalid',
+    };
+  }
+  if (Buffer.byteLength(humanEvidence.content.trim(), 'utf8') > MAX_SHADOW_HUMAN_EVIDENCE_BYTES) {
+    return {
+      shouldRespond: false,
+      humanEvidence: null,
+      humanEvidenceUnavailableReason: 'human_evidence_too_large',
+    };
+  }
+  return {
+    shouldRespond: false,
+    humanEvidence,
+    humanEvidenceUnavailableReason: null,
+  };
+}
 
 /**
  * Wait before responding in a thread, then re-check if a human has already replied.
- * Returns true if Addie should still respond, false if a human got there first.
+ * Returns the exact earliest attributable human reply when a human got there
+ * first. Oversized evidence is rejected, never truncated or replaced by a
+ * later reply.
  */
 async function shouldRespondAfterDelay(
   channelId: string,
   threadTs: string,
   triggerMessageTs: string,
   botUserId: string,
-): Promise<boolean> {
+): Promise<DelayedResponseDecision> {
   await new Promise(resolve => setTimeout(resolve, THREAD_RESPONSE_DELAY_MS));
 
   try {
     const freshMessages = await getThreadReplies(channelId, threadTs);
-    // Check if any human message arrived AFTER the triggering message.
-    // Slack timestamps are "epoch.sequence" strings — string comparison is chronologically correct.
-    const hasNewerHumanReply = freshMessages.some(
-      msg => msg.user
-        && msg.user !== botUserId
-        && msg.ts > triggerMessageTs
+    return selectDelayedResponseDecision(
+      freshMessages,
+      triggerMessageTs,
+      botUserId,
     );
-    return !hasNewerHumanReply;
   } catch (error) {
     logger.warn({ error, channelId, threadTs }, 'Addie Bolt: Failed to re-check thread after delay');
     // On error, respond anyway to avoid silently dropping messages
-    return true;
+    return {
+      shouldRespond: true,
+      humanEvidence: null,
+      humanEvidenceUnavailableReason: null,
+    };
   }
 }
 
@@ -3445,6 +3560,12 @@ async function queueSuppressedShadowEvaluation(input: {
   channelContext?: ThreadContext;
   plan: ChannelRespondPlan;
   siRetrievalResult: SIRetrievalResult | null;
+  humanEvidence?: AttributableHumanReply;
+  humanEvidenceUnavailableReason?:
+    | typeof HUMAN_EVIDENCE_UNATTRIBUTABLE_REASON
+    | 'human_evidence_invalid'
+    | 'human_evidence_not_substantive'
+    | 'human_evidence_too_large';
 }): Promise<void> {
   const threadService = getThreadService();
   if (!isOfficialDocsProfile(input.plan)) {
@@ -3472,6 +3593,10 @@ async function queueSuppressedShadowEvaluation(input: {
     await failAttempt('trace_capture_disabled');
     return;
   }
+  if (input.humanEvidenceUnavailableReason) {
+    await failAttempt(input.humanEvidenceUnavailableReason);
+    return;
+  }
   if (!input.sourceQuestionMessageId || input.sourceConfigVersionId == null) {
     await failAttempt('missing_attributable_source');
     return;
@@ -3535,6 +3660,7 @@ async function queueSuppressedShadowEvaluation(input: {
       docsCorpusFingerprint,
       providerWebSearchEnabled: claudeClient.isWebSearchEnabled()
         && traceProcessOptions.disableServerTools !== true,
+      humanEvidence: input.humanEvidence,
     });
   } catch {
     logger.warn(
@@ -4481,10 +4607,10 @@ async function handleChannelMessage({
       // Skip delay if user explicitly named Addie (they want her input).
       const explicitlyNamedAddie = /\baddie\b/i.test(messageText);
       if (!explicitlyNamedAddie && multiParty) {
-        const shouldRespond = await shouldRespondAfterDelay(
+        const delayed = await shouldRespondAfterDelay(
           channelId, threadTsForCheck, event.ts, context.botUserId
         );
-        if (!shouldRespond) {
+        if (!delayed.shouldRespond) {
           logger.info(
             { channelId, userId, threadTs: threadTsForCheck },
             'Addie Bolt: Skipping active thread reply — human replied during delay'
@@ -4683,9 +4809,9 @@ async function handleChannelMessage({
             { channelId, userId, threadTs, humanReplies: humanRepliesBeforeTrigger.length, confidence },
             'Addie Bolt: Skipping channel thread — humans already answering'
           );
-          // Flag high-confidence suppressions and queue shadow evaluation —
-          // Addie had a good answer but stayed silent. The shadow evaluator will
-          // generate what she would have said and compare with the human's answer.
+          // Flag high-confidence suppressions. Because the human reply predates
+          // this signed question, record an unattributable capture attempt and
+          // never reuse that reply as judgment evidence.
           if (confidence === 'high') {
             try {
               await threadService.flagThread(
@@ -4705,6 +4831,7 @@ async function handleChannelMessage({
                 memberContext,
                 channelContext,
                 siRetrievalResult,
+                humanEvidenceUnavailableReason: HUMAN_EVIDENCE_UNATTRIBUTABLE_REASON,
               });
             } catch {
               await threadService.patchThreadContext(
@@ -4724,10 +4851,10 @@ async function handleChannelMessage({
       }
 
       // Also delay and re-check for new human replies after the trigger
-      const shouldRespond = await shouldRespondAfterDelay(
+      const delayed = await shouldRespondAfterDelay(
         channelId, threadTs, event.ts, context.botUserId
       );
-      if (!shouldRespond) {
+      if (!delayed.shouldRespond) {
         const confidence = plan.action === 'respond' ? plan.confidence : undefined;
         logger.info(
           { channelId, userId, threadTs, confidence },
@@ -4752,6 +4879,9 @@ async function handleChannelMessage({
               memberContext,
               channelContext,
               siRetrievalResult,
+              humanEvidence: delayed.humanEvidence ?? undefined,
+              humanEvidenceUnavailableReason:
+                delayed.humanEvidenceUnavailableReason ?? undefined,
             });
           } catch {
             await threadService.patchThreadContext(

server/src/addie/config-version.ts

@@ -30,7 +30,7 @@ import { loadRules, loadResponseStyle } from './rules/index.js';
  * Format: YYYY.MM.N where N is incremented for multiple changes in a month
  * Example: 2025.01.1, 2025.01.2, 2025.02.1
  */
-export const CODE_VERSION = '2026.08.20';
+export const CODE_VERSION = '2026.08.21';
 
 // Types
 export interface ConfigVersion {