feat: enforce Convex retention policy

.agents/skills/convex-retention/SKILL.md

@@ -0,0 +1,35 @@
+---
+name: convex-retention
+description: Use when adding or changing ClawHub Convex tables, TTL fields, cleanup crons, retention policy, auth/session cleanup, metric dedupe cleanup, or deprecated table removal
+---
+
+# Convex Retention
+
+## Overview
+
+ClawHub retention is code-owned. Every current Convex table must be classified in
+`convex/lib/retentionPolicy.ts`, and ephemeral tables need an indexed, bounded cleanup path unless
+their lifecycle is handled by usage-time validation or a documented component.
+
+## Checklist
+
+- Read `convex/_generated/ai/guidelines.md` and the Convex ops rules in `AGENTS.md` first.
+- Add every new schema table to `RETENTION_POLICIES`; the `Record<TableNames, RetentionPolicy>` type
+  is the enforcement gate.
+- For ephemeral tables, prefer an explicit expiration field plus index, then prune with `.withIndex()`
+  and `.take(...)`.
+- For new generic TTL tables, prefer `expirationTime` to match Convex Auth. Keep existing `expiresAt`,
+  `dayStart`, and `processedAt` fields unless that table already needs a real migration.
+- Use `RETENTION_STANDARD_BATCH_SIZE` for ordinary retention jobs. Keep incident-tested special cases,
+  such as `skillStatEvents`, on their documented caps.
+- Cron jobs should schedule bounded cleanup entrypoints only. Large one-off production migrations or
+  destructive backfills still start with `convex-migration-helper`.
+- Do not bulk-clear active auth state. Expired `authSessions` and `authRefreshTokens` are pruned by
+  `convex/retention.ts`.
+
+## Verification
+
+- Add or update focused tests for policy classification and cleanup behavior.
+- Run the focused Vitest slice for touched cleanup modules.
+- Run `bunx convex codegen` after schema/API changes.
+- Run a real Convex runtime check such as `bunx convex dev --once --typecheck=disable`.

AGENTS.md

@@ -108,6 +108,7 @@ Specialized corpus, scanner, security-worker, UI proof, proof publishing, Crabbo
 ## Convex Migrations & Backfills
 
 - Any Convex production data migration, backfill, destructive cleanup, schema narrowing, or table reshaping must start with the `convex-migration-helper` skill. Default to `@convex-dev/migrations` for production data changes because it provides batching, dry runs, resume/progress tracking, and safer operator UX. Exceptions require an explicit note explaining why the component is unnecessary, plus equivalent dry-run support, cursor batching, resume/progress behavior, confirmation for destructive writes, and real Convex runtime validation.
+- When adding or changing Convex tables, TTL fields, cleanup crons, retention policy, auth/session cleanup, metric dedupe cleanup, or deprecated table removal, use the repo-local `convex-retention` skill and update `convex/lib/retentionPolicy.ts`.
 - Use `convex/migrations.ts` for component-backed table-wide backfills; keep custom repairs, admin-gated operations, and incident-specific workflows in `convex/maintenance.ts`.
 - After a migration or cleanup is verified complete, remove temporary migration functions/code in a follow-up PR unless they are intentionally retained as ongoing maintenance tooling.
 

convex/_generated/api.d.ts

@@ -97,6 +97,7 @@ import type * as lib_registryArtifactBackup from "../lib/registryArtifactBackup.
 import type * as lib_reporting from "../lib/reporting.js";
 import type * as lib_reservedHandles from "../lib/reservedHandles.js";
 import type * as lib_reservedSlugs from "../lib/reservedSlugs.js";
+import type * as lib_retentionPolicy from "../lib/retentionPolicy.js";
 import type * as lib_searchText from "../lib/searchText.js";
 import type * as lib_securityPrompt from "../lib/securityPrompt.js";
 import type * as lib_skillBackfill from "../lib/skillBackfill.js";
@@ -134,6 +135,7 @@ import type * as registryArtifactBackups from "../registryArtifactBackups.js";
 import type * as registryArtifactBackupsNode from "../registryArtifactBackupsNode.js";
 import type * as registryArtifactRestore from "../registryArtifactRestore.js";
 import type * as registryArtifactRestoreMutations from "../registryArtifactRestoreMutations.js";
+import type * as retention from "../retention.js";
 import type * as search from "../search.js";
 import type * as securityDataset from "../securityDataset.js";
 import type * as securityDatasetNode from "../securityDatasetNode.js";
@@ -247,6 +249,7 @@ declare const fullApi: ApiFromModules<{
   "lib/reporting": typeof lib_reporting;
   "lib/reservedHandles": typeof lib_reservedHandles;
   "lib/reservedSlugs": typeof lib_reservedSlugs;
+  "lib/retentionPolicy": typeof lib_retentionPolicy;
   "lib/searchText": typeof lib_searchText;
   "lib/securityPrompt": typeof lib_securityPrompt;
   "lib/skillBackfill": typeof lib_skillBackfill;
@@ -284,6 +287,7 @@ declare const fullApi: ApiFromModules<{
   registryArtifactBackupsNode: typeof registryArtifactBackupsNode;
   registryArtifactRestore: typeof registryArtifactRestore;
   registryArtifactRestoreMutations: typeof registryArtifactRestoreMutations;
+  retention: typeof retention;
   search: typeof search;
   securityDataset: typeof securityDataset;
   securityDatasetNode: typeof securityDatasetNode;

convex/crons.test.ts

@@ -8,13 +8,17 @@ const mocks = vi.hoisted(() => {
   const installTelemetryDedupePruneRef = Symbol("install-telemetry-dedupe-prune");
   const rateLimitCountersPruneRef = Symbol("rate-limit-counters-prune");
   const skillStatEventPruneRef = Symbol("skill-stat-event-prune");
+  const authSessionsPruneRef = Symbol("auth-sessions-prune");
+  const authRefreshTokensPruneRef = Symbol("auth-refresh-tokens-prune");
   return {
     interval,
     githubSkillSyncRef,
     registryArtifactBackupRetryRef,
     installTelemetryDedupePruneRef,
     rateLimitCountersPruneRef,
     skillStatEventPruneRef,
+    authSessionsPruneRef,
+    authRefreshTokensPruneRef,
   };
 });
 
@@ -63,6 +67,10 @@ vi.mock("./_generated/api", () => ({
     rateLimits: {
       pruneRateLimitCountersInternal: mocks.rateLimitCountersPruneRef,
     },
+    retention: {
+      pruneExpiredAuthSessionsInternal: mocks.authSessionsPruneRef,
+      pruneExpiredAuthRefreshTokensInternal: mocks.authRefreshTokensPruneRef,
+    },
   },
 }));
 
@@ -140,6 +148,23 @@ describe("crons", () => {
     );
   });
 
+  it("prunes expired auth sessions and refresh tokens with the standard batch size", async () => {
+    await import("./crons");
+
+    expect(mocks.interval).toHaveBeenCalledWith(
+      "auth-session-retention-prune",
+      { hours: 1 },
+      mocks.authSessionsPruneRef,
+      { batchSize: 500 },
+    );
+    expect(mocks.interval).toHaveBeenCalledWith(
+      "auth-refresh-token-retention-prune",
+      { hours: 6 },
+      mocks.authRefreshTokensPruneRef,
+      { batchSize: 500 },
+    );
+  });
+
   it("prunes processed skill stat events daily with a seven-day retention window", async () => {
     await import("./crons");
 

convex/crons.ts

@@ -1,5 +1,6 @@
 import { cronJobs } from "convex/server";
 import { internal } from "./_generated/api";
+import { RETENTION_STANDARD_BATCH_SIZE } from "./lib/retentionPolicy";
 
 const crons = cronJobs();
 
@@ -134,11 +135,25 @@ if (process.env.CLAWHUB_DISABLE_CRONS !== "1") {
     {},
   );
 
+  crons.interval(
+    "auth-session-retention-prune",
+    { hours: 1 },
+    internal.retention.pruneExpiredAuthSessionsInternal,
+    { batchSize: RETENTION_STANDARD_BATCH_SIZE },
+  );
+
+  crons.interval(
+    "auth-refresh-token-retention-prune",
+    { hours: 6 },
+    internal.retention.pruneExpiredAuthRefreshTokensInternal,
+    { batchSize: RETENTION_STANDARD_BATCH_SIZE },
+  );
+
   crons.interval(
     "rate-limit-counters-prune",
     { minutes: 15 },
     internal.rateLimits.pruneRateLimitCountersInternal,
-    { batchSize: 500 },
+    { batchSize: RETENTION_STANDARD_BATCH_SIZE },
   );
 }
 

convex/downloadMetrics.test.ts

@@ -227,7 +227,7 @@ describe("download metric helpers", () => {
     expect(result).toEqual({ deleted: 2, hasMore: false });
     expect(indexCalls[0]?.table).toBe("downloadMetricDedupes");
     expect(indexCalls[0]?.indexName).toBe("by_day");
-    expect(take).toHaveBeenCalledWith(200);
+    expect(take).toHaveBeenCalledWith(500);
     expect(delete_).toHaveBeenCalledWith("downloadMetricDedupes:one");
     expect(delete_).toHaveBeenCalledWith("downloadMetricDedupes:two");
   });
@@ -258,16 +258,16 @@ describe("download metric helpers", () => {
 
   it("reschedules stale dedupe pruning when one bounded batch fills", async () => {
     vi.setSystemTime(30 * 86_400_000);
-    const rows = Array.from({ length: 200 }, (_, index) => ({
+    const rows = Array.from({ length: 500 }, (_, index) => ({
       _id: `downloadMetricDedupes:${index}`,
     }));
     const { db, delete_ } = makeDb({}, { downloadMetricDedupes: rows });
     const runAfter = vi.fn();
 
     const result = await pruneDownloadMetricDedupesHandler({ db, scheduler: { runAfter } }, {});
 
-    expect(result).toEqual({ deleted: 200, hasMore: true });
-    expect(delete_).toHaveBeenCalledTimes(200);
+    expect(result).toEqual({ deleted: 500, hasMore: true });
+    expect(delete_).toHaveBeenCalledTimes(500);
     expect(runAfter).toHaveBeenCalledWith(0, expect.anything(), {});
   });
 });

convex/downloadMetrics.ts

@@ -3,12 +3,13 @@ import { internal } from "./_generated/api";
 import type { Id } from "./_generated/dataModel";
 import { internalMutation } from "./functions";
 import { getClientIp } from "./lib/httpRateLimit";
+import { RETENTION_STANDARD_BATCH_SIZE } from "./lib/retentionPolicy";
 import { hashToken } from "./lib/tokens";
 import { insertStatEvent } from "./skillStatEvents";
 
 const DAY_MS = 86_400_000;
 const DEDUPE_RETENTION_MS = 14 * DAY_MS;
-const PRUNE_BATCH_SIZE = 200;
+const PRUNE_BATCH_SIZE = RETENTION_STANDARD_BATCH_SIZE;
 
 const identityKindValidator = v.union(v.literal("user"), v.literal("ip"));
 

convex/lib/retentionPolicy.test.ts

@@ -0,0 +1,42 @@
+/* @vitest-environment node */
+import { describe, expect, it } from "vitest";
+import {
+  RETENTION_POLICIES,
+  RETENTION_STANDARD_BATCH_SIZE,
+  getRetentionPolicy,
+} from "./retentionPolicy";
+
+describe("retention policies", () => {
+  it("classifies auth session tables as indexed expiring tables", () => {
+    expect(getRetentionPolicy("authSessions")).toMatchObject({
+      classification: "ephemeral",
+      expirationField: "expirationTime",
+      expirationIndex: "by_expiration_time",
+      prune: "retention.pruneExpiredAuthSessionsInternal",
+    });
+    expect(getRetentionPolicy("authRefreshTokens")).toMatchObject({
+      classification: "ephemeral",
+      expirationField: "expirationTime",
+      expirationIndex: "by_expiration_time",
+      prune: "retention.pruneExpiredAuthRefreshTokensInternal",
+    });
+  });
+
+  it("uses one standard batch size for retention jobs", () => {
+    expect(RETENTION_STANDARD_BATCH_SIZE).toBe(500);
+    const batchSizes = Object.values(RETENTION_POLICIES)
+      .filter((policy) => policy.classification === "ephemeral")
+      .map((policy) => policy.standardBatchSize);
+
+    expect(batchSizes.length).toBeGreaterThan(0);
+    expect(new Set(batchSizes)).toEqual(new Set([RETENTION_STANDARD_BATCH_SIZE]));
+  });
+
+  it("documents active expiring operational tables", () => {
+    expect(getRetentionPolicy("rateLimitCounters")).toMatchObject({
+      classification: "ephemeral",
+      expirationField: "expiresAt",
+      expirationIndex: "by_expires_at",
+    });
+  });
+});