feat(abtest): hypothesis metadata for pre-registration (Change Set A)

.sampo/changesets/abtest-hypothesis.md

@@ -0,0 +1,5 @@
+---
+npm/@listmonk-ops/abtest: minor (Added)
+---
+
+Add hypothesis metadata for A/B test pre-registration: structured objective, primary metric, expected lift, owner, and experiment scope with canonical checksum locking. AbTest gains optional hypothesis and assignmentProvenance fields.

.sampo/changesets/abtest-stratification.md

@@ -0,0 +1,5 @@
+---
+npm/@listmonk-ops/abtest: minor (Added)
+---
+
+Add recipient-domain stratification for A/B test assignment: classify subscribers into provider strata and compute a constrained quota matrix where row sums match stratum sizes and column sums match exact variant/holdout counts. Includes the DEFAULT_STRATIFICATION_POLICY, normalizeDomain, classifyStratum, and a paired-swap computeStratifiedQuotas solver.

README.md

@@ -465,6 +465,33 @@ send results. Summary of the current behavior:
 See [`packages/abtest/README.md`](packages/abtest/README.md) for the
 underlying Listmonk API behavior and spike rationale.
 
+### Hypothesis pre-registration and recipient-domain stratification
+
+`abtest create` accepts two advanced experimentation inputs:
+
+- `--hypothesis '{...}'` — a pre-registration hypothesis (objective, primary
+  metric, expected lift, owner, experiment scope). The service locks it
+  (SHA-256 checksum) before recipient assignment so the metadata cannot
+  change after recipients are set.
+- `--enable-stratification` — classify subscribers by email-domain provider
+  and compute a constrained quota matrix so each provider stratum gets a
+  proportional share of every variant/holdout group. The quota matrix is
+  computed and stored on the test for reporting/validation; applying it to
+  the actual assignment slices is deferred to a follow-up change set.
+
+```bash
+listmonk-cli abtest create \
+  --name "Subject Line Test" \
+  --campaign-id 1 \
+  --variants '[...]' --lists 1,2 \
+  --enable-stratification \
+  --hypothesis '{"objective":"Increase CTR","hypothesis":"Shorter subject lifts CTR","primary_metric":{"type":"click_rate","direction":"maximize"},"expected_lift":{"kind":"relative","value":0.1},"owner":{"id":"user-1"},"experiment_scope":{"channel":"email","experiment_family_key":"onboarding.welcome","attribution_window_hours":72,"exclusion_window_hours":168}}'
+```
+
+See [`packages/abtest/README.md`](packages/abtest/README.md) for the full
+validation rules, the stratified quota solver, and bilingual (EN/KO)
+guidance.
+
 ## Ops Automation Commands
 
 ```bash

README_ko.md

@@ -460,6 +460,31 @@ A/B 테스트 도메인은 발송 결과를 왜곡할 수 있는 여러 정확
 자세한 Listmonk API 동작과 spike 근거는
 [`packages/abtest/README.md`](packages/abtest/README.md)를 참고하세요.
 
+### 가설 사전 등록 및 수신자 도메인 층화
+
+`abtest create`는 고급 실험 입력 두 가지를 받습니다:
+
+- `--hypothesis '{...}'` — 사전 등록 가설(목표, 주요 지표, 기대 효과,
+  담당자, 실험 범위). 서비스가 수신자 할당 전에 잠금(SHA-256 체크섬)
+  처리하므로 수신자가 정해진 뒤에는 가설을 변경할 수 없습니다.
+- `--enable-stratification` — 구독자를 이메일 도메인 제공자별로 분류하고
+  제약된 할당량 행렬을 계산하여 각 제공자 층(stratum)이 모든 변형/홀드아웃
+  그룹의 비례 배분을 받도록 합니다. 할당량 행렬은 보고/검증을 위해 계산되어
+  테스트에 저장되며, 실제 할당 슬라이스에 적용하는 것은 후속 변경 세트로
+  연기됩니다.
+
+```bash
+listmonk-cli abtest create \
+  --name "Subject Line Test" \
+  --campaign-id 1 \
+  --variants '[...]' --lists 1,2 \
+  --enable-stratification \
+  --hypothesis '{"objective":"CTR 향상","hypothesis":"짧은 제목이 CTR을 높인다","primary_metric":{"type":"click_rate","direction":"maximize"},"expected_lift":{"kind":"relative","value":0.1},"owner":{"id":"user-1"},"experiment_scope":{"channel":"email","experiment_family_key":"onboarding.welcome","attribution_window_hours":72,"exclusion_window_hours":168}}'
+```
+
+전체 검증 규칙, 층화 할당량 솔버, 한영(EN/KO) 가이드는
+[`packages/abtest/README.md`](packages/abtest/README.md)를 참고하세요.
+
 ## 운영 자동화 명령
 
 ```bash

apps/cli/src/commands/abtest.ts

@@ -26,6 +26,7 @@ import {
 	invokeRunAbTestOperation,
 	invokeStopAbTestOperation,
 	invokeTickAbTestsOperation,
+	validateHypothesisMetadata,
 	validateStoredAbTestStore,
 } from "@listmonk-ops/abtest";
 import { OutputUtils } from "@listmonk-ops/common";
@@ -166,6 +167,8 @@ export function buildCreateInputFromFlags(flags: {
 	"test-group-percentage"?: number;
 	"auto-deploy-winner": boolean;
 	"ignore-sample-size-warnings": boolean;
+	"enable-stratification"?: boolean;
+	hypothesis?: string;
 }): CreateAbTestInput {
 	const parsedVariants = parseJson<VariantInput[]>(flags.variants, "variants");
 	if (!Array.isArray(parsedVariants)) {
@@ -180,6 +183,27 @@ export function buildCreateInputFromFlags(flags: {
 	const baseSubject = flags.subject?.trim() ?? "";
 	const baseBody = flags.body?.trim() ?? "";
 
+	// The hypothesis is a JSON document matching the CreateAbTestInput
+	// hypothesis shape (objective, primary_metric, expected_lift, owner,
+	// experiment_scope). Parsed here so CLI/MCP share the same contract.
+	// Reject non-object JSON (e.g. "null") so the shared schema validates it
+	// the same way MCP does, rather than silently dropping it.
+	let hypothesis: CreateAbTestInput["hypothesis"] | undefined;
+	if (flags.hypothesis !== undefined) {
+		const parsed = parseJson<CreateAbTestInput["hypothesis"]>(
+			flags.hypothesis,
+			"hypothesis",
+		);
+		if (parsed === null || typeof parsed !== "object" || Array.isArray(
+			parsed,
+		)) {
+			throw new Error(
+				"hypothesis must be a JSON object matching the pre-registration shape",
+			);
+		}
+		hypothesis = parsed;
+	}
+
 	return {
 		name: flags.name,
 		campaign_id: String(flags["campaign-id"]),
@@ -196,6 +220,8 @@ export function buildCreateInputFromFlags(flags: {
 		test_group_percentage: testGroupPercentage,
 		auto_deploy_winner: flags["auto-deploy-winner"],
 		ignore_sample_size_warnings: flags["ignore-sample-size-warnings"],
+		enable_stratification: flags["enable-stratification"] ?? undefined,
+		hypothesis,
 	};
 }
 
@@ -476,6 +502,29 @@ async function promptInteractiveInput(
 		throw new Error("Prompt cancelled by user");
 	}
 
+	const stratifyResult = await clack.confirm({
+		message:
+			"Enable recipient-domain stratification during holdout provisioning?",
+		initialValue: false,
+	});
+	if (clack.isCancel(stratifyResult)) {
+		clack.cancel("Cancelled");
+		throw new Error("Prompt cancelled by user");
+	}
+
+	// Optional pre-registration hypothesis as a JSON document. Empty input
+	// skips it; the shared service locks whatever is provided.
+	const hypothesisResult = await clack.text({
+		message:
+			"Pre-registration hypothesis JSON (leave empty to skip)?",
+		placeholder: '{"objective": "...", "primary_metric": {...}, ...}',
+		defaultValue: "",
+	});
+	if (clack.isCancel(hypothesisResult)) {
+		clack.cancel("Cancelled");
+		throw new Error("Prompt cancelled by user");
+	}
+
 	const input = buildCreateInputFromFlags({
 		name: nameResult,
 		"campaign-id": Number(campaignIdResult),
@@ -487,8 +536,49 @@ async function promptInteractiveInput(
 		"test-group-percentage": Number(testGroupResult),
 		"auto-deploy-winner": autoDeployResult,
 		"ignore-sample-size-warnings": ignoreWarningsResult,
+		"enable-stratification": stratifyResult,
+		hypothesis:
+			hypothesisResult.trim().length > 0 ? hypothesisResult.trim() : undefined,
 	});
 
+	// Validate the hypothesis shape before showing the summary so malformed
+	// input fails early with a clear error rather than after confirmation.
+	if (input.hypothesis) {
+		validateHypothesisMetadata(
+			{
+				objective: input.hypothesis.objective,
+				hypothesis: input.hypothesis.hypothesis,
+				primaryMetric: {
+					type: input.hypothesis.primary_metric.type,
+					direction: input.hypothesis.primary_metric.direction,
+				},
+				expectedLift:
+					input.hypothesis.expected_lift.kind === "relative"
+						? {
+								kind: "relative",
+								value: input.hypothesis.expected_lift.value,
+							}
+						: {
+								kind: "absolute",
+								value: input.hypothesis.expected_lift.value,
+								unit: input.hypothesis.expected_lift.unit,
+							},
+				owner: { id: input.hypothesis.owner.id },
+				experimentScope: {
+					channel: input.hypothesis.experiment_scope.channel,
+					experimentFamilyKey:
+						input.hypothesis.experiment_scope.experiment_family_key,
+					attributionWindowHours:
+						input.hypothesis.experiment_scope.attribution_window_hours,
+					exclusionWindowHours:
+						input.hypothesis.experiment_scope.exclusion_window_hours,
+				},
+				createdAt: new Date().toISOString(),
+			},
+			true,
+		);
+	}
+
 	clack.note(
 		JSON.stringify(
 			{
@@ -502,6 +592,16 @@ async function promptInteractiveInput(
 				testingMode: input.testing_mode,
 				testGroupPercentage: input.test_group_percentage,
 				autoDeployWinner: input.auto_deploy_winner,
+				enableStratification: input.enable_stratification ?? false,
+				hypothesis: input.hypothesis
+					? {
+							objective: input.hypothesis.objective,
+							primaryMetric: input.hypothesis.primary_metric.type,
+							direction: input.hypothesis.primary_metric.direction,
+							familyKey:
+								input.hypothesis.experiment_scope.experiment_family_key,
+						}
+					: undefined,
 			},
 			null,
 			2,
@@ -638,6 +738,14 @@ export default defineGroup({
 						description: "Ignore sample-size warnings",
 					},
 				),
+				"enable-stratification": option(z.coerce.boolean().default(false), {
+					description:
+						"Enable recipient-domain stratification during holdout provisioning",
+				}),
+				hypothesis: option(z.string().optional(), {
+					description:
+						"Pre-registration hypothesis as JSON (objective, primary_metric, expected_lift, owner, experiment_scope)",
+				}),
 			},
 			handler: async ({ flags, ...args }) => {
 				try {

packages/abtest/README.md

@@ -661,3 +661,155 @@ assignment and chunked bulk list membership:
 ## License
 
 MIT License - see LICENSE file for details.
+
+## Hypothesis pre-registration (advanced experimentation)
+
+An A/B test can carry a **pre-registered hypothesis** that is locked
+(checksummed) before recipient assignment. After locking, the hypothesis
+cannot change without discarding the assignment manifest and provisioning.
+This prevents post-hoc hypothesis adjustment (p-hacking) and gives reports a
+stable reference.
+
+### Locking and verification
+
+```typescript
+import {
+  lockHypothesis,
+  verifyHypothesisChecksum,
+  validateHypothesisMetadata,
+} from "@listmonk-ops/abtest";
+
+const metadata = {
+  objective: "Increase click-through rate on the welcome email",
+  hypothesis: "A shorter subject line will increase CTR by 10%",
+  primaryMetric: { type: "click_rate", direction: "maximize" },
+  expectedLift: { kind: "relative", value: 0.1 },
+  owner: { id: "user-1", displayName: "Test User" },
+  experimentScope: {
+    channel: "email",
+    experimentFamilyKey: "onboarding.welcome.subject",
+    attributionWindowHours: 72,
+    exclusionWindowHours: 168,
+  },
+  createdAt: new Date().toISOString(),
+};
+
+validateHypothesisMetadata(metadata, true); // strict launch check
+const locked = lockHypothesis(metadata);
+verifyHypothesisChecksum(locked); // true
+```
+
+The checksum recursively canonicalizes nested fields
+(`primaryMetric`, `expectedLift`, `owner`, `experimentScope`, `createdAt`),
+so any change after locking invalidates it.
+
+### Wiring through creation
+
+Pass a `hypothesis` field to `createAbTest`. The service locks it before
+provisioning, so the assignment manifest is always bound to a frozen
+hypothesis:
+
+```typescript
+const test = await abTestExecutors.createAbTest({
+  name: "Subject Line Test",
+  variants: [/* ... */],
+  lists: [1, 2],
+  hypothesis: {
+    objective: "Increase CTR",
+    hypothesis: "Shorter subject lifts CTR",
+    primary_metric: { type: "click_rate", direction: "maximize" },
+    expected_lift: { kind: "relative", value: 0.1 },
+    owner: { id: "user-1" },
+    experiment_scope: {
+      channel: "email",
+      experiment_family_key: "onboarding.welcome",
+      attribution_window_hours: 72,
+      exclusion_window_hours: 168,
+    },
+  },
+});
+```
+
+### Validation rules
+
+- `createdAt` and `lockedAt` must be strict ISO 8601 timestamps (the year-zero
+  string `"0"`, localized formats like `"01/02/03"`, and overflowed dates like
+  `"2026-02-30"` are rejected).
+- `primaryMetric.type` ∈ `click_rate | conversion_rate | revenue_per_recipient`.
+- `primaryMetric.direction` ∈ `maximize | minimize`.
+- `expectedLift.kind` ∈ `relative | absolute`. Absolute lifts require a `unit`
+  ∈ `percentage_point | currency_per_recipient`.
+- Metric/unit coupling: `revenue_per_recipient` requires
+  `currency_per_recipient` absolute lift; `click_rate`/`conversion_rate`
+  require `percentage_point`. Relative lift is unit-agnostic.
+- `experimentScope.experimentFamilyKey` must be lowercase alphanumeric segments
+  joined by single `.` / `_` / `-` separators (e.g.
+  `onboarding.activation.day1`, `cart-recovery_24h`); `.`, `foo.`, and
+  `foo..bar` are rejected.
+
+### 가설 사전 등록 (Korean)
+
+A/B 테스트에 **사전 등록된 가설**을 설정하면 수신자 할당 전에 잠금(체크섬)
+처리됩니다. 잠금 후에는 할당 매니페스트와 프로비저닝을 폐기하지 않는 한
+가설을 변경할 수 없습니다. 이는 사후 가설 조정(p-hacking)을 방지하고
+보고서에 안정적인 기준점을 제공합니다.
+
+- `createAbTest`에 `hypothesis` 필드를 전달하면 서비스가 프로비저닝 전에
+  잠금 처리합니다.
+- 체크섬은 중첩 필드(`primaryMetric`, `expectedLift`, `owner`,
+  `experimentScope`, `createdAt`)를 재귀적으로 정규화하므로 잠금 후 어떤
+  변경도 무효화됩니다.
+- `createdAt`/`lockedAt`은 엄격한 ISO 8601이어야 합니다.
+- `experimentFamilyKey`는 `.` / `_` / `-` 로 구분된 소문자 영숫자 세그먼트여야 합니다.
+
+## Recipient-domain stratification (advanced experimentation)
+
+Stratification classifies subscribers by email-domain provider and computes a
+**constrained quota matrix** so each provider stratum gets a proportional share
+of every variant/holdout group. The quota matrix is computed and stored for
+reporting and validation. Note: applying these quotas to the actual recipient
+assignment slices is a planned follow-up; today the assignment itself remains
+the deterministic largest-remainder manifest, and the quota matrix documents
+the target proportional allocation.
+
+```typescript
+import {
+  classifyStratum,
+  computeStratifiedQuotas,
+  DEFAULT_STRATIFICATION_POLICY,
+} from "@listmonk-ops/abtest";
+
+const policy = { ...DEFAULT_STRATIFICATION_POLICY, enabled: true };
+const stratum = classifyStratum("user@gmail.com", policy); // "gmail"
+
+const result = computeStratifiedQuotas({
+  stratumSizes: { gmail: 600, naver: 300, other: 100 },
+  groupExactCounts: { "variant:A": 500, "variant:B": 500 },
+  groupOrder: ["variant:A", "variant:B"],
+  totalAudience: 1000,
+});
+```
+
+The solver uses the largest-remainder method per stratum row, then a paired-swap
+column correction that preserves row sums while matching exact group column
+counts. Configured domains in the provider map are normalized with the same
+rules applied to subscriber emails, so mixed-case entries like `"GMAIL.COM"`
+match correctly.
+
+During holdout provisioning, when a stratification policy is enabled and the
+resolved audience carries emails, the quota matrix is computed and stored on
+the `AbTest.stratification` field for reporting and validation.
+
+### 수신자 도메인 층화 (Korean)
+
+층화는 구독자를 이메일 도메인 제공자별로 분류하고, 각 제공자 층(stratum)이
+모든 변형/홀드아웃 그룹의 비례 배분을 받도록 **제약된 할당량 행렬**을
+계산합니다. 할당량 행렬은 보고/검증을 위해 계산되어 저장됩니다. 참고:
+이 할당량을 실제 수신자 할당 슬라이스에 적용하는 것은 후속 작업이며,
+현재 할당 자체는 결정론적 largest-remainder 매니페스트를 그대로 사용하고
+할당량 행렬은 목표 비례 배분을 문서화합니다.
+
+- `classifyStratum`으로 구독자를 분류하고, `computeStratifiedQuotas`로
+  할당량 행렬을 계산합니다.
+- 홀드아웃 프로비저닝 시 층화 정책이 활성화되어 있으면 할당량 행렬이
+  `AbTest.stratification`에 저장되어 보고/검증에 사용됩니다.