TypeBox => Zod

AGENTS.md

@@ -45,11 +45,26 @@ Key modules under `src/`:
 - **Events** (`src/services/events/`): `publisher.ts` / `publisherUtils.ts` publish raw page hits to Pub/Sub; `subscriber.ts` consumes from a subscription. Uses `@google-cloud/pubsub`.
 - **Batch worker** (`src/services/batch-worker/`): subscribes, transforms each message, batches, and flushes to Tinybird (`BATCH_SIZE`, `BATCH_FLUSH_INTERVAL_MS`).
 - **Tinybird** (`src/services/tinybird/`): `client.ts` posts single or NDJSON-batch events to `{PROXY_TARGET}/v0/events?name=analytics_events`.
-- **Transformations / schemas** (`src/transformations/page-hit-transformations.ts`, `src/schemas/v1/`): build the raw payload from the request and transform raw → processed (user-agent parsing via `ua-parser-js`, referrer parsing via `@tryghost/referrer-parser`, user signature, bot filtering).
+- **Transformations / schemas** (`src/transformations/page-hit-transformations.ts`, `src/schemas/v1/`): Zod schemas for the request, raw and processed events; build the raw payload from the request and transform raw → processed (user-agent parsing via `ua-parser-js`, referrer parsing via `@tryghost/referrer-parser`, user signature, bot filtering).
+- **Validation** (`src/schemas/validation.ts`): compiles the Zod schemas to ajv validators, and provides the Fastify type provider.
 - **Salt store** (`src/services/salt-store/`): adapter pattern (`memory`, `file`, `firestore`) behind `ISaltStore`, selected by `SALT_STORE_TYPE`.
 - **User signature** (`src/services/user-signature/`): SHA-256 of daily-rotating salt + site UUID + IP + user agent.
 - **Instrumentation** (`src/utils/instrumentation.ts`): OpenTelemetry setup — Jaeger (default) or Google Cloud Trace.
 
+### Schemas & Validation
+
+Schemas are written in Zod (`src/schemas/v1/`), but nothing validates with Zod at runtime. `src/schemas/validation.ts` converts each schema to JSON Schema once at boot via `z.toJSONSchema` and compiles it with ajv, so the per-request path is generated code rather than Zod's interpreter. Zod is there for authoring and type inference (`z.infer`, and the `ZodTypeProvider` that types route handlers).
+
+Two ajv instances, because they are used for different things:
+- `requestAjv` — route validation. Mirrors Fastify's own options (`coerceTypes: 'array'`, `useDefaults`, `removeAdditional`), since HTTP query params and headers arrive as strings and need coercing.
+- `dataAjv` — `createValidator`, used off the HTTP path (the batch worker validating Pub/Sub messages). Coercion is off: that data is already typed JSON, and coercing it rewrites a null into an empty string to satisfy the string branch of a union.
+
+Things to know when editing a schema:
+- **Never use `.transform()` or `.pipe()` in a schema.** JSON Schema cannot express them and `z.toJSONSchema` drops them without error, so the transform silently stops running. Do that work in a preHandler instead — `resolveEventId` in `page-hit-request.ts` is the pattern. `.default()` is fine; it survives as `default` and ajv's `useDefaults` applies it.
+- **`z.guid()`, not `z.uuid()`.** We only require UUID-shaped values; `z.uuid()` enforces RFC version and variant nibbles and would reject IDs real Ghost sites send.
+- **`z.iso.datetime({precision: 3})`.** Without the precision, offsets and second-granularity timestamps are accepted; we only store the canonical `toISOString()` shape.
+- `test/unit/schemas/validation.test.ts` guards the Zod ↔ ajv projection. Nothing in the type system keeps the two in step, so add a case there when adding a schema construct that is new to the codebase.
+
 ### Salt Store
 
 Adapter pattern behind `ISaltStore`, selected by `SALT_STORE_TYPE` (see `SaltStoreFactory.ts`):
@@ -118,6 +133,7 @@ Tests use Vitest and follow the same directory structure as the source code. Whe
 ## Development Notes
 
 - The project uses Fastify for high-performance HTTP handling
+- Zod for schemas, compiled to ajv validators (see Schemas & Validation above)
 - TypeScript with strict mode enabled
 - Docker-first development approach
 - All external dependencies are kept in package.json (not bundled in build)

package.json

@@ -59,7 +59,6 @@
   "dependencies": {
     "@fastify/cors": "11.3.0",
     "@fastify/reply-from": "12.6.4",
-    "@fastify/type-provider-typebox": "5.2.0",
     "@google-cloud/firestore": "8.7.0",
     "@google-cloud/opentelemetry-cloud-trace-exporter": "3.0.0",
     "@google-cloud/pino-logging-gcp-config": "1.3.5",
@@ -72,13 +71,15 @@
     "@opentelemetry/sdk-node": "0.221.0",
     "@opentelemetry/sdk-trace-base": "2.10.0",
     "@opentelemetry/semantic-conventions": "1.43.0",
-    "@sinclair/typebox": "0.34.52",
     "@tryghost/errors": "3.3.6",
     "@tryghost/referrer-parser": "0.1.21",
-    "@tryghost/validator": "3.2.6",
+    "ajv": "8.20.0",
+    "ajv-formats": "3.0.1",
+    "fast-json-stringify": "7.0.1",
     "fastify": "5.10.0",
     "fastify-plugin": "6.0.0",
     "pino": "10.3.1",
-    "ua-parser-js": "1.0.41"
+    "ua-parser-js": "1.0.41",
+    "zod": "4.4.3"
   }
 }

src/app.ts

@@ -1,6 +1,6 @@
 // Main module file
 import fastify from 'fastify';
-import {TypeBoxTypeProvider} from '@fastify/type-provider-typebox';
+import {serializerCompiler, validatorCompiler, type ZodTypeProvider} from './schemas';
 import loggingPlugin from './plugins/logging';
 import timestampPlugin from './plugins/timestamp';
 import corsPlugin from './plugins/cors';
@@ -15,7 +15,11 @@ const app = fastify({
     logger: getLoggerConfig(),
     disableRequestLogging: true,
     trustProxy: process.env.TRUST_PROXY !== 'false'
-}).withTypeProvider<TypeBoxTypeProvider>();
+}).withTypeProvider<ZodTypeProvider>();
+
+// Route schemas are Zod; compile them to ajv rather than running Zod's parser per request
+app.setValidatorCompiler(validatorCompiler);
+app.setSerializerCompiler(serializerCompiler);
 
 // Global error handler
 app.setErrorHandler(errorHandler());

src/handlers/page-hit-handlers.ts

@@ -1,5 +1,5 @@
-import {FastifyReply, FastifyRequest} from 'fastify';
-import {PageHitRequestBodySchema, PageHitRequestHeadersSchema, PageHitRequestQueryParamsSchema, PageHitRequestType, populateAndTransformPageHitRequest, transformPageHitRawToProcessed, type PageHitRequestBodyType, type PageHitRequestHeadersType, type PageHitRequestQueryParamsType} from '../schemas';
+import type {FastifyReply, FastifyRequest} from 'fastify';
+import {PageHitRequestType, PageHitRequestSchema, populateAndTransformPageHitRequest, transformPageHitRawToProcessed, type PageHitRequestBodyType, type PageHitRequestHeadersType, type PageHitRequestQueryParamsType} from '../schemas';
 import {publishPageHitRaw} from '../services/events/publisherUtils';
 import {pageHitRawPayloadFromRequest} from '../transformations/page-hit-transformations';
 
@@ -127,11 +127,7 @@ export const pageHitRequestHandler = async (request: FastifyRequest<{
 
 export const pageHitRouteOptions = {
     bodyLimit: MAX_BODY_SIZE_BYTES,
-    schema: {
-        querystring: PageHitRequestQueryParamsSchema,
-        headers: PageHitRequestHeadersSchema,
-        body: PageHitRequestBodySchema
-    },
+    schema: PageHitRequestSchema.shape,
     preHandler: populateAndTransformPageHitRequest,
     handler: pageHitRequestHandler
 };

src/schemas/index.ts

@@ -1,10 +1,8 @@
-import {registerFormatValidators} from './format-registry';
-
-// Ensure format validators are registered before any schema usage
-registerFormatValidators();
+// Compilers and the type provider that turn the Zod schemas below into ajv validators
+export * from './validation';
 
 // Export current version (v1)
 export * from './v1';
 
 // Version-specific exports for explicit imports
-export * as v1 from './v1';
+export * as v1 from './v1';

src/schemas/v1/page-hit-processed.ts

@@ -1,4 +1,4 @@
-import {Type, Static} from '@sinclair/typebox';
+import {z} from 'zod';
 import {PageHitRaw} from './page-hit-raw';
 import type {ParsedReferrer} from './page-hit-raw';
 import uap from 'ua-parser-js';
@@ -7,49 +7,54 @@ import {userSignatureService} from '../../services/user-signature';
 
 const referrerParser = new ReferrerParser();
 
+// See page-hit-request.ts: UUID-shaped is enough, RFC compliance is not required.
+const UUIDSchema = z.guid();
+const ISO8601DateTimeSchema = z.iso.datetime({precision: 3});
+const NullableString = z.string().nullable();
+
 // Complete page hit processed schema
-export const PageHitProcessedSchema = Type.Object({
-    timestamp: Type.String({format: 'date-time'}),
-    action: Type.Literal('page_hit'),
-    version: Type.Literal('1'),
-    site_uuid: Type.String({format: 'uuid'}),
-    session_id: Type.String(),
-    payload: Type.Object({
-        event_id: Type.String({format: 'uuid'}),
-        site_uuid: Type.String({format: 'uuid'}),
-        member_uuid: Type.Union([Type.String({format: 'uuid'}), Type.Literal('undefined')]),
-        member_status: Type.Union([Type.String({minLength: 1}), Type.Literal('undefined')]),
-        post_uuid: Type.Union([Type.String({format: 'uuid'}), Type.Literal('undefined')]),
-        post_type: Type.Union([Type.Literal('null'), Type.Literal('post'), Type.Literal('page')]),
-        gift_link: Type.Optional(Type.Union([Type.String(), Type.Null()])),
-        locale: Type.String({minLength: 1}),
-        location: Type.Union([Type.String({minLength: 1}), Type.Null()]),
-        pathname: Type.String({minLength: 1}),
-        href: Type.String(),
-        os: Type.String(),
-        browser: Type.String(),
-        device: Type.String(),
-        parsedReferrer: Type.Optional(Type.Object({
-            url: Type.Union([Type.String(), Type.Null()]),
-            source: Type.Union([Type.String(), Type.Null()]),
-            medium: Type.Union([Type.String(), Type.Null()])
-        })),
-        referrerUrl: Type.Optional(Type.Union([Type.String(), Type.Null()])),
-        referrerSource: Type.Optional(Type.Union([Type.String(), Type.Null()])),
-        referrerMedium: Type.Optional(Type.Union([Type.String(), Type.Null()])),
-        utm_source: Type.Optional(Type.Union([Type.String(), Type.Null()])),
-        utm_medium: Type.Optional(Type.Union([Type.String(), Type.Null()])),
-        utm_campaign: Type.Optional(Type.Union([Type.String(), Type.Null()])),
-        utm_term: Type.Optional(Type.Union([Type.String(), Type.Null()])),
-        utm_content: Type.Optional(Type.Union([Type.String(), Type.Null()])),
-        'user-agent': Type.String(),
-        meta: Type.Object({
-            received_timestamp: Type.Union([Type.String({format: 'date-time'}), Type.Null()])
+export const PageHitProcessedSchema = z.object({
+    timestamp: ISO8601DateTimeSchema,
+    action: z.literal('page_hit'),
+    version: z.literal('1'),
+    site_uuid: UUIDSchema,
+    session_id: z.string(),
+    payload: z.object({
+        event_id: UUIDSchema,
+        site_uuid: UUIDSchema,
+        member_uuid: z.union([UUIDSchema, z.literal('undefined')]),
+        member_status: z.union([z.string().min(1), z.literal('undefined')]),
+        post_uuid: z.union([UUIDSchema, z.literal('undefined')]),
+        post_type: z.enum(['null', 'post', 'page']),
+        gift_link: NullableString.optional(),
+        locale: z.string().min(1),
+        location: z.string().min(1).nullable(),
+        pathname: z.string().min(1),
+        href: z.string(),
+        os: z.string(),
+        browser: z.string(),
+        device: z.string(),
+        parsedReferrer: z.object({
+            url: NullableString,
+            source: NullableString,
+            medium: NullableString
+        }).optional(),
+        referrerUrl: NullableString.optional(),
+        referrerSource: NullableString.optional(),
+        referrerMedium: NullableString.optional(),
+        utm_source: NullableString.optional(),
+        utm_medium: NullableString.optional(),
+        utm_campaign: NullableString.optional(),
+        utm_term: NullableString.optional(),
+        utm_content: NullableString.optional(),
+        'user-agent': z.string(),
+        meta: z.object({
+            received_timestamp: ISO8601DateTimeSchema.nullable()
         })
     })
 });
 
-export type PageHitProcessed = Static<typeof PageHitProcessedSchema>;
+export type PageHitProcessed = z.infer<typeof PageHitProcessedSchema>;
 
 // Transform functions
 // NOTE: These functions are deliberately duplicated from the proxy service /processors

src/schemas/v1/page-hit-raw.ts

@@ -1,58 +1,57 @@
-import {Type, Static} from '@sinclair/typebox';
+import {z} from 'zod';
 
 // Common types
-const StringSchema = Type.String();
-const NonEmptyStringSchema = Type.String({minLength: 1});
-const UUIDSchema = Type.String({format: 'uuid'});
-const ISO8601DateTimeSchema = Type.String({
-    format: 'date-time'
-});
+const StringSchema = z.string();
+const NonEmptyStringSchema = z.string().min(1);
+// See page-hit-request.ts: UUID-shaped is enough, RFC compliance is not required.
+const UUIDSchema = z.guid();
+const ISO8601DateTimeSchema = z.iso.datetime({precision: 3});
 
 // Enum types for page hit raw events
-const ActionSchema = Type.Literal('page_hit');
-const VersionSchema = Type.Literal('1');
+const ActionSchema = z.literal('page_hit');
+const VersionSchema = z.literal('1');
 
 // Parsed referrer schema
-const ParsedReferrerSchema = Type.Object({
-    source: Type.Union([StringSchema, Type.Null()]),
-    medium: Type.Union([StringSchema, Type.Null()]),
-    url: Type.Union([StringSchema, Type.Null()])
+const ParsedReferrerSchema = z.object({
+    source: StringSchema.nullable(),
+    medium: StringSchema.nullable(),
+    url: StringSchema.nullable()
 });
 
-export type ParsedReferrer = Static<typeof ParsedReferrerSchema>;
+export type ParsedReferrer = z.infer<typeof ParsedReferrerSchema>;
 
 // Payload schema for page hit raw events
-const PayloadSchema = Type.Object({
-    event_id: Type.Optional(StringSchema),
-    member_uuid: Type.Union([UUIDSchema, Type.Literal('undefined')]),
-    member_status: Type.Union([NonEmptyStringSchema, Type.Literal('undefined')]),
-    post_uuid: Type.Union([UUIDSchema, Type.Literal('undefined')]),
-    post_type: Type.Union([Type.Literal('null'), Type.Literal('post'), Type.Literal('page')]),
-    gift_link: Type.Optional(Type.Union([StringSchema, Type.Null()])),
+const PayloadSchema = z.object({
+    event_id: StringSchema.optional(),
+    member_uuid: z.union([UUIDSchema, z.literal('undefined')]),
+    member_status: z.union([NonEmptyStringSchema, z.literal('undefined')]),
+    post_uuid: z.union([UUIDSchema, z.literal('undefined')]),
+    post_type: z.enum(['null', 'post', 'page']),
+    gift_link: StringSchema.nullable().optional(),
     locale: NonEmptyStringSchema,
-    location: Type.Union([NonEmptyStringSchema, Type.Null()]),
-    referrer: Type.Optional(Type.Union([StringSchema, Type.Null()])),
-    parsedReferrer: Type.Optional(ParsedReferrerSchema),
+    location: NonEmptyStringSchema.nullable(),
+    referrer: StringSchema.nullable().optional(),
+    parsedReferrer: ParsedReferrerSchema.optional(),
     pathname: NonEmptyStringSchema,
-    href: Type.String(),
-    utm_source: Type.Optional(Type.Union([StringSchema, Type.Null()])),
-    utm_medium: Type.Optional(Type.Union([StringSchema, Type.Null()])),
-    utm_campaign: Type.Optional(Type.Union([StringSchema, Type.Null()])),
-    utm_term: Type.Optional(Type.Union([StringSchema, Type.Null()])),
-    utm_content: Type.Optional(Type.Union([StringSchema, Type.Null()])),
-    meta: Type.Object({
-        received_timestamp: Type.Union([ISO8601DateTimeSchema, Type.Null()])
+    href: StringSchema,
+    utm_source: StringSchema.nullable().optional(),
+    utm_medium: StringSchema.nullable().optional(),
+    utm_campaign: StringSchema.nullable().optional(),
+    utm_term: StringSchema.nullable().optional(),
+    utm_content: StringSchema.nullable().optional(),
+    meta: z.object({
+        received_timestamp: ISO8601DateTimeSchema.nullable()
     })
 });
 
 // Meta schema for page hit raw events
-const MetaSchema = Type.Object({
+const MetaSchema = z.object({
     ip: NonEmptyStringSchema,
     'user-agent': NonEmptyStringSchema
 });
 
 // Complete page hit raw schema
-export const PageHitRawSchema = Type.Object({
+export const PageHitRawSchema = z.object({
     timestamp: ISO8601DateTimeSchema,
     action: ActionSchema,
     version: VersionSchema,
@@ -61,4 +60,4 @@ export const PageHitRawSchema = Type.Object({
     meta: MetaSchema
 });
 
-export type PageHitRaw = Static<typeof PageHitRawSchema>;
+export type PageHitRaw = z.infer<typeof PageHitRawSchema>;