diff --git a/AGENTS.md b/AGENTS.md index 26266b8d..5898f192 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,11 +45,26 @@ Key modules under `src/`: - **Events** (`src/services/events/`): `publisher.ts` / `publisherUtils.ts` publish raw page hits to Pub/Sub; `subscriber.ts` consumes from a subscription. Uses `@google-cloud/pubsub`. - **Batch worker** (`src/services/batch-worker/`): subscribes, transforms each message, batches, and flushes to Tinybird (`BATCH_SIZE`, `BATCH_FLUSH_INTERVAL_MS`). - **Tinybird** (`src/services/tinybird/`): `client.ts` posts single or NDJSON-batch events to `{PROXY_TARGET}/v0/events?name=analytics_events`. -- **Transformations / schemas** (`src/transformations/page-hit-transformations.ts`, `src/schemas/v1/`): build the raw payload from the request and transform raw → processed (user-agent parsing via `ua-parser-js`, referrer parsing via `@tryghost/referrer-parser`, user signature, bot filtering). +- **Transformations / schemas** (`src/transformations/page-hit-transformations.ts`, `src/schemas/v1/`): Zod schemas for the request, raw and processed events; build the raw payload from the request and transform raw → processed (user-agent parsing via `ua-parser-js`, referrer parsing via `@tryghost/referrer-parser`, user signature, bot filtering). +- **Validation** (`src/schemas/validation.ts`): compiles the Zod schemas to ajv validators, and provides the Fastify type provider. - **Salt store** (`src/services/salt-store/`): adapter pattern (`memory`, `file`, `firestore`) behind `ISaltStore`, selected by `SALT_STORE_TYPE`. - **User signature** (`src/services/user-signature/`): SHA-256 of daily-rotating salt + site UUID + IP + user agent. - **Instrumentation** (`src/utils/instrumentation.ts`): OpenTelemetry setup — Jaeger (default) or Google Cloud Trace. +### Schemas & Validation + +Schemas are written in Zod (`src/schemas/v1/`), but nothing validates with Zod at runtime. `src/schemas/validation.ts` converts each schema to JSON Schema once at boot via `z.toJSONSchema` and compiles it with ajv, so the per-request path is generated code rather than Zod's interpreter. Zod is there for authoring and type inference (`z.infer`, and the `ZodTypeProvider` that types route handlers). + +Two ajv instances, because they are used for different things: +- `requestAjv` — route validation. Mirrors Fastify's own options (`coerceTypes: 'array'`, `useDefaults`, `removeAdditional`), since HTTP query params and headers arrive as strings and need coercing. +- `dataAjv` — `createValidator`, used off the HTTP path (the batch worker validating Pub/Sub messages). Coercion is off: that data is already typed JSON, and coercing it rewrites a null into an empty string to satisfy the string branch of a union. + +Things to know when editing a schema: +- **Never use `.transform()` or `.pipe()` in a schema.** JSON Schema cannot express them and `z.toJSONSchema` drops them without error, so the transform silently stops running. Do that work in a preHandler instead — `resolveEventId` in `page-hit-request.ts` is the pattern. `.default()` is fine; it survives as `default` and ajv's `useDefaults` applies it. +- **`z.guid()`, not `z.uuid()`.** We only require UUID-shaped values; `z.uuid()` enforces RFC version and variant nibbles and would reject IDs real Ghost sites send. +- **`z.iso.datetime({precision: 3})`.** Without the precision, offsets and second-granularity timestamps are accepted; we only store the canonical `toISOString()` shape. +- `test/unit/schemas/validation.test.ts` guards the Zod ↔ ajv projection. Nothing in the type system keeps the two in step, so add a case there when adding a schema construct that is new to the codebase. + ### Salt Store Adapter pattern behind `ISaltStore`, selected by `SALT_STORE_TYPE` (see `SaltStoreFactory.ts`): @@ -119,6 +134,7 @@ Tests use Vitest and follow the same directory structure as the source code. Whe ## Development Notes - The project uses Fastify for high-performance HTTP handling +- Zod for schemas, compiled to ajv validators (see Schemas & Validation above) - TypeScript with strict mode enabled - Docker-first development approach - All external dependencies are kept in package.json (not bundled in build) diff --git a/package.json b/package.json index 19cf8566..56e5f24d 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,6 @@ "@fastify/cors": "11.3.0", "@fastify/otel": "0.20.1", "@fastify/reply-from": "12.6.4", - "@fastify/type-provider-typebox": "5.2.0", "@google-cloud/firestore": "8.7.1", "@google-cloud/opentelemetry-cloud-trace-exporter": "3.0.0", "@google-cloud/pino-logging-gcp-config": "1.3.5", @@ -77,13 +76,15 @@ "@opentelemetry/sdk-node": "0.221.0", "@opentelemetry/sdk-trace-base": "2.10.0", "@opentelemetry/semantic-conventions": "1.43.0", - "@sinclair/typebox": "0.34.52", "@tryghost/errors": "3.3.6", "@tryghost/referrer-parser": "0.1.21", - "@tryghost/validator": "3.2.6", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "fast-json-stringify": "7.0.1", "fastify": "5.11.3", "fastify-plugin": "6.0.0", "pino": "10.3.1", - "ua-parser-js": "1.0.41" + "ua-parser-js": "1.0.41", + "zod": "4.4.3" } } diff --git a/src/app.ts b/src/app.ts index 6940fc1b..1c422454 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,6 +1,6 @@ // Main module file import fastify from 'fastify'; -import {TypeBoxTypeProvider} from '@fastify/type-provider-typebox'; +import {serializerCompiler, validatorCompiler, type ZodTypeProvider} from './schemas'; import loggingPlugin from './plugins/logging'; import timestampPlugin from './plugins/timestamp'; import corsPlugin from './plugins/cors'; @@ -16,7 +16,11 @@ const app = fastify({ logger: getLoggerConfig(), disableRequestLogging: true, trustProxy: process.env.TRUST_PROXY !== 'false' -}).withTypeProvider(); +}).withTypeProvider(); + +// Route schemas are Zod; compile them to ajv rather than running Zod's parser per request +app.setValidatorCompiler(validatorCompiler); +app.setSerializerCompiler(serializerCompiler); // Tracing first, so its hooks wrap everything registered below app.register(fastifyOtelInstrumentation.plugin()); diff --git a/src/handlers/page-hit-handlers.ts b/src/handlers/page-hit-handlers.ts index 3e21e5a0..b8bd780e 100644 --- a/src/handlers/page-hit-handlers.ts +++ b/src/handlers/page-hit-handlers.ts @@ -1,5 +1,5 @@ -import {FastifyReply, FastifyRequest} from 'fastify'; -import {PageHitRequestBodySchema, PageHitRequestHeadersSchema, PageHitRequestQueryParamsSchema, PageHitRequestType, populateAndTransformPageHitRequest, transformPageHitRawToProcessed, type PageHitRequestBodyType, type PageHitRequestHeadersType, type PageHitRequestQueryParamsType} from '../schemas'; +import type {FastifyReply, FastifyRequest} from 'fastify'; +import {PageHitRequestType, PageHitRequestSchema, populateAndTransformPageHitRequest, transformPageHitRawToProcessed, type PageHitRequestBodyType, type PageHitRequestHeadersType, type PageHitRequestQueryParamsType} from '../schemas'; import {publishPageHitRaw} from '../services/events/publisherUtils'; import {pageHitRawPayloadFromRequest} from '../transformations/page-hit-transformations'; import {PAGE_HIT_ACCEPTED_RESPONSE} from '../utils/page-hit-response'; @@ -118,11 +118,7 @@ export const pageHitRequestHandler = async (request: FastifyRequest<{ export const pageHitRouteOptions = { bodyLimit: MAX_BODY_SIZE_BYTES, - schema: { - querystring: PageHitRequestQueryParamsSchema, - headers: PageHitRequestHeadersSchema, - body: PageHitRequestBodySchema - }, + schema: PageHitRequestSchema.shape, preHandler: populateAndTransformPageHitRequest, handler: pageHitRequestHandler }; diff --git a/src/schemas/format-registry.ts b/src/schemas/format-registry.ts deleted file mode 100644 index 563668d6..00000000 --- a/src/schemas/format-registry.ts +++ /dev/null @@ -1,26 +0,0 @@ -import {FormatRegistry} from '@sinclair/typebox'; -import validator from '@tryghost/validator'; - -/** - * Registers all format validators used by schemas. - * This should be called once at application startup to ensure - * format validators are available before any schema usage. - */ -export function registerFormatValidators(): void { - FormatRegistry.Set('uuid', (value) => { - // We only require UUID-shaped values here; they do not need to be fully RFC compliant. - return validator.isUUID(value, 'loose'); - }); - - // URI format validator using @tryghost/validator - FormatRegistry.Set('uri', (value) => { - return validator.isURL(value); - }); - - // ISO8601 date-time format validator - FormatRegistry.Set('date-time', (value) => { - // Use native Date parsing which handles ISO8601 formats properly - const date = new Date(value); - return !isNaN(date.getTime()) && date.toISOString() === value; - }); -} diff --git a/src/schemas/index.ts b/src/schemas/index.ts index 2ad8f789..12967054 100644 --- a/src/schemas/index.ts +++ b/src/schemas/index.ts @@ -1,10 +1,8 @@ -import {registerFormatValidators} from './format-registry'; - -// Ensure format validators are registered before any schema usage -registerFormatValidators(); +// Compilers and the type provider that turn the Zod schemas below into ajv validators +export * from './validation'; // Export current version (v1) export * from './v1'; // Version-specific exports for explicit imports -export * as v1 from './v1'; \ No newline at end of file +export * as v1 from './v1'; diff --git a/src/schemas/v1/page-hit-processed.ts b/src/schemas/v1/page-hit-processed.ts index 3e4fa651..7c8c2631 100644 --- a/src/schemas/v1/page-hit-processed.ts +++ b/src/schemas/v1/page-hit-processed.ts @@ -1,4 +1,4 @@ -import {Type, Static} from '@sinclair/typebox'; +import {z} from 'zod'; import {PageHitRaw} from './page-hit-raw'; import type {ParsedReferrer} from './page-hit-raw'; import uap from 'ua-parser-js'; @@ -8,49 +8,54 @@ import {isBot} from '../../utils/bot-detection'; const referrerParser = new ReferrerParser(); +// See page-hit-request.ts: UUID-shaped is enough, RFC compliance is not required. +const UUIDSchema = z.guid(); +const ISO8601DateTimeSchema = z.iso.datetime({precision: 3}); +const NullableString = z.string().nullable(); + // Complete page hit processed schema -export const PageHitProcessedSchema = Type.Object({ - timestamp: Type.String({format: 'date-time'}), - action: Type.Literal('page_hit'), - version: Type.Literal('1'), - site_uuid: Type.String({format: 'uuid'}), - session_id: Type.String(), - payload: Type.Object({ - event_id: Type.String({format: 'uuid'}), - site_uuid: Type.String({format: 'uuid'}), - member_uuid: Type.Union([Type.String({format: 'uuid'}), Type.Literal('undefined')]), - member_status: Type.Union([Type.String({minLength: 1}), Type.Literal('undefined')]), - post_uuid: Type.Union([Type.String({format: 'uuid'}), Type.Literal('undefined')]), - post_type: Type.Union([Type.Literal('null'), Type.Literal('post'), Type.Literal('page')]), - gift_link: Type.Optional(Type.Union([Type.String(), Type.Null()])), - locale: Type.String({minLength: 1}), - location: Type.Union([Type.String({minLength: 1}), Type.Null()]), - pathname: Type.String({minLength: 1}), - href: Type.String(), - os: Type.String(), - browser: Type.String(), - device: Type.String(), - parsedReferrer: Type.Optional(Type.Object({ - url: Type.Union([Type.String(), Type.Null()]), - source: Type.Union([Type.String(), Type.Null()]), - medium: Type.Union([Type.String(), Type.Null()]) - })), - referrerUrl: Type.Optional(Type.Union([Type.String(), Type.Null()])), - referrerSource: Type.Optional(Type.Union([Type.String(), Type.Null()])), - referrerMedium: Type.Optional(Type.Union([Type.String(), Type.Null()])), - utm_source: Type.Optional(Type.Union([Type.String(), Type.Null()])), - utm_medium: Type.Optional(Type.Union([Type.String(), Type.Null()])), - utm_campaign: Type.Optional(Type.Union([Type.String(), Type.Null()])), - utm_term: Type.Optional(Type.Union([Type.String(), Type.Null()])), - utm_content: Type.Optional(Type.Union([Type.String(), Type.Null()])), - 'user-agent': Type.String(), - meta: Type.Object({ - received_timestamp: Type.Union([Type.String({format: 'date-time'}), Type.Null()]) +export const PageHitProcessedSchema = z.object({ + timestamp: ISO8601DateTimeSchema, + action: z.literal('page_hit'), + version: z.literal('1'), + site_uuid: UUIDSchema, + session_id: z.string(), + payload: z.object({ + event_id: UUIDSchema, + site_uuid: UUIDSchema, + member_uuid: z.union([UUIDSchema, z.literal('undefined')]), + member_status: z.union([z.string().min(1), z.literal('undefined')]), + post_uuid: z.union([UUIDSchema, z.literal('undefined')]), + post_type: z.enum(['null', 'post', 'page']), + gift_link: NullableString.optional(), + locale: z.string().min(1), + location: z.string().min(1).nullable(), + pathname: z.string().min(1), + href: z.string(), + os: z.string(), + browser: z.string(), + device: z.string(), + parsedReferrer: z.object({ + url: NullableString, + source: NullableString, + medium: NullableString + }).optional(), + referrerUrl: NullableString.optional(), + referrerSource: NullableString.optional(), + referrerMedium: NullableString.optional(), + utm_source: NullableString.optional(), + utm_medium: NullableString.optional(), + utm_campaign: NullableString.optional(), + utm_term: NullableString.optional(), + utm_content: NullableString.optional(), + 'user-agent': z.string(), + meta: z.object({ + received_timestamp: ISO8601DateTimeSchema.nullable() }) }) }); -export type PageHitProcessed = Static; +export type PageHitProcessed = z.infer; // Transform functions // NOTE: These functions are deliberately duplicated from the proxy service /processors diff --git a/src/schemas/v1/page-hit-raw.ts b/src/schemas/v1/page-hit-raw.ts index baa2907f..c995b190 100644 --- a/src/schemas/v1/page-hit-raw.ts +++ b/src/schemas/v1/page-hit-raw.ts @@ -1,58 +1,57 @@ -import {Type, Static} from '@sinclair/typebox'; +import {z} from 'zod'; // Common types -const StringSchema = Type.String(); -const NonEmptyStringSchema = Type.String({minLength: 1}); -const UUIDSchema = Type.String({format: 'uuid'}); -const ISO8601DateTimeSchema = Type.String({ - format: 'date-time' -}); +const StringSchema = z.string(); +const NonEmptyStringSchema = z.string().min(1); +// See page-hit-request.ts: UUID-shaped is enough, RFC compliance is not required. +const UUIDSchema = z.guid(); +const ISO8601DateTimeSchema = z.iso.datetime({precision: 3}); // Enum types for page hit raw events -const ActionSchema = Type.Literal('page_hit'); -const VersionSchema = Type.Literal('1'); +const ActionSchema = z.literal('page_hit'); +const VersionSchema = z.literal('1'); // Parsed referrer schema -const ParsedReferrerSchema = Type.Object({ - source: Type.Union([StringSchema, Type.Null()]), - medium: Type.Union([StringSchema, Type.Null()]), - url: Type.Union([StringSchema, Type.Null()]) +const ParsedReferrerSchema = z.object({ + source: StringSchema.nullable(), + medium: StringSchema.nullable(), + url: StringSchema.nullable() }); -export type ParsedReferrer = Static; +export type ParsedReferrer = z.infer; // Payload schema for page hit raw events -const PayloadSchema = Type.Object({ - event_id: Type.Optional(StringSchema), - member_uuid: Type.Union([UUIDSchema, Type.Literal('undefined')]), - member_status: Type.Union([NonEmptyStringSchema, Type.Literal('undefined')]), - post_uuid: Type.Union([UUIDSchema, Type.Literal('undefined')]), - post_type: Type.Union([Type.Literal('null'), Type.Literal('post'), Type.Literal('page')]), - gift_link: Type.Optional(Type.Union([StringSchema, Type.Null()])), +const PayloadSchema = z.object({ + event_id: StringSchema.optional(), + member_uuid: z.union([UUIDSchema, z.literal('undefined')]), + member_status: z.union([NonEmptyStringSchema, z.literal('undefined')]), + post_uuid: z.union([UUIDSchema, z.literal('undefined')]), + post_type: z.enum(['null', 'post', 'page']), + gift_link: StringSchema.nullable().optional(), locale: NonEmptyStringSchema, - location: Type.Union([NonEmptyStringSchema, Type.Null()]), - referrer: Type.Optional(Type.Union([StringSchema, Type.Null()])), - parsedReferrer: Type.Optional(ParsedReferrerSchema), + location: NonEmptyStringSchema.nullable(), + referrer: StringSchema.nullable().optional(), + parsedReferrer: ParsedReferrerSchema.optional(), pathname: NonEmptyStringSchema, - href: Type.String(), - utm_source: Type.Optional(Type.Union([StringSchema, Type.Null()])), - utm_medium: Type.Optional(Type.Union([StringSchema, Type.Null()])), - utm_campaign: Type.Optional(Type.Union([StringSchema, Type.Null()])), - utm_term: Type.Optional(Type.Union([StringSchema, Type.Null()])), - utm_content: Type.Optional(Type.Union([StringSchema, Type.Null()])), - meta: Type.Object({ - received_timestamp: Type.Union([ISO8601DateTimeSchema, Type.Null()]) + href: StringSchema, + utm_source: StringSchema.nullable().optional(), + utm_medium: StringSchema.nullable().optional(), + utm_campaign: StringSchema.nullable().optional(), + utm_term: StringSchema.nullable().optional(), + utm_content: StringSchema.nullable().optional(), + meta: z.object({ + received_timestamp: ISO8601DateTimeSchema.nullable() }) }); // Meta schema for page hit raw events -const MetaSchema = Type.Object({ +const MetaSchema = z.object({ ip: NonEmptyStringSchema, 'user-agent': NonEmptyStringSchema }); // Complete page hit raw schema -export const PageHitRawSchema = Type.Object({ +export const PageHitRawSchema = z.object({ timestamp: ISO8601DateTimeSchema, action: ActionSchema, version: VersionSchema, @@ -61,4 +60,4 @@ export const PageHitRawSchema = Type.Object({ meta: MetaSchema }); -export type PageHitRaw = Static; +export type PageHitRaw = z.infer; diff --git a/src/schemas/v1/page-hit-request.ts b/src/schemas/v1/page-hit-request.ts index 3c87a4d5..56a4b161 100644 --- a/src/schemas/v1/page-hit-request.ts +++ b/src/schemas/v1/page-hit-request.ts @@ -1,128 +1,134 @@ -import {Static, Type} from '@sinclair/typebox'; -import {Value} from '@sinclair/typebox/value'; +import {z} from 'zod'; import {randomUUID} from 'crypto'; import {FastifyRequest} from 'fastify'; // Common types -const StringSchema = Type.String(); -const NonEmptyStringSchema = Type.String({ - minLength: 1, - pattern: '^.*\\S.*$' // At least one non-whitespace character -}); -const UUIDSchema = Type.String({format: 'uuid'}); -const ISO8601DateTimeSchema = Type.String({ - format: 'date-time' -}); -const VersionSchema = Type.Literal('1'); +const StringSchema = z.string(); +const NonEmptyStringSchema = z.string().min(1).regex(/^.*\S.*$/); // At least one non-whitespace character +// `guid`, not `uuid`: we only require UUID-shaped values, not RFC-compliant version and +// variant nibbles. Sites in the wild send IDs that fail the stricter check. +const UUIDSchema = z.guid(); +// `precision: 3` pins this to the canonical `Date.prototype.toISOString()` shape, which is +// what we store. Without it, offsets and second-precision timestamps would be accepted. +const ISO8601DateTimeSchema = z.iso.datetime({precision: 3}); +const VersionSchema = z.literal('1'); // Enum types -const AnalyticsEventNameSchema = Type.Union([ - Type.Literal('analytics_events'), - Type.Literal('analytics_events_test') -]); -const ActionSchema = Type.Literal('page_hit'); -const ContentTypeSchema = Type.Literal('application/json'); - -// Accept any value, but transform it to a UUID if it's not a string -// Allows non-valid UUIDs to be passed as long as they are a string -export const EventIdSchema = Type.Transform(Type.Any()) - .Decode((value: unknown) => { - // If it's a string and non-empty, use it - if (typeof value === 'string' && value.length > 0) { - return value; - } - // If it's an empty string, undefined, null, or any other value, generate a new UUID - return randomUUID(); - }) - .Encode((value: string) => value); +const AnalyticsEventNameSchema = z.enum(['analytics_events', 'analytics_events_test']); +const ActionSchema = z.literal('page_hit'); +const ContentTypeSchema = z.literal('application/json'); + +// Accept any value. Clients send all sorts of things here, and anything unusable is +// replaced by resolveEventId rather than rejected. +export const EventIdSchema = z.any(); + +/** + * Resolve the client-supplied event ID to the one we store. + * + * Any non-empty string is kept as-is, so IDs that are not valid UUIDs still round-trip. + * Empty strings, missing values and non-strings get a fresh UUID. + */ +export const resolveEventId = (value: unknown): string => { + if (typeof value === 'string' && value.length > 0) { + return value; + } + + return randomUUID(); +}; // Query parameters schema -export const PageHitRequestQueryParamsSchema = Type.Object({ - token: Type.Optional(NonEmptyStringSchema), +export const PageHitRequestQueryParamsSchema = z.object({ + token: NonEmptyStringSchema.optional(), name: AnalyticsEventNameSchema -}, { - additionalProperties: Type.String() -}); +}).catchall(StringSchema); -export type PageHitRequestQueryParamsType = Static; +export type PageHitRequestQueryParamsType = z.infer; // Headers schema -export const PageHitRequestHeadersSchema = Type.Object({ +export const PageHitRequestHeadersSchema = z.object({ 'x-site-uuid': UUIDSchema, 'content-type': ContentTypeSchema, 'user-agent': NonEmptyStringSchema, - 'x-ghost-analytics-start': Type.Optional(StringSchema), - referer: Type.Optional(StringSchema) -}, { - additionalProperties: Type.Union([StringSchema, Type.Array(StringSchema)]) -}); + 'x-ghost-analytics-start': StringSchema.optional(), + referer: StringSchema.optional() +}).catchall(z.union([StringSchema, z.array(StringSchema)])); -export type PageHitRequestHeadersType = Static; +export type PageHitRequestHeadersType = z.infer; // Parsed referrer schema -const ParsedReferrerSchema = Type.Object({ - source: Type.Union([StringSchema, Type.Null()]), - medium: Type.Union([StringSchema, Type.Null()]), - url: Type.Union([StringSchema, Type.Null()]) +const ParsedReferrerSchema = z.object({ + source: StringSchema.nullable(), + medium: StringSchema.nullable(), + url: StringSchema.nullable() }); // Payload schema -export const PageHitRequestPayloadSchema = Type.Object({ - event_id: Type.Optional(EventIdSchema), +// `looseObject` allows processors to add os, browser, device, etc. +export const PageHitRequestPayloadSchema = z.looseObject({ + event_id: EventIdSchema.optional(), 'user-agent': NonEmptyStringSchema, locale: NonEmptyStringSchema, - location: Type.Union([NonEmptyStringSchema, Type.Null()]), - referrer: Type.Optional(Type.Union([StringSchema, Type.Null()])), - parsedReferrer: Type.Optional(ParsedReferrerSchema), + location: NonEmptyStringSchema.nullable(), + referrer: StringSchema.nullable().optional(), + parsedReferrer: ParsedReferrerSchema.optional(), pathname: NonEmptyStringSchema, - href: Type.String(), + href: StringSchema, site_uuid: UUIDSchema, - post_uuid: Type.Union([UUIDSchema, Type.Literal('undefined')]), - post_type: Type.Union([Type.Literal('null'), Type.Literal('post'), Type.Literal('page')]), - gift_link: Type.Optional(Type.Union([StringSchema, Type.Null()])), - member_uuid: Type.Union([UUIDSchema, Type.Literal('undefined')]), - member_status: Type.Union([NonEmptyStringSchema, Type.Literal('undefined')]), - utm_source: Type.Optional(Type.Union([StringSchema, Type.Null()])), - utm_medium: Type.Optional(Type.Union([StringSchema, Type.Null()])), - utm_campaign: Type.Optional(Type.Union([StringSchema, Type.Null()])), - utm_term: Type.Optional(Type.Union([StringSchema, Type.Null()])), - utm_content: Type.Optional(Type.Union([StringSchema, Type.Null()])) -}, { - additionalProperties: true // Allow processors to add os, browser, device, etc. + post_uuid: z.union([UUIDSchema, z.literal('undefined')]), + post_type: z.enum(['null', 'post', 'page']), + gift_link: StringSchema.nullable().optional(), + member_uuid: z.union([UUIDSchema, z.literal('undefined')]), + member_status: z.union([NonEmptyStringSchema, z.literal('undefined')]), + utm_source: StringSchema.nullable().optional(), + utm_medium: StringSchema.nullable().optional(), + utm_campaign: StringSchema.nullable().optional(), + utm_term: StringSchema.nullable().optional(), + utm_content: StringSchema.nullable().optional() }); // Request body schema -export const PageHitRequestBodySchema = Type.Object({ +export const PageHitRequestBodySchema = z.object({ timestamp: ISO8601DateTimeSchema, action: ActionSchema, version: VersionSchema, - session_id: Type.Optional(StringSchema), + session_id: StringSchema.optional(), payload: PageHitRequestPayloadSchema }); -export type PageHitRequestBodyType = Static; +export type PageHitRequestBodyType = z.infer; // Complete request schema -export const PageHitRequestSchema = Type.Object({ +export const PageHitRequestSchema = z.object({ querystring: PageHitRequestQueryParamsSchema, headers: PageHitRequestHeadersSchema, body: PageHitRequestBodySchema }); -export interface PageHitRequestType extends FastifyRequest { - query: Static; - headers: Static; - body: Static; -} +// Built from FastifyRequest's own generic rather than hand-written, so it matches the +// request type Fastify infers for the route exactly - including the way it merges declared +// headers with IncomingHttpHeaders. +export type PageHitRequestType = FastifyRequest<{ + Querystring: PageHitRequestQueryParamsType; + Headers: PageHitRequestHeadersType; + Body: PageHitRequestBodyType; +}>; +/** + * Apply payload defaults and settle the event ID. + * + * Fastify has already validated the body, query and headers against the schemas above + * using ajv, so this does not revalidate them - it only does the two things a JSON + * Schema cannot express. + */ export const populateAndTransformPageHitRequest = async (request: PageHitRequestType): Promise => { - request.body.payload = { + const payload = { ...PageHitRequestPayloadDefaults, ...request.body.payload }; - request.body = Value.Decode(PageHitRequestBodySchema, request.body); - request.query = Value.Decode(PageHitRequestQueryParamsSchema, request.query); - request.headers = Value.Decode(PageHitRequestHeadersSchema, request.headers); + + payload.event_id = resolveEventId(payload.event_id); + request.body.payload = payload; + return request; }; diff --git a/src/schemas/validation.ts b/src/schemas/validation.ts new file mode 100644 index 00000000..a7ac45f7 --- /dev/null +++ b/src/schemas/validation.ts @@ -0,0 +1,92 @@ +import Ajv, {type ValidateFunction} from 'ajv'; +import addFormats from 'ajv-formats'; +import fastJson from 'fast-json-stringify'; +import type {FastifySchemaCompiler, FastifySerializerCompiler, FastifyTypeProvider} from 'fastify'; +import {z, type ZodType} from 'zod'; + +/** + * Schemas are authored in Zod, but validated by ajv-generated code. + * + * Zod's own `parse` walks the schema tree on every call; ajv compiles a schema once into + * straight-line JavaScript. Converting at boot gets us Zod's authoring and type inference + * without paying an interpreter per request. + */ + +// Collecting every error lets a hostile payload burn CPU, which is why Fastify pins this +// off on its own instance too. +const allErrors = false; + +// The same options Fastify hands its own ajv instance, so replacing the compiler does not +// silently change coercion, default filling, or additionalProperties stripping. +// See @fastify/ajv-compiler/lib/default-ajv-options. +const requestAjv = addFormats(new Ajv({ + coerceTypes: 'array', + useDefaults: true, + removeAdditional: true, + addUsedSchema: false, + allErrors +})); + +// Everything off the HTTP path already arrives as typed JSON, so coercion would only +// destroy information - it rewrites a null utm_source to an empty string to satisfy the +// string branch of a union, for instance. +const dataAjv = addFormats(new Ajv({ + coerceTypes: false, + useDefaults: false, + removeAdditional: false, + addUsedSchema: false, + allErrors +})); + +function toJsonSchema(schema: ZodType, io: 'input' | 'output'): object { + const json = z.toJSONSchema(schema, {target: 'draft-7', io, unrepresentable: 'any'}) as Record; + + // ajv is configured for draft-07 already and rejects the 2020-12 dialect marker Zod + // emits by default. + delete json.$schema; + + return json; +} + +/** + * Compile a Zod schema into a validator, for use outside the request lifecycle. + * + * Throws on invalid input, so callers can treat the return value as the parsed event. + * Note this validates without transforming: the value is checked in place and returned + * as-is, so it must already be the right shape. + */ +export function createValidator(schema: T): (data: unknown) => z.output { + const validate: ValidateFunction = dataAjv.compile(toJsonSchema(schema, 'input')); + + return (data: unknown) => { + if (!validate(data)) { + throw new Error(dataAjv.errorsText(validate.errors)); + } + + return data as z.output; + }; +} + +// Returns ajv's compiled function directly rather than wrapping it. Fastify reads +// `.errors` off it on failure, which is what gives validation failures their +// `body/timestamp must ...` messages and the structured `validation` array the error +// handler logs. +export const validatorCompiler: FastifySchemaCompiler = ({schema}) => { + return requestAjv.compile(toJsonSchema(schema, 'input')); +}; + +export const serializerCompiler: FastifySerializerCompiler = ({schema}) => { + return fastJson(toJsonSchema(schema, 'output') as Parameters[0]); +}; + +/** + * Types come from the Zod schema; validation comes from ajv. + * + * `input` and `output` only diverge where a schema has a transform, which JSON Schema + * cannot express and `toJSONSchema` drops silently - keep those out of schemas and do the + * work in a preHandler instead. + */ +export interface ZodTypeProvider extends FastifyTypeProvider { + validator: this['schema'] extends ZodType ? z.input : unknown; + serializer: this['schema'] extends ZodType ? z.output : unknown; +} diff --git a/src/services/batch-worker/BatchWorker.ts b/src/services/batch-worker/BatchWorker.ts index fca31994..a39852e5 100644 --- a/src/services/batch-worker/BatchWorker.ts +++ b/src/services/batch-worker/BatchWorker.ts @@ -1,10 +1,12 @@ import {Message} from '@google-cloud/pubsub'; import {EventSubscriber} from '../events/subscriber'; -import {PageHitRaw, PageHitRawSchema, PageHitProcessed, transformPageHitRawToProcessed} from '../../schemas'; -import {Value} from '@sinclair/typebox/value'; +import {PageHitRaw, PageHitRawSchema, PageHitProcessed, transformPageHitRawToProcessed, createValidator} from '../../schemas'; import {TinybirdClient} from '../tinybird/client'; import logger from '../../utils/logger'; +// Compiled once at module load rather than per message. +const validatePageHitRaw = createValidator(PageHitRawSchema); + interface BatchWorkerConfig { batchSize?: number; flushInterval?: number; @@ -115,7 +117,7 @@ class BatchWorker { try { const messageData = message.data.toString(); const parsedMessageData = JSON.parse(messageData); - return Value.Parse(PageHitRawSchema, parsedMessageData); + return validatePageHitRaw(parsedMessageData); } catch (err) { logger.error({ event: 'WorkerMessageParsingFailed', diff --git a/src/transformations/page-hit-transformations.ts b/src/transformations/page-hit-transformations.ts index 570762f6..86db1f79 100644 --- a/src/transformations/page-hit-transformations.ts +++ b/src/transformations/page-hit-transformations.ts @@ -1,5 +1,4 @@ -import {randomUUID} from 'crypto'; -import {PageHitRequestType, PageHitRaw} from '../schemas'; +import {PageHitRequestType, PageHitRaw, resolveEventId} from '../schemas'; export const pageHitRawPayloadFromRequest = (request: PageHitRequestType): PageHitRaw => { const parseReceivedTimestamp = (value: string | undefined) => { @@ -20,7 +19,9 @@ export const pageHitRawPayloadFromRequest = (request: PageHitRequestType): PageH version: request.body.version, site_uuid: request.headers['x-site-uuid'], payload: { - event_id: request.body.payload.event_id && request.body.payload.event_id.length > 0 ? request.body.payload.event_id : randomUUID(), + // Already settled by the preHandler; called again so this stays correct for + // callers that build a payload without going through it. + event_id: resolveEventId(request.body.payload.event_id), member_uuid: request.body.payload.member_uuid, member_status: request.body.payload.member_status, post_uuid: request.body.payload.post_uuid, diff --git a/src/types/tryghost-validator.d.ts b/src/types/tryghost-validator.d.ts deleted file mode 100644 index cc1a6853..00000000 --- a/src/types/tryghost-validator.d.ts +++ /dev/null @@ -1,23 +0,0 @@ -declare module '@tryghost/validator' { - interface Validator { - isLength(str: string, options?: {min?: number; max?: number}): boolean; - isEmpty(str: string): boolean; - isURL(str: string): boolean; - isEmail(str: string): boolean; - isIn(str: string, values: string[]): boolean; - isUUID(str: string, version?: number | 'loose' | 'all' | 'nil' | 'max'): boolean; - isBoolean(str: string): boolean; - isInt(str: string, options?: {min?: number; max?: number}): boolean; - isLowercase(str: string): boolean; - equals(str: string, comparison: string): boolean; - matches(str: string, pattern: RegExp | string, modifiers?: string): boolean; - isTimezone(str: string): boolean; - isEmptyOrURL(str: string): boolean; - isSlug(str: string): boolean; - } - - export function validate(schema: Record, data: Record): {isValid: boolean; errors: Record[]}; - - const validator: Validator; - export default validator; -} diff --git a/test/integration/app.test.ts b/test/integration/app.test.ts index b34ab6df..13d597e9 100644 --- a/test/integration/app.test.ts +++ b/test/integration/app.test.ts @@ -354,6 +354,34 @@ describe('Fastify App', () => { .send(payloadWithWeirdHref) .expect(202); }); + + // Timestamps that are valid RFC 3339 but not what Date.prototype.toISOString() + // emits. These used to pass schema validation and then blow up in the preHandler, + // returning a 500; they belong in the same 400 bucket as any other bad field. + const nonCanonicalTimestamps = [ + ['no milliseconds', '2025-04-14T22:16:06Z'], + ['a UTC offset instead of Z', '2025-04-14T22:16:06.095+02:00'], + ['more than millisecond precision', '2025-04-14T22:16:06.095123Z'], + ['a date that does not exist', '2025-02-30T22:16:06.095Z'] + ] as const; + + for (const [label, timestamp] of nonCanonicalTimestamps) { + it(`should reject a timestamp with ${label}`, async function () { + await request(proxyServer) + .post(path) + .query({token: 'abc123', name: 'analytics_events_test'}) + .set('Content-Type', 'application/json') + .set('x-site-uuid', '940b73e9-4952-4752-b23d-9486f999c47e') + .set('User-Agent', 'Mozilla/5.0 Test Browser') + .send({...eventPayload, timestamp}) + .expect(400) + .expect(function (res) { + assert.ok(res.body.message.includes('body/timestamp')); + }); + + expect(targetRequests.length).toBe(0); + }); + } }); describe('event id transformation', function () { @@ -393,6 +421,31 @@ describe('Fastify App', () => { const targetRequest = targetRequests[0]; expect(targetRequest.body.payload.event_id).toBe(eventId); }); + + // The schema accepts any type here on purpose, so junk is replaced rather than + // rejected. Guards against event_id being tightened to a string in the schema. + const junkEventIds = [ + ['an empty string', ''], + ['null', null], + ['a number', 123], + ['a boolean', true] + ] as const; + + for (const [label, eventId] of junkEventIds) { + it(`should generate an event id when given ${label}`, async function () { + await request(proxyServer) + .post(path) + .query({token: 'abc123', name: 'analytics_events_test'}) + .set('Content-Type', 'application/json') + .set('x-site-uuid', '940b73e9-4952-4752-b23d-9486f999c47e') + .set('User-Agent', 'Mozilla/5.0 Test Browser') + .send({...eventPayload, payload: {...eventPayload.payload, event_id: eventId}}) + .expect(202); + + const targetRequest = targetRequests[0]; + expect(targetRequest.body.payload.event_id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/); + }); + } }); describe('user agent parsing', function () { diff --git a/test/integration/validation-error-logging.test.ts b/test/integration/validation-error-logging.test.ts index a679c3b0..34b10317 100644 --- a/test/integration/validation-error-logging.test.ts +++ b/test/integration/validation-error-logging.test.ts @@ -151,7 +151,9 @@ describe('Validation Error Logging', () => { validation: expect.arrayContaining([ expect.objectContaining({ instancePath: '/x-site-uuid', - message: expect.stringContaining('format') + // z.guid() emits both `format: uuid` and an equivalent `pattern`; + // ajv stops at the first failure, so either may be reported. + message: expect.stringMatching(/pattern|format/) }) ]) }); @@ -180,7 +182,10 @@ describe('Validation Error Logging', () => { validation: expect.arrayContaining([ expect.objectContaining({ instancePath: '/timestamp', - message: expect.stringContaining('date-time') + // The schema constrains the timestamp with both `pattern` and + // `format`; ajv stops at the first failure, so either may be + // reported depending on the input. + message: expect.stringMatching(/pattern|format/) }) ]) }); diff --git a/test/unit/schemas/v1/page-hit-processed.test.ts b/test/unit/schemas/v1/page-hit-processed.test.ts index f2c6c7cf..1b2c2cf2 100644 --- a/test/unit/schemas/v1/page-hit-processed.test.ts +++ b/test/unit/schemas/v1/page-hit-processed.test.ts @@ -1,5 +1,4 @@ import {describe, it, expect, vi, beforeEach} from 'vitest'; -import {Value} from '@sinclair/typebox/value'; import { PageHitProcessedSchema, transformUserAgent, @@ -107,13 +106,13 @@ describe('PageHitProcessedSchema v1', () => { }; it('should validate valid page hit processed data', () => { - expect(Value.Check(PageHitProcessedSchema, validPageHitProcessed)).toBe(true); + expect(PageHitProcessedSchema.safeParse(validPageHitProcessed).success).toBe(true); }); it('should require session_id field', () => { // eslint-disable-next-line @typescript-eslint/no-unused-vars, camelcase const {session_id, ...invalidData} = validPageHitProcessed; - expect(Value.Check(PageHitProcessedSchema, invalidData)).toBe(false); + expect(PageHitProcessedSchema.safeParse(invalidData).success).toBe(false); }); it('should not be too strict about the href value', async () => { @@ -139,7 +138,7 @@ describe('PageHitProcessedSchema v1', () => { os: undefined } }; - expect(Value.Check(PageHitProcessedSchema, invalidData)).toBe(false); + expect(PageHitProcessedSchema.safeParse(invalidData).success).toBe(false); }); it('should require browser field', () => { @@ -150,7 +149,7 @@ describe('PageHitProcessedSchema v1', () => { browser: undefined } }; - expect(Value.Check(PageHitProcessedSchema, invalidData)).toBe(false); + expect(PageHitProcessedSchema.safeParse(invalidData).success).toBe(false); }); it('should require device field', () => { @@ -161,7 +160,7 @@ describe('PageHitProcessedSchema v1', () => { device: undefined } }; - expect(Value.Check(PageHitProcessedSchema, invalidData)).toBe(false); + expect(PageHitProcessedSchema.safeParse(invalidData).success).toBe(false); }); it('should require user-agent field', () => { @@ -172,7 +171,7 @@ describe('PageHitProcessedSchema v1', () => { 'user-agent': undefined } }; - expect(Value.Check(PageHitProcessedSchema, invalidData)).toBe(false); + expect(PageHitProcessedSchema.safeParse(invalidData).success).toBe(false); }); it('should allow optional referrer fields', () => { @@ -185,7 +184,7 @@ describe('PageHitProcessedSchema v1', () => { referrer_medium: undefined } }; - expect(Value.Check(PageHitProcessedSchema, validData)).toBe(true); + expect(PageHitProcessedSchema.safeParse(validData).success).toBe(true); }); it('should validate with all referrer fields present', () => { @@ -198,7 +197,7 @@ describe('PageHitProcessedSchema v1', () => { referrerMedium: 'social' } }; - expect(Value.Check(PageHitProcessedSchema, validData)).toBe(true); + expect(PageHitProcessedSchema.safeParse(validData).success).toBe(true); }); }); describe('transformUserAgent', () => { @@ -534,7 +533,7 @@ describe('PageHitProcessedSchema v1', () => { it('should produce valid PageHitProcessed schema', async () => { const result = await transformPageHitRawToProcessed(validPageHitRaw); - expect(Value.Check(PageHitProcessedSchema, result)).toBe(true); + expect(PageHitProcessedSchema.safeParse(result).success).toBe(true); }); }); }); diff --git a/test/unit/schemas/v1/page-hit-raw.test.ts b/test/unit/schemas/v1/page-hit-raw.test.ts index 8f6c305a..d768ad80 100644 --- a/test/unit/schemas/v1/page-hit-raw.test.ts +++ b/test/unit/schemas/v1/page-hit-raw.test.ts @@ -1,5 +1,4 @@ import {describe, it, expect} from 'vitest'; -import {Value} from '@sinclair/typebox/value'; import {PageHitRawSchema} from '../../../../src/schemas'; describe('PageHitRawSchema v1', () => { @@ -30,7 +29,7 @@ describe('PageHitRawSchema v1', () => { }; it('should validate valid page hit raw data', () => { - expect(Value.Check(PageHitRawSchema, validPageHitRaw)).toBe(true); + expect(PageHitRawSchema.safeParse(validPageHitRaw).success).toBe(true); }); describe('timestamp validation', () => { @@ -39,7 +38,7 @@ describe('PageHitRawSchema v1', () => { ...validPageHitRaw, timestamp: '2024-12-25T15:30:45.123Z' }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should reject invalid timestamp format', () => { @@ -47,7 +46,7 @@ describe('PageHitRawSchema v1', () => { ...validPageHitRaw, timestamp: '2024-01-01' }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); it('should reject non-ISO8601 timestamp', () => { @@ -55,7 +54,7 @@ describe('PageHitRawSchema v1', () => { ...validPageHitRaw, timestamp: 'January 1, 2024' }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); }); @@ -65,7 +64,7 @@ describe('PageHitRawSchema v1', () => { ...validPageHitRaw, action: 'page_hit' }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should reject invalid action', () => { @@ -73,7 +72,7 @@ describe('PageHitRawSchema v1', () => { ...validPageHitRaw, action: 'click_event' }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); }); @@ -83,7 +82,7 @@ describe('PageHitRawSchema v1', () => { ...validPageHitRaw, version: '1' }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should reject invalid version', () => { @@ -91,7 +90,7 @@ describe('PageHitRawSchema v1', () => { ...validPageHitRaw, version: '2' }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); }); @@ -101,7 +100,7 @@ describe('PageHitRawSchema v1', () => { ...validPageHitRaw, site_uuid: 'a1b2c3d4-e5f6-7890-abcd-ef1234567890' }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should validate UUID-shaped values without requiring RFC version and variant bits', () => { @@ -109,7 +108,7 @@ describe('PageHitRawSchema v1', () => { ...validPageHitRaw, site_uuid: '12345678-1234-1234-1234-123456789012' }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should reject invalid UUID format', () => { @@ -117,7 +116,7 @@ describe('PageHitRawSchema v1', () => { ...validPageHitRaw, site_uuid: 'invalid-uuid' }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); }); @@ -130,7 +129,7 @@ describe('PageHitRawSchema v1', () => { member_uuid: '12345678-1234-1234-1234-123456789012' } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should validate with "undefined" member_uuid', () => { @@ -141,7 +140,7 @@ describe('PageHitRawSchema v1', () => { member_uuid: 'undefined' } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should validate with "undefined" member_status', () => { @@ -152,7 +151,7 @@ describe('PageHitRawSchema v1', () => { member_status: 'undefined' } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should validate with UUID post_uuid', () => { @@ -163,7 +162,7 @@ describe('PageHitRawSchema v1', () => { post_uuid: '12345678-1234-1234-1234-123456789012' } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should validate with "undefined" post_uuid', () => { @@ -174,7 +173,7 @@ describe('PageHitRawSchema v1', () => { post_uuid: 'undefined' } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should validate all post_type values', () => { @@ -188,7 +187,7 @@ describe('PageHitRawSchema v1', () => { post_type: postType } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); }); @@ -200,7 +199,7 @@ describe('PageHitRawSchema v1', () => { post_type: 'article' } }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); it('should validate with null location', () => { @@ -211,7 +210,7 @@ describe('PageHitRawSchema v1', () => { location: null } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should validate with null referrer', () => { @@ -222,7 +221,7 @@ describe('PageHitRawSchema v1', () => { referrer: null } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should validate without referrer field (optional)', () => { @@ -232,7 +231,7 @@ describe('PageHitRawSchema v1', () => { ...validPageHitRaw, payload: payloadWithoutReferrer }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should validate with parsedReferrer object with all string values', () => { @@ -247,7 +246,7 @@ describe('PageHitRawSchema v1', () => { } } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should validate with parsedReferrer object with null values', () => { @@ -262,7 +261,7 @@ describe('PageHitRawSchema v1', () => { } } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should validate with parsedReferrer object with mixed null and string values', () => { @@ -277,7 +276,7 @@ describe('PageHitRawSchema v1', () => { } } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should validate without parsedReferrer field (optional)', () => { @@ -288,7 +287,7 @@ describe('PageHitRawSchema v1', () => { // parsedReferrer field omitted } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should reject parsedReferrer with missing required fields', () => { @@ -303,7 +302,7 @@ describe('PageHitRawSchema v1', () => { } } }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); it('should reject parsedReferrer with invalid field types', () => { @@ -318,7 +317,7 @@ describe('PageHitRawSchema v1', () => { } } }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); it('should not be too strict about the href value', () => { @@ -329,7 +328,7 @@ describe('PageHitRawSchema v1', () => { href: 'not-a-url' } }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(true); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(true); }); it('should reject empty pathname', () => { @@ -340,7 +339,7 @@ describe('PageHitRawSchema v1', () => { pathname: '' } }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); it('should reject empty locale', () => { @@ -351,7 +350,7 @@ describe('PageHitRawSchema v1', () => { locale: '' } }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); it('should reject invalid member_uuid', () => { @@ -362,7 +361,7 @@ describe('PageHitRawSchema v1', () => { member_uuid: 'invalid-uuid' } }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); it('should reject invalid post_uuid', () => { @@ -373,7 +372,7 @@ describe('PageHitRawSchema v1', () => { post_uuid: 'invalid-uuid' } }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); }); @@ -386,7 +385,7 @@ describe('PageHitRawSchema v1', () => { 'user-agent': 'Chrome/91.0.4472.124' } }; - expect(Value.Check(PageHitRawSchema, validData)).toBe(true); + expect(PageHitRawSchema.safeParse(validData).success).toBe(true); }); it('should reject empty ip', () => { @@ -397,7 +396,7 @@ describe('PageHitRawSchema v1', () => { ip: '' } }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); it('should reject empty user-agent', () => { @@ -408,7 +407,7 @@ describe('PageHitRawSchema v1', () => { 'user-agent': '' } }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); it('should reject missing ip', () => { @@ -418,7 +417,7 @@ describe('PageHitRawSchema v1', () => { 'user-agent': validPageHitRaw.meta['user-agent'] } }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); it('should reject missing user-agent', () => { @@ -428,7 +427,7 @@ describe('PageHitRawSchema v1', () => { ip: validPageHitRaw.meta.ip } }; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); }); @@ -436,25 +435,25 @@ describe('PageHitRawSchema v1', () => { it('should reject missing timestamp', () => { // eslint-disable-next-line @typescript-eslint/no-unused-vars const {timestamp, ...invalidData} = validPageHitRaw; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); it('should reject missing payload', () => { // eslint-disable-next-line @typescript-eslint/no-unused-vars const {payload, ...invalidData} = validPageHitRaw; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); it('should reject missing meta', () => { // eslint-disable-next-line @typescript-eslint/no-unused-vars const {meta, ...invalidData} = validPageHitRaw; - expect(Value.Check(PageHitRawSchema, invalidData)).toBe(false); + expect(PageHitRawSchema.safeParse(invalidData).success).toBe(false); }); }); describe('event_id validation', () => { it('should validate when event_id is present', () => { - expect(Value.Check(PageHitRawSchema, validPageHitRaw)).toBe(true); + expect(PageHitRawSchema.safeParse(validPageHitRaw).success).toBe(true); }); it('should validate when event_id is missing (optional field)', () => { @@ -464,7 +463,7 @@ describe('PageHitRawSchema v1', () => { ...validPageHitRaw, payload: payloadWithoutEventId }; - expect(Value.Check(PageHitRawSchema, validDataWithoutEventId)).toBe(true); + expect(PageHitRawSchema.safeParse(validDataWithoutEventId).success).toBe(true); }); it('should accept any string as event_id (validation happens during processing)', () => { @@ -475,7 +474,7 @@ describe('PageHitRawSchema v1', () => { event_id: 'not-a-uuid' } }; - expect(Value.Check(PageHitRawSchema, dataWithInvalidEventId)).toBe(true); + expect(PageHitRawSchema.safeParse(dataWithInvalidEventId).success).toBe(true); }); }); @@ -507,7 +506,7 @@ describe('PageHitRawSchema v1', () => { } }; - expect(Value.Check(PageHitRawSchema, realWorldPayload)).toBe(true); + expect(PageHitRawSchema.safeParse(realWorldPayload).success).toBe(true); }); it('should validate payload with parsedReferrer', () => { @@ -542,7 +541,7 @@ describe('PageHitRawSchema v1', () => { } }; - expect(Value.Check(PageHitRawSchema, realWorldPayloadWithParsedReferrer)).toBe(true); + expect(PageHitRawSchema.safeParse(realWorldPayloadWithParsedReferrer).success).toBe(true); }); }); }); diff --git a/test/unit/schemas/v1/page-hit-request.test.ts b/test/unit/schemas/v1/page-hit-request.test.ts index f7b73c91..89c321ed 100644 --- a/test/unit/schemas/v1/page-hit-request.test.ts +++ b/test/unit/schemas/v1/page-hit-request.test.ts @@ -1,64 +1,59 @@ import {describe, it, expect} from 'vitest'; -import {Value} from '@sinclair/typebox/value'; import { PageHitRequestQueryParamsSchema, PageHitRequestHeadersSchema, PageHitRequestPayloadSchema, PageHitRequestBodySchema, PageHitRequestSchema, - EventIdSchema + EventIdSchema, + resolveEventId } from '../../../../src/schemas'; import assert from 'node:assert/strict'; +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + describe('PageHitRequestSchema v1', () => { describe('EventIdSchema', () => { - it('should validate with an undefined event ID', () => { - assert.ok(Value.Check(EventIdSchema, undefined), 'Event ID can be undefined'); - }); - - it('should validate with a null event ID', () => { - assert.ok(Value.Check(EventIdSchema, null), 'Event ID can be null'); - }); - - it('should validate with a string event ID', () => { - assert.ok(Value.Check(EventIdSchema, '12345678-1234-1234-1234-123456789012'), 'Event ID can be a string'); - }); - - it('should validate with a number event ID', () => { - assert.ok(Value.Check(EventIdSchema, 123), 'Event ID can be a number'); - }); - - it('should validate with a boolean event ID', () => { - assert.ok(Value.Check(EventIdSchema, true), 'Event ID can be a boolean'); - }); - - it('should transform undefined to a UUID', () => { - const result = Value.Decode(EventIdSchema, undefined); - expect(typeof result).toBe('string'); - expect(result).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i); - }); + // The schema deliberately accepts anything, so that a junk event ID is replaced by + // resolveEventId rather than turning the whole request into a 400. + const anyValues = [ + ['undefined', undefined], + ['null', null], + ['a string', '12345678-1234-1234-1234-123456789012'], + ['a number', 123], + ['a boolean', true] + ] as const; + + for (const [label, value] of anyValues) { + it(`should validate with ${label} as the event ID`, () => { + assert.ok(EventIdSchema.safeParse(value).success, `Event ID can be ${label}`); + }); + } + }); - it('should transform null to a UUID', () => { - const result = Value.Decode(EventIdSchema, null); - expect(typeof result).toBe('string'); - expect(result).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i); + describe('resolveEventId', () => { + it('should keep a non-empty string as-is, even when it is not a valid UUID', () => { + expect(resolveEventId('12345678-1234-1234-1234-123456789012')).toBe('12345678-1234-1234-1234-123456789012'); + expect(resolveEventId('not-a-uuid')).toBe('not-a-uuid'); }); - it('should transform a string event ID to itself', () => { - const result = Value.Decode(EventIdSchema, '12345678-1234-1234-1234-123456789012'); - expect(result).toBe('12345678-1234-1234-1234-123456789012'); - }); + const generatedCases = [ + ['undefined', undefined], + ['null', null], + ['an empty string', ''], + ['a number', 123], + ['a boolean', true], + ['an object', {}] + ] as const; - it('should transform an empty string to a UUID', () => { - const result = Value.Decode(EventIdSchema, ''); - expect(typeof result).toBe('string'); - expect(result).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i); - }); + for (const [label, value] of generatedCases) { + it(`should generate a UUID for ${label}`, () => { + expect(resolveEventId(value)).toMatch(UUID_PATTERN); + }); + } - it('should transform a non-string value to a UUID', () => { - const result = Value.Decode(EventIdSchema, 123); - expect(typeof result).toBe('string'); - expect(result).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i); + it('should generate a different UUID on each call', () => { + expect(resolveEventId(undefined)).not.toBe(resolveEventId(undefined)); }); }); @@ -69,7 +64,7 @@ describe('PageHitRequestSchema v1', () => { name: 'analytics_events' }; - expect(Value.Check(PageHitRequestQueryParamsSchema, validParams)).toBe(true); + expect(PageHitRequestQueryParamsSchema.safeParse(validParams).success).toBe(true); }); it('should validate analytics_events_test name', () => { @@ -78,7 +73,7 @@ describe('PageHitRequestSchema v1', () => { name: 'analytics_events_test' }; - expect(Value.Check(PageHitRequestQueryParamsSchema, validParams)).toBe(true); + expect(PageHitRequestQueryParamsSchema.safeParse(validParams).success).toBe(true); }); it('should reject invalid name values', () => { @@ -87,7 +82,7 @@ describe('PageHitRequestSchema v1', () => { name: 'invalid_event_name' }; - expect(Value.Check(PageHitRequestQueryParamsSchema, invalidParams)).toBe(false); + expect(PageHitRequestQueryParamsSchema.safeParse(invalidParams).success).toBe(false); }); it('should validate without token (optional)', () => { @@ -95,7 +90,7 @@ describe('PageHitRequestSchema v1', () => { name: 'analytics_events' }; - expect(Value.Check(PageHitRequestQueryParamsSchema, validParams)).toBe(true); + expect(PageHitRequestQueryParamsSchema.safeParse(validParams).success).toBe(true); }); it('should reject empty token when provided', () => { @@ -104,7 +99,7 @@ describe('PageHitRequestSchema v1', () => { name: 'analytics_events' }; - expect(Value.Check(PageHitRequestQueryParamsSchema, invalidParams)).toBe(false); + expect(PageHitRequestQueryParamsSchema.safeParse(invalidParams).success).toBe(false); }); it('should allow additional properties', () => { @@ -113,7 +108,7 @@ describe('PageHitRequestSchema v1', () => { additional: 'property' }; - expect(Value.Check(PageHitRequestQueryParamsSchema, validParams)).toBe(true); + expect(PageHitRequestQueryParamsSchema.safeParse(validParams).success).toBe(true); }); }); @@ -125,7 +120,7 @@ describe('PageHitRequestSchema v1', () => { 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36' }; - expect(Value.Check(PageHitRequestHeadersSchema, validHeaders)).toBe(true); + expect(PageHitRequestHeadersSchema.safeParse(validHeaders).success).toBe(true); }); it('should validate with optional referer', () => { @@ -136,7 +131,7 @@ describe('PageHitRequestSchema v1', () => { referer: 'https://example.com' }; - expect(Value.Check(PageHitRequestHeadersSchema, validHeaders)).toBe(true); + expect(PageHitRequestHeadersSchema.safeParse(validHeaders).success).toBe(true); }); it('should reject invalid UUID format', () => { @@ -146,7 +141,7 @@ describe('PageHitRequestSchema v1', () => { 'user-agent': 'Mozilla/5.0' }; - expect(Value.Check(PageHitRequestHeadersSchema, invalidHeaders)).toBe(false); + expect(PageHitRequestHeadersSchema.safeParse(invalidHeaders).success).toBe(false); }); it('should reject invalid content-type', () => { @@ -156,7 +151,7 @@ describe('PageHitRequestSchema v1', () => { 'user-agent': 'Mozilla/5.0' }; - expect(Value.Check(PageHitRequestHeadersSchema, invalidHeaders)).toBe(false); + expect(PageHitRequestHeadersSchema.safeParse(invalidHeaders).success).toBe(false); }); it('should reject missing required headers', () => { @@ -165,7 +160,7 @@ describe('PageHitRequestSchema v1', () => { // Missing content-type and user-agent }; - expect(Value.Check(PageHitRequestHeadersSchema, invalidHeaders)).toBe(false); + expect(PageHitRequestHeadersSchema.safeParse(invalidHeaders).success).toBe(false); }); }); @@ -185,7 +180,7 @@ describe('PageHitRequestSchema v1', () => { }; it('should validate valid payload', () => { - expect(Value.Check(PageHitRequestPayloadSchema, validPayload)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(validPayload).success).toBe(true); }); it('should validate with null referrer', () => { @@ -194,14 +189,14 @@ describe('PageHitRequestSchema v1', () => { referrer: null }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithNullReferrer)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithNullReferrer).success).toBe(true); }); it('should validate without referrer field (optional)', () => { // eslint-disable-next-line @typescript-eslint/no-unused-vars const {referrer, ...payloadWithoutReferrer} = validPayload; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithoutReferrer)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithoutReferrer).success).toBe(true); }); it('should validate with empty string referrer', () => { @@ -210,7 +205,7 @@ describe('PageHitRequestSchema v1', () => { referrer: '' }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithEmptyReferrer)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithEmptyReferrer).success).toBe(true); }); it('should validate with UUID post_uuid', () => { @@ -219,7 +214,7 @@ describe('PageHitRequestSchema v1', () => { post_uuid: '12345678-1234-1234-1234-123456789012' }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithUUIDPost)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithUUIDPost).success).toBe(true); }); it('should validate with UUID member_uuid', () => { @@ -228,7 +223,7 @@ describe('PageHitRequestSchema v1', () => { member_uuid: '12345678-1234-1234-1234-123456789012' }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithUUIDMember)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithUUIDMember).success).toBe(true); }); it('should validate all post_type values', () => { @@ -239,7 +234,7 @@ describe('PageHitRequestSchema v1', () => { ...validPayload, post_type: postType }; - expect(Value.Check(PageHitRequestPayloadSchema, payload)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payload).success).toBe(true); }); }); @@ -249,7 +244,7 @@ describe('PageHitRequestSchema v1', () => { post_type: 'article' }; - expect(Value.Check(PageHitRequestPayloadSchema, invalidPayload)).toBe(false); + expect(PageHitRequestPayloadSchema.safeParse(invalidPayload).success).toBe(false); }); it('should not be too strict about the href value', () => { @@ -258,7 +253,7 @@ describe('PageHitRequestSchema v1', () => { href: 'not-a-url' }; - expect(Value.Check(PageHitRequestPayloadSchema, invalidPayload)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(invalidPayload).success).toBe(true); }); it('should reject empty required strings', () => { @@ -267,7 +262,7 @@ describe('PageHitRequestSchema v1', () => { 'user-agent': '' }; - expect(Value.Check(PageHitRequestPayloadSchema, invalidPayload)).toBe(false); + expect(PageHitRequestPayloadSchema.safeParse(invalidPayload).success).toBe(false); }); it('should reject invalid site_uuid', () => { @@ -276,7 +271,7 @@ describe('PageHitRequestSchema v1', () => { site_uuid: 'invalid-uuid' }; - expect(Value.Check(PageHitRequestPayloadSchema, invalidPayload)).toBe(false); + expect(PageHitRequestPayloadSchema.safeParse(invalidPayload).success).toBe(false); }); it('should validate real healthcheck payload with null location and undefined member_status', () => { @@ -299,7 +294,7 @@ describe('PageHitRequestSchema v1', () => { member_status: 'undefined' }; - expect(Value.Check(PageHitRequestPayloadSchema, healthcheckPayload)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(healthcheckPayload).success).toBe(true); }); it('should validate with parsedReferrer object with all string values', () => { @@ -312,7 +307,7 @@ describe('PageHitRequestSchema v1', () => { } }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithParsedReferrer)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithParsedReferrer).success).toBe(true); }); it('should validate with parsedReferrer object with mixed null and string values', () => { @@ -325,7 +320,7 @@ describe('PageHitRequestSchema v1', () => { } }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithMixedParsedReferrer)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithMixedParsedReferrer).success).toBe(true); }); it('should validate without parsedReferrer field (optional)', () => { @@ -334,7 +329,7 @@ describe('PageHitRequestSchema v1', () => { // parsedReferrer field omitted }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithoutParsedReferrer)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithoutParsedReferrer).success).toBe(true); }); it('should reject parsedReferrer with missing required fields', () => { @@ -347,7 +342,7 @@ describe('PageHitRequestSchema v1', () => { } }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithIncompleteParsedReferrer)).toBe(false); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithIncompleteParsedReferrer).success).toBe(false); }); it('should validate parsedReferrer with UTM parameters', () => { @@ -365,7 +360,7 @@ describe('PageHitRequestSchema v1', () => { } }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithUTMParams)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithUTMParams).success).toBe(true); }); it('should validate parsedReferrer with partial UTM parameters', () => { @@ -382,7 +377,7 @@ describe('PageHitRequestSchema v1', () => { } }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithPartialUTM)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithPartialUTM).success).toBe(true); }); it('should validate parsedReferrer with null UTM parameters', () => { @@ -400,7 +395,7 @@ describe('PageHitRequestSchema v1', () => { } }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithNullUTM)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithNullUTM).success).toBe(true); }); it('should validate parsedReferrer without UTM parameters', () => { @@ -414,7 +409,7 @@ describe('PageHitRequestSchema v1', () => { } }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithoutUTM)).toBe(true); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithoutUTM).success).toBe(true); }); it('should reject parsedReferrer with invalid field types', () => { @@ -427,7 +422,7 @@ describe('PageHitRequestSchema v1', () => { } }; - expect(Value.Check(PageHitRequestPayloadSchema, payloadWithInvalidParsedReferrer)).toBe(false); + expect(PageHitRequestPayloadSchema.safeParse(payloadWithInvalidParsedReferrer).success).toBe(false); }); }); @@ -452,7 +447,7 @@ describe('PageHitRequestSchema v1', () => { }; it('should validate valid body', () => { - expect(Value.Check(PageHitRequestBodySchema, validBody)).toBe(true); + expect(PageHitRequestBodySchema.safeParse(validBody).success).toBe(true); }); it('should validate body without session_id (optional)', () => { @@ -464,7 +459,7 @@ describe('PageHitRequestSchema v1', () => { // session_id omitted }; - expect(Value.Check(PageHitRequestBodySchema, bodyWithoutSessionId)).toBe(true); + expect(PageHitRequestBodySchema.safeParse(bodyWithoutSessionId).success).toBe(true); }); it('should reject invalid timestamp format', () => { @@ -473,7 +468,7 @@ describe('PageHitRequestSchema v1', () => { timestamp: '2024-01-01' }; - expect(Value.Check(PageHitRequestBodySchema, invalidBody)).toBe(false); + expect(PageHitRequestBodySchema.safeParse(invalidBody).success).toBe(false); }); it('should reject invalid action', () => { @@ -482,7 +477,7 @@ describe('PageHitRequestSchema v1', () => { action: 'click_event' }; - expect(Value.Check(PageHitRequestBodySchema, invalidBody)).toBe(false); + expect(PageHitRequestBodySchema.safeParse(invalidBody).success).toBe(false); }); }); @@ -517,7 +512,7 @@ describe('PageHitRequestSchema v1', () => { }; it('should validate complete valid request', () => { - expect(Value.Check(PageHitRequestSchema, validRequest)).toBe(true); + expect(PageHitRequestSchema.safeParse(validRequest).success).toBe(true); }); it('should reject request with invalid query params', () => { @@ -528,7 +523,7 @@ describe('PageHitRequestSchema v1', () => { } }; - expect(Value.Check(PageHitRequestSchema, invalidRequest)).toBe(false); + expect(PageHitRequestSchema.safeParse(invalidRequest).success).toBe(false); }); it('should reject request with invalid headers', () => { @@ -541,7 +536,7 @@ describe('PageHitRequestSchema v1', () => { } }; - expect(Value.Check(PageHitRequestSchema, invalidRequest)).toBe(false); + expect(PageHitRequestSchema.safeParse(invalidRequest).success).toBe(false); }); it('should reject request with invalid body', () => { @@ -553,7 +548,7 @@ describe('PageHitRequestSchema v1', () => { } }; - expect(Value.Check(PageHitRequestSchema, invalidRequest)).toBe(false); + expect(PageHitRequestSchema.safeParse(invalidRequest).success).toBe(false); }); }); }); \ No newline at end of file diff --git a/test/unit/schemas/validation.test.ts b/test/unit/schemas/validation.test.ts new file mode 100644 index 00000000..fab16418 --- /dev/null +++ b/test/unit/schemas/validation.test.ts @@ -0,0 +1,155 @@ +import {describe, it, expect} from 'vitest'; +import {z} from 'zod'; +import { + createValidator, + validatorCompiler, + PageHitRawSchema, + PageHitRequestBodySchema, + PageHitRequestHeadersSchema, + PageHitRequestQueryParamsSchema +} from '../../../src/schemas'; + +const UUID = '940b73e9-4952-4752-b23d-9486f999c47e'; + +function validRawEvent(): Record & {payload: Record} { + return { + timestamp: '2025-04-14T22:16:06.095Z', + action: 'page_hit', + version: '1', + site_uuid: UUID, + payload: { + event_id: UUID, + member_uuid: 'undefined', + member_status: 'free', + post_uuid: 'undefined', + post_type: 'null', + locale: 'en-US', + location: 'US', + pathname: '/', + href: 'https://example.com/', + meta: {received_timestamp: null} + }, + meta: {ip: '127.0.0.1', 'user-agent': 'Mozilla/5.0 Test Browser'} + }; +} + +function validRequestBody() { + return { + timestamp: '2025-04-14T22:16:06.095Z', + action: 'page_hit', + version: '1', + payload: { + 'user-agent': 'Mozilla/5.0 Test Browser', + locale: 'en-US', + location: 'US', + pathname: '/', + href: 'https://example.com/', + site_uuid: UUID, + post_uuid: 'undefined', + post_type: 'null', + member_uuid: 'undefined', + member_status: 'free' + } + }; +} + +function accepts(schema: z.ZodType, value: unknown): boolean { + try { + createValidator(schema)(value); + return true; + } catch { + return false; + } +} + +describe('schema validation', () => { + // Schemas are written in Zod but enforced by ajv, against a JSON Schema projection of + // them. Nothing in the type system keeps the two in step, and `toJSONSchema` drops what + // it cannot express without complaining - so check that they actually agree. + describe('the ajv projection agrees with Zod', () => { + // Built lazily so each case is constructed inside its own `it`. + const cases: Array<[string, z.ZodType, () => unknown]> = [ + ['a valid raw event', PageHitRawSchema, () => validRawEvent()], + ['a raw event with a bad site_uuid', PageHitRawSchema, () => ({...validRawEvent(), site_uuid: 'nope'})], + ['a raw event with a non-canonical timestamp', PageHitRawSchema, () => ({...validRawEvent(), timestamp: '2025-04-14T22:16:06Z'})], + ['a raw event missing meta', PageHitRawSchema, () => ({...validRawEvent(), meta: undefined})], + ['a valid request body', PageHitRequestBodySchema, () => validRequestBody()], + ['a request body with an unknown payload key', PageHitRequestBodySchema, () => ({ + ...validRequestBody(), + payload: {...validRequestBody().payload, os: 'macOS'} + })], + ['a request body with a whitespace-only locale', PageHitRequestBodySchema, () => ({ + ...validRequestBody(), + payload: {...validRequestBody().payload, locale: ' '} + })], + ['a request body with the wrong action', PageHitRequestBodySchema, () => ({...validRequestBody(), action: 'nope'})], + ['valid query params', PageHitRequestQueryParamsSchema, () => ({name: 'analytics_events', token: 'abc'})], + ['query params with an unknown key', PageHitRequestQueryParamsSchema, () => ({name: 'analytics_events', extra: 'x'})], + ['query params with a bad name', PageHitRequestQueryParamsSchema, () => ({name: 'nope'})], + ['valid headers', PageHitRequestHeadersSchema, () => ({ + 'x-site-uuid': UUID, + 'content-type': 'application/json', + 'user-agent': 'Mozilla/5.0 Test Browser' + })], + ['headers missing a user-agent', PageHitRequestHeadersSchema, () => ({ + 'x-site-uuid': UUID, + 'content-type': 'application/json' + })] + ]; + + for (const [label, schema, build] of cases) { + it(`should reach the same verdict for ${label}`, () => { + // ajv checks in place, so give each side its own copy. + expect(accepts(schema, build())).toBe(schema.safeParse(build()).success); + }); + } + }); + + describe('validatorCompiler', () => { + it('should return ajv errors rather than throwing, so Fastify can format them', () => { + const validate = validatorCompiler({ + schema: PageHitRequestQueryParamsSchema, + method: 'POST', + url: '/api/v1/page_hit', + httpPart: 'querystring' + }); + + expect(validate({name: 'analytics_events'})).toBe(true); + expect(validate({name: 'nope'})).toBe(false); + // Fastify reads `.errors` off the compiled function to build its 400 response. + expect(validate.errors).toEqual([expect.objectContaining({instancePath: '/name'})]); + }); + }); + + describe('createValidator', () => { + it('should return the value when it is valid', () => { + const event = validRawEvent(); + + expect(createValidator(PageHitRawSchema)(event)).toBe(event); + }); + + it('should throw a message naming the offending field', () => { + const validate = createValidator(PageHitRawSchema); + + expect(() => validate({...validRawEvent(), site_uuid: 'nope'})).toThrow(/site_uuid/); + }); + + it('should not coerce values the way the request validator does', () => { + // Pub/Sub messages arrive as typed JSON. Coercion here would rewrite a null + // utm_source into an empty string to satisfy the string branch of its union. + const event = validRawEvent(); + event.payload = {...event.payload, utm_source: null}; + + expect(createValidator(PageHitRawSchema)(event).payload.utm_source).toBeNull(); + }); + + it('should drop a transform silently, which is why schemas must not use them', () => { + // Guards the assumption `validation.ts` documents: JSON Schema cannot express a + // transform, so one added to a schema would stop running with no error anywhere. + const schema = z.object({n: z.string().transform(value => value.toUpperCase())}); + + expect(schema.parse({n: 'a'})).toEqual({n: 'A'}); + expect(createValidator(schema)({n: 'a'})).toEqual({n: 'a'}); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index ea5ceeb5..f9ec315e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -289,11 +289,6 @@ toad-cache "^3.7.0" undici "^7.0.0" -"@fastify/type-provider-typebox@5.2.0": - version "5.2.0" - resolved "https://registry.yarnpkg.com/@fastify/type-provider-typebox/-/type-provider-typebox-5.2.0.tgz#d04b7ce431aa217cf085e3aabc3f051c764acc01" - integrity sha512-RoUFTQNYlaVM/gXosFqlrUAD/JHC+OXLcj4DxNoMOag2GI7OydfCt+3vdT+6D2daJwhGAdkpxB0wLNqS7gf4CQ== - "@glimmer/interfaces@0.94.6": version "0.94.6" resolved "https://registry.yarnpkg.com/@glimmer/interfaces/-/interfaces-0.94.6.tgz#a4a2877730f37587326cab361de81cc0da71a823" @@ -1101,11 +1096,6 @@ resolved "https://registry.yarnpkg.com/@simple-dom/interface/-/interface-1.4.0.tgz#e8feea579232017f89b0138e2726facda6fbb71f" integrity sha512-l5qumKFWU0S+4ZzMaLXFU8tQZsicHEMEyAxI5kDFGhJsRqDwe0a7/iPA/GdxlGyDKseQQAgIz5kzU7eXTrlSpA== -"@sinclair/typebox@0.34.52": - version "0.34.52" - resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.34.52.tgz#62f8a686e4ab28a8944902e2ad2d648312ef11cb" - integrity sha512-XiMQh7qqVlxZzcVD+kkGMNGMzcTrDMLWI7S4x7z1MkCkbDPrekpZXEUK0eZqZFMuHQg2a2DZOcDIh9o5v3Gonw== - "@standard-schema/spec@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@standard-schema/spec/-/spec-1.1.0.tgz#a79b55dbaf8604812f52d140b2c9ab41bc150bb8" @@ -1126,22 +1116,6 @@ resolved "https://registry.yarnpkg.com/@tryghost/referrer-parser/-/referrer-parser-0.1.21.tgz#516ff8cbf4b10093ff95e9c3c7fe1553df1c533b" integrity sha512-Uz7bj8IadLah1j9GgieGQ0VGytdOG6mIlYdbpNu1r8/o9pjR2IS4bt88tvwpGSPNb47P9NrGntGn+Le1W/90jg== -"@tryghost/tpl@2.3.6": - version "2.3.6" - resolved "https://registry.yarnpkg.com/@tryghost/tpl/-/tpl-2.3.6.tgz#c417dce9cf7215f3545337b98a0c305a718797cd" - integrity sha512-EcSqPN0Dl6BBKANfhONDFOE3iyiyTdumB1+tyq8bnkJEAItkzmigSeAwDLocEjnPaVCHLwhPSZcgmC+dpMyuOw== - -"@tryghost/validator@3.2.6": - version "3.2.6" - resolved "https://registry.yarnpkg.com/@tryghost/validator/-/validator-3.2.6.tgz#a0ad39b25d896141a1fd335c584a87d63c2f7b0a" - integrity sha512-Pn+aFS/NybjxRghNzN+6Cg/BjnKuDf7Hv9awc5r1ULlgiUap1jFwVo+vC4FlLrBpW1sU2GGcWxINic62BCN76A== - dependencies: - "@tryghost/errors" "3.3.6" - "@tryghost/tpl" "2.3.6" - lodash "4.18.1" - moment-timezone "^0.5.23" - validator "13.15.35" - "@types/caseless@*": version "0.12.5" resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.5.tgz#db9468cb1b1b5a925b8f34822f1669df0c5472f5" @@ -1458,13 +1432,23 @@ agent-base@^7.1.0, agent-base@^7.1.2: resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.4.tgz#e3cd76d4c548ee895d3c3fd8dc1f6c5b9032e7a8" integrity sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ== -ajv-formats@^3.0.1: +ajv-formats@3.0.1, ajv-formats@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-3.0.1.tgz#3d5dc762bca17679c3c2ea7e90ad6b7532309578" integrity sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ== dependencies: ajv "^8.0.0" +ajv@8.20.0, ajv@^8.0.0, ajv@^8.12.0: + version "8.20.0" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.20.0.tgz#304b3636add88ba7d936760dd50ece006dea95f9" + integrity sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA== + dependencies: + fast-deep-equal "^3.1.3" + fast-uri "^3.0.1" + json-schema-traverse "^1.0.0" + require-from-string "^2.0.2" + ajv@^6.12.4: version "6.15.0" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.15.0.tgz#07e982c74626167aa7a2495c53817892d7139492" @@ -1475,16 +1459,6 @@ ajv@^6.12.4: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ajv@^8.0.0, ajv@^8.12.0: - version "8.20.0" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.20.0.tgz#304b3636add88ba7d936760dd50ece006dea95f9" - integrity sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA== - dependencies: - fast-deep-equal "^3.1.3" - fast-uri "^3.0.1" - json-schema-traverse "^1.0.0" - require-from-string "^2.0.2" - ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" @@ -2267,7 +2241,7 @@ fast-json-stable-stringify@^2.0.0: resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== -fast-json-stringify@^7.0.0: +fast-json-stringify@7.0.1, fast-json-stringify@^7.0.0: version "7.0.1" resolved "https://registry.yarnpkg.com/fast-json-stringify/-/fast-json-stringify-7.0.1.tgz#e528a11a6196fa54edd7e51369c30764e0839d13" integrity sha512-eRSayARSbbwlBjpP4vnTTIRD5QPcIrmihPxDeN1DtKnHPg66UuJLx+8hlK1kaFdjvzyQ/dzALoi4vwAQ+T+iZA== @@ -3193,7 +3167,7 @@ lodash.upperfirst@4.3.1: resolved "https://registry.yarnpkg.com/lodash.upperfirst/-/lodash.upperfirst-4.3.1.tgz#1365edf431480481ef0d1c68957a5ed99d49f7ce" integrity sha512-sReKOYJIJf74dhJONhU4e0/shzi1trVbSWDOhKYE5XV2O+H7Sb2Dihwuc7xWxVl+DgFPyTqIN3zMfT9cq5iWDg== -lodash@4.18.1, lodash@^4.17.21: +lodash@^4.17.21: version "4.18.1" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.18.1.tgz#ff2b66c1f6326d59513de2407bf881439812771c" integrity sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q== @@ -3316,18 +3290,6 @@ module-details-from-path@^1.0.3, module-details-from-path@^1.0.4: resolved "https://registry.yarnpkg.com/module-details-from-path/-/module-details-from-path-1.0.4.tgz#b662fdcd93f6c83d3f25289da0ce81c8d9685b94" integrity sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w== -moment-timezone@^0.5.23: - version "0.5.48" - resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.48.tgz#111727bb274734a518ae154b5ca589283f058967" - integrity sha512-f22b8LV1gbTO2ms2j2z13MuPogNoh5UzxL3nzNAYKGraILnbGc9NEE6dyiiiLv46DGRb8A4kg8UKWLjPthxBHw== - dependencies: - moment "^2.29.4" - -moment@^2.29.4: - version "2.30.1" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.30.1.tgz#f8c91c07b7a786e30c59926df530b4eac96974ae" - integrity sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how== - ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" @@ -4346,11 +4308,6 @@ validate-npm-package-license@^3.0.1: spdx-correct "^3.0.0" spdx-expression-parse "^3.0.0" -validator@13.15.35: - version "13.15.35" - resolved "https://registry.yarnpkg.com/validator/-/validator-13.15.35.tgz#81cf455c51f15b69d8d340be5914f3fab00dbf7f" - integrity sha512-TQ5pAGhd5whStmqWvYF4OjQROlmv9SMFVt37qoCBdqRffuuklWYQlCNnEs2ZaIBD1kZRNnikiZOS1eqgkar0iw== - vite@8.2.1, "vite@^6.0.0 || ^7.0.0 || ^8.0.0": version "8.2.1" resolved "https://registry.yarnpkg.com/vite/-/vite-8.2.1.tgz#6fc8d8bb843bd52353091fac978e194d4de5b31d" @@ -4492,3 +4449,8 @@ yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== + +zod@4.4.3: + version "4.4.3" + resolved "https://registry.yarnpkg.com/zod/-/zod-4.4.3.tgz#b680f172885d18bbebf21a834ea25e55a1bbf356" + integrity sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==