From 60f40a3816fc0ddb8da7b55b849019c0f03754d5 Mon Sep 17 00:00:00 2001 From: Pete F Date: Tue, 17 Feb 2026 17:15:18 +0000 Subject: [PATCH 1/2] Refactor processCategoryCodes params There are a lot of params with the same type, including a few optional params. At a certain point this becomes an antipattern as it's easy to mix up the order of params inadvertently with little chance of the type checker picking up on the issue. --- ingestion-lambda/src/categoryCodes.test.ts | 11 ++++-- ingestion-lambda/src/processContentObject.ts | 39 ++++++++++++-------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/ingestion-lambda/src/categoryCodes.test.ts b/ingestion-lambda/src/categoryCodes.test.ts index 8a2e27901..07afacdfa 100644 --- a/ingestion-lambda/src/categoryCodes.test.ts +++ b/ingestion-lambda/src/categoryCodes.test.ts @@ -352,9 +352,14 @@ describe('processCategoryCodes', () => { it('should filter out empty category codes', () => { const content = 'US and EU leaders meet in Paris to discuss international trade agreements.'; - expect(processCategoryCodes('MINOR_AGENCIES', [''], [], content)).toEqual( - [], - ); + expect( + processCategoryCodes({ + supplier: 'MINOR_AGENCIES', + subjectCodes: [''], + destinationCodes: [], + bodyText: content, + }), + ).toEqual([]); }); }); diff --git a/ingestion-lambda/src/processContentObject.ts b/ingestion-lambda/src/processContentObject.ts index 42a4a4751..8e6a12628 100644 --- a/ingestion-lambda/src/processContentObject.ts +++ b/ingestion-lambda/src/processContentObject.ts @@ -62,14 +62,21 @@ export const processKeywords = ( return cleanAndDedupeKeywords(keywords.split('+')); }; -export const processCategoryCodes = ( - supplier: string, - subjectCodes: string[], - destinationCodes: string[], - bodyText?: string, - priority?: string, - mediaCatCodes?: string, -) => { +export function processCategoryCodes({ + supplier, + subjectCodes, + destinationCodes, + bodyText, + priority, + mediaCatCodes, +}: { + supplier: string; + subjectCodes: string[]; + destinationCodes: string[]; + bodyText?: string; + priority?: string; + mediaCatCodes?: string; +}) { const catCodes: string[] = priority === '1' ? ['HIGH_PRIORITY'] : []; const regionCodes = inferGeographicalCategoriesFromText(bodyText); @@ -127,7 +134,7 @@ export const processCategoryCodes = ( ...regionCodes, ]; } -}; +} export const decodeBodyTextContent = ( text: string | undefined, @@ -221,14 +228,14 @@ export function processFingerpostJsonContent( const supplier = lookupSupplier(content['source-feed']) ?? 'Unknown'; const categoryCodes = dedupeStrings( - processCategoryCodes( + processCategoryCodes({ supplier, - content.subjects?.code ?? [], - content.destinations?.code ?? [], - `${content.headline ?? ''} ${content.abstract ?? ''} ${content.body_text}`, - content.priority, - content.mediaCatCodes, - ), + subjectCodes: content.subjects?.code ?? [], + destinationCodes: content.destinations?.code ?? [], + bodyText: `${content.headline ?? ''} ${content.abstract ?? ''} ${content.body_text}`, + priority: content.priority, + mediaCatCodes: content.mediaCatCodes, + }), ); return { status: 'success' as const, From 44233546365912f1f5b7195d46b7be6143af69db Mon Sep 17 00:00:00 2001 From: Pete F Date: Tue, 17 Feb 2026 17:42:48 +0000 Subject: [PATCH 2/2] Add agencyMetaData and extract media topics from it for PA API --- ingestion-lambda/src/categoryCodes.ts | 12 +- .../src/processContentObject.test.ts | 2 + ingestion-lambda/src/processContentObject.ts | 10 +- ingestion-lambda/test/fixtures/PA_API.json | 115 +++++++++++++++++- shared/src/types.ts | 17 +++ 5 files changed, 152 insertions(+), 4 deletions(-) diff --git a/ingestion-lambda/src/categoryCodes.ts b/ingestion-lambda/src/categoryCodes.ts index fc4340d24..21d9aca21 100644 --- a/ingestion-lambda/src/categoryCodes.ts +++ b/ingestion-lambda/src/categoryCodes.ts @@ -1,4 +1,5 @@ import nlp from 'compromise'; +import type { AgencyMetadata } from 'newswires-shared/types'; import { worldTopicCodes } from '../topicCodes'; import { alpha2CountriesMap, @@ -156,12 +157,19 @@ export function processFingerpostPACategoryCodes(original: string[]) { export function processFingerpostPAAPICategoryCodes( original: string[], mediaCatCodes?: string, + agencyMetadata?: AgencyMetadata, ) { + const mediaTopicCodes = (agencyMetadata?.subject ?? []) + .filter((_) => _.code.startsWith('medtop:')) + .map((_) => _.code); + const originalWithMediaTopics = [...original, ...mediaTopicCodes]; + if (mediaCatCodes) { - return [...original, `paCat:${mediaCatCodes}`]; + return [...originalWithMediaTopics, `paCat:${mediaCatCodes}`]; } - return [...original]; + return [...originalWithMediaTopics]; } + export function processUnknownFingerpostCategoryCodes( original: string[], supplier: string, diff --git a/ingestion-lambda/src/processContentObject.test.ts b/ingestion-lambda/src/processContentObject.test.ts index 7056f40a1..cbcc01a77 100644 --- a/ingestion-lambda/src/processContentObject.test.ts +++ b/ingestion-lambda/src/processContentObject.test.ts @@ -180,6 +180,8 @@ describe('processFingerpostJsonContent', () => { 'news:uk', 'politics', 'news:scotland', + 'medtop:11000000', + 'medtop:04000000', 'paCat:SCN', ], }); diff --git a/ingestion-lambda/src/processContentObject.ts b/ingestion-lambda/src/processContentObject.ts index 8e6a12628..b06a3e466 100644 --- a/ingestion-lambda/src/processContentObject.ts +++ b/ingestion-lambda/src/processContentObject.ts @@ -1,5 +1,6 @@ import { getErrorMessage } from '@guardian/libs'; import type { + AgencyMetadata, IngestorInputBody, OperationResult, ProcessedObject, @@ -69,6 +70,7 @@ export function processCategoryCodes({ bodyText, priority, mediaCatCodes, + agencyMetadata, }: { supplier: string; subjectCodes: string[]; @@ -76,6 +78,7 @@ export function processCategoryCodes({ bodyText?: string; priority?: string; mediaCatCodes?: string; + agencyMetadata?: AgencyMetadata; }) { const catCodes: string[] = priority === '1' ? ['HIGH_PRIORITY'] : []; const regionCodes = inferGeographicalCategoriesFromText(bodyText); @@ -116,7 +119,11 @@ export function processCategoryCodes({ case 'PAAPI': return [ ...catCodes, - ...processFingerpostPAAPICategoryCodes(subjectCodes, mediaCatCodes), + ...processFingerpostPAAPICategoryCodes( + subjectCodes, + mediaCatCodes, + agencyMetadata, + ), ]; case 'MINOR_AGENCIES': { const updatedSubjectCodes = [ @@ -235,6 +242,7 @@ export function processFingerpostJsonContent( bodyText: `${content.headline ?? ''} ${content.abstract ?? ''} ${content.body_text}`, priority: content.priority, mediaCatCodes: content.mediaCatCodes, + agencyMetadata: content.agencyMetadata, }), ); return { diff --git a/ingestion-lambda/test/fixtures/PA_API.json b/ingestion-lambda/test/fixtures/PA_API.json index 3198dcf24..ef8a5dd69 100644 --- a/ingestion-lambda/test/fixtures/PA_API.json +++ b/ingestion-lambda/test/fixtures/PA_API.json @@ -30,5 +30,118 @@ "usage": "", "location": "", "abstract": "POL", - "body_text": "Anas Sarwar has said..." + "body_text": "Anas Sarwar has said...", + "agencyMetadata": { + "subject": [ + { + "code": "paservice:news", + "name": "News", + "profile": "paservice", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "partOf" + }, + { + "code": "paservice:news:uk", + "name": "UK", + "profile": "paservice", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "partOf" + }, + { + "code": "paservice:politics", + "name": "Politics", + "profile": "paservice", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "partOf" + }, + { + "code": "paservice:esg", + "name": "ESG", + "profile": "paservice", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "partOf" + }, + { + "code": "paservice:esg:social", + "name": "Social", + "profile": "paservice", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "partOf" + }, + { + "code": "patopic:commons", + "name": "Commons", + "profile": "patopic", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "about" + }, + { + "code": "pakeyword:lgbt", + "name": "LGBT", + "profile": "pakeyword", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "about" + }, + { + "code": "paterritory:uk", + "name": "UK", + "profile": "paterritory", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "intendedFor" + }, + { + "code": "legislature:westminster", + "name": "Westminster", + "profile": "legislature", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "about" + }, + { + "code": "legislature:westminster:commons", + "name": "Commons", + "profile": "legislature", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "about" + }, + { + "code": "contributor:pa", + "name": "PA", + "profile": "contributor", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "contributedBy" + }, + { + "code": "medtop:11000000", + "name": "politics", + "profile": "medtop", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "about" + }, + { + "code": "medtop:04000000", + "name": "economy, business and finance", + "profile": "medtop", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "about" + } + ], + "event": [ + { + "code": "paevent:3ff7f1381d0288aebb74f792ea1dfb145d4ca8fb9e5042513d60118a9c4a64d9", + "name": "Commons LGBT 12/02/2026", + "profile": "paevent", + "scheme": "https://content.api.pressassociation.io/v1/event", + "rel": "partOf" + } + ], + "place": [ + { + "code": "iso:code:3166:GB", + "name": "United Kingdom", + "profile": "isocountry", + "scheme": "https://www.iso.org", + "rel": "about" + } + ] + } } diff --git a/shared/src/types.ts b/shared/src/types.ts index cf1ba7d2a..2244c5b3c 100644 --- a/shared/src/types.ts +++ b/shared/src/types.ts @@ -13,6 +13,22 @@ const OptionalStringOrArrayAsArrayOfStrings = z return [val]; }); +const AgencyMetadataItemSchema = z.object({ + code: z.string(), + profile: z.string(), + name: z.string(), + scheme: z.string(), + rel: z.string(), +}); + +const AgencyMetadataSchema = z.object({ + subject: z.array(AgencyMetadataItemSchema).optional(), + event: z.array(AgencyMetadataItemSchema).optional(), + place: z.array(AgencyMetadataItemSchema).optional(), +}); + +export type AgencyMetadata = z.infer; + /** * looseObject because we want to preserve additional properties that are not defined in the schema * Useful to be able to test new fields @@ -61,6 +77,7 @@ const FingerpostFeedPayloadSchema = z.looseObject({ body_text: z.string().optional(), copyrightHolder: z.string().optional(), copyrightNotice: z.string().optional(), + agencyMetadata: AgencyMetadataSchema.optional(), // only expecting this for PA API via Fingerpost currently }); export const IngestorInputBodySchema = FingerpostFeedPayloadSchema.extend({