diff --git a/ingestion-lambda/src/categoryCodes.test.ts b/ingestion-lambda/src/categoryCodes.test.ts index 8a2e27901..07afacdfa 100644 --- a/ingestion-lambda/src/categoryCodes.test.ts +++ b/ingestion-lambda/src/categoryCodes.test.ts @@ -352,9 +352,14 @@ describe('processCategoryCodes', () => { it('should filter out empty category codes', () => { const content = 'US and EU leaders meet in Paris to discuss international trade agreements.'; - expect(processCategoryCodes('MINOR_AGENCIES', [''], [], content)).toEqual( - [], - ); + expect( + processCategoryCodes({ + supplier: 'MINOR_AGENCIES', + subjectCodes: [''], + destinationCodes: [], + bodyText: content, + }), + ).toEqual([]); }); }); diff --git a/ingestion-lambda/src/categoryCodes.ts b/ingestion-lambda/src/categoryCodes.ts index fc4340d24..21d9aca21 100644 --- a/ingestion-lambda/src/categoryCodes.ts +++ b/ingestion-lambda/src/categoryCodes.ts @@ -1,4 +1,5 @@ import nlp from 'compromise'; +import type { AgencyMetadata } from 'newswires-shared/types'; import { worldTopicCodes } from '../topicCodes'; import { alpha2CountriesMap, @@ -156,12 +157,19 @@ export function processFingerpostPACategoryCodes(original: string[]) { export function processFingerpostPAAPICategoryCodes( original: string[], mediaCatCodes?: string, + agencyMetadata?: AgencyMetadata, ) { + const mediaTopicCodes = (agencyMetadata?.subject ?? []) + .filter((_) => _.code.startsWith('medtop:')) + .map((_) => _.code); + const originalWithMediaTopics = [...original, ...mediaTopicCodes]; + if (mediaCatCodes) { - return [...original, `paCat:${mediaCatCodes}`]; + return [...originalWithMediaTopics, `paCat:${mediaCatCodes}`]; } - return [...original]; + return [...originalWithMediaTopics]; } + export function processUnknownFingerpostCategoryCodes( original: string[], supplier: string, diff --git a/ingestion-lambda/src/processContentObject.test.ts b/ingestion-lambda/src/processContentObject.test.ts index 7056f40a1..cbcc01a77 100644 --- a/ingestion-lambda/src/processContentObject.test.ts +++ b/ingestion-lambda/src/processContentObject.test.ts @@ -180,6 +180,8 @@ describe('processFingerpostJsonContent', () => { 'news:uk', 'politics', 'news:scotland', + 'medtop:11000000', + 'medtop:04000000', 'paCat:SCN', ], }); diff --git a/ingestion-lambda/src/processContentObject.ts b/ingestion-lambda/src/processContentObject.ts index 42a4a4751..b06a3e466 100644 --- a/ingestion-lambda/src/processContentObject.ts +++ b/ingestion-lambda/src/processContentObject.ts @@ -1,5 +1,6 @@ import { getErrorMessage } from '@guardian/libs'; import type { + AgencyMetadata, IngestorInputBody, OperationResult, ProcessedObject, @@ -62,14 +63,23 @@ export const processKeywords = ( return cleanAndDedupeKeywords(keywords.split('+')); }; -export const processCategoryCodes = ( - supplier: string, - subjectCodes: string[], - destinationCodes: string[], - bodyText?: string, - priority?: string, - mediaCatCodes?: string, -) => { +export function processCategoryCodes({ + supplier, + subjectCodes, + destinationCodes, + bodyText, + priority, + mediaCatCodes, + agencyMetadata, +}: { + supplier: string; + subjectCodes: string[]; + destinationCodes: string[]; + bodyText?: string; + priority?: string; + mediaCatCodes?: string; + agencyMetadata?: AgencyMetadata; +}) { const catCodes: string[] = priority === '1' ? ['HIGH_PRIORITY'] : []; const regionCodes = inferGeographicalCategoriesFromText(bodyText); @@ -109,7 +119,11 @@ export const processCategoryCodes = ( case 'PAAPI': return [ ...catCodes, - ...processFingerpostPAAPICategoryCodes(subjectCodes, mediaCatCodes), + ...processFingerpostPAAPICategoryCodes( + subjectCodes, + mediaCatCodes, + agencyMetadata, + ), ]; case 'MINOR_AGENCIES': { const updatedSubjectCodes = [ @@ -127,7 +141,7 @@ export const processCategoryCodes = ( ...regionCodes, ]; } -}; +} export const decodeBodyTextContent = ( text: string | undefined, @@ -221,14 +235,15 @@ export function processFingerpostJsonContent( const supplier = lookupSupplier(content['source-feed']) ?? 'Unknown'; const categoryCodes = dedupeStrings( - processCategoryCodes( + processCategoryCodes({ supplier, - content.subjects?.code ?? [], - content.destinations?.code ?? [], - `${content.headline ?? ''} ${content.abstract ?? ''} ${content.body_text}`, - content.priority, - content.mediaCatCodes, - ), + subjectCodes: content.subjects?.code ?? [], + destinationCodes: content.destinations?.code ?? [], + bodyText: `${content.headline ?? ''} ${content.abstract ?? ''} ${content.body_text}`, + priority: content.priority, + mediaCatCodes: content.mediaCatCodes, + agencyMetadata: content.agencyMetadata, + }), ); return { status: 'success' as const, diff --git a/ingestion-lambda/test/fixtures/PA_API.json b/ingestion-lambda/test/fixtures/PA_API.json index 3198dcf24..ef8a5dd69 100644 --- a/ingestion-lambda/test/fixtures/PA_API.json +++ b/ingestion-lambda/test/fixtures/PA_API.json @@ -30,5 +30,118 @@ "usage": "", "location": "", "abstract": "POL", - "body_text": "Anas Sarwar has said..." + "body_text": "Anas Sarwar has said...", + "agencyMetadata": { + "subject": [ + { + "code": "paservice:news", + "name": "News", + "profile": "paservice", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "partOf" + }, + { + "code": "paservice:news:uk", + "name": "UK", + "profile": "paservice", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "partOf" + }, + { + "code": "paservice:politics", + "name": "Politics", + "profile": "paservice", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "partOf" + }, + { + "code": "paservice:esg", + "name": "ESG", + "profile": "paservice", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "partOf" + }, + { + "code": "paservice:esg:social", + "name": "Social", + "profile": "paservice", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "partOf" + }, + { + "code": "patopic:commons", + "name": "Commons", + "profile": "patopic", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "about" + }, + { + "code": "pakeyword:lgbt", + "name": "LGBT", + "profile": "pakeyword", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "about" + }, + { + "code": "paterritory:uk", + "name": "UK", + "profile": "paterritory", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "intendedFor" + }, + { + "code": "legislature:westminster", + "name": "Westminster", + "profile": "legislature", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "about" + }, + { + "code": "legislature:westminster:commons", + "name": "Commons", + "profile": "legislature", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "about" + }, + { + "code": "contributor:pa", + "name": "PA", + "profile": "contributor", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "contributedBy" + }, + { + "code": "medtop:11000000", + "name": "politics", + "profile": "medtop", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "about" + }, + { + "code": "medtop:04000000", + "name": "economy, business and finance", + "profile": "medtop", + "scheme": "https://content.api.pressassociation.io/v1/subject", + "rel": "about" + } + ], + "event": [ + { + "code": "paevent:3ff7f1381d0288aebb74f792ea1dfb145d4ca8fb9e5042513d60118a9c4a64d9", + "name": "Commons LGBT 12/02/2026", + "profile": "paevent", + "scheme": "https://content.api.pressassociation.io/v1/event", + "rel": "partOf" + } + ], + "place": [ + { + "code": "iso:code:3166:GB", + "name": "United Kingdom", + "profile": "isocountry", + "scheme": "https://www.iso.org", + "rel": "about" + } + ] + } } diff --git a/shared/src/types.ts b/shared/src/types.ts index cf1ba7d2a..2244c5b3c 100644 --- a/shared/src/types.ts +++ b/shared/src/types.ts @@ -13,6 +13,22 @@ const OptionalStringOrArrayAsArrayOfStrings = z return [val]; }); +const AgencyMetadataItemSchema = z.object({ + code: z.string(), + profile: z.string(), + name: z.string(), + scheme: z.string(), + rel: z.string(), +}); + +const AgencyMetadataSchema = z.object({ + subject: z.array(AgencyMetadataItemSchema).optional(), + event: z.array(AgencyMetadataItemSchema).optional(), + place: z.array(AgencyMetadataItemSchema).optional(), +}); + +export type AgencyMetadata = z.infer; + /** * looseObject because we want to preserve additional properties that are not defined in the schema * Useful to be able to test new fields @@ -61,6 +77,7 @@ const FingerpostFeedPayloadSchema = z.looseObject({ body_text: z.string().optional(), copyrightHolder: z.string().optional(), copyrightNotice: z.string().optional(), + agencyMetadata: AgencyMetadataSchema.optional(), // only expecting this for PA API via Fingerpost currently }); export const IngestorInputBodySchema = FingerpostFeedPayloadSchema.extend({