Merge pull request #162 from guardian/aap-world

Add AAP World bucket
guardian · Feb 21, 2025 · 6847b26 · 6847b26
2 parents dcb6e59 + 78176b8
commit 6847b26
Show file tree

Hide file tree

Showing 5 changed files with 667 additions and 8 deletions.
diff --git a/ingestion-lambda/src/categoryCodes.test.ts b/ingestion-lambda/src/categoryCodes.test.ts
@@ -1,4 +1,4 @@
-import { processFingerpostAPCategoryCodes } from './categoryCodes';
+import {processFingerpostAAPCategoryCodes, processFingerpostAPCategoryCodes} from './categoryCodes';
 
 describe('processFingerpostAPCategoryCodes', () => {
 	it('should return an empty array if provided with an empty array', () => {
@@ -55,3 +55,38 @@ describe('processFingerpostAPCategoryCodes', () => {
 		).toEqual(['apCat:a', 'apCat:c']);
 	});
 });
+
+describe('processFingerpostAAPCategoryCodes', () => {
+	it('should return an empty array if provided with an empty array', () => {
+		expect(processFingerpostAAPCategoryCodes([])).toEqual([]);
+	});
+
+	it('should strip out non-valid news codes', () => {
+		expect(processFingerpostAAPCategoryCodes(['and'])).toEqual([]);
+	});
+
+	it('should process valid news codes', () => {
+		expect(
+			processFingerpostAAPCategoryCodes([
+				'04007003+food',
+				'goods|04013002+food',
+				'and',
+				'medtop:20000049'
+			]),
+		).toEqual(['medtop:20000049', 'subj:04007003', 'subj:04013002']);
+	});
+
+	it('should flatten embedded news codes', () => {
+		expect(
+			processFingerpostAAPCategoryCodes([
+				'11001000+defence|11001005+military',
+				'equipment|11002000+diplomacy|04015001+air',
+			]),
+		).toEqual([
+			'subj:11001000',
+			'subj:11001005',
+			'subj:11002000',
+			'subj:04015001',
+		]);
+	});
+});
diff --git a/ingestion-lambda/src/categoryCodes.ts b/ingestion-lambda/src/categoryCodes.ts
@@ -39,3 +39,20 @@ export function processFingerpostAPCategoryCodes(original: string[]): string[] {
 	const deduped = [...new Set(allCategoryCodes)];
 	return deduped;
 }
+
+export function processFingerpostAAPCategoryCodes(categoryCodes: string[]): string[] {
+	const allCategoryCodes = categoryCodes
+		.flatMap((categoryCode) => categoryCode.split('|'))
+
+	const mediaTopics = allCategoryCodes
+		.filter((_) => _.split(':').length > 1)
+
+	const legacySubjectCodes = allCategoryCodes
+		.filter((_) => _.split('+').length > 1)
+		.map((categoryCode) => {
+			const [ code, _label ] = categoryCode.split('+');
+			return `subj:${code}`
+		});
+
+	return [...mediaTopics, ...legacySubjectCodes]
+}
diff --git a/ingestion-lambda/src/handler.ts b/ingestion-lambda/src/handler.ts
@@ -8,7 +8,7 @@ import { createLogger } from '../../shared/lambda-logging';
 import { createDbConnection } from '../../shared/rds';
 import type { IngestorInputBody } from '../../shared/types';
 import { IngestorInputBodySchema } from '../../shared/types';
-import { processFingerpostAPCategoryCodes } from './categoryCodes';
+import {processFingerpostAAPCategoryCodes, processFingerpostAPCategoryCodes} from './categoryCodes';
 import { tableName } from './database';
 import { BUCKET_NAME, s3Client } from './s3';
 import { lookupSupplier } from './suppliers';
@@ -52,6 +52,21 @@ export const processKeywords = (
 	return cleanAndDedupeKeywords(keywords.split('+'));
 };
 
+const processCategoryCodes = (supplier: string | undefined, subjectCodes: string[]) => {
+	switch (supplier) {
+		case 'AP':
+			return processFingerpostAPCategoryCodes(
+				subjectCodes,
+			);
+		case 'AAP':
+			return processFingerpostAAPCategoryCodes(
+				subjectCodes,
+			);
+		default:
+			return [];
+	}
+}
+
 const safeBodyParse = (body: string): IngestorInputBody => {
 	try {
 		const json = JSON.parse(body) as Record<string, unknown>;
@@ -139,12 +154,7 @@ export const main = async (event: SQSEvent): Promise<SQSBatchResponse> => {
 
 						const supplier = lookupSupplier(snsMessageContent['source-feed']);
 
-						const categoryCodes =
-							supplier === 'AP'
-								? processFingerpostAPCategoryCodes(
-										snsMessageContent.subjects?.code ?? [],
-									)
-								: [];
+						const categoryCodes = processCategoryCodes(supplier, snsMessageContent.subjects?.code ?? [])
 
 						const result = await sql`
                             INSERT INTO ${sql(tableName)}