From 67660db2bbf2642621b5ad6546263b552c16919b Mon Sep 17 00:00:00 2001 From: Mark Ng'ang'a Date: Mon, 10 Aug 2026 13:07:31 +0300 Subject: [PATCH 1/5] Ghost ingest: reformat parser and tests with black and sorted imports Pure formatting ahead of the language-detection work, so that the functional diff which follows is readable rather than buried in reflowed decorators and call sites. black at its default line length (88) plus import sorting; no behaviour change. Verified by comparing the AST of each module before and after, ignoring import order. --- server/pesacheck/ingest/ghost_parser.py | 46 +++++-- server/tests/ingest/test_ghost_parser.py | 155 +++++++++++++++++------ 2 files changed, 150 insertions(+), 51 deletions(-) diff --git a/server/pesacheck/ingest/ghost_parser.py b/server/pesacheck/ingest/ghost_parser.py index 17ad01e93..3f7c78152 100644 --- a/server/pesacheck/ingest/ghost_parser.py +++ b/server/pesacheck/ingest/ghost_parser.py @@ -1,24 +1,29 @@ -import json import hashlib +import json import logging import random import time import unicodedata from copy import deepcopy -from urllib.parse import urlparse - from datetime import datetime, timezone +from urllib.parse import urlparse from superdesk.errors import ParserError from superdesk.etree import parse_html -from superdesk.metadata.utils import generate_guid from superdesk.io.feed_parsers import FileFeedParser from superdesk.io.registry import register_feed_parser from superdesk.media.renditions import update_renditions -from superdesk.metadata.item import FORMAT, GUID_FIELD, GUID_TAG, ITEM_TYPE, CONTENT_TYPE, FORMATS +from superdesk.metadata.item import ( + CONTENT_TYPE, + FORMAT, + FORMATS, + GUID_FIELD, + GUID_TAG, + ITEM_TYPE, +) +from superdesk.metadata.utils import generate_guid from superdesk.utc import utcnow - logger = logging.getLogger(__name__) # Ghost exports replace the site URL with this portable placeholder in @@ -203,7 +208,9 @@ def _fetch_renditions_with_retry(self, association, url): if attempt == policy["retries"]: break - delay = (policy["base_backoff"] * attempt) + random.uniform(0, _IMAGE_FETCH_JITTER_SECONDS) + delay = (policy["base_backoff"] * attempt) + random.uniform( + 0, _IMAGE_FETCH_JITTER_SECONDS + ) logger.warning( "Image fetch failed for %s (attempt %s/%s), retrying in %.2fs: %s", url, @@ -216,11 +223,16 @@ def _fetch_renditions_with_retry(self, association, url): if policy["failure_cooldown"] > 0: # After exhausting retries, cool down before the next image to reduce cascading failures. - time.sleep(policy["failure_cooldown"] + random.uniform(0, _IMAGE_FETCH_JITTER_SECONDS)) + time.sleep( + policy["failure_cooldown"] + + random.uniform(0, _IMAGE_FETCH_JITTER_SECONDS) + ) raise last_error - def _add_image(self, item, url, alt_text="", description_text="", is_featured=False): + def _add_image( + self, item, url, alt_text="", description_text="", is_featured=False + ): """Fetch image, attach it as an association, and return the local storage href (or None).""" associations = item.setdefault("associations", {}) association = { @@ -283,7 +295,9 @@ def _parse_inline_images(self, item, html): if local_href: url_rewrites[src] = local_href except Exception as e: - logger.warning("Failed to parse inline image %s: %s", img.get("src", "unknown"), e) + logger.warning( + "Failed to parse inline image %s: %s", img.get("src", "unknown"), e + ) if url_rewrites: body = item.get("body_html") or "" @@ -312,14 +326,18 @@ def _parse_date(self, value): def _parse_post(self, post, authors_by_post, tags_by_post): post_id = post.get("id", "") - authors = sorted(authors_by_post.get(post_id, []), key=lambda x: x["sort_order"]) + authors = sorted( + authors_by_post.get(post_id, []), key=lambda x: x["sort_order"] + ) byline = ", ".join(a["name"] for a in authors if a.get("name")) tags = sorted(tags_by_post.get(post_id, []), key=lambda x: x["sort_order"]) keywords = [t["name"] for t in tags if t.get("name")] firstcreated = self._parse_date(post.get("created_at")) - versioncreated = self._parse_date(post.get("published_at") or post.get("updated_at")) + versioncreated = self._parse_date( + post.get("published_at") or post.get("updated_at") + ) html = post.get("html") or "" if self._ghost_url and html: @@ -361,7 +379,9 @@ def iter_items(self, file_path, provider=None): """Parse a Ghost JSON export file and yield Superdesk items one at a time.""" self._image_assoc_cache = {} self._last_image_fetch_ts = 0.0 - self._ghost_url = ((provider or {}).get("config", {}).get("url") or "").rstrip("/") + self._ghost_url = ((provider or {}).get("config", {}).get("url") or "").rstrip( + "/" + ) try: with open(file_path, "r", encoding="utf-8") as f: data = json.load(f) diff --git a/server/tests/ingest/test_ghost_parser.py b/server/tests/ingest/test_ghost_parser.py index d868a5339..e72687360 100644 --- a/server/tests/ingest/test_ghost_parser.py +++ b/server/tests/ingest/test_ghost_parser.py @@ -1,13 +1,14 @@ -import os import json +import os import tempfile - from unittest.mock import patch -from superdesk.tests import TestCase -from pesacheck.ingest.ghost_parser import GhostParser +from pesacheck.ingest.ghost_parser import GhostParser +from superdesk.tests import TestCase -FIXTURE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../fixtures/ghost_export") +FIXTURE_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "../fixtures/ghost_export" +) FIXTURE_PATH = os.path.join(FIXTURE_DIR, "ghost_export.json") @@ -38,118 +39,196 @@ async def test_can_parse_rejects_missing_file(self): # parse — field mapping # ------------------------------------------------------------------ - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_returns_only_published_posts(self, _mock): items = await self.parser.parse(FIXTURE_PATH) # draft (post_003) and page (post_004) should be excluded self.assertEqual(len(items), 2) - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_headline_and_slugline(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertEqual(post1["headline"], "FAUX: This claim is false") self.assertEqual(post1["slugline"], "faux-this-claim-is-false") - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_abstract(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertEqual(post1["abstract"], "A short description of the article.") - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_body_html(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertIn("

This is the article body.

", post1["body_html"]) - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_dates(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertEqual(post1["firstcreated"].isoformat(), "2025-11-01T10:00:00+00:00") - self.assertEqual(post1["versioncreated"].isoformat(), "2025-11-01T11:00:00+00:00") - - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + self.assertEqual( + post1["versioncreated"].isoformat(), "2025-11-01T11:00:00+00:00" + ) + + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_source(self, _mock): items = await self.parser.parse(FIXTURE_PATH) for item in items: self.assertEqual(item["source"], "Ghost") - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_locale_mapped_to_language(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertEqual(post1["language"], "fr") - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_null_locale_guesses_language(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post2 = next(i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb") + post2 = next( + i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb" + ) self.assertIn("language", post2) # ------------------------------------------------------------------ # parse — authors → byline # ------------------------------------------------------------------ - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_byline_multiple_authors_sorted(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertEqual(post1["byline"], "Alice Reporter, Bob Editor") - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_byline_single_author(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post2 = next(i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb") + post2 = next( + i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb" + ) self.assertEqual(post2["byline"], "Alice Reporter") # ------------------------------------------------------------------ # parse — tags → keywords # ------------------------------------------------------------------ - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_keywords(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertEqual(post1["keywords"], ["Fact Check", "Africa"]) - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_keywords_single_tag(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post2 = next(i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb") + post2 = next( + i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb" + ) self.assertEqual(post2["keywords"], ["Fact Check"]) # ------------------------------------------------------------------ # parse — images # ------------------------------------------------------------------ - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_feature_image(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertIn("associations", post1) featuremedia = post1["associations"]["featuremedia"] self.assertEqual(featuremedia["type"], "picture") self.assertTrue(featuremedia["guid"].endswith("-image")) - self.assertEqual(featuremedia["renditions"]["original"]["href"], "https://example.com/feature.jpg") - - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + self.assertEqual( + featuremedia["renditions"]["original"]["href"], + "https://example.com/feature.jpg", + ) + + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_inline_image_added_as_embedded(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) associations = post1["associations"] embedded_keys = [k for k in associations if k.startswith("embedded")] self.assertEqual(len(embedded_keys), 1) embedded = associations[embedded_keys[0]] - self.assertEqual(embedded["renditions"]["original"]["href"], "https://example.com/inline.jpg") + self.assertEqual( + embedded["renditions"]["original"]["href"], "https://example.com/inline.jpg" + ) self.assertEqual(embedded["alt_text"], "An inline image") self.assertEqual(embedded["description_text"], "Image caption here") - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_no_feature_image_no_associations(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post2 = next(i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb") + post2 = next( + i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb" + ) self.assertNotIn("associations", post2) # ------------------------------------------------------------------ From 1277883c3d0f338047269c7b8724ab5fd91d3c51 Mon Sep 17 00:00:00 2001 From: Mark Ng'ang'a Date: Mon, 10 Aug 2026 13:07:46 +0300 Subject: [PATCH 2/5] Add a language detector for the six PesaCheck publishing languages Superdesk needs an ISO 639-1 code on every ingested item, and PesaCheck publishes in English, Kiswahili, French, Somali, Afaan Oromo and Amharic. Add a self-contained detector with two strategies: - detect_language_from_tags: recent posts carry an explicit language tag ("English", "Afaan Oromo", ...). This is editorial metadata, so it is authoritative and tried first. Matching is exact on the normalised name or slug so a language is never confused with the country tag beside it, which PesaCheck always sets: French/France, Somali/Somalia. Where two language tags appear the lowest sort_order wins, matching Ghost's primary tag. - detect_language_from_text: for posts predating that convention. Ethiopic script share identifies Amharic; the Latin-script languages are separated by calibrated function-word coverage. The classifier is deliberately closed over just these six languages rather than delegating to a general-purpose detector. langdetect has no model for Afaan Oromo and labels it Somali (both Cushitic, both using the doubled-vowel Latin orthography), and it raises outright on Amharic. Scoring only what we publish removes the failure mode where an article is lost to a seventh language that was never an option. Marker lists and the per-language coverage constants were built and measured against the 50-post reference export in ops/local/ingest/ghost. Bare "Oromo" is deliberately not accepted as a language tag: it is an ethnonym first and a plausible topic tag on an English article, and Oromo prose is recognised well by the text path, so an unmatched tag degrades safely. The module imports only the standard library. It deliberately lives in pesacheck/ rather than pesacheck/ingest/: the ingest package's __init__ eagerly imports the parsers, which pull in superdesk.io and the native libmagic dependency, so placing it there would mean this pure-stdlib logic -- and its tests -- could not be imported without the whole Superdesk stack. --- server/pesacheck/language.py | 266 +++++++++++++++ server/pesacheck/language_markers.py | 493 +++++++++++++++++++++++++++ server/tests/test_language.py | 224 ++++++++++++ 3 files changed, 983 insertions(+) create mode 100644 server/pesacheck/language.py create mode 100644 server/pesacheck/language_markers.py create mode 100644 server/tests/test_language.py diff --git a/server/pesacheck/language.py b/server/pesacheck/language.py new file mode 100644 index 000000000..6c2e1d00d --- /dev/null +++ b/server/pesacheck/language.py @@ -0,0 +1,266 @@ +"""Language identification for PesaCheck ingest. + +PesaCheck publishes in six languages, and Superdesk needs an ISO 639-1 code on +every ingested item. Two strategies are used, in order: + +1. :func:`detect_language_from_tags` — recent posts carry an explicit language + tag (``English``, ``Kiswahili``, ``Afaan Oromo``, ...). This is editorial + metadata, so it is authoritative and is always tried first. +2. :func:`detect_language_from_text` — older posts predate the tagging + convention, so the body text is classified directly. + +The text classifier is deliberately closed over just these six languages. A +general-purpose detector is a poor fit here: ``langdetect`` has no model for +Afaan Oromo at all and mislabels it as Somali (both are Cushitic and share the +doubled-vowel Latin orthography), and it raises on Amharic. Scoring only the +languages we actually publish in avoids losing articles to a seventh language +that was never an option. +""" + +import logging +import re +import unicodedata +from collections.abc import Iterable, Sequence +from typing import TypeAlias + +from .language_markers import ( + AMHARIC, + ENGLISH, + FRENCH, + KISWAHILI, + MARKERS, + OROMO, + SOMALI, + LanguageCode, +) + +logger = logging.getLogger(__name__) + + +# A ``(sort_order, label)`` pair as Ghost exports it, where ``label`` is a tag +# display name or slug. Both come straight from JSON, so both may be null and +# neither is guaranteed to be a string. +TagLabel: TypeAlias = tuple[int | None, object] + + +# The languages PesaCheck publishes in, as ISO 639-1 codes. +SUPPORTED_LANGUAGES: tuple[LanguageCode, ...] = ( + ENGLISH, + KISWAHILI, + FRENCH, + SOMALI, + OROMO, + AMHARIC, +) + +# Used when detection is impossible (no tags, empty body). English is the +# house language, so it is the least surprising thing to fall back to. +DEFAULT_LANGUAGE: LanguageCode = ENGLISH + + +# Ghost tag names/slugs that identify a language, normalised by +# ``_normalise_tag``. Both the English name and the endonym are accepted, along +# with ISO 639-1/639-2 codes in case a tag is ever created from a code. +# +# Matching is exact on the whole normalised string so that language tags are +# never confused with the country tags they sit next to: PesaCheck tags posts +# with both ``French``/``France`` and ``Somali``/``Somalia``. +# +# Deliberately absent: bare ``Oromo``. Unlike ``Amharic`` or ``Kiswahili`` it is +# an ethnonym before it is a language name, and it is a plausible topic tag on +# an English article about Oromo people. PesaCheck's convention is the +# unambiguous ``Afaan Oromo``, and Oromo prose is recognised well by the text +# classifier, so an unmatched tag degrades safely. +_LANGUAGE_TAGS: dict[str, LanguageCode] = { + "english": ENGLISH, + "en": ENGLISH, + "eng": ENGLISH, + "kiswahili": KISWAHILI, + "swahili": KISWAHILI, + "sw": KISWAHILI, + "swa": KISWAHILI, + "french": FRENCH, + "francais": FRENCH, + "français": FRENCH, + "fr": FRENCH, + "fra": FRENCH, + "somali": SOMALI, + "soomaali": SOMALI, + "af soomaali": SOMALI, + "so": SOMALI, + "som": SOMALI, + "afaan oromo": OROMO, + "afaan oromoo": OROMO, + "afan oromo": OROMO, + "oromiffa": OROMO, + "oromifa": OROMO, + "oromoo": OROMO, + "om": OROMO, + "orm": OROMO, + "amharic": AMHARIC, + "amarigna": AMHARIC, + "amharigna": AMHARIC, + "አማርኛ": AMHARIC, + "am": AMHARIC, + "amh": AMHARIC, +} + +# Runs of Ethiopic characters, covering the block plus its supplement and +# extensions (U+1200-U+139F merges Ethiopic and Ethiopic Supplement). Amharic is +# the only Ethiopic-script language PesaCheck publishes in. +# +# Both of these count characters by matching *runs* and summing their lengths, +# which keeps the scan in C: a per-character Python loop over a multi-kilobyte +# body cost ~1.2ms per post, around 90% of the whole text-detection path. +_ETHIOPIC_RUN_RE: re.Pattern[str] = re.compile( + "[" + "ሀ-᎟" # Ethiopic + Ethiopic Supplement + "ⶀ-⷟" # Ethiopic Extended + "꬀-꬯" # Ethiopic Extended-A + "\U0001e7e0-\U0001e7ff" # Ethiopic Extended-B + "]+" +) + +# Approximates ``str.isalpha`` per character: word characters excluding digits +# and underscore. +_ALPHABETIC_RUN_RE: re.Pattern[str] = re.compile(r"[^\W\d_]+", re.UNICODE) + +# Share of cased letters that must be Ethiopic before a post is called Amharic. +# Amharic posts in the reference export are >99% Ethiopic, and Latin-script +# posts contain none, so there is a wide margin here. A *share* rather than +# "contains any Ethiopic character" matters: English and Afaan Oromo articles +# about Ethiopia routinely quote an Amharic phrase, and one quote must not +# relabel the whole article. +_ETHIOPIC_SHARE_THRESHOLD: float = 0.15 + +# Coverage a native document is expected to reach for each language, used to +# put the raw scores on a comparable scale. Without this the argmax just +# favours whichever marker list happens to cover the most running text. +_EXPECTED_COVERAGE: dict[LanguageCode, float] = { + ENGLISH: 0.35, + FRENCH: 0.36, + KISWAHILI: 0.28, + SOMALI: 0.26, + OROMO: 0.25, +} + +# Tokens are lowercased runs of letters, keeping the apostrophes that Afaan +# Oromo uses for the glottal stop ("ta'e") so those stay single tokens. +_TOKEN_RE: re.Pattern[str] = re.compile(r"[^\W\d_]+(?:['’ʼ][^\W\d_]+)*", re.UNICODE) + +# Below this many tokens the marker counts are too sparse to separate the +# candidates, so text detection declines rather than guessing. +_MIN_TOKENS: int = 8 + +# The winner must beat the runner-up by this ratio. Fact checks quote the +# claim they are debunking, so a Somali or Oromo article always contains some +# English; requiring a margin keeps those quotes from tipping the result. +_MIN_MARGIN: float = 1.15 + + +def _normalise_tag(value: object) -> str: + """Casefold a tag and collapse punctuation/whitespace to single spaces.""" + if not value: + return "" + text = unicodedata.normalize("NFC", str(value)).casefold() + return re.sub(r"[\s\-_]+", " ", text).strip() + + +def normalise_language_code(value: object) -> LanguageCode | None: + """Normalise a Ghost ``locale`` to a language code, or return ``None``. + + Accepts anything BCP 47-ish (``en``, ``en-US``, ``am_ET``) and keeps only + the base subtag. Codes outside :data:`SUPPORTED_LANGUAGES` are passed + through rather than dropped, so a locale PesaCheck starts publishing in + later still reaches Superdesk. + """ + if not value: + return None + base = _normalise_tag(re.split(r"[-_]", str(value).strip())[0]) + if not base: + return None + return _LANGUAGE_TAGS.get(base, base) + + +def detect_language_from_tags(tags: Iterable[TagLabel]) -> LanguageCode | None: + """Return the language code named by a post's tags, or ``None``. + + ``tags`` is an iterable of ``(sort_order, name)`` pairs; Ghost exposes both + a display name and a slug per tag and either may be passed. When a post + carries more than one language tag the lowest ``sort_order`` wins, matching + Ghost's notion of a primary tag — in practice PesaCheck puts the language + first, so this is also what an editor would expect. + """ + matches: list[tuple[int, LanguageCode]] = [] + for sort_order, name in tags: + code = _LANGUAGE_TAGS.get(_normalise_tag(name)) + if code: + matches.append((sort_order if sort_order is not None else 0, code)) + + if not matches: + return None + return min(matches, key=lambda pair: pair[0])[1] + + +def _count_matched_chars(pattern: re.Pattern[str], text: str) -> int: + return sum(len(run) for run in pattern.findall(text)) + + +def _ethiopic_share(text: str) -> float: + """Fraction of the alphabetic characters in ``text`` that are Ethiopic.""" + alphabetic = _count_matched_chars(_ALPHABETIC_RUN_RE, text) + if not alphabetic: + return 0.0 + return _count_matched_chars(_ETHIOPIC_RUN_RE, text) / alphabetic + + +def _tokenize(text: str) -> list[str]: + return _TOKEN_RE.findall(text.casefold()) + + +def _score_markers(tokens: Sequence[str]) -> dict[LanguageCode, float]: + """Return ``{language: calibrated coverage}`` for the Latin-script set.""" + total = len(tokens) + scores: dict[LanguageCode, float] = {} + for language, markers in MARKERS.items(): + hits = sum(1 for token in tokens if token in markers) + scores[language] = (hits / total) / _EXPECTED_COVERAGE[language] + return scores + + +def detect_language_from_text(text: str | None) -> LanguageCode | None: + """Classify ``text`` as one of :data:`SUPPORTED_LANGUAGES`, or ``None``. + + ``None`` means "not enough evidence" — too little text, or no candidate + clearly ahead — and lets the caller decide what to do rather than having a + coin-flip recorded as a real language. + """ + if not text or not text.strip(): + return None + + if _ethiopic_share(text) >= _ETHIOPIC_SHARE_THRESHOLD: + return AMHARIC + + tokens = _tokenize(text) + if len(tokens) < _MIN_TOKENS: + return None + + scores = _score_markers(tokens) + ranked = sorted(scores.items(), key=lambda pair: pair[1], reverse=True) + (best, best_score), (_, runner_up_score) = ranked[0], ranked[1] + + if best_score <= 0: + return None + if runner_up_score > 0 and best_score / runner_up_score < _MIN_MARGIN: + logger.debug("Language detection inconclusive, scores=%s", ranked[:3]) + return None + return best + + +def detect_language( + tags: Iterable[TagLabel], + text: str | None = "", + default: LanguageCode | None = DEFAULT_LANGUAGE, +) -> LanguageCode | None: + """Best-effort language for a post: tags first, then text, then ``default``.""" + return detect_language_from_tags(tags) or detect_language_from_text(text) or default diff --git a/server/pesacheck/language_markers.py b/server/pesacheck/language_markers.py new file mode 100644 index 000000000..3f460f1b9 --- /dev/null +++ b/server/pesacheck/language_markers.py @@ -0,0 +1,493 @@ +from typing import TypeAlias + +# An ISO 639-1 language code. Not narrowed to a ``Literal`` of the six +# supported languages on purpose: :func:`normalise_language_code` passes +# unrecognised codes through, so the codes in flight are not a closed set. +LanguageCode: TypeAlias = str + + +ENGLISH: LanguageCode = "en" +KISWAHILI: LanguageCode = "sw" +FRENCH: LanguageCode = "fr" +SOMALI: LanguageCode = "so" +OROMO: LanguageCode = "om" +AMHARIC: LanguageCode = "am" + +# Latin-script detection +# ---------------------------------------------------------------------- + +# High-frequency function words per language. Content words are avoided except +# where a whole domain vocabulary is stable across PesaCheck's output ("video", +# "image", "claim"), because function words survive translation of the subject +# matter and are what actually separates these languages. +# +# Words shared across candidates are simply left out of both lists rather than +# down-weighted: Somali "in"/English "in", Somali "la"/French "la" and Swahili +# "au"/French "au" carry no signal, so including them only adds noise. +MARKERS: dict[LanguageCode, set[str]] = { + ENGLISH: { + "the", + "and", + "of", + "to", + "is", + "was", + "that", + "this", + "with", + "which", + "for", + "on", + "are", + "from", + "at", + "be", + "has", + "have", + "had", + "but", + "not", + "were", + "will", + "would", + "can", + "could", + "there", + "these", + "those", + "when", + "what", + "who", + "how", + "also", + "however", + "according", + "said", + "says", + "shows", + "showing", + "show", + "showed", + "image", + "video", + "photo", + "claim", + "claims", + "false", + "true", + "been", + "being", + "their", + "they", + "them", + "its", + "his", + "her", + "she", + "we", + "you", + "about", + "into", + "over", + "under", + "after", + "before", + "during", + "between", + "while", + "than", + "then", + "such", + "other", + "more", + "most", + "some", + "any", + "all", + "posted", + "post", + "shared", + "social", + "media", + "facebook", + "search", + "found", + "reverse", + "original", + "therefore", + "does", + "did", + "doesn", + "isn", + "wasn", + "authentic", + "misleading", + "verified", + "screenshot", + "caption", + }, + FRENCH: { + "le", + "les", + "de", + "des", + "du", + "une", + "et", + "est", + "ne", + "pas", + "que", + "qui", + "dans", + "pour", + "sur", + "cette", + "ce", + "ces", + "cet", + "par", + "avec", + "aux", + "plus", + "mais", + "son", + "sa", + "ses", + "leur", + "leurs", + "été", + "être", + "ont", + "ils", + "elles", + "elle", + "nous", + "vous", + "se", + "où", + "donc", + "alors", + "ainsi", + "comme", + "tout", + "tous", + "toute", + "toutes", + "même", + "aussi", + "selon", + "publiée", + "publié", + "publication", + "vidéo", + "image", + "photo", + "montre", + "montrant", + "fausse", + "faux", + "vrai", + "vraie", + "sans", + "sous", + "entre", + "après", + "avant", + "pendant", + "depuis", + "très", + "bien", + "encore", + "déjà", + "cependant", + "toutefois", + "néanmoins", + "effet", + "fait", + "faits", + "dit", + "déclaré", + "affirme", + "capture", + "écran", + "réseaux", + "sociaux", + "internaute", + "internautes", + "recherche", + "inversée", + "trompeur", + "trompeuse", + "vérification", + "montrent", + "était", + "étaient", + "sont", + "avait", + "avaient", + "cela", + "lui", + "nos", + "vos", + "notre", + "votre", + }, + KISWAHILI: { + "na", + "ya", + "wa", + "ni", + "kwa", + "katika", + "hii", + "huu", + "hiyo", + "huo", + "hizo", + "hao", + "si", + "kuwa", + "kwamba", + "ambayo", + "ambao", + "ambaye", + "ilikuwa", + "alikuwa", + "walikuwa", + "hakuna", + "lakini", + "hivyo", + "hata", + "pia", + "ili", + "ndani", + "juu", + "chini", + "baada", + "kabla", + "alisema", + "walisema", + "imeonyesha", + "inaonyesha", + "uchunguzi", + "madai", + "taarifa", + "mtandao", + "mitandao", + "kijamii", + "picha", + "mwaka", + "siku", + "watu", + "serikali", + "rais", + "wakati", + "kama", + "kwenye", + "za", + "cha", + "vya", + "wale", + "yake", + "yao", + "zake", + "huyu", + "wengi", + "sana", + "sasa", + "bado", + "tena", + "kila", + "hakika", + "kweli", + "uongo", + "ukweli", + "chapisho", + "machapisho", + "ujumbe", + "habari", + "kutoka", + "kuhusu", + "kufanya", + "ndiyo", + "yenye", + "hadi", + "zaidi", + "wako", + "yeye", + "mmoja", + "mbili", + "kwanza", + "video", + "ilionekana", + "sio", + "hapa", + "huku", + }, + SOMALI: { + "waxa", + "waxaa", + "waxaan", + "ayaa", + "ayay", + "ayuu", + "oo", + "iyo", + "ee", + "aha", + "inay", + "inuu", + "kale", + "sidoo", + "laakiin", + "wuxuu", + "waa", + "lagu", + "loo", + "uu", + "ay", + "sheegay", + "sheegtay", + "muuqaalka", + "muuqaalkan", + "sawirka", + "sawirkan", + "warbaahinta", + "xaqiijiyay", + "been", + "iyaga", + "kuwaas", + "taas", + "taasi", + "kaas", + "hadda", + "sida", + "marka", + "halkaas", + "dowladda", + "dawladda", + "madaxweynaha", + "shirkadda", + "bogga", + "barta", + "bulshada", + "xayeysiinta", + "qoraalka", + "wararka", + "markaana", + "sidaas", + "maaha", + "kuwa", + "wax", + "aad", + "badan", + "kama", + "ugu", + "isku", + "iyada", + "laakin", + "haddii", + "balse", + "warbixinta", + "baaritaan", + "baaritaanka", + "xogta", + "dhexe", + "soo", + "kordhay", + "muuqaal", + "sawir", + "faafiyay", + "baahiyay", + }, + OROMO: { + "kun", + "kana", + "kanaa", + "kanaaf", + "kanaan", + "irratti", + "irra", + "keessatti", + "keessa", + "akka", + "hin", + "waan", + "dha", + "jira", + "jiru", + "jedha", + "jedhu", + "jedhan", + "jedhame", + "isaa", + "isaan", + "ishee", + "namoota", + "namoonni", + "fi", + "yoo", + "malee", + "garuu", + "akkasumas", + "ykn", + "viidiyoo", + "suuraa", + "odeeffannoo", + "gabaasa", + "gabaafame", + "mootummaa", + "walitti", + "hedduu", + "immoo", + "sana", + "tokko", + "lama", + "waggaa", + "guyyaa", + "torban", + "bara", + "soba", + "dhugaa", + "miidiyaa", + "hawaasummaa", + "maxxansi", + "maxxansa", + "barreeffamni", + "barreeffama", + "agarsiisa", + "agarsiisu", + "raawwatame", + "argaman", + "ummata", + "uummata", + "naannoo", + "godina", + "aanaa", + "hojii", + "dhaabbata", + "itti", + "ittiin", + "isa", + "hunda", + "hundaa", + "erga", + "booda", + "dura", + "wayita", + "yeroo", + "qabu", + "qaba", + "taʼe", + "ta'e", + "taʼuu", + "ta'uu", + "hiriyaa", + "mirkaneesse", + "mirkaneeffame", + "seenaa", + "biyya", + "biyyoota", + }, +} diff --git a/server/tests/test_language.py b/server/tests/test_language.py new file mode 100644 index 000000000..882ef8f01 --- /dev/null +++ b/server/tests/test_language.py @@ -0,0 +1,224 @@ +import unittest + +from pesacheck import language as language_module +from pesacheck.language import ( + detect_language, + detect_language_from_tags, + detect_language_from_text, + normalise_language_code, +) + + +# Excerpts from the reference Ghost export in ops/local/ingest/ghost, which is +# the corpus these lists were built and verified against. +ENGLISH_TEXT: str = ( + "This poster claiming Hormuud Telecom is offering instant loans is a HOAX. The advert, " + "shared on Facebook on 22 June 2026, shows amounts ranging from $50 to $200,000 and " + "carries the company's logo and colours. However, a reverse image search shows that the " + "poster was fabricated and the company has confirmed it did not issue any such offer." +) + +FRENCH_TEXT: str = ( + "Cette vidéo qui montrerait des soldats maliens tués par l'armée est PARTIELLEMENT FAUSSE. " + "La séquence, publiée sur les réseaux sociaux, ne montre pas les faits allégués. Selon nos " + "vérifications, cette image a été prise dans un autre pays et n'a aucun lien avec les " + "événements décrits par les internautes." +) + +SOMALI_TEXT: str = ( + "Xayeysiintan lagu baahiyay Facebook ee lagu sheegay in shirkadda Hormuud ay bixinayso " + "deyn ayaa ah KHIYAANO. Xayeysiinta oo la baahiyay 22-kii June 2026 ayaa muujinaysa " + "xaddiga lacagaha, waxaana wehliya astaanta iyo midabada shirkadda. Warbaahinta ayaa " + "xaqiijiyay in sawirkan been abuur yahay, laakiin qoraalka wali waa la wadaagayaa." +) + +OROMO_TEXT: str = ( + "Maxxansi Facebook viidiyoo haleellaa lubbuu galaafate kan ummata Oromoo irratti " + "raawwatame agarsiisa jedhu kun SOBA. Barreeffamni Afaan Oromoo akkana jedha, garuu " + "odeeffannoon kun dhugaa hin qabu. Viidiyoon kun naannoo Oromiyaa keessatti waraabame " + "kan jedhu mirkaneeffame hin jiru, mootummaan waan kana ilaalchisee hin dubbanne." +) + +AMHARIC_TEXT: str = ( + "የኢትዮጵያ ኦርቶዶክስ ተዋህዶ ቤተክርስቲያን የሃይማኖት አባት በምርጫ ዘመቻ ላይ ሲሳተፉ ያሳያል በሚል ፌስቡክ ላይ ከምስል ጋር " + "የተጋራው ይህ ልጥፍ የተቀየረ ነው። ጉግልን በመጠቀም የተደረገው የምስል ፍለጋ በመጣራት ላይ ያለው ፎቶ እንደተቀየረ አረጋግጧል።" +) + +# No Kiswahili post exists in the reference export, so this sample is written to +# match the register PesaCheck publishes in rather than lifted from the corpus. +KISWAHILI_TEXT: str = ( + "Chapisho hili la Facebook linalodai kuwa serikali imetangaza siku ya mapumziko ni la " + "uongo. Uchunguzi wetu umeonyesha kwamba picha hiyo ilikuwa imechapishwa mwaka 2019 na " + "haihusiani na madai hayo. Taarifa hiyo haikuwa kwenye mitandao ya kijamii ya serikali, " + "na msemaji alisema hakuna tangazo lililotolewa kuhusu suala hilo." +) + + +class LanguageFromTagsTestCase(unittest.TestCase): + def test_detects_each_published_language(self) -> None: + for tag, expected in [ + ("English", "en"), + ("Kiswahili", "sw"), + ("French", "fr"), + ("Somali", "so"), + ("Afaan Oromo", "om"), + ("Amharic", "am"), + ]: + with self.subTest(tag=tag): + self.assertEqual(detect_language_from_tags([(0, tag)]), expected) + + def test_matches_slugs_as_well_as_display_names(self) -> None: + self.assertEqual(detect_language_from_tags([(0, "afaan-oromo")]), "om") + self.assertEqual(detect_language_from_tags([(0, "kiswahili")]), "sw") + + def test_matching_ignores_case_and_padding(self) -> None: + self.assertEqual(detect_language_from_tags([(0, " FRENCH ")]), "fr") + + def test_accepts_endonyms(self) -> None: + self.assertEqual(detect_language_from_tags([(0, "Soomaali")]), "so") + self.assertEqual(detect_language_from_tags([(0, "Français")]), "fr") + + def test_returns_none_without_a_language_tag(self) -> None: + self.assertIsNone(detect_language_from_tags([(0, "Kenya"), (1, "Short Form")])) + self.assertIsNone(detect_language_from_tags([])) + + def test_country_tags_are_not_read_as_languages(self) -> None: + """The country sitting next to the language tag must not win. + + PesaCheck tags posts with both ``Somali``/``Somalia`` and + ``French``/``France``, so these pairs are one substring away from + collapsing into each other. + """ + for tags, expected in [ + ([(0, "English"), (1, "Somalia")], "en"), + ([(0, "English"), (1, "France")], "en"), + ([(0, "English"), (1, "Ethiopia")], "en"), + ([(0, "Somali"), (1, "Somalia")], "so"), + ([(0, "French"), (1, "France")], "fr"), + ]: + with self.subTest(tags=tags): + self.assertEqual(detect_language_from_tags(tags), expected) + + def test_bare_oromo_ethnonym_is_not_treated_as_a_language(self) -> None: + # An English article about Oromo people may carry "Oromo" as a topic + # tag; only the unambiguous "Afaan Oromo" marks the language. + self.assertEqual( + detect_language_from_tags([(0, "English"), (1, "Oromo")]), "en" + ) + + def test_primary_tag_wins_when_two_languages_are_tagged(self) -> None: + self.assertEqual( + detect_language_from_tags([(0, "English"), (1, "Amharic")]), "en" + ) + # Order in the list must not matter — sort_order decides. + self.assertEqual( + detect_language_from_tags([(1, "Amharic"), (0, "English")]), "en" + ) + + def test_null_sort_order_is_tolerated(self) -> None: + self.assertEqual(detect_language_from_tags([(None, "Somali")]), "so") + + +class LanguageFromTextTestCase(unittest.TestCase): + def test_detects_each_published_language(self) -> None: + for text, expected in [ + (ENGLISH_TEXT, "en"), + (KISWAHILI_TEXT, "sw"), + (FRENCH_TEXT, "fr"), + (SOMALI_TEXT, "so"), + (OROMO_TEXT, "om"), + (AMHARIC_TEXT, "am"), + ]: + with self.subTest(expected=expected): + self.assertEqual(detect_language_from_text(text), expected) + + def test_separates_oromo_from_somali(self) -> None: + """Both are Cushitic with doubled-vowel Latin spelling; langdetect, + which has no Oromo model, labels Oromo prose as Somali.""" + self.assertEqual(detect_language_from_text(OROMO_TEXT), "om") + self.assertEqual(detect_language_from_text(SOMALI_TEXT), "so") + + def test_embedded_english_quotes_do_not_flip_the_result(self) -> None: + # Fact checks quote the claim being debunked, often in English. + quoted = ( + SOMALI_TEXT + ' Qoraalka waxaa lagu yiri "Easy application process" iyo ' + '"Instant approval" oo ah hadal been abuur ah.' + ) + self.assertEqual(detect_language_from_text(quoted), "so") + + def test_a_quoted_amharic_phrase_does_not_make_an_article_amharic(self) -> None: + # The previous detector returned "am" for any text containing a single + # Ethiopic character. + quoted = ( + ENGLISH_TEXT + + " The post, written in Amharic, reads “የመንግስት ሰራተኞች ደመዎዝ ጥማሪ”." + ) + self.assertEqual(detect_language_from_text(quoted), "en") + + def test_detects_from_a_headline_alone(self) -> None: + self.assertEqual( + detect_language_from_text( + "FAUX : Cette vidéo ne montre pas des billets dérobés" + ), + "fr", + ) + self.assertEqual( + detect_language_from_text( + "BEEN: Muuqaalkan ma aha hub lagu qabtay oo ku yaal Soomaaliya" + ), + "so", + ) + + def test_survives_html_markup(self) -> None: + self.assertEqual(detect_language_from_text("

%s

" % FRENCH_TEXT), "fr") + + def test_declines_when_no_language_is_clearly_ahead(self) -> None: + # Function words borrowed from several candidates at once: nothing + # clears the margin, so the inconclusive branch logs and returns None. + mixed = "the le kwa oo kun de and ya iyo akka" + with self.assertLogs(language_module.logger, level="DEBUG"): + self.assertIsNone(detect_language_from_text(mixed)) + + def test_declines_to_guess_without_usable_text(self) -> None: + for text in [ + "", + " ", + None, + "Hormuud", + "2026", + "$50 200,000 !!!", + "http://example.com/a", + ]: + with self.subTest(text=text): + self.assertIsNone(detect_language_from_text(text)) + + +class NormaliseLanguageCodeTestCase(unittest.TestCase): + def test_strips_region_subtags(self) -> None: + self.assertEqual(normalise_language_code("en-US"), "en") + self.assertEqual(normalise_language_code("am_ET"), "am") + + def test_maps_names_to_codes(self) -> None: + self.assertEqual(normalise_language_code("Kiswahili"), "sw") + + def test_passes_through_unsupported_codes(self) -> None: + # PesaCheck may add a language before this module knows about it. + self.assertEqual(normalise_language_code("pt-BR"), "pt") + + def test_returns_none_for_empty_values(self) -> None: + self.assertIsNone(normalise_language_code(None)) + self.assertIsNone(normalise_language_code("")) + + +class DetectLanguageTestCase(unittest.TestCase): + def test_tags_take_precedence_over_text(self) -> None: + self.assertEqual(detect_language([(0, "Kiswahili")], FRENCH_TEXT), "sw") + + def test_falls_back_to_text_when_untagged(self) -> None: + self.assertEqual(detect_language([(0, "Kenya")], FRENCH_TEXT), "fr") + + def test_falls_back_to_english_when_nothing_is_detectable(self) -> None: + self.assertEqual(detect_language([], ""), "en") + + def test_default_is_overridable(self) -> None: + self.assertIsNone(detect_language([], "", default=None)) From 36726ba68113ca4a1fb3cf5c79fde34f17906598 Mon Sep 17 00:00:00 2001 From: Mark Ng'ang'a Date: Mon, 10 Aug 2026 13:08:08 +0300 Subject: [PATCH 3/5] Ghost ingest: detect post language from tags with a text fallback Replace _guess_language, which only ever returned am, fr or en. Somali and Afaan Oromo both collapsed to English: langdetect was consulted but its result was trusted only for en/fr, and it has no Oromo model regardless. Measured against the 50-post reference export that was 92% overall but 0% on Somali and Oromo. _parse_language now resolves in three steps: Ghost's locale (normalised, so en-US becomes en), then the post's language tag, then the new text classifier. Tag records carry their slug so either the display name or the slug can name the language. Detection runs on Ghost's plaintext rendering rather than body_html, so markup no longer dilutes the word counts, and falls back to html when a post has no plaintext. Also coerce a null sort_order on tags: the existing sort would have raised TypeError comparing None to int, and sort_order now decides which language tag wins. Both paths reach 100% on the reference export -- via tags, and via text with the language tag stripped to simulate the older untagged posts. --- server/pesacheck/ingest/ghost_parser.py | 61 ++++---- .../ghost_export/ghost_export_languages.json | 133 ++++++++++++++++++ server/tests/ingest/test_ghost_parser.py | 44 +++++- 3 files changed, 209 insertions(+), 29 deletions(-) create mode 100644 server/tests/fixtures/ghost_export/ghost_export_languages.json diff --git a/server/pesacheck/ingest/ghost_parser.py b/server/pesacheck/ingest/ghost_parser.py index 3f7c78152..d3e40755d 100644 --- a/server/pesacheck/ingest/ghost_parser.py +++ b/server/pesacheck/ingest/ghost_parser.py @@ -3,7 +3,6 @@ import logging import random import time -import unicodedata from copy import deepcopy from datetime import datetime, timezone from urllib.parse import urlparse @@ -22,8 +21,11 @@ ITEM_TYPE, ) from superdesk.metadata.utils import generate_guid +from superdesk.text_utils import get_text from superdesk.utc import utcnow +from pesacheck.language import detect_language, normalise_language_code + logger = logging.getLogger(__name__) # Ghost exports replace the site URL with this portable placeholder in @@ -132,28 +134,27 @@ def _sleep_before_next_fetch(self, min_interval): def _mark_fetch_done(self): self._last_image_fetch_ts = time.monotonic() - def _guess_language(self, text): - """Guess language from text: returns 'am', 'fr', or 'en'. + def _parse_language(self, post, tags, text): + """Resolve the item language from ``locale``, then tags, then body text. - Ethiopic script detection is used first (langdetect has no Amharic model). - langdetect is then used to distinguish English from French; anything - unrecognised falls back to English. + Ghost's own ``locale`` is authoritative when set, but PesaCheck's export + leaves it null on every post, so in practice the language comes from the + post's language tag. Posts predating that tagging convention fall + through to text classification. """ - if not text: - return None - # Detect Amharic by Ethiopic Unicode block (U+1200–U+137F, etc.) - for ch in text: - if "ETHIOPIC" in unicodedata.name(ch, ""): - return "am" - try: - from langdetect import detect + locale = normalise_language_code(post.get("locale")) + if locale: + return locale - detected = detect(text) - if detected in ("en", "fr"): - return detected - except Exception: - pass - return "en" + # Offer both the display name and the slug: either may be the form that + # names the language ("Afaan Oromo" / "afaan-oromo"). + tag_labels = [ + (tag["sort_order"], label) + for tag in tags + for label in (tag["name"], tag["slug"]) + ] + + return detect_language(tag_labels, text) def can_parse(self, file_path): try: @@ -358,13 +359,12 @@ def _parse_post(self, post, authors_by_post, tags_by_post): "versioncreated": versioncreated, } - locale = post.get("locale") - if locale: - item["language"] = locale - else: - guessed = self._guess_language(html or post.get("title") or "") - if guessed: - item["language"] = guessed + # Ghost exports a markup-free rendering of the body; prefer it for + # language detection so HTML tag names don't dilute the word counts. + # Older exports omit it, so strip the markup ourselves in that case. + body_text = post.get("plaintext") or get_text(html, content="html") + sample = " ".join(part for part in (post.get("title") or "", body_text) if part) + item["language"] = self._parse_language(post, tags, sample) self._parse_feature_image(item, post) self._parse_inline_images(item, html) @@ -422,7 +422,12 @@ def iter_items(self, file_path, provider=None): tags_by_post.setdefault(pid, []).append( { "name": tag.get("name", ""), - "sort_order": pt.get("sort_order", 0), + # Ghost slugs are the stable identifier, and the language + # tag is matched on either form. + "slug": tag.get("slug", ""), + # Coerce a null sort_order so the sort below can't blow + # up comparing None to an int. + "sort_order": pt.get("sort_order") or 0, } ) diff --git a/server/tests/fixtures/ghost_export/ghost_export_languages.json b/server/tests/fixtures/ghost_export/ghost_export_languages.json new file mode 100644 index 000000000..69521dc47 --- /dev/null +++ b/server/tests/fixtures/ghost_export/ghost_export_languages.json @@ -0,0 +1,133 @@ +{ + "db": [ + { + "data": { + "posts": [ + { + "id": "lang_001", + "uuid": "11111111-0001-0001-0001-111111111111", + "title": "UONGO: Chapisho hili la Facebook ni la uongo", + "slug": "uongo-chapisho-hili", + "html": "

Chapisho hili la Facebook linalodai kuwa serikali imetangaza siku ya mapumziko ni la uongo.

", + "plaintext": "Chapisho hili la Facebook linalodai kuwa serikali imetangaza siku ya mapumziko ni la uongo. Uchunguzi wetu umeonyesha kwamba picha hiyo ilikuwa imechapishwa mwaka 2019 na haihusiani na madai hayo. Taarifa hiyo haikuwa kwenye mitandao ya kijamii ya serikali.", + "feature_image": null, + "type": "post", + "status": "published", + "locale": null, + "created_at": "2026-01-05T10:00:00.000Z", + "updated_at": "2026-01-05T11:00:00.000Z", + "published_at": "2026-01-05T11:00:00.000Z", + "custom_excerpt": null + }, + { + "id": "lang_002", + "uuid": "22222222-0002-0002-0002-222222222222", + "title": "BEEN: Muuqaalkan ma aha hub lagu qabtay", + "slug": "been-muuqaalkan-ma-aha", + "html": "

Xayeysiintan lagu baahiyay Facebook waa KHIYAANO.

", + "plaintext": "Xayeysiintan lagu baahiyay Facebook ee lagu sheegay in shirkadda Hormuud ay bixinayso deyn ayaa ah KHIYAANO. Warbaahinta ayaa xaqiijiyay in sawirkan been abuur yahay, laakiin qoraalka wali waa la wadaagayaa oo waxaa lagu sheegay iyo midabada shirkadda.", + "feature_image": null, + "type": "post", + "status": "published", + "locale": null, + "created_at": "2026-01-06T10:00:00.000Z", + "updated_at": "2026-01-06T11:00:00.000Z", + "published_at": "2026-01-06T11:00:00.000Z", + "custom_excerpt": null + }, + { + "id": "lang_003", + "uuid": "33333333-0003-0003-0003-333333333333", + "title": "SOBA: Maxxansi Facebook kun soba", + "slug": "soba-maxxansi-facebook-kun", + "html": "

Maxxansi Facebook kun SOBA.

", + "plaintext": "Maxxansi Facebook viidiyoo haleellaa ummata Oromoo irratti raawwatame agarsiisa jedhu kun SOBA. Barreeffamni Afaan Oromoo akkana jedha, garuu odeeffannoon kun dhugaa hin qabu. Viidiyoon kun naannoo Oromiyaa keessatti waraabame kan jedhu mirkaneeffame hin jiru.", + "feature_image": null, + "type": "post", + "status": "published", + "locale": null, + "created_at": "2026-01-07T10:00:00.000Z", + "updated_at": "2026-01-07T11:00:00.000Z", + "published_at": "2026-01-07T11:00:00.000Z", + "custom_excerpt": null + }, + { + "id": "lang_004", + "uuid": "44444444-0004-0004-0004-444444444444", + "title": "FAUX : Cette video ne montre pas des billets", + "slug": "faux-cette-video", + "html": "

Cette video ne montre pas les faits allegues.

", + "plaintext": "Cette video qui montrerait des soldats tues par l'armee est PARTIELLEMENT FAUSSE. La sequence, publiee sur les reseaux sociaux, ne montre pas les faits allegues selon nos verifications.", + "feature_image": null, + "type": "post", + "status": "published", + "locale": "en-US", + "created_at": "2026-01-08T10:00:00.000Z", + "updated_at": "2026-01-08T11:00:00.000Z", + "published_at": "2026-01-08T11:00:00.000Z", + "custom_excerpt": null + }, + { + "id": "lang_005", + "uuid": "55555555-0005-0005-0005-555555555555", + "title": "የተቀየረ: ይህ ምስል የተቀየረ ነው", + "slug": "yetekeyere-yih-mesel", + "html": "

ይህ ልጥፍ የተቀየረ ነው።

", + "plaintext": "የኢትዮጵያ ኦርቶዶክስ ተዋህዶ ቤተክርስቲያን የሃይማኖት አባት በምርጫ ዘመቻ ላይ ሲሳተፉ ያሳያል በሚል ፌስቡክ ላይ ከምስል ጋር የተጋራው ይህ ልጥፍ የተቀየረ ነው። ጉግልን በመጠቀም የተደረገው የምስል ፍለጋ በመጣራት ላይ ያለው ፎቶ እንደተቀየረ አረጋግጧል።", + "feature_image": null, + "type": "post", + "status": "published", + "locale": null, + "created_at": "2026-01-09T10:00:00.000Z", + "updated_at": "2026-01-09T11:00:00.000Z", + "published_at": "2026-01-09T11:00:00.000Z", + "custom_excerpt": null + }, + { + "id": "lang_006", + "uuid": "66666666-0006-0006-0006-666666666666", + "title": "HOAX: This poster offering instant loans is fake", + "slug": "hoax-this-poster", + "html": "

This poster claiming instant loans is a HOAX.

", + "plaintext": "This poster claiming Hormuud Telecom is offering instant loans is a HOAX. The advert, shared on Facebook, shows amounts ranging from $50 to $200,000. However, a reverse image search shows that the poster was fabricated and the company has confirmed it did not issue any such offer.", + "feature_image": null, + "type": "post", + "status": "published", + "locale": null, + "created_at": "2026-01-10T10:00:00.000Z", + "updated_at": "2026-01-10T11:00:00.000Z", + "published_at": "2026-01-10T11:00:00.000Z", + "custom_excerpt": null + } + ], + "users": [], + "posts_authors": [], + "tags": [ + {"id": "tag_sw", "name": "Kiswahili", "slug": "kiswahili"}, + {"id": "tag_om", "name": "Afaan Oromo", "slug": "afaan-oromo"}, + {"id": "tag_en", "name": "English", "slug": "english"}, + {"id": "tag_fr", "name": "French", "slug": "french"}, + {"id": "tag_kenya", "name": "Kenya", "slug": "kenya"}, + {"id": "tag_somalia", "name": "Somalia", "slug": "somalia"}, + {"id": "tag_ethiopia", "name": "Ethiopia", "slug": "ethiopia"}, + {"id": "tag_short", "name": "Short Form", "slug": "short-form"} + ], + "posts_tags": [ + {"post_id": "lang_001", "tag_id": "tag_sw", "sort_order": 0}, + {"post_id": "lang_001", "tag_id": "tag_kenya", "sort_order": 1}, + + {"post_id": "lang_002", "tag_id": "tag_somalia", "sort_order": 0}, + {"post_id": "lang_002", "tag_id": "tag_short", "sort_order": 1}, + + {"post_id": "lang_003", "tag_id": "tag_om", "sort_order": 0}, + {"post_id": "lang_003", "tag_id": "tag_ethiopia", "sort_order": 1}, + + {"post_id": "lang_004", "tag_id": "tag_fr", "sort_order": 0}, + + {"post_id": "lang_006", "tag_id": "tag_en", "sort_order": 0}, + {"post_id": "lang_006", "tag_id": "tag_somalia", "sort_order": 1} + ] + } + } + ] +} diff --git a/server/tests/ingest/test_ghost_parser.py b/server/tests/ingest/test_ghost_parser.py index e72687360..46befd72e 100644 --- a/server/tests/ingest/test_ghost_parser.py +++ b/server/tests/ingest/test_ghost_parser.py @@ -4,12 +4,14 @@ from unittest.mock import patch from pesacheck.ingest.ghost_parser import GhostParser +from pesacheck.language import SUPPORTED_LANGUAGES from superdesk.tests import TestCase FIXTURE_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "../fixtures/ghost_export" ) FIXTURE_PATH = os.path.join(FIXTURE_DIR, "ghost_export.json") +LANGUAGES_FIXTURE_PATH = os.path.join(FIXTURE_DIR, "ghost_export_languages.json") def _mock_update_renditions(item, url, old_item, **kwargs): @@ -125,7 +127,47 @@ async def test_parse_null_locale_guesses_language(self, _mock): post2 = next( i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb" ) - self.assertIn("language", post2) + self.assertEqual(post2["language"], "en") + + # ------------------------------------------------------------------ + # parse — language detection + # ------------------------------------------------------------------ + + async def _languages_by_guid(self): + items = await self.parser.parse(LANGUAGES_FIXTURE_PATH) + return {item["guid"]: item["language"] for item in items} + + async def test_parse_language_from_tag(self): + languages = await self._languages_by_guid() + self.assertEqual(languages["11111111-0001-0001-0001-111111111111"], "sw") + self.assertEqual(languages["33333333-0003-0003-0003-333333333333"], "om") + self.assertEqual(languages["66666666-0006-0006-0006-666666666666"], "en") + + async def test_parse_language_falls_back_to_text_when_untagged(self): + # lang_002 predates the language-tag convention: only "Somalia" and + # "Short Form" are tagged, so the body has to carry the language. + languages = await self._languages_by_guid() + self.assertEqual(languages["22222222-0002-0002-0002-222222222222"], "so") + + async def test_parse_language_falls_back_to_text_without_any_tags(self): + languages = await self._languages_by_guid() + self.assertEqual(languages["55555555-0005-0005-0005-555555555555"], "am") + + async def test_parse_language_prefers_locale_and_normalises_it(self): + # locale "en-US" outranks the French tag and is reduced to "en". + languages = await self._languages_by_guid() + self.assertEqual(languages["44444444-0004-0004-0004-444444444444"], "en") + + async def test_parse_language_ignores_adjacent_country_tag(self): + # lang_006 is tagged English + Somalia; "Somalia" must not read as Somali. + languages = await self._languages_by_guid() + self.assertEqual(languages["66666666-0006-0006-0006-666666666666"], "en") + + async def test_parse_language_set_on_every_item(self): + items = await self.parser.parse(LANGUAGES_FIXTURE_PATH) + self.assertEqual(len(items), 6) + for item in items: + self.assertIn(item["language"], SUPPORTED_LANGUAGES) # ------------------------------------------------------------------ # parse — authors → byline From 1578e080d0911f4a397eb88f5b37abf2e01f47d4 Mon Sep 17 00:00:00 2001 From: Mark Ng'ang'a Date: Mon, 10 Aug 2026 13:08:23 +0300 Subject: [PATCH 4/5] Drop the langdetect dependency langdetect was added as a direct requirement solely for Ghost ingest language detection, which no longer uses it. Nothing else in the tree imports it. six stays in requirements.txt: it is still required by flask-oidc-ex, oauth2client and python-dateutil. --- server/requirements.in | 1 - server/requirements.txt | 2 -- 2 files changed, 3 deletions(-) diff --git a/server/requirements.in b/server/requirements.in index 256e5b720..b8f9f94b6 100755 --- a/server/requirements.in +++ b/server/requirements.in @@ -1,6 +1,5 @@ honcho celery-redbeat -langdetect git+https://github.com/superdesk/superdesk-core.git@develop#egg=superdesk-core git+https://github.com/superdesk/superdesk-planning.git@develop#egg=superdesk-planning diff --git a/server/requirements.txt b/server/requirements.txt index ac2cbe5ee..70bf60e1d 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -185,8 +185,6 @@ jwcrypto==1.5.8 # python-jwt kombu[redis]==5.6.2 # via celery -langdetect==1.0.9 - # via -r requirements.in ldap3==2.9.1 # via superdesk-core lxml==5.3.2 From 53108d4dcbcb0879cd032a073e3e6515a4313c21 Mon Sep 17 00:00:00 2001 From: Mark Ng'ang'a Date: Mon, 10 Aug 2026 13:28:24 +0300 Subject: [PATCH 5/5] Run Black --- scripts/dev/extension-edit.py | 12 ++++++-- scripts/update-cores-references.py | 28 +++++++++-------- server/macros/replace_words.py | 4 ++- .../pesacheck/ingest/ghost_feeding_service.py | 24 +++++++++++---- server/pesacheck/ingest/medium_parser.py | 30 ++++++++++++++----- server/settings.py | 14 +++++++-- server/tests/ingest/test_medium_parser.py | 15 +++++++--- 7 files changed, 91 insertions(+), 36 deletions(-) diff --git a/scripts/dev/extension-edit.py b/scripts/dev/extension-edit.py index c7d7e6579..966a0440b 100755 --- a/scripts/dev/extension-edit.py +++ b/scripts/dev/extension-edit.py @@ -36,7 +36,9 @@ def main(): mode, name, path = sys.argv[1], sys.argv[2], sys.argv[3] if mode not in ("enable", "disable"): - print(f"ERROR: mode must be 'enable' or 'disable', got '{mode}'", file=sys.stderr) + print( + f"ERROR: mode must be 'enable' or 'disable', got '{mode}'", file=sys.stderr + ) sys.exit(2) with open(path) as f: @@ -63,7 +65,9 @@ def main(): # a line whose stripped content is '},'. Anything else (blank lines, # comments) is treated as a passthrough chunk between entries. region = lines[start_idx + 1 : end_idx] - entries = [] # list of (kind, value): kind is 'entry' (list of lines) or 'other' (single line) + entries = ( + [] + ) # list of (kind, value): kind is 'entry' (list of lines) or 'other' (single line) current = None for line in region: stripped = line.strip() @@ -100,7 +104,9 @@ def is_standard_template(entry_lines): if mode == "enable": for kind, value in entries: if kind == "entry" and entry_id(value) == name: - print(f"INFO: '{name}' is already enabled — no change.", file=sys.stderr) + print( + f"INFO: '{name}' is already enabled — no change.", file=sys.stderr + ) sys.exit(0) # Build a fresh entry and insert before // extensions:end. new_entry = [ diff --git a/scripts/update-cores-references.py b/scripts/update-cores-references.py index af86ca414..0fd8da012 100755 --- a/scripts/update-cores-references.py +++ b/scripts/update-cores-references.py @@ -6,33 +6,35 @@ TO_BE_UPDATED = [ # superdesk-core { - 'feed_url': 'https://github.com/superdesk/superdesk-core/commits/master.atom', - 'file_name': 'server/requirements.txt', - 'pattern': 'superdesk-core.git@([a-f0-9]*)' + "feed_url": "https://github.com/superdesk/superdesk-core/commits/master.atom", + "file_name": "server/requirements.txt", + "pattern": "superdesk-core.git@([a-f0-9]*)", }, # superdesk-client-core { - 'feed_url': 'https://github.com/superdesk/superdesk-client-core/commits/master.atom', - 'file_name': 'client/package.json', - 'pattern': 'superdesk-client-core#([a-f0-9]*)' - } + "feed_url": "https://github.com/superdesk/superdesk-client-core/commits/master.atom", + "file_name": "client/package.json", + "pattern": "superdesk-client-core#([a-f0-9]*)", + }, ] + def get_last_commit(url): feed = feedparser.parse(url) - return feed['entries'][0]['id'].split('/')[1][:9] + return feed["entries"][0]["id"].split("/")[1][:9] def replace_in_file(filename, search, new_value): - textfile = open(filename, 'r') + textfile = open(filename, "r") filetext = textfile.read() textfile.close() matches = re.findall(search, filetext) with fileinput.FileInput(filename, inplace=True) as file: for line in file: - print(line.replace(matches[0], new_value), end='') + print(line.replace(matches[0], new_value), end="") + -if __name__ == '__main__': +if __name__ == "__main__": for repo in TO_BE_UPDATED: - last_commit_hash = get_last_commit(repo['feed_url']) - replace_in_file(repo['file_name'], repo['pattern'], last_commit_hash) + last_commit_hash = get_last_commit(repo["feed_url"]) + replace_in_file(repo["file_name"], repo["pattern"], last_commit_hash) diff --git a/server/macros/replace_words.py b/server/macros/replace_words.py index 28246be2f..7847c6dcc 100644 --- a/server/macros/replace_words.py +++ b/server/macros/replace_words.py @@ -64,7 +64,9 @@ def do_find_replace(input_string, words_list): while re.search(pattern, input_string, flags=re.IGNORECASE): # get the original string from the input - original = re.search(pattern, input_string, flags=re.IGNORECASE).group(0) + original = re.search(pattern, input_string, flags=re.IGNORECASE).group( + 0 + ) replacement = repl(word.get("replacement", ""), original) if found_list.get(original): break diff --git a/server/pesacheck/ingest/ghost_feeding_service.py b/server/pesacheck/ingest/ghost_feeding_service.py index d38e79153..00c9051dc 100644 --- a/server/pesacheck/ingest/ghost_feeding_service.py +++ b/server/pesacheck/ingest/ghost_feeding_service.py @@ -57,7 +57,9 @@ async def _update(self, provider, update): ) return - for filename in await get_sorted_files(self.path, sort_by=FileSortAttributes.created): + for filename in await get_sorted_files( + self.path, sort_by=FileSortAttributes.created + ): last_updated = None try: file_path = os.path.join(self.path, filename) @@ -66,8 +68,12 @@ async def _update(self, provider, update): last_updated = self.get_last_updated(file_path) - if not self.is_latest_content(last_updated, provider.get("last_updated")): - await self.move_file(self.path, filename, provider=provider, success=False) + if not self.is_latest_content( + last_updated, provider.get("last_updated") + ): + await self.move_file( + self.path, filename, provider=provider, success=False + ) continue if await self.is_empty(file_path): @@ -87,12 +93,18 @@ async def _update(self, provider, update): break yield batch - await self.move_file(self.path, filename, provider=provider, success=True) + await self.move_file( + self.path, filename, provider=provider, success=True + ) except Exception as ex: if last_updated and self.is_old_content(last_updated): - await self.move_file(self.path, filename, provider=provider, success=False) - raise ParserError.parseFileError("{}-{}".format(provider["name"], self.NAME), filename, ex, provider) + await self.move_file( + self.path, filename, provider=provider, success=False + ) + raise ParserError.parseFileError( + "{}-{}".format(provider["name"], self.NAME), filename, ex, provider + ) push_notification("ingest:update") diff --git a/server/pesacheck/ingest/medium_parser.py b/server/pesacheck/ingest/medium_parser.py index 1056313fc..3cf18c8f8 100644 --- a/server/pesacheck/ingest/medium_parser.py +++ b/server/pesacheck/ingest/medium_parser.py @@ -11,7 +11,14 @@ from superdesk.io.feed_parsers import FileFeedParser from superdesk.io.registry import register_feed_parser from superdesk.media.renditions import update_renditions -from superdesk.metadata.item import FORMAT, GUID_FIELD, GUID_TAG, ITEM_TYPE, CONTENT_TYPE, FORMATS +from superdesk.metadata.item import ( + FORMAT, + GUID_FIELD, + GUID_TAG, + ITEM_TYPE, + CONTENT_TYPE, + FORMATS, +) logger = logging.getLogger(__name__) @@ -54,10 +61,13 @@ def can_parse(self, file_path): html_content = f.read().decode("utf-8") root = parse_html(html_content, "html") - exported_from_medium = 'Exported from Medium' in to_string( - root.find(".//footer"), method="html" + exported_from_medium = ( + 'Exported from Medium' + in to_string(root.find(".//footer"), method="html") + ) + article_body_found = ( + root.find(".//section[@data-field='body']") is not None ) - article_body_found = root.find(".//section[@data-field='body']") is not None return exported_from_medium and article_body_found except Exception: @@ -78,7 +88,9 @@ def _generate_image_guid(self, url): guid_hash = hashlib.sha1(url.encode("utf8")).hexdigest() return generate_guid(type=GUID_TAG, id=guid_hash + "-image") - def _add_image(self, item, url, alt_text="", description_text="", is_featured=False): + def _add_image( + self, item, url, alt_text="", description_text="", is_featured=False + ): """Add an image to the item's associations. :param item: The item dictionary to add the image to @@ -132,7 +144,9 @@ def parse_images(self, item, article): self._add_image(item, src, alt_text, description_text, is_featured) except Exception as e: - logger.warning(f"Failed to parse image {img.get('src', 'unknown')}: {e}") + logger.warning( + f"Failed to parse image {img.get('src', 'unknown')}: {e}" + ) continue async def parse(self, file_path, provider=None): @@ -168,7 +182,9 @@ async def parse(self, file_path, provider=None): firstcreated = utcnow() text_nodes = article.xpath(".//text()") - word_count = sum(len(text.strip().split()) for text in text_nodes if text.strip()) + word_count = sum( + len(text.strip().split()) for text in text_nodes if text.strip() + ) item = { ITEM_TYPE: CONTENT_TYPE.TEXT, diff --git a/server/settings.py b/server/settings.py index 678a95b3c..9f705cbb5 100644 --- a/server/settings.py +++ b/server/settings.py @@ -179,8 +179,18 @@ "picture": { "headline": {"order": 1, "sdWidth": "full", "editor3": True}, "alt_text": {"order": 2, "sdWidth": "full", "textarea": True, "editor3": True}, - "description_text": {"order": 3, "sdWidth": "full", "textarea": True, "editor3": True}, - "creditline": {"order": 4, "sdWidth": "full", "displayOnMediaEditor": True, "editor3": True}, + "description_text": { + "order": 3, + "sdWidth": "full", + "textarea": True, + "editor3": True, + }, + "creditline": { + "order": 4, + "sdWidth": "full", + "displayOnMediaEditor": True, + "editor3": True, + }, "copyrightholder": {"order": 5, "displayOnMediaEditor": True, "editor3": True}, "usageterms": {"order": 6, "displayOnMediaEditor": True}, "copyrightnotice": {"order": 7, "displayOnMediaEditor": True}, diff --git a/server/tests/ingest/test_medium_parser.py b/server/tests/ingest/test_medium_parser.py index 3514127e3..0fcef54ca 100644 --- a/server/tests/ingest/test_medium_parser.py +++ b/server/tests/ingest/test_medium_parser.py @@ -9,7 +9,9 @@ def setUp(self): super().setUp() dirname = os.path.dirname(os.path.realpath(__file__)) - fixture_path = os.path.normpath(os.path.join(dirname, "../fixtures/medium_export")) + fixture_path = os.path.normpath( + os.path.join(dirname, "../fixtures/medium_export") + ) self.file_path = os.path.join( fixture_path, "2022-02-27_HOAX--This-UNAIDS-job-advert-in-Uganda-is-fake-f8d269a3d85d.html", @@ -23,9 +25,12 @@ async def test_can_parse(self): async def test_parse_medium_html(self): item = await self.parser.parse(self.file_path) - self.assertEqual(item["headline"], "HOAX: This UNAIDS job advert in Uganda is fake") self.assertEqual( - item["abstract"], "A UNAIDS Communications officer told PesaCheck that the job advertisement is fake." + item["headline"], "HOAX: This UNAIDS job advert in Uganda is fake" + ) + self.assertEqual( + item["abstract"], + "A UNAIDS Communications officer told PesaCheck that the job advertisement is fake.", ) self.assertTrue(item["body_html"].startswith("