diff --git a/scripts/dev/extension-edit.py b/scripts/dev/extension-edit.py
index c7d7e6579a..966a0440b3 100755
--- a/scripts/dev/extension-edit.py
+++ b/scripts/dev/extension-edit.py
@@ -36,7 +36,9 @@ def main():
mode, name, path = sys.argv[1], sys.argv[2], sys.argv[3]
if mode not in ("enable", "disable"):
- print(f"ERROR: mode must be 'enable' or 'disable', got '{mode}'", file=sys.stderr)
+ print(
+ f"ERROR: mode must be 'enable' or 'disable', got '{mode}'", file=sys.stderr
+ )
sys.exit(2)
with open(path) as f:
@@ -63,7 +65,9 @@ def main():
# a line whose stripped content is '},'. Anything else (blank lines,
# comments) is treated as a passthrough chunk between entries.
region = lines[start_idx + 1 : end_idx]
- entries = [] # list of (kind, value): kind is 'entry' (list of lines) or 'other' (single line)
+ entries = (
+ []
+ ) # list of (kind, value): kind is 'entry' (list of lines) or 'other' (single line)
current = None
for line in region:
stripped = line.strip()
@@ -100,7 +104,9 @@ def is_standard_template(entry_lines):
if mode == "enable":
for kind, value in entries:
if kind == "entry" and entry_id(value) == name:
- print(f"INFO: '{name}' is already enabled — no change.", file=sys.stderr)
+ print(
+ f"INFO: '{name}' is already enabled — no change.", file=sys.stderr
+ )
sys.exit(0)
# Build a fresh entry and insert before // extensions:end.
new_entry = [
diff --git a/scripts/update-cores-references.py b/scripts/update-cores-references.py
index af86ca4142..0fd8da0122 100755
--- a/scripts/update-cores-references.py
+++ b/scripts/update-cores-references.py
@@ -6,33 +6,35 @@
TO_BE_UPDATED = [
# superdesk-core
{
- 'feed_url': 'https://github.com/superdesk/superdesk-core/commits/master.atom',
- 'file_name': 'server/requirements.txt',
- 'pattern': 'superdesk-core.git@([a-f0-9]*)'
+ "feed_url": "https://github.com/superdesk/superdesk-core/commits/master.atom",
+ "file_name": "server/requirements.txt",
+ "pattern": "superdesk-core.git@([a-f0-9]*)",
},
# superdesk-client-core
{
- 'feed_url': 'https://github.com/superdesk/superdesk-client-core/commits/master.atom',
- 'file_name': 'client/package.json',
- 'pattern': 'superdesk-client-core#([a-f0-9]*)'
- }
+ "feed_url": "https://github.com/superdesk/superdesk-client-core/commits/master.atom",
+ "file_name": "client/package.json",
+ "pattern": "superdesk-client-core#([a-f0-9]*)",
+ },
]
+
def get_last_commit(url):
feed = feedparser.parse(url)
- return feed['entries'][0]['id'].split('/')[1][:9]
+ return feed["entries"][0]["id"].split("/")[1][:9]
def replace_in_file(filename, search, new_value):
- textfile = open(filename, 'r')
+ textfile = open(filename, "r")
filetext = textfile.read()
textfile.close()
matches = re.findall(search, filetext)
with fileinput.FileInput(filename, inplace=True) as file:
for line in file:
- print(line.replace(matches[0], new_value), end='')
+ print(line.replace(matches[0], new_value), end="")
+
-if __name__ == '__main__':
+if __name__ == "__main__":
for repo in TO_BE_UPDATED:
- last_commit_hash = get_last_commit(repo['feed_url'])
- replace_in_file(repo['file_name'], repo['pattern'], last_commit_hash)
+ last_commit_hash = get_last_commit(repo["feed_url"])
+ replace_in_file(repo["file_name"], repo["pattern"], last_commit_hash)
diff --git a/server/macros/replace_words.py b/server/macros/replace_words.py
index 28246be2f1..7847c6dcca 100644
--- a/server/macros/replace_words.py
+++ b/server/macros/replace_words.py
@@ -64,7 +64,9 @@ def do_find_replace(input_string, words_list):
while re.search(pattern, input_string, flags=re.IGNORECASE):
# get the original string from the input
- original = re.search(pattern, input_string, flags=re.IGNORECASE).group(0)
+ original = re.search(pattern, input_string, flags=re.IGNORECASE).group(
+ 0
+ )
replacement = repl(word.get("replacement", ""), original)
if found_list.get(original):
break
diff --git a/server/pesacheck/ingest/ghost_feeding_service.py b/server/pesacheck/ingest/ghost_feeding_service.py
index d38e791535..00c9051dc7 100644
--- a/server/pesacheck/ingest/ghost_feeding_service.py
+++ b/server/pesacheck/ingest/ghost_feeding_service.py
@@ -57,7 +57,9 @@ async def _update(self, provider, update):
)
return
- for filename in await get_sorted_files(self.path, sort_by=FileSortAttributes.created):
+ for filename in await get_sorted_files(
+ self.path, sort_by=FileSortAttributes.created
+ ):
last_updated = None
try:
file_path = os.path.join(self.path, filename)
@@ -66,8 +68,12 @@ async def _update(self, provider, update):
last_updated = self.get_last_updated(file_path)
- if not self.is_latest_content(last_updated, provider.get("last_updated")):
- await self.move_file(self.path, filename, provider=provider, success=False)
+ if not self.is_latest_content(
+ last_updated, provider.get("last_updated")
+ ):
+ await self.move_file(
+ self.path, filename, provider=provider, success=False
+ )
continue
if await self.is_empty(file_path):
@@ -87,12 +93,18 @@ async def _update(self, provider, update):
break
yield batch
- await self.move_file(self.path, filename, provider=provider, success=True)
+ await self.move_file(
+ self.path, filename, provider=provider, success=True
+ )
except Exception as ex:
if last_updated and self.is_old_content(last_updated):
- await self.move_file(self.path, filename, provider=provider, success=False)
- raise ParserError.parseFileError("{}-{}".format(provider["name"], self.NAME), filename, ex, provider)
+ await self.move_file(
+ self.path, filename, provider=provider, success=False
+ )
+ raise ParserError.parseFileError(
+ "{}-{}".format(provider["name"], self.NAME), filename, ex, provider
+ )
push_notification("ingest:update")
diff --git a/server/pesacheck/ingest/ghost_parser.py b/server/pesacheck/ingest/ghost_parser.py
index 17ad01e935..d3e40755d4 100644
--- a/server/pesacheck/ingest/ghost_parser.py
+++ b/server/pesacheck/ingest/ghost_parser.py
@@ -1,23 +1,30 @@
-import json
import hashlib
+import json
import logging
import random
import time
-import unicodedata
from copy import deepcopy
-from urllib.parse import urlparse
-
from datetime import datetime, timezone
+from urllib.parse import urlparse
from superdesk.errors import ParserError
from superdesk.etree import parse_html
-from superdesk.metadata.utils import generate_guid
from superdesk.io.feed_parsers import FileFeedParser
from superdesk.io.registry import register_feed_parser
from superdesk.media.renditions import update_renditions
-from superdesk.metadata.item import FORMAT, GUID_FIELD, GUID_TAG, ITEM_TYPE, CONTENT_TYPE, FORMATS
+from superdesk.metadata.item import (
+ CONTENT_TYPE,
+ FORMAT,
+ FORMATS,
+ GUID_FIELD,
+ GUID_TAG,
+ ITEM_TYPE,
+)
+from superdesk.metadata.utils import generate_guid
+from superdesk.text_utils import get_text
from superdesk.utc import utcnow
+from pesacheck.language import detect_language, normalise_language_code
logger = logging.getLogger(__name__)
@@ -127,28 +134,27 @@ def _sleep_before_next_fetch(self, min_interval):
def _mark_fetch_done(self):
self._last_image_fetch_ts = time.monotonic()
- def _guess_language(self, text):
- """Guess language from text: returns 'am', 'fr', or 'en'.
+ def _parse_language(self, post, tags, text):
+ """Resolve the item language from ``locale``, then tags, then body text.
- Ethiopic script detection is used first (langdetect has no Amharic model).
- langdetect is then used to distinguish English from French; anything
- unrecognised falls back to English.
+ Ghost's own ``locale`` is authoritative when set, but PesaCheck's export
+ leaves it null on every post, so in practice the language comes from the
+ post's language tag. Posts predating that tagging convention fall
+ through to text classification.
"""
- if not text:
- return None
- # Detect Amharic by Ethiopic Unicode block (U+1200–U+137F, etc.)
- for ch in text:
- if "ETHIOPIC" in unicodedata.name(ch, ""):
- return "am"
- try:
- from langdetect import detect
+ locale = normalise_language_code(post.get("locale"))
+ if locale:
+ return locale
- detected = detect(text)
- if detected in ("en", "fr"):
- return detected
- except Exception:
- pass
- return "en"
+ # Offer both the display name and the slug: either may be the form that
+ # names the language ("Afaan Oromo" / "afaan-oromo").
+ tag_labels = [
+ (tag["sort_order"], label)
+ for tag in tags
+ for label in (tag["name"], tag["slug"])
+ ]
+
+ return detect_language(tag_labels, text)
def can_parse(self, file_path):
try:
@@ -203,7 +209,9 @@ def _fetch_renditions_with_retry(self, association, url):
if attempt == policy["retries"]:
break
- delay = (policy["base_backoff"] * attempt) + random.uniform(0, _IMAGE_FETCH_JITTER_SECONDS)
+ delay = (policy["base_backoff"] * attempt) + random.uniform(
+ 0, _IMAGE_FETCH_JITTER_SECONDS
+ )
logger.warning(
"Image fetch failed for %s (attempt %s/%s), retrying in %.2fs: %s",
url,
@@ -216,11 +224,16 @@ def _fetch_renditions_with_retry(self, association, url):
if policy["failure_cooldown"] > 0:
# After exhausting retries, cool down before the next image to reduce cascading failures.
- time.sleep(policy["failure_cooldown"] + random.uniform(0, _IMAGE_FETCH_JITTER_SECONDS))
+ time.sleep(
+ policy["failure_cooldown"]
+ + random.uniform(0, _IMAGE_FETCH_JITTER_SECONDS)
+ )
raise last_error
- def _add_image(self, item, url, alt_text="", description_text="", is_featured=False):
+ def _add_image(
+ self, item, url, alt_text="", description_text="", is_featured=False
+ ):
"""Fetch image, attach it as an association, and return the local storage href (or None)."""
associations = item.setdefault("associations", {})
association = {
@@ -283,7 +296,9 @@ def _parse_inline_images(self, item, html):
if local_href:
url_rewrites[src] = local_href
except Exception as e:
- logger.warning("Failed to parse inline image %s: %s", img.get("src", "unknown"), e)
+ logger.warning(
+ "Failed to parse inline image %s: %s", img.get("src", "unknown"), e
+ )
if url_rewrites:
body = item.get("body_html") or ""
@@ -312,14 +327,18 @@ def _parse_date(self, value):
def _parse_post(self, post, authors_by_post, tags_by_post):
post_id = post.get("id", "")
- authors = sorted(authors_by_post.get(post_id, []), key=lambda x: x["sort_order"])
+ authors = sorted(
+ authors_by_post.get(post_id, []), key=lambda x: x["sort_order"]
+ )
byline = ", ".join(a["name"] for a in authors if a.get("name"))
tags = sorted(tags_by_post.get(post_id, []), key=lambda x: x["sort_order"])
keywords = [t["name"] for t in tags if t.get("name")]
firstcreated = self._parse_date(post.get("created_at"))
- versioncreated = self._parse_date(post.get("published_at") or post.get("updated_at"))
+ versioncreated = self._parse_date(
+ post.get("published_at") or post.get("updated_at")
+ )
html = post.get("html") or ""
if self._ghost_url and html:
@@ -340,13 +359,12 @@ def _parse_post(self, post, authors_by_post, tags_by_post):
"versioncreated": versioncreated,
}
- locale = post.get("locale")
- if locale:
- item["language"] = locale
- else:
- guessed = self._guess_language(html or post.get("title") or "")
- if guessed:
- item["language"] = guessed
+ # Ghost exports a markup-free rendering of the body; prefer it for
+ # language detection so HTML tag names don't dilute the word counts.
+ # Older exports omit it, so strip the markup ourselves in that case.
+ body_text = post.get("plaintext") or get_text(html, content="html")
+ sample = " ".join(part for part in (post.get("title") or "", body_text) if part)
+ item["language"] = self._parse_language(post, tags, sample)
self._parse_feature_image(item, post)
self._parse_inline_images(item, html)
@@ -361,7 +379,9 @@ def iter_items(self, file_path, provider=None):
"""Parse a Ghost JSON export file and yield Superdesk items one at a time."""
self._image_assoc_cache = {}
self._last_image_fetch_ts = 0.0
- self._ghost_url = ((provider or {}).get("config", {}).get("url") or "").rstrip("/")
+ self._ghost_url = ((provider or {}).get("config", {}).get("url") or "").rstrip(
+ "/"
+ )
try:
with open(file_path, "r", encoding="utf-8") as f:
data = json.load(f)
@@ -402,7 +422,12 @@ def iter_items(self, file_path, provider=None):
tags_by_post.setdefault(pid, []).append(
{
"name": tag.get("name", ""),
- "sort_order": pt.get("sort_order", 0),
+ # Ghost slugs are the stable identifier, and the language
+ # tag is matched on either form.
+ "slug": tag.get("slug", ""),
+ # Coerce a null sort_order so the sort below can't blow
+ # up comparing None to an int.
+ "sort_order": pt.get("sort_order") or 0,
}
)
diff --git a/server/pesacheck/ingest/medium_parser.py b/server/pesacheck/ingest/medium_parser.py
index 1056313fc3..3cf18c8f8d 100644
--- a/server/pesacheck/ingest/medium_parser.py
+++ b/server/pesacheck/ingest/medium_parser.py
@@ -11,7 +11,14 @@
from superdesk.io.feed_parsers import FileFeedParser
from superdesk.io.registry import register_feed_parser
from superdesk.media.renditions import update_renditions
-from superdesk.metadata.item import FORMAT, GUID_FIELD, GUID_TAG, ITEM_TYPE, CONTENT_TYPE, FORMATS
+from superdesk.metadata.item import (
+ FORMAT,
+ GUID_FIELD,
+ GUID_TAG,
+ ITEM_TYPE,
+ CONTENT_TYPE,
+ FORMATS,
+)
logger = logging.getLogger(__name__)
@@ -54,10 +61,13 @@ def can_parse(self, file_path):
html_content = f.read().decode("utf-8")
root = parse_html(html_content, "html")
- exported_from_medium = 'Exported from Medium' in to_string(
- root.find(".//footer"), method="html"
+ exported_from_medium = (
+ 'Exported from Medium'
+ in to_string(root.find(".//footer"), method="html")
+ )
+ article_body_found = (
+ root.find(".//section[@data-field='body']") is not None
)
- article_body_found = root.find(".//section[@data-field='body']") is not None
return exported_from_medium and article_body_found
except Exception:
@@ -78,7 +88,9 @@ def _generate_image_guid(self, url):
guid_hash = hashlib.sha1(url.encode("utf8")).hexdigest()
return generate_guid(type=GUID_TAG, id=guid_hash + "-image")
- def _add_image(self, item, url, alt_text="", description_text="", is_featured=False):
+ def _add_image(
+ self, item, url, alt_text="", description_text="", is_featured=False
+ ):
"""Add an image to the item's associations.
:param item: The item dictionary to add the image to
@@ -132,7 +144,9 @@ def parse_images(self, item, article):
self._add_image(item, src, alt_text, description_text, is_featured)
except Exception as e:
- logger.warning(f"Failed to parse image {img.get('src', 'unknown')}: {e}")
+ logger.warning(
+ f"Failed to parse image {img.get('src', 'unknown')}: {e}"
+ )
continue
async def parse(self, file_path, provider=None):
@@ -168,7 +182,9 @@ async def parse(self, file_path, provider=None):
firstcreated = utcnow()
text_nodes = article.xpath(".//text()")
- word_count = sum(len(text.strip().split()) for text in text_nodes if text.strip())
+ word_count = sum(
+ len(text.strip().split()) for text in text_nodes if text.strip()
+ )
item = {
ITEM_TYPE: CONTENT_TYPE.TEXT,
diff --git a/server/pesacheck/language.py b/server/pesacheck/language.py
new file mode 100644
index 0000000000..6c2e1d00dd
--- /dev/null
+++ b/server/pesacheck/language.py
@@ -0,0 +1,266 @@
+"""Language identification for PesaCheck ingest.
+
+PesaCheck publishes in six languages, and Superdesk needs an ISO 639-1 code on
+every ingested item. Two strategies are used, in order:
+
+1. :func:`detect_language_from_tags` — recent posts carry an explicit language
+ tag (``English``, ``Kiswahili``, ``Afaan Oromo``, ...). This is editorial
+ metadata, so it is authoritative and is always tried first.
+2. :func:`detect_language_from_text` — older posts predate the tagging
+ convention, so the body text is classified directly.
+
+The text classifier is deliberately closed over just these six languages. A
+general-purpose detector is a poor fit here: ``langdetect`` has no model for
+Afaan Oromo at all and mislabels it as Somali (both are Cushitic and share the
+doubled-vowel Latin orthography), and it raises on Amharic. Scoring only the
+languages we actually publish in avoids losing articles to a seventh language
+that was never an option.
+"""
+
+import logging
+import re
+import unicodedata
+from collections.abc import Iterable, Sequence
+from typing import TypeAlias
+
+from .language_markers import (
+ AMHARIC,
+ ENGLISH,
+ FRENCH,
+ KISWAHILI,
+ MARKERS,
+ OROMO,
+ SOMALI,
+ LanguageCode,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# A ``(sort_order, label)`` pair as Ghost exports it, where ``label`` is a tag
+# display name or slug. Both come straight from JSON, so both may be null and
+# neither is guaranteed to be a string.
+TagLabel: TypeAlias = tuple[int | None, object]
+
+
+# The languages PesaCheck publishes in, as ISO 639-1 codes.
+SUPPORTED_LANGUAGES: tuple[LanguageCode, ...] = (
+ ENGLISH,
+ KISWAHILI,
+ FRENCH,
+ SOMALI,
+ OROMO,
+ AMHARIC,
+)
+
+# Used when detection is impossible (no tags, empty body). English is the
+# house language, so it is the least surprising thing to fall back to.
+DEFAULT_LANGUAGE: LanguageCode = ENGLISH
+
+
+# Ghost tag names/slugs that identify a language, normalised by
+# ``_normalise_tag``. Both the English name and the endonym are accepted, along
+# with ISO 639-1/639-2 codes in case a tag is ever created from a code.
+#
+# Matching is exact on the whole normalised string so that language tags are
+# never confused with the country tags they sit next to: PesaCheck tags posts
+# with both ``French``/``France`` and ``Somali``/``Somalia``.
+#
+# Deliberately absent: bare ``Oromo``. Unlike ``Amharic`` or ``Kiswahili`` it is
+# an ethnonym before it is a language name, and it is a plausible topic tag on
+# an English article about Oromo people. PesaCheck's convention is the
+# unambiguous ``Afaan Oromo``, and Oromo prose is recognised well by the text
+# classifier, so an unmatched tag degrades safely.
+_LANGUAGE_TAGS: dict[str, LanguageCode] = {
+ "english": ENGLISH,
+ "en": ENGLISH,
+ "eng": ENGLISH,
+ "kiswahili": KISWAHILI,
+ "swahili": KISWAHILI,
+ "sw": KISWAHILI,
+ "swa": KISWAHILI,
+ "french": FRENCH,
+ "francais": FRENCH,
+ "français": FRENCH,
+ "fr": FRENCH,
+ "fra": FRENCH,
+ "somali": SOMALI,
+ "soomaali": SOMALI,
+ "af soomaali": SOMALI,
+ "so": SOMALI,
+ "som": SOMALI,
+ "afaan oromo": OROMO,
+ "afaan oromoo": OROMO,
+ "afan oromo": OROMO,
+ "oromiffa": OROMO,
+ "oromifa": OROMO,
+ "oromoo": OROMO,
+ "om": OROMO,
+ "orm": OROMO,
+ "amharic": AMHARIC,
+ "amarigna": AMHARIC,
+ "amharigna": AMHARIC,
+ "አማርኛ": AMHARIC,
+ "am": AMHARIC,
+ "amh": AMHARIC,
+}
+
+# Runs of Ethiopic characters, covering the block plus its supplement and
+# extensions (U+1200-U+139F merges Ethiopic and Ethiopic Supplement). Amharic is
+# the only Ethiopic-script language PesaCheck publishes in.
+#
+# Both of these count characters by matching *runs* and summing their lengths,
+# which keeps the scan in C: a per-character Python loop over a multi-kilobyte
+# body cost ~1.2ms per post, around 90% of the whole text-detection path.
+_ETHIOPIC_RUN_RE: re.Pattern[str] = re.compile(
+ "["
+ "ሀ-" # Ethiopic + Ethiopic Supplement
+ "ⶀ-" # Ethiopic Extended
+ "-" # Ethiopic Extended-A
+ "\U0001e7e0-\U0001e7ff" # Ethiopic Extended-B
+ "]+"
+)
+
+# Approximates ``str.isalpha`` per character: word characters excluding digits
+# and underscore.
+_ALPHABETIC_RUN_RE: re.Pattern[str] = re.compile(r"[^\W\d_]+", re.UNICODE)
+
+# Share of cased letters that must be Ethiopic before a post is called Amharic.
+# Amharic posts in the reference export are >99% Ethiopic, and Latin-script
+# posts contain none, so there is a wide margin here. A *share* rather than
+# "contains any Ethiopic character" matters: English and Afaan Oromo articles
+# about Ethiopia routinely quote an Amharic phrase, and one quote must not
+# relabel the whole article.
+_ETHIOPIC_SHARE_THRESHOLD: float = 0.15
+
+# Coverage a native document is expected to reach for each language, used to
+# put the raw scores on a comparable scale. Without this the argmax just
+# favours whichever marker list happens to cover the most running text.
+_EXPECTED_COVERAGE: dict[LanguageCode, float] = {
+ ENGLISH: 0.35,
+ FRENCH: 0.36,
+ KISWAHILI: 0.28,
+ SOMALI: 0.26,
+ OROMO: 0.25,
+}
+
+# Tokens are lowercased runs of letters, keeping the apostrophes that Afaan
+# Oromo uses for the glottal stop ("ta'e") so those stay single tokens.
+_TOKEN_RE: re.Pattern[str] = re.compile(r"[^\W\d_]+(?:['’ʼ][^\W\d_]+)*", re.UNICODE)
+
+# Below this many tokens the marker counts are too sparse to separate the
+# candidates, so text detection declines rather than guessing.
+_MIN_TOKENS: int = 8
+
+# The winner must beat the runner-up by this ratio. Fact checks quote the
+# claim they are debunking, so a Somali or Oromo article always contains some
+# English; requiring a margin keeps those quotes from tipping the result.
+_MIN_MARGIN: float = 1.15
+
+
+def _normalise_tag(value: object) -> str:
+ """Casefold a tag and collapse punctuation/whitespace to single spaces."""
+ if not value:
+ return ""
+ text = unicodedata.normalize("NFC", str(value)).casefold()
+ return re.sub(r"[\s\-_]+", " ", text).strip()
+
+
+def normalise_language_code(value: object) -> LanguageCode | None:
+ """Normalise a Ghost ``locale`` to a language code, or return ``None``.
+
+ Accepts anything BCP 47-ish (``en``, ``en-US``, ``am_ET``) and keeps only
+ the base subtag. Codes outside :data:`SUPPORTED_LANGUAGES` are passed
+ through rather than dropped, so a locale PesaCheck starts publishing in
+ later still reaches Superdesk.
+ """
+ if not value:
+ return None
+ base = _normalise_tag(re.split(r"[-_]", str(value).strip())[0])
+ if not base:
+ return None
+ return _LANGUAGE_TAGS.get(base, base)
+
+
+def detect_language_from_tags(tags: Iterable[TagLabel]) -> LanguageCode | None:
+ """Return the language code named by a post's tags, or ``None``.
+
+ ``tags`` is an iterable of ``(sort_order, name)`` pairs; Ghost exposes both
+ a display name and a slug per tag and either may be passed. When a post
+ carries more than one language tag the lowest ``sort_order`` wins, matching
+ Ghost's notion of a primary tag — in practice PesaCheck puts the language
+ first, so this is also what an editor would expect.
+ """
+ matches: list[tuple[int, LanguageCode]] = []
+ for sort_order, name in tags:
+ code = _LANGUAGE_TAGS.get(_normalise_tag(name))
+ if code:
+ matches.append((sort_order if sort_order is not None else 0, code))
+
+ if not matches:
+ return None
+ return min(matches, key=lambda pair: pair[0])[1]
+
+
+def _count_matched_chars(pattern: re.Pattern[str], text: str) -> int:
+ return sum(len(run) for run in pattern.findall(text))
+
+
+def _ethiopic_share(text: str) -> float:
+ """Fraction of the alphabetic characters in ``text`` that are Ethiopic."""
+ alphabetic = _count_matched_chars(_ALPHABETIC_RUN_RE, text)
+ if not alphabetic:
+ return 0.0
+ return _count_matched_chars(_ETHIOPIC_RUN_RE, text) / alphabetic
+
+
+def _tokenize(text: str) -> list[str]:
+ return _TOKEN_RE.findall(text.casefold())
+
+
+def _score_markers(tokens: Sequence[str]) -> dict[LanguageCode, float]:
+ """Return ``{language: calibrated coverage}`` for the Latin-script set."""
+ total = len(tokens)
+ scores: dict[LanguageCode, float] = {}
+ for language, markers in MARKERS.items():
+ hits = sum(1 for token in tokens if token in markers)
+ scores[language] = (hits / total) / _EXPECTED_COVERAGE[language]
+ return scores
+
+
+def detect_language_from_text(text: str | None) -> LanguageCode | None:
+ """Classify ``text`` as one of :data:`SUPPORTED_LANGUAGES`, or ``None``.
+
+ ``None`` means "not enough evidence" — too little text, or no candidate
+ clearly ahead — and lets the caller decide what to do rather than having a
+ coin-flip recorded as a real language.
+ """
+ if not text or not text.strip():
+ return None
+
+ if _ethiopic_share(text) >= _ETHIOPIC_SHARE_THRESHOLD:
+ return AMHARIC
+
+ tokens = _tokenize(text)
+ if len(tokens) < _MIN_TOKENS:
+ return None
+
+ scores = _score_markers(tokens)
+ ranked = sorted(scores.items(), key=lambda pair: pair[1], reverse=True)
+ (best, best_score), (_, runner_up_score) = ranked[0], ranked[1]
+
+ if best_score <= 0:
+ return None
+ if runner_up_score > 0 and best_score / runner_up_score < _MIN_MARGIN:
+ logger.debug("Language detection inconclusive, scores=%s", ranked[:3])
+ return None
+ return best
+
+
+def detect_language(
+ tags: Iterable[TagLabel],
+ text: str | None = "",
+ default: LanguageCode | None = DEFAULT_LANGUAGE,
+) -> LanguageCode | None:
+ """Best-effort language for a post: tags first, then text, then ``default``."""
+ return detect_language_from_tags(tags) or detect_language_from_text(text) or default
diff --git a/server/pesacheck/language_markers.py b/server/pesacheck/language_markers.py
new file mode 100644
index 0000000000..3f460f1b95
--- /dev/null
+++ b/server/pesacheck/language_markers.py
@@ -0,0 +1,493 @@
+from typing import TypeAlias
+
+# An ISO 639-1 language code. Not narrowed to a ``Literal`` of the six
+# supported languages on purpose: :func:`normalise_language_code` passes
+# unrecognised codes through, so the codes in flight are not a closed set.
+LanguageCode: TypeAlias = str
+
+
+ENGLISH: LanguageCode = "en"
+KISWAHILI: LanguageCode = "sw"
+FRENCH: LanguageCode = "fr"
+SOMALI: LanguageCode = "so"
+OROMO: LanguageCode = "om"
+AMHARIC: LanguageCode = "am"
+
+# Latin-script detection
+# ----------------------------------------------------------------------
+
+# High-frequency function words per language. Content words are avoided except
+# where a whole domain vocabulary is stable across PesaCheck's output ("video",
+# "image", "claim"), because function words survive translation of the subject
+# matter and are what actually separates these languages.
+#
+# Words shared across candidates are simply left out of both lists rather than
+# down-weighted: Somali "in"/English "in", Somali "la"/French "la" and Swahili
+# "au"/French "au" carry no signal, so including them only adds noise.
+MARKERS: dict[LanguageCode, set[str]] = {
+ ENGLISH: {
+ "the",
+ "and",
+ "of",
+ "to",
+ "is",
+ "was",
+ "that",
+ "this",
+ "with",
+ "which",
+ "for",
+ "on",
+ "are",
+ "from",
+ "at",
+ "be",
+ "has",
+ "have",
+ "had",
+ "but",
+ "not",
+ "were",
+ "will",
+ "would",
+ "can",
+ "could",
+ "there",
+ "these",
+ "those",
+ "when",
+ "what",
+ "who",
+ "how",
+ "also",
+ "however",
+ "according",
+ "said",
+ "says",
+ "shows",
+ "showing",
+ "show",
+ "showed",
+ "image",
+ "video",
+ "photo",
+ "claim",
+ "claims",
+ "false",
+ "true",
+ "been",
+ "being",
+ "their",
+ "they",
+ "them",
+ "its",
+ "his",
+ "her",
+ "she",
+ "we",
+ "you",
+ "about",
+ "into",
+ "over",
+ "under",
+ "after",
+ "before",
+ "during",
+ "between",
+ "while",
+ "than",
+ "then",
+ "such",
+ "other",
+ "more",
+ "most",
+ "some",
+ "any",
+ "all",
+ "posted",
+ "post",
+ "shared",
+ "social",
+ "media",
+ "facebook",
+ "search",
+ "found",
+ "reverse",
+ "original",
+ "therefore",
+ "does",
+ "did",
+ "doesn",
+ "isn",
+ "wasn",
+ "authentic",
+ "misleading",
+ "verified",
+ "screenshot",
+ "caption",
+ },
+ FRENCH: {
+ "le",
+ "les",
+ "de",
+ "des",
+ "du",
+ "une",
+ "et",
+ "est",
+ "ne",
+ "pas",
+ "que",
+ "qui",
+ "dans",
+ "pour",
+ "sur",
+ "cette",
+ "ce",
+ "ces",
+ "cet",
+ "par",
+ "avec",
+ "aux",
+ "plus",
+ "mais",
+ "son",
+ "sa",
+ "ses",
+ "leur",
+ "leurs",
+ "été",
+ "être",
+ "ont",
+ "ils",
+ "elles",
+ "elle",
+ "nous",
+ "vous",
+ "se",
+ "où",
+ "donc",
+ "alors",
+ "ainsi",
+ "comme",
+ "tout",
+ "tous",
+ "toute",
+ "toutes",
+ "même",
+ "aussi",
+ "selon",
+ "publiée",
+ "publié",
+ "publication",
+ "vidéo",
+ "image",
+ "photo",
+ "montre",
+ "montrant",
+ "fausse",
+ "faux",
+ "vrai",
+ "vraie",
+ "sans",
+ "sous",
+ "entre",
+ "après",
+ "avant",
+ "pendant",
+ "depuis",
+ "très",
+ "bien",
+ "encore",
+ "déjà",
+ "cependant",
+ "toutefois",
+ "néanmoins",
+ "effet",
+ "fait",
+ "faits",
+ "dit",
+ "déclaré",
+ "affirme",
+ "capture",
+ "écran",
+ "réseaux",
+ "sociaux",
+ "internaute",
+ "internautes",
+ "recherche",
+ "inversée",
+ "trompeur",
+ "trompeuse",
+ "vérification",
+ "montrent",
+ "était",
+ "étaient",
+ "sont",
+ "avait",
+ "avaient",
+ "cela",
+ "lui",
+ "nos",
+ "vos",
+ "notre",
+ "votre",
+ },
+ KISWAHILI: {
+ "na",
+ "ya",
+ "wa",
+ "ni",
+ "kwa",
+ "katika",
+ "hii",
+ "huu",
+ "hiyo",
+ "huo",
+ "hizo",
+ "hao",
+ "si",
+ "kuwa",
+ "kwamba",
+ "ambayo",
+ "ambao",
+ "ambaye",
+ "ilikuwa",
+ "alikuwa",
+ "walikuwa",
+ "hakuna",
+ "lakini",
+ "hivyo",
+ "hata",
+ "pia",
+ "ili",
+ "ndani",
+ "juu",
+ "chini",
+ "baada",
+ "kabla",
+ "alisema",
+ "walisema",
+ "imeonyesha",
+ "inaonyesha",
+ "uchunguzi",
+ "madai",
+ "taarifa",
+ "mtandao",
+ "mitandao",
+ "kijamii",
+ "picha",
+ "mwaka",
+ "siku",
+ "watu",
+ "serikali",
+ "rais",
+ "wakati",
+ "kama",
+ "kwenye",
+ "za",
+ "cha",
+ "vya",
+ "wale",
+ "yake",
+ "yao",
+ "zake",
+ "huyu",
+ "wengi",
+ "sana",
+ "sasa",
+ "bado",
+ "tena",
+ "kila",
+ "hakika",
+ "kweli",
+ "uongo",
+ "ukweli",
+ "chapisho",
+ "machapisho",
+ "ujumbe",
+ "habari",
+ "kutoka",
+ "kuhusu",
+ "kufanya",
+ "ndiyo",
+ "yenye",
+ "hadi",
+ "zaidi",
+ "wako",
+ "yeye",
+ "mmoja",
+ "mbili",
+ "kwanza",
+ "video",
+ "ilionekana",
+ "sio",
+ "hapa",
+ "huku",
+ },
+ SOMALI: {
+ "waxa",
+ "waxaa",
+ "waxaan",
+ "ayaa",
+ "ayay",
+ "ayuu",
+ "oo",
+ "iyo",
+ "ee",
+ "aha",
+ "inay",
+ "inuu",
+ "kale",
+ "sidoo",
+ "laakiin",
+ "wuxuu",
+ "waa",
+ "lagu",
+ "loo",
+ "uu",
+ "ay",
+ "sheegay",
+ "sheegtay",
+ "muuqaalka",
+ "muuqaalkan",
+ "sawirka",
+ "sawirkan",
+ "warbaahinta",
+ "xaqiijiyay",
+ "been",
+ "iyaga",
+ "kuwaas",
+ "taas",
+ "taasi",
+ "kaas",
+ "hadda",
+ "sida",
+ "marka",
+ "halkaas",
+ "dowladda",
+ "dawladda",
+ "madaxweynaha",
+ "shirkadda",
+ "bogga",
+ "barta",
+ "bulshada",
+ "xayeysiinta",
+ "qoraalka",
+ "wararka",
+ "markaana",
+ "sidaas",
+ "maaha",
+ "kuwa",
+ "wax",
+ "aad",
+ "badan",
+ "kama",
+ "ugu",
+ "isku",
+ "iyada",
+ "laakin",
+ "haddii",
+ "balse",
+ "warbixinta",
+ "baaritaan",
+ "baaritaanka",
+ "xogta",
+ "dhexe",
+ "soo",
+ "kordhay",
+ "muuqaal",
+ "sawir",
+ "faafiyay",
+ "baahiyay",
+ },
+ OROMO: {
+ "kun",
+ "kana",
+ "kanaa",
+ "kanaaf",
+ "kanaan",
+ "irratti",
+ "irra",
+ "keessatti",
+ "keessa",
+ "akka",
+ "hin",
+ "waan",
+ "dha",
+ "jira",
+ "jiru",
+ "jedha",
+ "jedhu",
+ "jedhan",
+ "jedhame",
+ "isaa",
+ "isaan",
+ "ishee",
+ "namoota",
+ "namoonni",
+ "fi",
+ "yoo",
+ "malee",
+ "garuu",
+ "akkasumas",
+ "ykn",
+ "viidiyoo",
+ "suuraa",
+ "odeeffannoo",
+ "gabaasa",
+ "gabaafame",
+ "mootummaa",
+ "walitti",
+ "hedduu",
+ "immoo",
+ "sana",
+ "tokko",
+ "lama",
+ "waggaa",
+ "guyyaa",
+ "torban",
+ "bara",
+ "soba",
+ "dhugaa",
+ "miidiyaa",
+ "hawaasummaa",
+ "maxxansi",
+ "maxxansa",
+ "barreeffamni",
+ "barreeffama",
+ "agarsiisa",
+ "agarsiisu",
+ "raawwatame",
+ "argaman",
+ "ummata",
+ "uummata",
+ "naannoo",
+ "godina",
+ "aanaa",
+ "hojii",
+ "dhaabbata",
+ "itti",
+ "ittiin",
+ "isa",
+ "hunda",
+ "hundaa",
+ "erga",
+ "booda",
+ "dura",
+ "wayita",
+ "yeroo",
+ "qabu",
+ "qaba",
+ "taʼe",
+ "ta'e",
+ "taʼuu",
+ "ta'uu",
+ "hiriyaa",
+ "mirkaneesse",
+ "mirkaneeffame",
+ "seenaa",
+ "biyya",
+ "biyyoota",
+ },
+}
diff --git a/server/requirements.in b/server/requirements.in
index 256e5b7202..b8f9f94b6d 100755
--- a/server/requirements.in
+++ b/server/requirements.in
@@ -1,6 +1,5 @@
honcho
celery-redbeat
-langdetect
git+https://github.com/superdesk/superdesk-core.git@develop#egg=superdesk-core
git+https://github.com/superdesk/superdesk-planning.git@develop#egg=superdesk-planning
diff --git a/server/requirements.txt b/server/requirements.txt
index ac2cbe5eed..70bf60e1dd 100644
--- a/server/requirements.txt
+++ b/server/requirements.txt
@@ -185,8 +185,6 @@ jwcrypto==1.5.8
# python-jwt
kombu[redis]==5.6.2
# via celery
-langdetect==1.0.9
- # via -r requirements.in
ldap3==2.9.1
# via superdesk-core
lxml==5.3.2
diff --git a/server/settings.py b/server/settings.py
index 678a95b3c7..9f705cbb56 100644
--- a/server/settings.py
+++ b/server/settings.py
@@ -179,8 +179,18 @@
"picture": {
"headline": {"order": 1, "sdWidth": "full", "editor3": True},
"alt_text": {"order": 2, "sdWidth": "full", "textarea": True, "editor3": True},
- "description_text": {"order": 3, "sdWidth": "full", "textarea": True, "editor3": True},
- "creditline": {"order": 4, "sdWidth": "full", "displayOnMediaEditor": True, "editor3": True},
+ "description_text": {
+ "order": 3,
+ "sdWidth": "full",
+ "textarea": True,
+ "editor3": True,
+ },
+ "creditline": {
+ "order": 4,
+ "sdWidth": "full",
+ "displayOnMediaEditor": True,
+ "editor3": True,
+ },
"copyrightholder": {"order": 5, "displayOnMediaEditor": True, "editor3": True},
"usageterms": {"order": 6, "displayOnMediaEditor": True},
"copyrightnotice": {"order": 7, "displayOnMediaEditor": True},
diff --git a/server/tests/fixtures/ghost_export/ghost_export_languages.json b/server/tests/fixtures/ghost_export/ghost_export_languages.json
new file mode 100644
index 0000000000..69521dc472
--- /dev/null
+++ b/server/tests/fixtures/ghost_export/ghost_export_languages.json
@@ -0,0 +1,133 @@
+{
+ "db": [
+ {
+ "data": {
+ "posts": [
+ {
+ "id": "lang_001",
+ "uuid": "11111111-0001-0001-0001-111111111111",
+ "title": "UONGO: Chapisho hili la Facebook ni la uongo",
+ "slug": "uongo-chapisho-hili",
+ "html": "Chapisho hili la Facebook linalodai kuwa serikali imetangaza siku ya mapumziko ni la uongo.
",
+ "plaintext": "Chapisho hili la Facebook linalodai kuwa serikali imetangaza siku ya mapumziko ni la uongo. Uchunguzi wetu umeonyesha kwamba picha hiyo ilikuwa imechapishwa mwaka 2019 na haihusiani na madai hayo. Taarifa hiyo haikuwa kwenye mitandao ya kijamii ya serikali.",
+ "feature_image": null,
+ "type": "post",
+ "status": "published",
+ "locale": null,
+ "created_at": "2026-01-05T10:00:00.000Z",
+ "updated_at": "2026-01-05T11:00:00.000Z",
+ "published_at": "2026-01-05T11:00:00.000Z",
+ "custom_excerpt": null
+ },
+ {
+ "id": "lang_002",
+ "uuid": "22222222-0002-0002-0002-222222222222",
+ "title": "BEEN: Muuqaalkan ma aha hub lagu qabtay",
+ "slug": "been-muuqaalkan-ma-aha",
+ "html": "Xayeysiintan lagu baahiyay Facebook waa KHIYAANO.
",
+ "plaintext": "Xayeysiintan lagu baahiyay Facebook ee lagu sheegay in shirkadda Hormuud ay bixinayso deyn ayaa ah KHIYAANO. Warbaahinta ayaa xaqiijiyay in sawirkan been abuur yahay, laakiin qoraalka wali waa la wadaagayaa oo waxaa lagu sheegay iyo midabada shirkadda.",
+ "feature_image": null,
+ "type": "post",
+ "status": "published",
+ "locale": null,
+ "created_at": "2026-01-06T10:00:00.000Z",
+ "updated_at": "2026-01-06T11:00:00.000Z",
+ "published_at": "2026-01-06T11:00:00.000Z",
+ "custom_excerpt": null
+ },
+ {
+ "id": "lang_003",
+ "uuid": "33333333-0003-0003-0003-333333333333",
+ "title": "SOBA: Maxxansi Facebook kun soba",
+ "slug": "soba-maxxansi-facebook-kun",
+ "html": "Maxxansi Facebook kun SOBA.
",
+ "plaintext": "Maxxansi Facebook viidiyoo haleellaa ummata Oromoo irratti raawwatame agarsiisa jedhu kun SOBA. Barreeffamni Afaan Oromoo akkana jedha, garuu odeeffannoon kun dhugaa hin qabu. Viidiyoon kun naannoo Oromiyaa keessatti waraabame kan jedhu mirkaneeffame hin jiru.",
+ "feature_image": null,
+ "type": "post",
+ "status": "published",
+ "locale": null,
+ "created_at": "2026-01-07T10:00:00.000Z",
+ "updated_at": "2026-01-07T11:00:00.000Z",
+ "published_at": "2026-01-07T11:00:00.000Z",
+ "custom_excerpt": null
+ },
+ {
+ "id": "lang_004",
+ "uuid": "44444444-0004-0004-0004-444444444444",
+ "title": "FAUX : Cette video ne montre pas des billets",
+ "slug": "faux-cette-video",
+ "html": "Cette video ne montre pas les faits allegues.
",
+ "plaintext": "Cette video qui montrerait des soldats tues par l'armee est PARTIELLEMENT FAUSSE. La sequence, publiee sur les reseaux sociaux, ne montre pas les faits allegues selon nos verifications.",
+ "feature_image": null,
+ "type": "post",
+ "status": "published",
+ "locale": "en-US",
+ "created_at": "2026-01-08T10:00:00.000Z",
+ "updated_at": "2026-01-08T11:00:00.000Z",
+ "published_at": "2026-01-08T11:00:00.000Z",
+ "custom_excerpt": null
+ },
+ {
+ "id": "lang_005",
+ "uuid": "55555555-0005-0005-0005-555555555555",
+ "title": "የተቀየረ: ይህ ምስል የተቀየረ ነው",
+ "slug": "yetekeyere-yih-mesel",
+ "html": "ይህ ልጥፍ የተቀየረ ነው።
",
+ "plaintext": "የኢትዮጵያ ኦርቶዶክስ ተዋህዶ ቤተክርስቲያን የሃይማኖት አባት በምርጫ ዘመቻ ላይ ሲሳተፉ ያሳያል በሚል ፌስቡክ ላይ ከምስል ጋር የተጋራው ይህ ልጥፍ የተቀየረ ነው። ጉግልን በመጠቀም የተደረገው የምስል ፍለጋ በመጣራት ላይ ያለው ፎቶ እንደተቀየረ አረጋግጧል።",
+ "feature_image": null,
+ "type": "post",
+ "status": "published",
+ "locale": null,
+ "created_at": "2026-01-09T10:00:00.000Z",
+ "updated_at": "2026-01-09T11:00:00.000Z",
+ "published_at": "2026-01-09T11:00:00.000Z",
+ "custom_excerpt": null
+ },
+ {
+ "id": "lang_006",
+ "uuid": "66666666-0006-0006-0006-666666666666",
+ "title": "HOAX: This poster offering instant loans is fake",
+ "slug": "hoax-this-poster",
+ "html": "This poster claiming instant loans is a HOAX.
",
+ "plaintext": "This poster claiming Hormuud Telecom is offering instant loans is a HOAX. The advert, shared on Facebook, shows amounts ranging from $50 to $200,000. However, a reverse image search shows that the poster was fabricated and the company has confirmed it did not issue any such offer.",
+ "feature_image": null,
+ "type": "post",
+ "status": "published",
+ "locale": null,
+ "created_at": "2026-01-10T10:00:00.000Z",
+ "updated_at": "2026-01-10T11:00:00.000Z",
+ "published_at": "2026-01-10T11:00:00.000Z",
+ "custom_excerpt": null
+ }
+ ],
+ "users": [],
+ "posts_authors": [],
+ "tags": [
+ {"id": "tag_sw", "name": "Kiswahili", "slug": "kiswahili"},
+ {"id": "tag_om", "name": "Afaan Oromo", "slug": "afaan-oromo"},
+ {"id": "tag_en", "name": "English", "slug": "english"},
+ {"id": "tag_fr", "name": "French", "slug": "french"},
+ {"id": "tag_kenya", "name": "Kenya", "slug": "kenya"},
+ {"id": "tag_somalia", "name": "Somalia", "slug": "somalia"},
+ {"id": "tag_ethiopia", "name": "Ethiopia", "slug": "ethiopia"},
+ {"id": "tag_short", "name": "Short Form", "slug": "short-form"}
+ ],
+ "posts_tags": [
+ {"post_id": "lang_001", "tag_id": "tag_sw", "sort_order": 0},
+ {"post_id": "lang_001", "tag_id": "tag_kenya", "sort_order": 1},
+
+ {"post_id": "lang_002", "tag_id": "tag_somalia", "sort_order": 0},
+ {"post_id": "lang_002", "tag_id": "tag_short", "sort_order": 1},
+
+ {"post_id": "lang_003", "tag_id": "tag_om", "sort_order": 0},
+ {"post_id": "lang_003", "tag_id": "tag_ethiopia", "sort_order": 1},
+
+ {"post_id": "lang_004", "tag_id": "tag_fr", "sort_order": 0},
+
+ {"post_id": "lang_006", "tag_id": "tag_en", "sort_order": 0},
+ {"post_id": "lang_006", "tag_id": "tag_somalia", "sort_order": 1}
+ ]
+ }
+ }
+ ]
+}
diff --git a/server/tests/ingest/test_ghost_parser.py b/server/tests/ingest/test_ghost_parser.py
index d868a53394..46befd72eb 100644
--- a/server/tests/ingest/test_ghost_parser.py
+++ b/server/tests/ingest/test_ghost_parser.py
@@ -1,14 +1,17 @@
-import os
import json
+import os
import tempfile
-
from unittest.mock import patch
-from superdesk.tests import TestCase
-from pesacheck.ingest.ghost_parser import GhostParser
+from pesacheck.ingest.ghost_parser import GhostParser
+from pesacheck.language import SUPPORTED_LANGUAGES
+from superdesk.tests import TestCase
-FIXTURE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../fixtures/ghost_export")
+FIXTURE_DIR = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)), "../fixtures/ghost_export"
+)
FIXTURE_PATH = os.path.join(FIXTURE_DIR, "ghost_export.json")
+LANGUAGES_FIXTURE_PATH = os.path.join(FIXTURE_DIR, "ghost_export_languages.json")
def _mock_update_renditions(item, url, old_item, **kwargs):
@@ -38,118 +41,236 @@ async def test_can_parse_rejects_missing_file(self):
# parse — field mapping
# ------------------------------------------------------------------
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_returns_only_published_posts(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
# draft (post_003) and page (post_004) should be excluded
self.assertEqual(len(items), 2)
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_headline_and_slugline(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa")
+ post1 = next(
+ i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa"
+ )
self.assertEqual(post1["headline"], "FAUX: This claim is false")
self.assertEqual(post1["slugline"], "faux-this-claim-is-false")
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_abstract(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa")
+ post1 = next(
+ i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa"
+ )
self.assertEqual(post1["abstract"], "A short description of the article.")
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_body_html(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa")
+ post1 = next(
+ i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa"
+ )
self.assertIn("This is the article body.
", post1["body_html"])
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_dates(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa")
+ post1 = next(
+ i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa"
+ )
self.assertEqual(post1["firstcreated"].isoformat(), "2025-11-01T10:00:00+00:00")
- self.assertEqual(post1["versioncreated"].isoformat(), "2025-11-01T11:00:00+00:00")
-
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ self.assertEqual(
+ post1["versioncreated"].isoformat(), "2025-11-01T11:00:00+00:00"
+ )
+
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_source(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
for item in items:
self.assertEqual(item["source"], "Ghost")
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_locale_mapped_to_language(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa")
+ post1 = next(
+ i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa"
+ )
self.assertEqual(post1["language"], "fr")
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_null_locale_guesses_language(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post2 = next(i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb")
- self.assertIn("language", post2)
+ post2 = next(
+ i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb"
+ )
+ self.assertEqual(post2["language"], "en")
+
+ # ------------------------------------------------------------------
+ # parse — language detection
+ # ------------------------------------------------------------------
+
+ async def _languages_by_guid(self):
+ items = await self.parser.parse(LANGUAGES_FIXTURE_PATH)
+ return {item["guid"]: item["language"] for item in items}
+
+ async def test_parse_language_from_tag(self):
+ languages = await self._languages_by_guid()
+ self.assertEqual(languages["11111111-0001-0001-0001-111111111111"], "sw")
+ self.assertEqual(languages["33333333-0003-0003-0003-333333333333"], "om")
+ self.assertEqual(languages["66666666-0006-0006-0006-666666666666"], "en")
+
+ async def test_parse_language_falls_back_to_text_when_untagged(self):
+ # lang_002 predates the language-tag convention: only "Somalia" and
+ # "Short Form" are tagged, so the body has to carry the language.
+ languages = await self._languages_by_guid()
+ self.assertEqual(languages["22222222-0002-0002-0002-222222222222"], "so")
+
+ async def test_parse_language_falls_back_to_text_without_any_tags(self):
+ languages = await self._languages_by_guid()
+ self.assertEqual(languages["55555555-0005-0005-0005-555555555555"], "am")
+
+ async def test_parse_language_prefers_locale_and_normalises_it(self):
+ # locale "en-US" outranks the French tag and is reduced to "en".
+ languages = await self._languages_by_guid()
+ self.assertEqual(languages["44444444-0004-0004-0004-444444444444"], "en")
+
+ async def test_parse_language_ignores_adjacent_country_tag(self):
+ # lang_006 is tagged English + Somalia; "Somalia" must not read as Somali.
+ languages = await self._languages_by_guid()
+ self.assertEqual(languages["66666666-0006-0006-0006-666666666666"], "en")
+
+ async def test_parse_language_set_on_every_item(self):
+ items = await self.parser.parse(LANGUAGES_FIXTURE_PATH)
+ self.assertEqual(len(items), 6)
+ for item in items:
+ self.assertIn(item["language"], SUPPORTED_LANGUAGES)
# ------------------------------------------------------------------
# parse — authors → byline
# ------------------------------------------------------------------
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_byline_multiple_authors_sorted(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa")
+ post1 = next(
+ i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa"
+ )
self.assertEqual(post1["byline"], "Alice Reporter, Bob Editor")
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_byline_single_author(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post2 = next(i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb")
+ post2 = next(
+ i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb"
+ )
self.assertEqual(post2["byline"], "Alice Reporter")
# ------------------------------------------------------------------
# parse — tags → keywords
# ------------------------------------------------------------------
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_keywords(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa")
+ post1 = next(
+ i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa"
+ )
self.assertEqual(post1["keywords"], ["Fact Check", "Africa"])
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_keywords_single_tag(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post2 = next(i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb")
+ post2 = next(
+ i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb"
+ )
self.assertEqual(post2["keywords"], ["Fact Check"])
# ------------------------------------------------------------------
# parse — images
# ------------------------------------------------------------------
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_feature_image(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa")
+ post1 = next(
+ i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa"
+ )
self.assertIn("associations", post1)
featuremedia = post1["associations"]["featuremedia"]
self.assertEqual(featuremedia["type"], "picture")
self.assertTrue(featuremedia["guid"].endswith("-image"))
- self.assertEqual(featuremedia["renditions"]["original"]["href"], "https://example.com/feature.jpg")
-
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ self.assertEqual(
+ featuremedia["renditions"]["original"]["href"],
+ "https://example.com/feature.jpg",
+ )
+
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_inline_image_added_as_embedded(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa")
+ post1 = next(
+ i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa"
+ )
associations = post1["associations"]
embedded_keys = [k for k in associations if k.startswith("embedded")]
self.assertEqual(len(embedded_keys), 1)
embedded = associations[embedded_keys[0]]
- self.assertEqual(embedded["renditions"]["original"]["href"], "https://example.com/inline.jpg")
+ self.assertEqual(
+ embedded["renditions"]["original"]["href"], "https://example.com/inline.jpg"
+ )
self.assertEqual(embedded["alt_text"], "An inline image")
self.assertEqual(embedded["description_text"], "Image caption here")
- @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions)
+ @patch(
+ "pesacheck.ingest.ghost_parser.update_renditions",
+ side_effect=_mock_update_renditions,
+ )
async def test_parse_no_feature_image_no_associations(self, _mock):
items = await self.parser.parse(FIXTURE_PATH)
- post2 = next(i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb")
+ post2 = next(
+ i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb"
+ )
self.assertNotIn("associations", post2)
# ------------------------------------------------------------------
diff --git a/server/tests/ingest/test_medium_parser.py b/server/tests/ingest/test_medium_parser.py
index 3514127e3a..0fcef54ca9 100644
--- a/server/tests/ingest/test_medium_parser.py
+++ b/server/tests/ingest/test_medium_parser.py
@@ -9,7 +9,9 @@ def setUp(self):
super().setUp()
dirname = os.path.dirname(os.path.realpath(__file__))
- fixture_path = os.path.normpath(os.path.join(dirname, "../fixtures/medium_export"))
+ fixture_path = os.path.normpath(
+ os.path.join(dirname, "../fixtures/medium_export")
+ )
self.file_path = os.path.join(
fixture_path,
"2022-02-27_HOAX--This-UNAIDS-job-advert-in-Uganda-is-fake-f8d269a3d85d.html",
@@ -23,9 +25,12 @@ async def test_can_parse(self):
async def test_parse_medium_html(self):
item = await self.parser.parse(self.file_path)
- self.assertEqual(item["headline"], "HOAX: This UNAIDS job advert in Uganda is fake")
self.assertEqual(
- item["abstract"], "A UNAIDS Communications officer told PesaCheck that the job advertisement is fake."
+ item["headline"], "HOAX: This UNAIDS job advert in Uganda is fake"
+ )
+ self.assertEqual(
+ item["abstract"],
+ "A UNAIDS Communications officer told PesaCheck that the job advertisement is fake.",
)
self.assertTrue(item["body_html"].startswith(" None:
+ for tag, expected in [
+ ("English", "en"),
+ ("Kiswahili", "sw"),
+ ("French", "fr"),
+ ("Somali", "so"),
+ ("Afaan Oromo", "om"),
+ ("Amharic", "am"),
+ ]:
+ with self.subTest(tag=tag):
+ self.assertEqual(detect_language_from_tags([(0, tag)]), expected)
+
+ def test_matches_slugs_as_well_as_display_names(self) -> None:
+ self.assertEqual(detect_language_from_tags([(0, "afaan-oromo")]), "om")
+ self.assertEqual(detect_language_from_tags([(0, "kiswahili")]), "sw")
+
+ def test_matching_ignores_case_and_padding(self) -> None:
+ self.assertEqual(detect_language_from_tags([(0, " FRENCH ")]), "fr")
+
+ def test_accepts_endonyms(self) -> None:
+ self.assertEqual(detect_language_from_tags([(0, "Soomaali")]), "so")
+ self.assertEqual(detect_language_from_tags([(0, "Français")]), "fr")
+
+ def test_returns_none_without_a_language_tag(self) -> None:
+ self.assertIsNone(detect_language_from_tags([(0, "Kenya"), (1, "Short Form")]))
+ self.assertIsNone(detect_language_from_tags([]))
+
+ def test_country_tags_are_not_read_as_languages(self) -> None:
+ """The country sitting next to the language tag must not win.
+
+ PesaCheck tags posts with both ``Somali``/``Somalia`` and
+ ``French``/``France``, so these pairs are one substring away from
+ collapsing into each other.
+ """
+ for tags, expected in [
+ ([(0, "English"), (1, "Somalia")], "en"),
+ ([(0, "English"), (1, "France")], "en"),
+ ([(0, "English"), (1, "Ethiopia")], "en"),
+ ([(0, "Somali"), (1, "Somalia")], "so"),
+ ([(0, "French"), (1, "France")], "fr"),
+ ]:
+ with self.subTest(tags=tags):
+ self.assertEqual(detect_language_from_tags(tags), expected)
+
+ def test_bare_oromo_ethnonym_is_not_treated_as_a_language(self) -> None:
+ # An English article about Oromo people may carry "Oromo" as a topic
+ # tag; only the unambiguous "Afaan Oromo" marks the language.
+ self.assertEqual(
+ detect_language_from_tags([(0, "English"), (1, "Oromo")]), "en"
+ )
+
+ def test_primary_tag_wins_when_two_languages_are_tagged(self) -> None:
+ self.assertEqual(
+ detect_language_from_tags([(0, "English"), (1, "Amharic")]), "en"
+ )
+ # Order in the list must not matter — sort_order decides.
+ self.assertEqual(
+ detect_language_from_tags([(1, "Amharic"), (0, "English")]), "en"
+ )
+
+ def test_null_sort_order_is_tolerated(self) -> None:
+ self.assertEqual(detect_language_from_tags([(None, "Somali")]), "so")
+
+
+class LanguageFromTextTestCase(unittest.TestCase):
+ def test_detects_each_published_language(self) -> None:
+ for text, expected in [
+ (ENGLISH_TEXT, "en"),
+ (KISWAHILI_TEXT, "sw"),
+ (FRENCH_TEXT, "fr"),
+ (SOMALI_TEXT, "so"),
+ (OROMO_TEXT, "om"),
+ (AMHARIC_TEXT, "am"),
+ ]:
+ with self.subTest(expected=expected):
+ self.assertEqual(detect_language_from_text(text), expected)
+
+ def test_separates_oromo_from_somali(self) -> None:
+ """Both are Cushitic with doubled-vowel Latin spelling; langdetect,
+ which has no Oromo model, labels Oromo prose as Somali."""
+ self.assertEqual(detect_language_from_text(OROMO_TEXT), "om")
+ self.assertEqual(detect_language_from_text(SOMALI_TEXT), "so")
+
+ def test_embedded_english_quotes_do_not_flip_the_result(self) -> None:
+ # Fact checks quote the claim being debunked, often in English.
+ quoted = (
+ SOMALI_TEXT + ' Qoraalka waxaa lagu yiri "Easy application process" iyo '
+ '"Instant approval" oo ah hadal been abuur ah.'
+ )
+ self.assertEqual(detect_language_from_text(quoted), "so")
+
+ def test_a_quoted_amharic_phrase_does_not_make_an_article_amharic(self) -> None:
+ # The previous detector returned "am" for any text containing a single
+ # Ethiopic character.
+ quoted = (
+ ENGLISH_TEXT
+ + " The post, written in Amharic, reads “የመንግስት ሰራተኞች ደመዎዝ ጥማሪ”."
+ )
+ self.assertEqual(detect_language_from_text(quoted), "en")
+
+ def test_detects_from_a_headline_alone(self) -> None:
+ self.assertEqual(
+ detect_language_from_text(
+ "FAUX : Cette vidéo ne montre pas des billets dérobés"
+ ),
+ "fr",
+ )
+ self.assertEqual(
+ detect_language_from_text(
+ "BEEN: Muuqaalkan ma aha hub lagu qabtay oo ku yaal Soomaaliya"
+ ),
+ "so",
+ )
+
+ def test_survives_html_markup(self) -> None:
+ self.assertEqual(detect_language_from_text("%s
" % FRENCH_TEXT), "fr")
+
+ def test_declines_when_no_language_is_clearly_ahead(self) -> None:
+ # Function words borrowed from several candidates at once: nothing
+ # clears the margin, so the inconclusive branch logs and returns None.
+ mixed = "the le kwa oo kun de and ya iyo akka"
+ with self.assertLogs(language_module.logger, level="DEBUG"):
+ self.assertIsNone(detect_language_from_text(mixed))
+
+ def test_declines_to_guess_without_usable_text(self) -> None:
+ for text in [
+ "",
+ " ",
+ None,
+ "Hormuud",
+ "2026",
+ "$50 200,000 !!!",
+ "http://example.com/a",
+ ]:
+ with self.subTest(text=text):
+ self.assertIsNone(detect_language_from_text(text))
+
+
+class NormaliseLanguageCodeTestCase(unittest.TestCase):
+ def test_strips_region_subtags(self) -> None:
+ self.assertEqual(normalise_language_code("en-US"), "en")
+ self.assertEqual(normalise_language_code("am_ET"), "am")
+
+ def test_maps_names_to_codes(self) -> None:
+ self.assertEqual(normalise_language_code("Kiswahili"), "sw")
+
+ def test_passes_through_unsupported_codes(self) -> None:
+ # PesaCheck may add a language before this module knows about it.
+ self.assertEqual(normalise_language_code("pt-BR"), "pt")
+
+ def test_returns_none_for_empty_values(self) -> None:
+ self.assertIsNone(normalise_language_code(None))
+ self.assertIsNone(normalise_language_code(""))
+
+
+class DetectLanguageTestCase(unittest.TestCase):
+ def test_tags_take_precedence_over_text(self) -> None:
+ self.assertEqual(detect_language([(0, "Kiswahili")], FRENCH_TEXT), "sw")
+
+ def test_falls_back_to_text_when_untagged(self) -> None:
+ self.assertEqual(detect_language([(0, "Kenya")], FRENCH_TEXT), "fr")
+
+ def test_falls_back_to_english_when_nothing_is_detectable(self) -> None:
+ self.assertEqual(detect_language([], ""), "en")
+
+ def test_default_is_overridable(self) -> None:
+ self.assertIsNone(detect_language([], "", default=None))