Improve Ghost ingest language detection

scripts/dev/extension-edit.py

@@ -36,7 +36,9 @@ def main():
     mode, name, path = sys.argv[1], sys.argv[2], sys.argv[3]
 
     if mode not in ("enable", "disable"):
-        print(f"ERROR: mode must be 'enable' or 'disable', got '{mode}'", file=sys.stderr)
+        print(
+            f"ERROR: mode must be 'enable' or 'disable', got '{mode}'", file=sys.stderr
+        )
         sys.exit(2)
 
     with open(path) as f:
@@ -63,7 +65,9 @@ def main():
     # a line whose stripped content is '},'. Anything else (blank lines,
     # comments) is treated as a passthrough chunk between entries.
     region = lines[start_idx + 1 : end_idx]
-    entries = []  # list of (kind, value): kind is 'entry' (list of lines) or 'other' (single line)
+    entries = (
+        []
+    )  # list of (kind, value): kind is 'entry' (list of lines) or 'other' (single line)
     current = None
     for line in region:
         stripped = line.strip()
@@ -100,7 +104,9 @@ def is_standard_template(entry_lines):
     if mode == "enable":
         for kind, value in entries:
             if kind == "entry" and entry_id(value) == name:
-                print(f"INFO: '{name}' is already enabled — no change.", file=sys.stderr)
+                print(
+                    f"INFO: '{name}' is already enabled — no change.", file=sys.stderr
+                )
                 sys.exit(0)
         # Build a fresh entry and insert before // extensions:end.
         new_entry = [

scripts/update-cores-references.py

@@ -6,33 +6,35 @@
 TO_BE_UPDATED = [
     # superdesk-core
     {
-        'feed_url': 'https://github.com/superdesk/superdesk-core/commits/master.atom',
-        'file_name': 'server/requirements.txt',
-        'pattern': 'superdesk-core.git@([a-f0-9]*)'
+        "feed_url": "https://github.com/superdesk/superdesk-core/commits/master.atom",
+        "file_name": "server/requirements.txt",
+        "pattern": "superdesk-core.git@([a-f0-9]*)",
     },
     # superdesk-client-core
     {
-        'feed_url': 'https://github.com/superdesk/superdesk-client-core/commits/master.atom',
-        'file_name': 'client/package.json',
-        'pattern': 'superdesk-client-core#([a-f0-9]*)'
-    }
+        "feed_url": "https://github.com/superdesk/superdesk-client-core/commits/master.atom",
+        "file_name": "client/package.json",
+        "pattern": "superdesk-client-core#([a-f0-9]*)",
+    },
 ]
 
+
 def get_last_commit(url):
     feed = feedparser.parse(url)
-    return feed['entries'][0]['id'].split('/')[1][:9]
+    return feed["entries"][0]["id"].split("/")[1][:9]
 
 
 def replace_in_file(filename, search, new_value):
-    textfile = open(filename, 'r')
+    textfile = open(filename, "r")
     filetext = textfile.read()
     textfile.close()
     matches = re.findall(search, filetext)
     with fileinput.FileInput(filename, inplace=True) as file:
         for line in file:
-            print(line.replace(matches[0], new_value), end='')
+            print(line.replace(matches[0], new_value), end="")
+
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     for repo in TO_BE_UPDATED:
-        last_commit_hash = get_last_commit(repo['feed_url'])
-        replace_in_file(repo['file_name'], repo['pattern'], last_commit_hash)
+        last_commit_hash = get_last_commit(repo["feed_url"])
+        replace_in_file(repo["file_name"], repo["pattern"], last_commit_hash)

server/macros/replace_words.py

@@ -64,7 +64,9 @@ def do_find_replace(input_string, words_list):
 
             while re.search(pattern, input_string, flags=re.IGNORECASE):
                 # get the original string from the input
-                original = re.search(pattern, input_string, flags=re.IGNORECASE).group(0)
+                original = re.search(pattern, input_string, flags=re.IGNORECASE).group(
+                    0
+                )
                 replacement = repl(word.get("replacement", ""), original)
                 if found_list.get(original):
                     break

server/pesacheck/ingest/ghost_feeding_service.py

@@ -57,7 +57,9 @@ async def _update(self, provider, update):
             )
             return
 
-        for filename in await get_sorted_files(self.path, sort_by=FileSortAttributes.created):
+        for filename in await get_sorted_files(
+            self.path, sort_by=FileSortAttributes.created
+        ):
             last_updated = None
             try:
                 file_path = os.path.join(self.path, filename)
@@ -66,8 +68,12 @@ async def _update(self, provider, update):
 
                 last_updated = self.get_last_updated(file_path)
 
-                if not self.is_latest_content(last_updated, provider.get("last_updated")):
-                    await self.move_file(self.path, filename, provider=provider, success=False)
+                if not self.is_latest_content(
+                    last_updated, provider.get("last_updated")
+                ):
+                    await self.move_file(
+                        self.path, filename, provider=provider, success=False
+                    )
                     continue
 
                 if await self.is_empty(file_path):
@@ -87,12 +93,18 @@ async def _update(self, provider, update):
                         break
                     yield batch
 
-                await self.move_file(self.path, filename, provider=provider, success=True)
+                await self.move_file(
+                    self.path, filename, provider=provider, success=True
+                )
 
             except Exception as ex:
                 if last_updated and self.is_old_content(last_updated):
-                    await self.move_file(self.path, filename, provider=provider, success=False)
-                raise ParserError.parseFileError("{}-{}".format(provider["name"], self.NAME), filename, ex, provider)
+                    await self.move_file(
+                        self.path, filename, provider=provider, success=False
+                    )
+                raise ParserError.parseFileError(
+                    "{}-{}".format(provider["name"], self.NAME), filename, ex, provider
+                )
 
         push_notification("ingest:update")
 

server/pesacheck/ingest/ghost_parser.py

@@ -1,23 +1,30 @@
-import json
 import hashlib
+import json
 import logging
 import random
 import time
-import unicodedata
 from copy import deepcopy
-from urllib.parse import urlparse
-
 from datetime import datetime, timezone
+from urllib.parse import urlparse
 
 from superdesk.errors import ParserError
 from superdesk.etree import parse_html
-from superdesk.metadata.utils import generate_guid
 from superdesk.io.feed_parsers import FileFeedParser
 from superdesk.io.registry import register_feed_parser
 from superdesk.media.renditions import update_renditions
-from superdesk.metadata.item import FORMAT, GUID_FIELD, GUID_TAG, ITEM_TYPE, CONTENT_TYPE, FORMATS
+from superdesk.metadata.item import (
+    CONTENT_TYPE,
+    FORMAT,
+    FORMATS,
+    GUID_FIELD,
+    GUID_TAG,
+    ITEM_TYPE,
+)
+from superdesk.metadata.utils import generate_guid
+from superdesk.text_utils import get_text
 from superdesk.utc import utcnow
 
+from pesacheck.language import detect_language, normalise_language_code
 
 logger = logging.getLogger(__name__)
 
@@ -127,28 +134,27 @@ def _sleep_before_next_fetch(self, min_interval):
     def _mark_fetch_done(self):
         self._last_image_fetch_ts = time.monotonic()
 
-    def _guess_language(self, text):
-        """Guess language from text: returns 'am', 'fr', or 'en'.
+    def _parse_language(self, post, tags, text):
+        """Resolve the item language from ``locale``, then tags, then body text.
 
-        Ethiopic script detection is used first (langdetect has no Amharic model).
-        langdetect is then used to distinguish English from French; anything
-        unrecognised falls back to English.
+        Ghost's own ``locale`` is authoritative when set, but PesaCheck's export
+        leaves it null on every post, so in practice the language comes from the
+        post's language tag. Posts predating that tagging convention fall
+        through to text classification.
         """
-        if not text:
-            return None
-        # Detect Amharic by Ethiopic Unicode block (U+1200–U+137F, etc.)
-        for ch in text:
-            if "ETHIOPIC" in unicodedata.name(ch, ""):
-                return "am"
-        try:
-            from langdetect import detect
+        locale = normalise_language_code(post.get("locale"))
+        if locale:
+            return locale
 
-            detected = detect(text)
-            if detected in ("en", "fr"):
-                return detected
-        except Exception:
-            pass
-        return "en"
+        # Offer both the display name and the slug: either may be the form that
+        # names the language ("Afaan Oromo" / "afaan-oromo").
+        tag_labels = [
+            (tag["sort_order"], label)
+            for tag in tags
+            for label in (tag["name"], tag["slug"])
+        ]
+
+        return detect_language(tag_labels, text)
 
     def can_parse(self, file_path):
         try:
@@ -203,7 +209,9 @@ def _fetch_renditions_with_retry(self, association, url):
                 if attempt == policy["retries"]:
                     break
 
-                delay = (policy["base_backoff"] * attempt) + random.uniform(0, _IMAGE_FETCH_JITTER_SECONDS)
+                delay = (policy["base_backoff"] * attempt) + random.uniform(
+                    0, _IMAGE_FETCH_JITTER_SECONDS
+                )
                 logger.warning(
                     "Image fetch failed for %s (attempt %s/%s), retrying in %.2fs: %s",
                     url,
@@ -216,11 +224,16 @@ def _fetch_renditions_with_retry(self, association, url):
 
         if policy["failure_cooldown"] > 0:
             # After exhausting retries, cool down before the next image to reduce cascading failures.
-            time.sleep(policy["failure_cooldown"] + random.uniform(0, _IMAGE_FETCH_JITTER_SECONDS))
+            time.sleep(
+                policy["failure_cooldown"]
+                + random.uniform(0, _IMAGE_FETCH_JITTER_SECONDS)
+            )
 
         raise last_error
 
-    def _add_image(self, item, url, alt_text="", description_text="", is_featured=False):
+    def _add_image(
+        self, item, url, alt_text="", description_text="", is_featured=False
+    ):
         """Fetch image, attach it as an association, and return the local storage href (or None)."""
         associations = item.setdefault("associations", {})
         association = {
@@ -283,7 +296,9 @@ def _parse_inline_images(self, item, html):
                 if local_href:
                     url_rewrites[src] = local_href
             except Exception as e:
-                logger.warning("Failed to parse inline image %s: %s", img.get("src", "unknown"), e)
+                logger.warning(
+                    "Failed to parse inline image %s: %s", img.get("src", "unknown"), e
+                )
 
         if url_rewrites:
             body = item.get("body_html") or ""
@@ -312,14 +327,18 @@ def _parse_date(self, value):
     def _parse_post(self, post, authors_by_post, tags_by_post):
         post_id = post.get("id", "")
 
-        authors = sorted(authors_by_post.get(post_id, []), key=lambda x: x["sort_order"])
+        authors = sorted(
+            authors_by_post.get(post_id, []), key=lambda x: x["sort_order"]
+        )
         byline = ", ".join(a["name"] for a in authors if a.get("name"))
 
         tags = sorted(tags_by_post.get(post_id, []), key=lambda x: x["sort_order"])
         keywords = [t["name"] for t in tags if t.get("name")]
 
         firstcreated = self._parse_date(post.get("created_at"))
-        versioncreated = self._parse_date(post.get("published_at") or post.get("updated_at"))
+        versioncreated = self._parse_date(
+            post.get("published_at") or post.get("updated_at")
+        )
 
         html = post.get("html") or ""
         if self._ghost_url and html:
@@ -340,13 +359,12 @@ def _parse_post(self, post, authors_by_post, tags_by_post):
             "versioncreated": versioncreated,
         }
 
-        locale = post.get("locale")
-        if locale:
-            item["language"] = locale
-        else:
-            guessed = self._guess_language(html or post.get("title") or "")
-            if guessed:
-                item["language"] = guessed
+        # Ghost exports a markup-free rendering of the body; prefer it for
+        # language detection so HTML tag names don't dilute the word counts.
+        # Older exports omit it, so strip the markup ourselves in that case.
+        body_text = post.get("plaintext") or get_text(html, content="html")
+        sample = " ".join(part for part in (post.get("title") or "", body_text) if part)
+        item["language"] = self._parse_language(post, tags, sample)
 
         self._parse_feature_image(item, post)
         self._parse_inline_images(item, html)
@@ -361,7 +379,9 @@ def iter_items(self, file_path, provider=None):
         """Parse a Ghost JSON export file and yield Superdesk items one at a time."""
         self._image_assoc_cache = {}
         self._last_image_fetch_ts = 0.0
-        self._ghost_url = ((provider or {}).get("config", {}).get("url") or "").rstrip("/")
+        self._ghost_url = ((provider or {}).get("config", {}).get("url") or "").rstrip(
+            "/"
+        )
         try:
             with open(file_path, "r", encoding="utf-8") as f:
                 data = json.load(f)
@@ -402,7 +422,12 @@ def iter_items(self, file_path, provider=None):
                 tags_by_post.setdefault(pid, []).append(
                     {
                         "name": tag.get("name", ""),
-                        "sort_order": pt.get("sort_order", 0),
+                        # Ghost slugs are the stable identifier, and the language
+                        # tag is matched on either form.
+                        "slug": tag.get("slug", ""),
+                        # Coerce a null sort_order so the sort below can't blow
+                        # up comparing None to an int.
+                        "sort_order": pt.get("sort_order") or 0,
                     }
                 )
 

server/pesacheck/ingest/medium_parser.py

@@ -11,7 +11,14 @@
 from superdesk.io.feed_parsers import FileFeedParser
 from superdesk.io.registry import register_feed_parser
 from superdesk.media.renditions import update_renditions
-from superdesk.metadata.item import FORMAT, GUID_FIELD, GUID_TAG, ITEM_TYPE, CONTENT_TYPE, FORMATS
+from superdesk.metadata.item import (
+    FORMAT,
+    GUID_FIELD,
+    GUID_TAG,
+    ITEM_TYPE,
+    CONTENT_TYPE,
+    FORMATS,
+)
 
 
 logger = logging.getLogger(__name__)
@@ -54,10 +61,13 @@ def can_parse(self, file_path):
                 html_content = f.read().decode("utf-8")
                 root = parse_html(html_content, "html")
 
-                exported_from_medium = 'Exported from <a href="https://medium.com">Medium</a>' in to_string(
-                    root.find(".//footer"), method="html"
+                exported_from_medium = (
+                    'Exported from <a href="https://medium.com">Medium</a>'
+                    in to_string(root.find(".//footer"), method="html")
+                )
+                article_body_found = (
+                    root.find(".//section[@data-field='body']") is not None
                 )
-                article_body_found = root.find(".//section[@data-field='body']") is not None
 
                 return exported_from_medium and article_body_found
         except Exception:
@@ -78,7 +88,9 @@ def _generate_image_guid(self, url):
         guid_hash = hashlib.sha1(url.encode("utf8")).hexdigest()
         return generate_guid(type=GUID_TAG, id=guid_hash + "-image")
 
-    def _add_image(self, item, url, alt_text="", description_text="", is_featured=False):
+    def _add_image(
+        self, item, url, alt_text="", description_text="", is_featured=False
+    ):
         """Add an image to the item's associations.
 
         :param item: The item dictionary to add the image to
@@ -132,7 +144,9 @@ def parse_images(self, item, article):
                 self._add_image(item, src, alt_text, description_text, is_featured)
 
             except Exception as e:
-                logger.warning(f"Failed to parse image {img.get('src', 'unknown')}: {e}")
+                logger.warning(
+                    f"Failed to parse image {img.get('src', 'unknown')}: {e}"
+                )
                 continue
 
     async def parse(self, file_path, provider=None):
@@ -168,7 +182,9 @@ async def parse(self, file_path, provider=None):
                 firstcreated = utcnow()
 
             text_nodes = article.xpath(".//text()")
-            word_count = sum(len(text.strip().split()) for text in text_nodes if text.strip())
+            word_count = sum(
+                len(text.strip().split()) for text in text_nodes if text.strip()
+            )
 
             item = {
                 ITEM_TYPE: CONTENT_TYPE.TEXT,