From 7934057efb33b0c846b6f572832245b313fee8cb Mon Sep 17 00:00:00 2001 From: Mark Ng'ang'a Date: Mon, 10 Aug 2026 14:59:58 +0300 Subject: [PATCH 1/6] Delete dead docker-compose file --- .github/docker-compose.yml | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 .github/docker-compose.yml diff --git a/.github/docker-compose.yml b/.github/docker-compose.yml deleted file mode 100644 index 6c237be754..0000000000 --- a/.github/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: "3.2" - -services: - elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:7.10.1 - ports: - - "9200:9200" - environment: - - discovery.type=single-node - tmpfs: - - /usr/share/elasticsearch/data - - redis: - image: redis:alpine - ports: - - "6379:6379" - - mongo: - image: mongo:4.4 - ports: - - "27017:27017" - tmpfs: - - /data/db - From 05e2b20ba0572399e91c969bc8cf8048d549f389 Mon Sep 17 00:00:00 2001 From: Mark Ng'ang'a Date: Mon, 10 Aug 2026 15:00:31 +0300 Subject: [PATCH 2/6] Update dependabot schedule --- .github/dependabot.yml | 40 +++++++++++++--------------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 779a1d62ef..95ca951e7b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,34 +1,20 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: # https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file - +# +# This fork deliberately does NOT run npm or pip version updates. +# +# Upstream superdesk/superdesk owns the client and server dependency trees; we +# take their bumps by merging upstream. +# Running Dependabot on the same manifests here produced daily PRs against a very +# large dependency graph — each one firing the full CI suite - and every merged +# bump became another conflict to resolve on the next upstream merge. It cost +# a lot of CI and moved nothing. +# +# Note that Dependabot *security* updates are a separate feature, configured in +# the repository's Settings → Code security, and are unaffected by this file. +# Leave those on — they are the reason dropping version updates is safe. version: 2 updates: - - package-ecosystem: "npm" # See documentation for possible values - directory: "client" # Location of package manifests - schedule: - interval: "daily" - - - package-ecosystem: "pip" # See documentation for possible values - directory: "server" # Location of package manifests - schedule: - interval: "daily" - - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" - - # keep release/3 up to date - - package-ecosystem: "npm" - directory: "client" - target-branch: "release/3" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "server" - target-branch: "release/3" - schedule: - interval: "daily" From c64751eb17e407713f91ee7e0699c2c36fd506c7 Mon Sep 17 00:00:00 2001 From: Mark Ng'ang'a Date: Mon, 10 Aug 2026 15:01:09 +0300 Subject: [PATCH 3/6] Consolidate workflows into two files * Drop unnecessary jobs and optimise jobs with shared roots --- .github/workflows/behave.yml | 58 ------------ .github/workflows/ci.yml | 157 ++++++++++++++++++++++++++++++++ .github/workflows/client.yml | 33 ------- .github/workflows/docker.yml | 28 ------ .github/workflows/dockerhub.yml | 25 ----- .github/workflows/publish.yml | 37 ++++++++ .github/workflows/pytest.yml | 59 ------------ .github/workflows/server.yml | 93 ------------------- .github/workflows/tests.yml | 46 ---------- 9 files changed, 194 insertions(+), 342 deletions(-) delete mode 100644 .github/workflows/behave.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/client.yml delete mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/dockerhub.yml create mode 100644 .github/workflows/publish.yml delete mode 100644 .github/workflows/pytest.yml delete mode 100644 .github/workflows/server.yml delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/behave.yml b/.github/workflows/behave.yml deleted file mode 100644 index a5b63bfc50..0000000000 --- a/.github/workflows/behave.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: "Behave" - -on: - workflow_call: - inputs: - python-version: - description: "Python version" - required: false - default: "3.12" - type: string - -permissions: - contents: read - -jobs: - behave: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: server - - services: - redis: - image: redis:alpine - ports: - - "6379:6379" - - mongo: - image: mongo:4 - ports: - - "27017:27017" - - elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.25 - ports: - - "9200:9200" - env: - discovery.type: single-node - - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-python@v7 - with: - python-version: ${{ inputs.python-version }} - cache: pip - - - run: | - sudo apt-get update - sudo apt-get -y install libxml2-dev libxmlsec1-dev libxmlsec1-openssl libexempi-dev - - - run: | - python -m pip install -U pip wheel setuptools - python -m pip install -Ur dev-requirements.txt - - - run: behave - diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..c7546248ec --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,157 @@ +name: "CI" + +on: + push: + branches: [pesacheck] + pull_request: + +permissions: + contents: read + +# Superseded runs are pointless: a newer commit on the same branch invalidates +# them. pesacheck itself is exempt because its runs gate the image builds, and +# cancelling those loses the only signal that main-line HEAD is buildable. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/pesacheck' }} + +env: + PYTHON_VERSION: "3.12" + NODE_VERSION: "22" + +jobs: + # black, flake8 and mypy used to be three jobs, each installing the whole of + # dev-requirements.txt (superdesk-core, lxml, xmlsec, …) to run one command. + # None of them need the application: black and flake8 never import it, and + # setup.cfg sets mypy's ignore_missing_imports, so unresolved third-party + # imports degrade to Any rather than erroring. Installing just the three + # tools takes seconds and needs none of the apt libraries. + # + # The trade-off is real but pre-existing: mypy sees external types as Any, so + # it checks less than a full install would. That is what ignore_missing_imports + # already opted into — this only makes it cheap. + lint: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: server + + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + cache-dependency-path: server/dev-requirements.txt + + # Versions come from dev-requirements.txt so there is one place to bump. + - run: pip install $(grep -E '^(black|flake8|mypy)==' dev-requirements.txt) + + # `!cancelled()` so one failing linter still reports the other two, instead + # of making you push a fix to find out what else is broken. + - run: black --check . + + - run: flake8 + if: ${{ !cancelled() }} + + - run: mypy . + if: ${{ !cancelled() }} + + # pytest and behave were separate jobs repeating an identical, expensive + # setup: three service containers, an apt install of the xmlsec/exempi + # headers, and all of dev-requirements.txt — to run a handful of parser tests + # and one smoke scenario. They share the environment, so they share the job. + # + # Service versions match docker-compose.yml (mongo 6, redis 8, elastic + # 7.17.29). CI previously tested against mongo 4 while the stack runs 6. + test: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: server + + services: + redis: + image: redis:8 + ports: + - "6379:6379" + + mongo: + image: mongo:6 + ports: + - "27017:27017" + + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.29 + ports: + - "9200:9200" + env: + discovery.type: single-node + + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + cache-dependency-path: server/dev-requirements.txt + + - run: | + sudo apt-get update + sudo apt-get -y install libxml2-dev libxmlsec1-dev libxmlsec1-openssl libexempi-dev + + - run: | + python -m pip install -U pip wheel setuptools + python -m pip install -Ur dev-requirements.txt + + - run: pytest + + # One scenario (GET / returns the auth link), but it is the only thing that + # proves the app boots with our settings.py and INSTALLED_APPS. Near-free + # now that it reuses the install above, and it runs even when pytest fails + # so a broken parser test cannot hide a boot failure. + - run: behave + if: ${{ !cancelled() }} + + client: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: client + + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-node@v7 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: client/package-lock.json + + - run: npm ci + - run: npm run build + + # Gated behind the other three jobs: there is no point spending twenty + # minutes on an image whose code we already know is broken. + images: + needs: [lint, test, client] + if: github.event_name == 'push' + + runs-on: ubuntu-latest + timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + component: [server, client] + + steps: + - uses: actions/checkout@v7 + + - run: docker build . + working-directory: ./${{ matrix.component }} diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml deleted file mode 100644 index ded805689d..0000000000 --- a/.github/workflows/client.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: "Client" - -on: - workflow_call: - inputs: - node-version: - description: "Node version" - required: false - default: "22" - type: string - -permissions: - contents: read - -jobs: - install: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: client - - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-node@v7 - with: - node-version: ${{ inputs.node-version }} - cache: npm - cache-dependency-path: client/package-lock.json - - - run: npm ci - - run: npm run build diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 55143a9f18..0000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: "Docker" - -on: - workflow_call: - -permissions: - contents: read - -jobs: - server: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v7 - - - run: docker build . - working-directory: ./server - timeout-minutes: 10 - - client: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v7 - - - run: docker build . - working-directory: ./client - timeout-minutes: 20 diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml deleted file mode 100644 index f3811ce9dd..0000000000 --- a/.github/workflows/dockerhub.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Publish Docker Images - -on: - release: - types: [released] - -permissions: - contents: read - -jobs: - push_to_dockerhub: - name: Push Docker images to Docker Hub - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v7 - - - name: Login to DockerHub - uses: docker/login-action@v4.5.2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push image - run: docker compose build && docker compose push diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000000..a563ea594a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,37 @@ +name: "Publish images" + +on: + release: + types: [released] + +permissions: + contents: read + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + component: [server, client] + + steps: + - uses: actions/checkout@v7 + + - uses: docker/login-action@v4.5.2 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + env: + IMAGE: codeforafrica/superdesk-pesacheck-${{ matrix.component }} + TAG: ${{ github.event.release.tag_name }} + run: | + docker build \ + --tag "$IMAGE:$TAG" \ + --tag "$IMAGE:latest" \ + ./${{ matrix.component }} + docker push --all-tags "$IMAGE" diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml deleted file mode 100644 index 7e394039d9..0000000000 --- a/.github/workflows/pytest.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: "Pytest" - -on: - workflow_call: - inputs: - python-version: - description: "Python version" - required: false - default: "3.12" - type: string - -permissions: - contents: read - -jobs: - - pytest: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: server - - services: - redis: - image: redis:alpine - ports: - - "6379:6379" - - mongo: - image: mongo:4 - ports: - - "27017:27017" - - elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.25 - ports: - - "9200:9200" - env: - discovery.type: single-node - - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-python@v7 - with: - python-version: ${{ inputs.python-version }} - cache: pip - - - run: | - sudo apt-get update - sudo apt-get -y install libxml2-dev libxmlsec1-dev libxmlsec1-openssl libexempi-dev - - - run: | - python -m pip install -U pip wheel setuptools - python -m pip install -Ur dev-requirements.txt - - - run: pytest - diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml deleted file mode 100644 index 5d2eb50e31..0000000000 --- a/.github/workflows/server.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: "Server" - -on: - workflow_call: - inputs: - python-version: - description: "Python version" - required: false - default: "3.12" - type: string - -permissions: - contents: read - -defaults: - run: - working-directory: server - -jobs: - install: - runs-on: ubuntu-latest - - services: - redis: - image: redis:alpine - ports: - - "6379:6379" - - mongo: - image: mongo:4 - ports: - - "27017:27017" - - elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.25 - ports: - - "9200:9200" - env: - discovery.type: single-node - - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-python@v7 - with: - python-version: ${{ inputs.python-version }} - cache: 'pip' - - - run: | - sudo apt-get update - sudo apt-get -y install libxml2-dev libxmlsec1-dev libxmlsec1-openssl libexempi-dev - - - run: | - python -m pip install -U pip wheel setuptools - python -m pip install -Ur dev-requirements.txt - - - name: init - run: | - honcho run python manage.py app:initialize_data - if: ${{ inputs.python-version != '3.12' }} # There is some issue with the 3.12 version atm - - black: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: ${{ inputs.python-version }} - cache: 'pip' - - run: pip install -Ur dev-requirements.txt - - run: black --check . - - flake8: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: ${{ inputs.python-version }} - cache: 'pip' - - run: pip install -Ur dev-requirements.txt - - run: flake8 - - mypy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: ${{ inputs.python-version }} - cache: 'pip' - - run: pip install -Ur dev-requirements.txt - - run: mypy . diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 55ecaaf2d1..0000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: "CI" - -on: [push, pull_request] - -permissions: - contents: read - -jobs: - server: - strategy: - fail-fast: false - matrix: - python-version: ['3.12'] - uses: ./.github/workflows/server.yml - with: - python-version: ${{ matrix.python-version }} - - pytest: - strategy: - fail-fast: false - matrix: - python-version: ['3.12'] - uses: ./.github/workflows/pytest.yml - with: - python-version: ${{ matrix.python-version }} - - behave: - strategy: - fail-fast: false - matrix: - python-version: ['3.12'] - uses: ./.github/workflows/behave.yml - with: - python-version: ${{ matrix.python-version }} - - client: - strategy: - fail-fast: false - matrix: - node-version: ['22'] - uses: ./.github/workflows/client.yml - with: - node-version: ${{ matrix.node-version }} - - docker: - uses: ./.github/workflows/docker.yml From f16a45912ad9eb840d95fbfeda48921c3fe942e8 Mon Sep 17 00:00:00 2001 From: Mark Ng'ang'a Date: Mon, 10 Aug 2026 15:02:39 +0300 Subject: [PATCH 4/6] Update README with new CI badge * Also updated links to images (images are not yet actually published) --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 276932a2af..742bdf49d7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # Superdesk -[![Test](https://github.com/superdesk/superdesk/actions/workflows/tests.yml/badge.svg)](https://github.com/superdesk/superdesk/actions/workflows/tests.yml) -[![Lint](https://github.com/superdesk/superdesk/actions/workflows/lint.yml/badge.svg)](https://github.com/superdesk/superdesk/actions/workflows/lint.yml) +[![CI](https://github.com/CodeForAfrica/superdesk-pesacheck/actions/workflows/ci.yml/badge.svg?branch=pesacheck)](https://github.com/CodeForAfrica/superdesk-pesacheck/actions/workflows/ci.yml) Superdesk is an open source end-to-end news creation, production, curation, distribution and publishing platform developed and maintained by Sourcefabric @@ -36,7 +35,7 @@ $ docker compose exec superdesk-server python manage.py users:create -u admin -p Then you can login with admin:admin credentials. -The Docker images are hosted on Dockerhub for the [client](https://hub.docker.com/r/sourcefabricoss/superdesk-client) and [server](https://hub.docker.com/r/sourcefabricoss/superdesk-server). +This fork publishes its own images on release, to Dockerhub for the [client](https://hub.docker.com/r/codeforafrica/superdesk-pesacheck-client) and [server](https://hub.docker.com/r/codeforafrica/superdesk-pesacheck-server). Upstream's images (`sourcefabricoss/superdesk-client`, `sourcefabricoss/superdesk-server`) do not carry our ingest module or content config. ## Local development (frontend / extension work) From 38cde003b789feec02deacaf343f211db54c2fc7 Mon Sep 17 00:00:00 2001 From: Mark Ng'ang'a Date: Mon, 10 Aug 2026 15:03:51 +0300 Subject: [PATCH 5/6] Run Black on server/ * This is unrelated to the changes on this branch and the formatting issues predate the fork --- server/macros/replace_words.py | 4 +- .../pesacheck/ingest/ghost_feeding_service.py | 24 ++- server/pesacheck/ingest/ghost_parser.py | 38 ++++- server/pesacheck/ingest/medium_parser.py | 30 +++- server/settings.py | 14 +- server/tests/ingest/test_ghost_parser.py | 148 ++++++++++++++---- server/tests/ingest/test_medium_parser.py | 15 +- 7 files changed, 211 insertions(+), 62 deletions(-) diff --git a/server/macros/replace_words.py b/server/macros/replace_words.py index 28246be2f1..7847c6dcca 100644 --- a/server/macros/replace_words.py +++ b/server/macros/replace_words.py @@ -64,7 +64,9 @@ def do_find_replace(input_string, words_list): while re.search(pattern, input_string, flags=re.IGNORECASE): # get the original string from the input - original = re.search(pattern, input_string, flags=re.IGNORECASE).group(0) + original = re.search(pattern, input_string, flags=re.IGNORECASE).group( + 0 + ) replacement = repl(word.get("replacement", ""), original) if found_list.get(original): break diff --git a/server/pesacheck/ingest/ghost_feeding_service.py b/server/pesacheck/ingest/ghost_feeding_service.py index d38e791535..00c9051dc7 100644 --- a/server/pesacheck/ingest/ghost_feeding_service.py +++ b/server/pesacheck/ingest/ghost_feeding_service.py @@ -57,7 +57,9 @@ async def _update(self, provider, update): ) return - for filename in await get_sorted_files(self.path, sort_by=FileSortAttributes.created): + for filename in await get_sorted_files( + self.path, sort_by=FileSortAttributes.created + ): last_updated = None try: file_path = os.path.join(self.path, filename) @@ -66,8 +68,12 @@ async def _update(self, provider, update): last_updated = self.get_last_updated(file_path) - if not self.is_latest_content(last_updated, provider.get("last_updated")): - await self.move_file(self.path, filename, provider=provider, success=False) + if not self.is_latest_content( + last_updated, provider.get("last_updated") + ): + await self.move_file( + self.path, filename, provider=provider, success=False + ) continue if await self.is_empty(file_path): @@ -87,12 +93,18 @@ async def _update(self, provider, update): break yield batch - await self.move_file(self.path, filename, provider=provider, success=True) + await self.move_file( + self.path, filename, provider=provider, success=True + ) except Exception as ex: if last_updated and self.is_old_content(last_updated): - await self.move_file(self.path, filename, provider=provider, success=False) - raise ParserError.parseFileError("{}-{}".format(provider["name"], self.NAME), filename, ex, provider) + await self.move_file( + self.path, filename, provider=provider, success=False + ) + raise ParserError.parseFileError( + "{}-{}".format(provider["name"], self.NAME), filename, ex, provider + ) push_notification("ingest:update") diff --git a/server/pesacheck/ingest/ghost_parser.py b/server/pesacheck/ingest/ghost_parser.py index 17ad01e935..2b082eca2b 100644 --- a/server/pesacheck/ingest/ghost_parser.py +++ b/server/pesacheck/ingest/ghost_parser.py @@ -15,7 +15,14 @@ from superdesk.io.feed_parsers import FileFeedParser from superdesk.io.registry import register_feed_parser from superdesk.media.renditions import update_renditions -from superdesk.metadata.item import FORMAT, GUID_FIELD, GUID_TAG, ITEM_TYPE, CONTENT_TYPE, FORMATS +from superdesk.metadata.item import ( + FORMAT, + GUID_FIELD, + GUID_TAG, + ITEM_TYPE, + CONTENT_TYPE, + FORMATS, +) from superdesk.utc import utcnow @@ -203,7 +210,9 @@ def _fetch_renditions_with_retry(self, association, url): if attempt == policy["retries"]: break - delay = (policy["base_backoff"] * attempt) + random.uniform(0, _IMAGE_FETCH_JITTER_SECONDS) + delay = (policy["base_backoff"] * attempt) + random.uniform( + 0, _IMAGE_FETCH_JITTER_SECONDS + ) logger.warning( "Image fetch failed for %s (attempt %s/%s), retrying in %.2fs: %s", url, @@ -216,11 +225,16 @@ def _fetch_renditions_with_retry(self, association, url): if policy["failure_cooldown"] > 0: # After exhausting retries, cool down before the next image to reduce cascading failures. - time.sleep(policy["failure_cooldown"] + random.uniform(0, _IMAGE_FETCH_JITTER_SECONDS)) + time.sleep( + policy["failure_cooldown"] + + random.uniform(0, _IMAGE_FETCH_JITTER_SECONDS) + ) raise last_error - def _add_image(self, item, url, alt_text="", description_text="", is_featured=False): + def _add_image( + self, item, url, alt_text="", description_text="", is_featured=False + ): """Fetch image, attach it as an association, and return the local storage href (or None).""" associations = item.setdefault("associations", {}) association = { @@ -283,7 +297,9 @@ def _parse_inline_images(self, item, html): if local_href: url_rewrites[src] = local_href except Exception as e: - logger.warning("Failed to parse inline image %s: %s", img.get("src", "unknown"), e) + logger.warning( + "Failed to parse inline image %s: %s", img.get("src", "unknown"), e + ) if url_rewrites: body = item.get("body_html") or "" @@ -312,14 +328,18 @@ def _parse_date(self, value): def _parse_post(self, post, authors_by_post, tags_by_post): post_id = post.get("id", "") - authors = sorted(authors_by_post.get(post_id, []), key=lambda x: x["sort_order"]) + authors = sorted( + authors_by_post.get(post_id, []), key=lambda x: x["sort_order"] + ) byline = ", ".join(a["name"] for a in authors if a.get("name")) tags = sorted(tags_by_post.get(post_id, []), key=lambda x: x["sort_order"]) keywords = [t["name"] for t in tags if t.get("name")] firstcreated = self._parse_date(post.get("created_at")) - versioncreated = self._parse_date(post.get("published_at") or post.get("updated_at")) + versioncreated = self._parse_date( + post.get("published_at") or post.get("updated_at") + ) html = post.get("html") or "" if self._ghost_url and html: @@ -361,7 +381,9 @@ def iter_items(self, file_path, provider=None): """Parse a Ghost JSON export file and yield Superdesk items one at a time.""" self._image_assoc_cache = {} self._last_image_fetch_ts = 0.0 - self._ghost_url = ((provider or {}).get("config", {}).get("url") or "").rstrip("/") + self._ghost_url = ((provider or {}).get("config", {}).get("url") or "").rstrip( + "/" + ) try: with open(file_path, "r", encoding="utf-8") as f: data = json.load(f) diff --git a/server/pesacheck/ingest/medium_parser.py b/server/pesacheck/ingest/medium_parser.py index 1056313fc3..3cf18c8f8d 100644 --- a/server/pesacheck/ingest/medium_parser.py +++ b/server/pesacheck/ingest/medium_parser.py @@ -11,7 +11,14 @@ from superdesk.io.feed_parsers import FileFeedParser from superdesk.io.registry import register_feed_parser from superdesk.media.renditions import update_renditions -from superdesk.metadata.item import FORMAT, GUID_FIELD, GUID_TAG, ITEM_TYPE, CONTENT_TYPE, FORMATS +from superdesk.metadata.item import ( + FORMAT, + GUID_FIELD, + GUID_TAG, + ITEM_TYPE, + CONTENT_TYPE, + FORMATS, +) logger = logging.getLogger(__name__) @@ -54,10 +61,13 @@ def can_parse(self, file_path): html_content = f.read().decode("utf-8") root = parse_html(html_content, "html") - exported_from_medium = 'Exported from Medium' in to_string( - root.find(".//footer"), method="html" + exported_from_medium = ( + 'Exported from Medium' + in to_string(root.find(".//footer"), method="html") + ) + article_body_found = ( + root.find(".//section[@data-field='body']") is not None ) - article_body_found = root.find(".//section[@data-field='body']") is not None return exported_from_medium and article_body_found except Exception: @@ -78,7 +88,9 @@ def _generate_image_guid(self, url): guid_hash = hashlib.sha1(url.encode("utf8")).hexdigest() return generate_guid(type=GUID_TAG, id=guid_hash + "-image") - def _add_image(self, item, url, alt_text="", description_text="", is_featured=False): + def _add_image( + self, item, url, alt_text="", description_text="", is_featured=False + ): """Add an image to the item's associations. :param item: The item dictionary to add the image to @@ -132,7 +144,9 @@ def parse_images(self, item, article): self._add_image(item, src, alt_text, description_text, is_featured) except Exception as e: - logger.warning(f"Failed to parse image {img.get('src', 'unknown')}: {e}") + logger.warning( + f"Failed to parse image {img.get('src', 'unknown')}: {e}" + ) continue async def parse(self, file_path, provider=None): @@ -168,7 +182,9 @@ async def parse(self, file_path, provider=None): firstcreated = utcnow() text_nodes = article.xpath(".//text()") - word_count = sum(len(text.strip().split()) for text in text_nodes if text.strip()) + word_count = sum( + len(text.strip().split()) for text in text_nodes if text.strip() + ) item = { ITEM_TYPE: CONTENT_TYPE.TEXT, diff --git a/server/settings.py b/server/settings.py index 678a95b3c7..9f705cbb56 100644 --- a/server/settings.py +++ b/server/settings.py @@ -179,8 +179,18 @@ "picture": { "headline": {"order": 1, "sdWidth": "full", "editor3": True}, "alt_text": {"order": 2, "sdWidth": "full", "textarea": True, "editor3": True}, - "description_text": {"order": 3, "sdWidth": "full", "textarea": True, "editor3": True}, - "creditline": {"order": 4, "sdWidth": "full", "displayOnMediaEditor": True, "editor3": True}, + "description_text": { + "order": 3, + "sdWidth": "full", + "textarea": True, + "editor3": True, + }, + "creditline": { + "order": 4, + "sdWidth": "full", + "displayOnMediaEditor": True, + "editor3": True, + }, "copyrightholder": {"order": 5, "displayOnMediaEditor": True, "editor3": True}, "usageterms": {"order": 6, "displayOnMediaEditor": True}, "copyrightnotice": {"order": 7, "displayOnMediaEditor": True}, diff --git a/server/tests/ingest/test_ghost_parser.py b/server/tests/ingest/test_ghost_parser.py index d868a53394..513c5e90e6 100644 --- a/server/tests/ingest/test_ghost_parser.py +++ b/server/tests/ingest/test_ghost_parser.py @@ -7,7 +7,9 @@ from pesacheck.ingest.ghost_parser import GhostParser -FIXTURE_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../fixtures/ghost_export") +FIXTURE_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "../fixtures/ghost_export" +) FIXTURE_PATH = os.path.join(FIXTURE_DIR, "ghost_export.json") @@ -38,118 +40,196 @@ async def test_can_parse_rejects_missing_file(self): # parse — field mapping # ------------------------------------------------------------------ - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_returns_only_published_posts(self, _mock): items = await self.parser.parse(FIXTURE_PATH) # draft (post_003) and page (post_004) should be excluded self.assertEqual(len(items), 2) - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_headline_and_slugline(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertEqual(post1["headline"], "FAUX: This claim is false") self.assertEqual(post1["slugline"], "faux-this-claim-is-false") - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_abstract(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertEqual(post1["abstract"], "A short description of the article.") - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_body_html(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertIn("

This is the article body.

", post1["body_html"]) - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_dates(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertEqual(post1["firstcreated"].isoformat(), "2025-11-01T10:00:00+00:00") - self.assertEqual(post1["versioncreated"].isoformat(), "2025-11-01T11:00:00+00:00") - - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + self.assertEqual( + post1["versioncreated"].isoformat(), "2025-11-01T11:00:00+00:00" + ) + + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_source(self, _mock): items = await self.parser.parse(FIXTURE_PATH) for item in items: self.assertEqual(item["source"], "Ghost") - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_locale_mapped_to_language(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertEqual(post1["language"], "fr") - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_null_locale_guesses_language(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post2 = next(i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb") + post2 = next( + i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb" + ) self.assertIn("language", post2) # ------------------------------------------------------------------ # parse — authors → byline # ------------------------------------------------------------------ - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_byline_multiple_authors_sorted(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertEqual(post1["byline"], "Alice Reporter, Bob Editor") - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_byline_single_author(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post2 = next(i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb") + post2 = next( + i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb" + ) self.assertEqual(post2["byline"], "Alice Reporter") # ------------------------------------------------------------------ # parse — tags → keywords # ------------------------------------------------------------------ - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_keywords(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertEqual(post1["keywords"], ["Fact Check", "Africa"]) - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_keywords_single_tag(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post2 = next(i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb") + post2 = next( + i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb" + ) self.assertEqual(post2["keywords"], ["Fact Check"]) # ------------------------------------------------------------------ # parse — images # ------------------------------------------------------------------ - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_feature_image(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) self.assertIn("associations", post1) featuremedia = post1["associations"]["featuremedia"] self.assertEqual(featuremedia["type"], "picture") self.assertTrue(featuremedia["guid"].endswith("-image")) - self.assertEqual(featuremedia["renditions"]["original"]["href"], "https://example.com/feature.jpg") - - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + self.assertEqual( + featuremedia["renditions"]["original"]["href"], + "https://example.com/feature.jpg", + ) + + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_inline_image_added_as_embedded(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post1 = next(i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa") + post1 = next( + i for i in items if i["guid"] == "aaaaaaaa-0001-0001-0001-aaaaaaaaaaaa" + ) associations = post1["associations"] embedded_keys = [k for k in associations if k.startswith("embedded")] self.assertEqual(len(embedded_keys), 1) embedded = associations[embedded_keys[0]] - self.assertEqual(embedded["renditions"]["original"]["href"], "https://example.com/inline.jpg") + self.assertEqual( + embedded["renditions"]["original"]["href"], "https://example.com/inline.jpg" + ) self.assertEqual(embedded["alt_text"], "An inline image") self.assertEqual(embedded["description_text"], "Image caption here") - @patch("pesacheck.ingest.ghost_parser.update_renditions", side_effect=_mock_update_renditions) + @patch( + "pesacheck.ingest.ghost_parser.update_renditions", + side_effect=_mock_update_renditions, + ) async def test_parse_no_feature_image_no_associations(self, _mock): items = await self.parser.parse(FIXTURE_PATH) - post2 = next(i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb") + post2 = next( + i for i in items if i["guid"] == "bbbbbbbb-0002-0002-0002-bbbbbbbbbbbb" + ) self.assertNotIn("associations", post2) # ------------------------------------------------------------------ diff --git a/server/tests/ingest/test_medium_parser.py b/server/tests/ingest/test_medium_parser.py index 3514127e3a..0fcef54ca9 100644 --- a/server/tests/ingest/test_medium_parser.py +++ b/server/tests/ingest/test_medium_parser.py @@ -9,7 +9,9 @@ def setUp(self): super().setUp() dirname = os.path.dirname(os.path.realpath(__file__)) - fixture_path = os.path.normpath(os.path.join(dirname, "../fixtures/medium_export")) + fixture_path = os.path.normpath( + os.path.join(dirname, "../fixtures/medium_export") + ) self.file_path = os.path.join( fixture_path, "2022-02-27_HOAX--This-UNAIDS-job-advert-in-Uganda-is-fake-f8d269a3d85d.html", @@ -23,9 +25,12 @@ async def test_can_parse(self): async def test_parse_medium_html(self): item = await self.parser.parse(self.file_path) - self.assertEqual(item["headline"], "HOAX: This UNAIDS job advert in Uganda is fake") self.assertEqual( - item["abstract"], "A UNAIDS Communications officer told PesaCheck that the job advertisement is fake." + item["headline"], "HOAX: This UNAIDS job advert in Uganda is fake" + ) + self.assertEqual( + item["abstract"], + "A UNAIDS Communications officer told PesaCheck that the job advertisement is fake.", ) self.assertTrue(item["body_html"].startswith(" Date: Mon, 10 Aug 2026 15:13:00 +0300 Subject: [PATCH 6/6] Pin Github actions to commit SHAs instead of tags --- .github/workflows/ci.yml | 19 ++++++++++++------- .github/workflows/publish.yml | 4 ++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7546248ec..e73aa8b550 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,10 @@ name: "CI" +# Actions are pinned to commit SHAs, not tags: a tag is mutable, so `@v7` means +# "whatever the owner points it at today" — including after an account +# compromise. The trailing `# vX.Y.Z` comment is not decoration; Dependabot +# reads it to know which release the SHA corresponds to, and rewrites both the +# SHA and the comment together when it opens a bump PR. Do not drop it. on: push: branches: [pesacheck] @@ -38,9 +43,9 @@ jobs: working-directory: server steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@v7 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} cache: pip @@ -92,9 +97,9 @@ jobs: discovery.type: single-node steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@v7 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} cache: pip @@ -125,9 +130,9 @@ jobs: working-directory: client steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-node@v7 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: ${{ env.NODE_VERSION }} cache: npm @@ -151,7 +156,7 @@ jobs: component: [server, client] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: docker build . working-directory: ./${{ matrix.component }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a563ea594a..fb87aa0d8f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,9 +18,9 @@ jobs: component: [server, client] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: docker/login-action@v4.5.2 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}