diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 779a1d62ef..95ca951e7b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,34 +1,20 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: # https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file - +# +# This fork deliberately does NOT run npm or pip version updates. +# +# Upstream superdesk/superdesk owns the client and server dependency trees; we +# take their bumps by merging upstream. +# Running Dependabot on the same manifests here produced daily PRs against a very +# large dependency graph — each one firing the full CI suite - and every merged +# bump became another conflict to resolve on the next upstream merge. It cost +# a lot of CI and moved nothing. +# +# Note that Dependabot *security* updates are a separate feature, configured in +# the repository's Settings → Code security, and are unaffected by this file. +# Leave those on — they are the reason dropping version updates is safe. version: 2 updates: - - package-ecosystem: "npm" # See documentation for possible values - directory: "client" # Location of package manifests - schedule: - interval: "daily" - - - package-ecosystem: "pip" # See documentation for possible values - directory: "server" # Location of package manifests - schedule: - interval: "daily" - - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" - - # keep release/3 up to date - - package-ecosystem: "npm" - directory: "client" - target-branch: "release/3" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "server" - target-branch: "release/3" - schedule: - interval: "daily" diff --git a/.github/docker-compose.yml b/.github/docker-compose.yml deleted file mode 100644 index 6c237be754..0000000000 --- a/.github/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: "3.2" - -services: - elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:7.10.1 - ports: - - "9200:9200" - environment: - - discovery.type=single-node - tmpfs: - - /usr/share/elasticsearch/data - - redis: - image: redis:alpine - ports: - - "6379:6379" - - mongo: - image: mongo:4.4 - ports: - - "27017:27017" - tmpfs: - - /data/db - diff --git a/.github/workflows/behave.yml b/.github/workflows/behave.yml deleted file mode 100644 index a5b63bfc50..0000000000 --- a/.github/workflows/behave.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: "Behave" - -on: - workflow_call: - inputs: - python-version: - description: "Python version" - required: false - default: "3.12" - type: string - -permissions: - contents: read - -jobs: - behave: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: server - - services: - redis: - image: redis:alpine - ports: - - "6379:6379" - - mongo: - image: mongo:4 - ports: - - "27017:27017" - - elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.25 - ports: - - "9200:9200" - env: - discovery.type: single-node - - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-python@v7 - with: - python-version: ${{ inputs.python-version }} - cache: pip - - - run: | - sudo apt-get update - sudo apt-get -y install libxml2-dev libxmlsec1-dev libxmlsec1-openssl libexempi-dev - - - run: | - python -m pip install -U pip wheel setuptools - python -m pip install -Ur dev-requirements.txt - - - run: behave - diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..e73aa8b550 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,162 @@ +name: "CI" + +# Actions are pinned to commit SHAs, not tags: a tag is mutable, so `@v7` means +# "whatever the owner points it at today" — including after an account +# compromise. The trailing `# vX.Y.Z` comment is not decoration; Dependabot +# reads it to know which release the SHA corresponds to, and rewrites both the +# SHA and the comment together when it opens a bump PR. Do not drop it. +on: + push: + branches: [pesacheck] + pull_request: + +permissions: + contents: read + +# Superseded runs are pointless: a newer commit on the same branch invalidates +# them. pesacheck itself is exempt because its runs gate the image builds, and +# cancelling those loses the only signal that main-line HEAD is buildable. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/pesacheck' }} + +env: + PYTHON_VERSION: "3.12" + NODE_VERSION: "22" + +jobs: + # black, flake8 and mypy used to be three jobs, each installing the whole of + # dev-requirements.txt (superdesk-core, lxml, xmlsec, …) to run one command. + # None of them need the application: black and flake8 never import it, and + # setup.cfg sets mypy's ignore_missing_imports, so unresolved third-party + # imports degrade to Any rather than erroring. Installing just the three + # tools takes seconds and needs none of the apt libraries. + # + # The trade-off is real but pre-existing: mypy sees external types as Any, so + # it checks less than a full install would. That is what ignore_missing_imports + # already opted into — this only makes it cheap. + lint: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: server + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + cache-dependency-path: server/dev-requirements.txt + + # Versions come from dev-requirements.txt so there is one place to bump. + - run: pip install $(grep -E '^(black|flake8|mypy)==' dev-requirements.txt) + + # `!cancelled()` so one failing linter still reports the other two, instead + # of making you push a fix to find out what else is broken. + - run: black --check . + + - run: flake8 + if: ${{ !cancelled() }} + + - run: mypy . + if: ${{ !cancelled() }} + + # pytest and behave were separate jobs repeating an identical, expensive + # setup: three service containers, an apt install of the xmlsec/exempi + # headers, and all of dev-requirements.txt — to run a handful of parser tests + # and one smoke scenario. They share the environment, so they share the job. + # + # Service versions match docker-compose.yml (mongo 6, redis 8, elastic + # 7.17.29). CI previously tested against mongo 4 while the stack runs 6. + test: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: server + + services: + redis: + image: redis:8 + ports: + - "6379:6379" + + mongo: + image: mongo:6 + ports: + - "27017:27017" + + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.29 + ports: + - "9200:9200" + env: + discovery.type: single-node + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + cache-dependency-path: server/dev-requirements.txt + + - run: | + sudo apt-get update + sudo apt-get -y install libxml2-dev libxmlsec1-dev libxmlsec1-openssl libexempi-dev + + - run: | + python -m pip install -U pip wheel setuptools + python -m pip install -Ur dev-requirements.txt + + - run: pytest + + # One scenario (GET / returns the auth link), but it is the only thing that + # proves the app boots with our settings.py and INSTALLED_APPS. Near-free + # now that it reuses the install above, and it runs even when pytest fails + # so a broken parser test cannot hide a boot failure. + - run: behave + if: ${{ !cancelled() }} + + client: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: client + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: client/package-lock.json + + - run: npm ci + - run: npm run build + + # Gated behind the other three jobs: there is no point spending twenty + # minutes on an image whose code we already know is broken. + images: + needs: [lint, test, client] + if: github.event_name == 'push' + + runs-on: ubuntu-latest + timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + component: [server, client] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - run: docker build . + working-directory: ./${{ matrix.component }} diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml deleted file mode 100644 index ded805689d..0000000000 --- a/.github/workflows/client.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: "Client" - -on: - workflow_call: - inputs: - node-version: - description: "Node version" - required: false - default: "22" - type: string - -permissions: - contents: read - -jobs: - install: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: client - - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-node@v7 - with: - node-version: ${{ inputs.node-version }} - cache: npm - cache-dependency-path: client/package-lock.json - - - run: npm ci - - run: npm run build diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 55143a9f18..0000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: "Docker" - -on: - workflow_call: - -permissions: - contents: read - -jobs: - server: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v7 - - - run: docker build . - working-directory: ./server - timeout-minutes: 10 - - client: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v7 - - - run: docker build . - working-directory: ./client - timeout-minutes: 20 diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml deleted file mode 100644 index f3811ce9dd..0000000000 --- a/.github/workflows/dockerhub.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Publish Docker Images - -on: - release: - types: [released] - -permissions: - contents: read - -jobs: - push_to_dockerhub: - name: Push Docker images to Docker Hub - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v7 - - - name: Login to DockerHub - uses: docker/login-action@v4.5.2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push image - run: docker compose build && docker compose push diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000000..fb87aa0d8f --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,37 @@ +name: "Publish images" + +on: + release: + types: [released] + +permissions: + contents: read + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + component: [server, client] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build and push + env: + IMAGE: codeforafrica/superdesk-pesacheck-${{ matrix.component }} + TAG: ${{ github.event.release.tag_name }} + run: | + docker build \ + --tag "$IMAGE:$TAG" \ + --tag "$IMAGE:latest" \ + ./${{ matrix.component }} + docker push --all-tags "$IMAGE" diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml deleted file mode 100644 index 7e394039d9..0000000000 --- a/.github/workflows/pytest.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: "Pytest" - -on: - workflow_call: - inputs: - python-version: - description: "Python version" - required: false - default: "3.12" - type: string - -permissions: - contents: read - -jobs: - - pytest: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: server - - services: - redis: - image: redis:alpine - ports: - - "6379:6379" - - mongo: - image: mongo:4 - ports: - - "27017:27017" - - elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.25 - ports: - - "9200:9200" - env: - discovery.type: single-node - - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-python@v7 - with: - python-version: ${{ inputs.python-version }} - cache: pip - - - run: | - sudo apt-get update - sudo apt-get -y install libxml2-dev libxmlsec1-dev libxmlsec1-openssl libexempi-dev - - - run: | - python -m pip install -U pip wheel setuptools - python -m pip install -Ur dev-requirements.txt - - - run: pytest - diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml deleted file mode 100644 index 5d2eb50e31..0000000000 --- a/.github/workflows/server.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: "Server" - -on: - workflow_call: - inputs: - python-version: - description: "Python version" - required: false - default: "3.12" - type: string - -permissions: - contents: read - -defaults: - run: - working-directory: server - -jobs: - install: - runs-on: ubuntu-latest - - services: - redis: - image: redis:alpine - ports: - - "6379:6379" - - mongo: - image: mongo:4 - ports: - - "27017:27017" - - elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.25 - ports: - - "9200:9200" - env: - discovery.type: single-node - - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-python@v7 - with: - python-version: ${{ inputs.python-version }} - cache: 'pip' - - - run: | - sudo apt-get update - sudo apt-get -y install libxml2-dev libxmlsec1-dev libxmlsec1-openssl libexempi-dev - - - run: | - python -m pip install -U pip wheel setuptools - python -m pip install -Ur dev-requirements.txt - - - name: init - run: | - honcho run python manage.py app:initialize_data - if: ${{ inputs.python-version != '3.12' }} # There is some issue with the 3.12 version atm - - black: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: ${{ inputs.python-version }} - cache: 'pip' - - run: pip install -Ur dev-requirements.txt - - run: black --check . - - flake8: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: ${{ inputs.python-version }} - cache: 'pip' - - run: pip install -Ur dev-requirements.txt - - run: flake8 - - mypy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: ${{ inputs.python-version }} - cache: 'pip' - - run: pip install -Ur dev-requirements.txt - - run: mypy . diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 55ecaaf2d1..0000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: "CI" - -on: [push, pull_request] - -permissions: - contents: read - -jobs: - server: - strategy: - fail-fast: false - matrix: - python-version: ['3.12'] - uses: ./.github/workflows/server.yml - with: - python-version: ${{ matrix.python-version }} - - pytest: - strategy: - fail-fast: false - matrix: - python-version: ['3.12'] - uses: ./.github/workflows/pytest.yml - with: - python-version: ${{ matrix.python-version }} - - behave: - strategy: - fail-fast: false - matrix: - python-version: ['3.12'] - uses: ./.github/workflows/behave.yml - with: - python-version: ${{ matrix.python-version }} - - client: - strategy: - fail-fast: false - matrix: - node-version: ['22'] - uses: ./.github/workflows/client.yml - with: - node-version: ${{ matrix.node-version }} - - docker: - uses: ./.github/workflows/docker.yml diff --git a/README.md b/README.md index 276932a2af..742bdf49d7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # Superdesk -[![Test](https://github.com/superdesk/superdesk/actions/workflows/tests.yml/badge.svg)](https://github.com/superdesk/superdesk/actions/workflows/tests.yml) -[![Lint](https://github.com/superdesk/superdesk/actions/workflows/lint.yml/badge.svg)](https://github.com/superdesk/superdesk/actions/workflows/lint.yml) +[![CI](https://github.com/CodeForAfrica/superdesk-pesacheck/actions/workflows/ci.yml/badge.svg?branch=pesacheck)](https://github.com/CodeForAfrica/superdesk-pesacheck/actions/workflows/ci.yml) Superdesk is an open source end-to-end news creation, production, curation, distribution and publishing platform developed and maintained by Sourcefabric @@ -36,7 +35,7 @@ $ docker compose exec superdesk-server python manage.py users:create -u admin -p Then you can login with admin:admin credentials. -The Docker images are hosted on Dockerhub for the [client](https://hub.docker.com/r/sourcefabricoss/superdesk-client) and [server](https://hub.docker.com/r/sourcefabricoss/superdesk-server). +This fork publishes its own images on release, to Dockerhub for the [client](https://hub.docker.com/r/codeforafrica/superdesk-pesacheck-client) and [server](https://hub.docker.com/r/codeforafrica/superdesk-pesacheck-server). Upstream's images (`sourcefabricoss/superdesk-client`, `sourcefabricoss/superdesk-server`) do not carry our ingest module or content config. ## Local development (frontend / extension work)