diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index bf40eee50..fc7b69a7e 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -15,7 +15,7 @@ - Chart pregenerator service: `chart_pregenerator/` - Templates: each app has its own `templates/` directory - Compiled bundles output: `build/static/bundles/` -- DevOps/Docker: `devops/docker/`, `devops/scripts/` +- Container builds and CI helper scripts: `ci/` (`ci/containers/Containerfile`, `ci/scripts/`) - Requirements: `requirements.txt`, `dev-requirements.txt`, `ci-requirements.txt` (compiled from `.in` files via pip-compile) ## Tech Stack @@ -31,9 +31,8 @@ ## Development -- `make dev-init` for initial setup (creates `.env` with UID) -- `docker compose up` to start services (postgresql, django, node, node-chart-pregenerator, selenium) -- `docker compose exec django ./manage.py createdevdata` to seed dev data +- `just dev` (or `just dev-init` then `docker compose up`) to start services (postgresql, django, node, node-chart-pregenerator, selenium) +- `just createdevdata` to seed dev data - Web app at `http://localhost:8000` - Wagtail admin at `/admin` (dev credentials: `test` / `test`) - API docs at `/api/schema/swagger-ui/` @@ -42,26 +41,28 @@ ## Testing -- Django tests: `make dev-tests` (runs with coverage via `coverage run ./manage.py test --noinput --failfast`) -- Jest tests: `make dev-jest-tests` (runs both main frontend and chart pregenerator tests) -- Migration check: `make check-migrations` +- Django tests: `just test` (runs with coverage via `coverage run ./manage.py test --noinput`) +- Jest tests: `just test-js` (runs both main frontend and chart pregenerator tests) +- Migration check: `just check-migrations` - CI enforces **100% coverage on changed lines** using `diff-cover` against `origin/develop` - Tests use a custom `SeededDiscoveryRunner` with fixed seed (`876394101`) for reproducibility - Tests live in `tests/` subdirectories within each Django app (e.g., `incident/tests/`) ## Linting and Code Quality -- `make ruff` — linter/formatter -- `make bandit` — security static analysis -- `make eslint` — JavaScript linting (airbnb config) -- `make stylelint` — SCSS linting (sass-guidelines config) +- `just ruff` — linter/formatter (`just ruff-fix` applies fixes) +- `just bandit` — security static analysis +- `just eslint` — JavaScript linting (airbnb config) +- `just stylelint` — SCSS linting (sass-guidelines config) +- `just lint` — all of the above plus the migration check - Ruff configured in `pyproject.toml`: `select = ["I", "F4"]` (isort + unused imports), target py312, Django/Wagtail-aware import section ordering ## Dependency Management - Uses pip-tools (`pip-compile`) for reproducible, hash-verified pinning -- Edit `.in` files, then `make compile-pip-dependencies` to recompile -- `make pip-update PACKAGE=name` to upgrade a specific package +- Edit `.in` files, then `just pip-compile` to recompile +- `just pip-compile --upgrade-package=name` to upgrade a specific package +- `just pip-check` fails if the lockfiles drift from the `.in` files (CI runs it) - pip-compile runs inside a Docker container matching the production Python version ## Frontend Build System @@ -76,8 +77,8 @@ ## Database - Dev credentials: user `tracker`, password `trackerpassword`, db `trackerdb`, port `5432` -- `make dev-import-db` — import a database dump (place as `import.db` in repo root) -- `make dev-save-db` / `make dev-restore-db` — save/restore DB snapshots per branch +- `just import-db` — import a database dump (place as `import.db` in repo root) +- `just save-db` / `just restore-db` — save/restore DB snapshots per branch ## Debugging diff --git a/.dockerignore b/.dockerignore index 9e5109b10..64f0a4773 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,66 @@ +# We ignore all files in the repo by default, +# then selectively permit paths for inclusion. +# We do this to avoid non-source-code files, such as +# .env, *.db, db-snapshots/, coverage.xml, etc. from +# making it into the built artifact. + + +# ignore everything +* + +# permit Django apps and the project package +!blog +!charts +!cloudflare +!common +!dashboard +!emails +!forms +!geonames +!home +!incident +!menus +!statistics +!styleguide +!tracker +# `build` is an INSTALLED_APP (a namespace package) as well as webpack's +# output directory, so Django will not start without it. +!build + +# permit backend packaging tooling +!manage.py +!requirements.txt +!dev-requirements.txt +!pyproject.toml +!scripts +# read at request time by statistics/views.py +!STATISTICS.rst + +# permit frontend packaging tooling +!client +!package.json +!package-lock.json +!webpack.config.js +!babel.config.js +!postcss.config.js +# the chart pregenerator is its own npm project; the `chartgen` stage copies it wholesale +!chart_pregenerator + +# permit general build tooling, +# e.g. for git version info +!ci +!.git + +# re-exclude unwanted artifact cruft, possibly included by the grants above +**/__pycache__ **/*.pyc -.env -client/build/ -db-snapshots -logs -media/ -node_modules -npm-debug.log +**/node_modules tracker/settings/local.py -.versioninfo +# Stale dev bundles from a host `npm run start` would otherwise survive the +# prod asset build and break collectstatic's manifest rewriting. +build/static/bundles +# esbuild output and jest coverage are rebuilt in-image; `client` here is a +# gitignored convenience symlink some developers keep. +chart_pregenerator/build +chart_pregenerator/coverage +chart_pregenerator/client diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index f27058590..d17b51841 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -5,16 +5,15 @@ on: pull_request: types: ['opened', 'synchronize'] push: - branches: ['prod', 'develop'] + branches: ['develop', 'prod'] merge_group: env: + CONTAINER_ENGINE: docker + VERSION_DOCKER: v29.6.2 COMPOSE_FILE_DEV: docker-compose.yaml COMPOSE_FILE_PROD: prod-docker-compose.yaml - # See here for valid options: - # https://github.com/docker/setup-docker-action?tab=readme-ov-file#inputs.version - VERSION_DOCKER: latest - # Must match the version specified in devops/docker/DevDjangoDockerfile + # Must match PYTHON_IMAGE in ci/containers/Containerfile VERSION_PYTHON: 3.14.6 permissions: @@ -31,9 +30,12 @@ jobs: uses: freedomofpress/actionslib/.github/workflows/oci-build.yaml@main with: context: '.' - containerfile: devops/docker/DevDjangoDockerfile - build-args: | - USERID=1001 + containerfile: ci/containers/Containerfile + target: dev + # All five build jobs share one Containerfile, so hadolint only needs to + # run in one of them. Honours .hadolint.yaml at the repo root. + lint: true + build-args: USERID=1001 registry: localhost/pressfreedomtracker-us-dev-django build-dev-node: @@ -41,9 +43,9 @@ jobs: uses: freedomofpress/actionslib/.github/workflows/oci-build.yaml@main with: context: '.' - containerfile: devops/docker/NodeDockerfile - build-args: | - USERID=1001 + containerfile: ci/containers/Containerfile + target: node-dev + build-args: USERID=1001 registry: localhost/pressfreedomtracker-us-dev-node build-dev-chartgen: @@ -51,9 +53,9 @@ jobs: uses: freedomofpress/actionslib/.github/workflows/oci-build.yaml@main with: context: '.' - containerfile: devops/docker/NodeChartPregeneratorDevDockerfile - build-args: | - USERID=1001 + containerfile: ci/containers/Containerfile + target: chartgen-dev + build-args: USERID=1001 registry: localhost/pressfreedomtracker-us-dev-chartgen build-prod-django: @@ -61,9 +63,11 @@ jobs: uses: freedomofpress/actionslib/.github/workflows/oci-build.yaml@main with: context: '.' - containerfile: devops/docker/ProdDjangoDockerfile - build-args: | - USERID=1000 + containerfile: ci/containers/Containerfile + target: prod + # No USERID override: the Containerfile defaults to 1000, matching both + # publish.yaml and the k8s securityContext, so the image tested here is + # the same artifact that gets deployed. registry: localhost/pressfreedomtracker-us requires-deep-history: true @@ -72,28 +76,68 @@ jobs: uses: freedomofpress/actionslib/.github/workflows/oci-build.yaml@main with: context: '.' - containerfile: devops/docker/NodeChartPregeneratorDockerfile - build-args: | - USERID=1001 + containerfile: ci/containers/Containerfile + target: chartgen + # Likewise: the chartgen stages default to 1001, the uid publish.yaml ships. registry: localhost/pressfreedomtracker-us-chartgen + lint-lockfiles: + # Standalone, with no `needs`, so it runs alongside the image builds rather + # than waiting on them: it uses a throwaway pip-tools container, not the dev + # image. The dev image installs the checked-in dev-requirements.txt, so + # nothing else in CI would notice the lockfiles drifting from the .in files. + name: Lint:Lockfiles + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - name: Install just + uses: taiki-e/install-action@6cd13508893c0e7eab5f273c2575d3859bd7229a # v2.86.6 + with: + tool: just + + - name: Recompile lockfiles and fail on drift + run: just pip-check + test: - name: Test:${{ matrix.target }} + # This job name is also the branch-protection check name, so it is decoupled + # from the just recipe: `name` pins the published check where a recipe has + # since been renamed, defaulting to the recipe name otherwise. Renaming a + # pinned check means updating the repo's required checks in the same breath. + name: Test:${{ matrix.name || matrix.target }} runs-on: ubuntu-latest needs: - build-dev-django - build-dev-node - build-dev-chartgen strategy: + # A failing linter shouldn't cancel the Django suite's feedback. + fail-fast: false matrix: - target: - - check-migrations - - dev-tests - - dev-jest-tests - - ruff - - bandit - - eslint - - stylelint + include: + # These run in a one-shot container via `compose run --no-deps`, so + # they need no service stack and the compose-up below is skipped. + - target: check-migrations + stack: false + - target: bandit + stack: false + - target: ruff + stack: false + - target: eslint + stack: false + - target: stylelint + stack: false + - target: test-js + name: dev-jest-tests + stack: false + # The Django suite needs live postgres, selenium, the built asset + # bundles and the chart pregenerator. + - target: test + name: dev-tests + stack: true steps: - name: Install Docker uses: docker/setup-docker-action@v5.4.0 @@ -103,8 +147,7 @@ jobs: - name: Checkout uses: actions/checkout@v7.0.1 with: - persist-credentials: true - fetch-depth: 0 + persist-credentials: false - name: Download dev django image uses: actions/download-artifact@v8 @@ -142,17 +185,26 @@ jobs: docker image tag "$NODE_IMAGE_URL" localhost/pressfreedomtracker-us-node docker image tag "$CHARTGEN_IMAGE_URL" localhost/pressfreedomtracker-us-node-chart-pregenerator + - name: Install just + uses: taiki-e/install-action@6cd13508893c0e7eab5f273c2575d3859bd7229a # v2.86.6 + with: + tool: just + - name: Create env config - run: make dev-init + run: just dev-init + # --no-build asserts the images restored above are the ones under test: + # compose errors out if a tag is missing rather than silently rebuilding + # from source, which would mask a broken artifact hand-off. - name: Start + if: ${{ matrix.stack }} run: docker compose --file="$COMPOSE_FILE_DEV" up --wait --wait-timeout=180 --detach --pull=missing --no-build - - name: Make target '${{ matrix.target }}' - run: make ${{ matrix.target }} + - name: Run target '${{ matrix.target }}' + run: just ${{ matrix.target }} - name: Collect logs on failure - if: ${{ failure() }} + if: ${{ failure() && matrix.stack }} run: | docker compose --file="$COMPOSE_FILE_DEV" logs django docker compose --file="$COMPOSE_FILE_DEV" logs node-chart-pregenerator @@ -160,9 +212,9 @@ jobs: - name: Save HTML coverage uses: actions/upload-artifact@v7 - if: ${{ matrix.target == 'dev-tests' }} + if: ${{ matrix.target == 'test' }} with: - name: test-${{ matrix.target }}-${{ github.run_id }}-htmlcov + name: test-${{ matrix.name }}-${{ github.run_id }}-htmlcov path: htmlcov if-no-files-found: error overwrite: true @@ -170,9 +222,9 @@ jobs: - name: Save PyTest coverage uses: actions/upload-artifact@v7 - if: ${{ matrix.target == 'dev-tests'}} + if: ${{ matrix.target == 'test' }} with: - name: test-${{ matrix.target }}-${{ github.run_id }}-pytestcov + name: test-${{ matrix.name }}-${{ github.run_id }}-pytestcov path: coverage.xml if-no-files-found: error overwrite: true @@ -262,8 +314,8 @@ jobs: docker image load --input "$CHARTGEN_LOAD_FILE" # This tag needs to match whatever is in the prod compose file - docker image tag "$DJANGO_IMAGE_URL" quay.io/freedomofpress/pressfreedomtrackerus - docker image tag "$CHARTGEN_IMAGE_URL" quay.io/freedomofpress/pft-chart-pregenerator + docker image tag "$DJANGO_IMAGE_URL" ghcr.io/freedomofpress/pressfreedomtracker-us + docker image tag "$CHARTGEN_IMAGE_URL" ghcr.io/freedomofpress/pressfreedomtracker-us-chartgen - name: Start run: docker compose --file="$COMPOSE_FILE_PROD" up --wait --wait-timeout=180 --detach --pull=missing --no-build @@ -273,13 +325,10 @@ jobs: docker compose --file="$COMPOSE_FILE_PROD" exec django \ /bin/bash -c "./manage.py createdevdata --no-download" + # Against the published port, from outside the container: the artifact + # under test is not modified to test it. - name: Verify site skeleton - run: | - docker compose --file="$COMPOSE_FILE_PROD" exec django \ - pip install --requirement=/django/requirements.txt --require-hashes --no-color --disable-pip-version-check - - docker compose --file="$COMPOSE_FILE_PROD" exec django \ - ./scripts/pytest + run: curl --fail --silent --show-error http://localhost:8000/ | grep -q 'Press Freedom Tracker' - name: Collect logs on failure if: ${{ failure() }} @@ -298,10 +347,10 @@ jobs: - build-prod-chartgen - test - test-prod - if: ${{ always() }} permissions: contents: read actions: write + if: ${{ always() }} steps: - name: Delete dev django image if: ${{ needs.build-dev-django.result == 'success' && always() }} @@ -328,7 +377,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Delete prod django image - if: ${{ needs.build-prod-dejango.result == 'success' && always() }} + if: ${{ needs.build-prod-django.result == 'success' && always() }} uses: freedomofpress/actionslib/act/delete-artifact@main with: artifact: ${{ needs.build-prod-django.outputs.artifact-name }} diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index da7c6e54a..27e47a96a 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -22,9 +22,9 @@ jobs: packages: write with: context: '.' - containerfile: devops/docker/ProdDjangoDockerfile - build-args: | - USERID=1000 + containerfile: ci/containers/Containerfile + target: prod + build-args: USERID=1000 registry: ghcr.io/freedomofpress/pressfreedomtracker-us requires-deep-history: true add-branch-tags: true @@ -38,16 +38,16 @@ jobs: packages: write with: context: '.' - containerfile: devops/docker/NodeChartPregeneratorDockerfile - build-args: | - USERID=1001 + containerfile: ci/containers/Containerfile + target: chartgen + build-args: USERID=1001 registry: ghcr.io/freedomofpress/pressfreedomtracker-us-chartgen add-branch-tags: true notify-django: uses: freedomofpress/actionslib/.github/workflows/update-k8s-trigger.yaml@main needs: - - build-django + - build-django if: ${{ needs.build-django.result == 'success' }} permissions: contents: read @@ -63,7 +63,7 @@ jobs: notify-chartgen: uses: freedomofpress/actionslib/.github/workflows/update-k8s-trigger.yaml@main needs: - - build-chartgen + - build-chartgen if: ${{ needs.build-chartgen.result == 'success' }} permissions: contents: read diff --git a/.gitignore b/.gitignore index aa32191f8..a7ce47a92 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ db.sqlite3* /pressfreedom/settings/local.py *.pyc __pycache__ -.pytest_cache .coverage coverage.xml @@ -20,17 +19,11 @@ webpack-stats.json .manage_cmds_ran/ logs/ -# devops virtualenv -.molecule .cache -*.retry # Functional tests geckodriver.log -# pip+docker install artifacts -src/ - # developer postgres testing import.db db-snapshots/ @@ -43,9 +36,6 @@ test-results # docker-compose .env -# Ignore files created to communicate system deployment info -.versioninfo - # Jetbrains IDE .idea diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 000000000..71e9b3351 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,11 @@ +ignored: + # Base images are already pinned by digest (see ci/containers/Containerfile), + # so the apt package set for a given build is already fixed in practice. + # Hand-pinning individual package versions on top of that would just break + # builds silently whenever Debian trixie rotates a version, for no real + # reproducibility gain. + - DL3008 + # We ensure that containers are run in prod with `runAsUser: 1000`, + # and also port dev UIDs into the container build, so we're sufficiently + # defensive about UIDs within the container. + - DL3066 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 44665e4b4..b225c54e7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,10 +30,10 @@ To get started working with the code, clone the repo and follow the instructions ### Code guidelines -- Python code should follow [PEP8](https://www.python.org/dev/peps/pep-0008) where possible. One exception to this is line lengths beyond 79 characters are allowed. You can check your compliance with the `make lint` command. +- Python code should follow [PEP8](https://www.python.org/dev/peps/pep-0008) where possible. One exception to this is line lengths beyond 79 characters are allowed. You can check your compliance with the `just lint` command. - Writing tests is strongly encouraged, especially if significant behavior is added or changed. - - Django-based tests are found in the `tests/` subdirectories of each separate app. Run the entire test suite with `make dev-tests` - - JavaScript, jest-based tests are found in the `client/common/js/tests/` directory. Run this test suite with `make dev-jest-tests` + - Django-based tests are found in the `tests/` subdirectories of each separate app. Run the entire test suite with `just test` + - JavaScript, jest-based tests are found in the `client/common/js/tests/` directory. Run this test suite with `just test-js` - JavaScript, CSS/SASS, and HTML markup should mimic the styles and patterns in the existing codebase. ## Conduct and communication diff --git a/Makefile b/Makefile deleted file mode 100644 index 31d8c2d17..000000000 --- a/Makefile +++ /dev/null @@ -1,146 +0,0 @@ -.DEFAULT_GOAL := help -DIR := ${CURDIR} -UID := $(shell id -u) -RAND_PORT := ${RAND_PORT} -GIT_REV := $(shell git rev-parse HEAD | cut -c1-10) -GIT_BR := $(shell git rev-parse --abbrev-ref HEAD) -REMOTE_IMAGE := quay.io/freedomofpress/tracker.us - -.PHONY: lint -lint: ruff - -.PHONY: ruff -ruff: ## Runs ruff linting in Python3 container. - @docker compose run --rm -T django /bin/bash -c "ruff check && ruff format --check" - -.PHONY: dev-init -dev-init: ## Initialize docker environment for developer workflow - echo UID=$(UID) > .env - -.PHONY: open-browser -open-browser: ## Opens a web-browser pointing to the compose env - @./devops/scripts/browser-open.sh - -.PHONY: dev-import-db -dev-import-db: ## Import a postgres export file located at import.db - docker compose exec -it postgresql bash -c "cat /django/import.db | sed 's/OWNER\ TO\ [a-z]*/OWNER\ TO\ postgres/g' | psql securedropdb -U postgres &> /dev/null" - -.PHONY: save-db -dev-save-db: ## Export developer db to file - ./devops/scripts/savedb.sh - -.PHONY: ci-go -ci-go: ## Stands up a prod like environment under one docker container - @molecule test -s ci - -.PHONY: ci-tests -ci-tests: ## Runs testinfra against a pre-running CI container (useful for debugging) - @molecule verify -s ci - -.PHONY: dev-tests -dev-tests: ## Run django tests against developer environment - docker compose exec django /bin/bash -ec \ - "coverage run ./manage.py test --noinput ; \ - coverage html ; \ - coverage xml ; \ - coverage report" - -.PHONY: dev-jest-tests -dev-jest-tests: ## Run django tests against developer environment - docker compose exec node npm test - docker compose exec node-chart-pregenerator npm run test - -.PHONY: compile-pip-dependencies -compile-pip-dependencies: ## Uses pip-compile to update requirements.txt -# It is critical that we run pip-compile via the same Python version -# that we're generating requirements for, otherwise the versions may -# be resolved differently. - docker run --rm -v "$(DIR):/code" -w /code -it python:3.14.6-slim-trixie \ - bash -c 'apt-get update && apt-get install gcc libpq-dev -y && \ - pip install pip-tools && \ - pip-compile --generate-hashes --no-header --allow-unsafe --output-file requirements.txt requirements.in && \ - pip-compile --generate-hashes --no-header --allow-unsafe --output-file ci-requirements.txt ci-requirements.in && \ - pip-compile --generate-hashes --no-header --allow-unsafe --output-file dev-requirements.txt dev-requirements.in' - -.PHONY: pip-update -pip-update: ## Uses pip-compile to update requirements.txt for upgrading a specific package -# It is critical that we run pip-compile via the same Python version -# that we're generating requirements for, otherwise the versions may -# be resolved differently. - docker run --rm -v "$(DIR):/code" -w /code -it python:3.14.6-slim-trixie \ - bash -c 'apt-get update && apt-get install gcc libpq-dev -y && \ - pip install pip-tools && \ - pip-compile --generate-hashes --no-header --allow-unsafe --upgrade-package $(PACKAGE) --output-file requirements.txt requirements.in && \ - pip-compile --generate-hashes --no-header --allow-unsafe --upgrade-package $(PACKAGE) --output-file ci-requirements.txt ci-requirements.in && \ - pip-compile --generate-hashes --no-header --allow-unsafe --upgrade-package $(PACKAGE) --output-file dev-requirements.txt dev-requirements.in' - -.PHONY: pip-upgrade -pip-upgrade: ## Uses pip-compile to update all requirements that are not pinned -# in requirements.in - docker run --rm -v "$(DIR):/code" -w /code -it python:3.14.6-slim-trixie \ - bash -c 'apt-get update && apt-get install gcc libpq-dev -y && \ - pip install pip-tools && \ - pip-compile --generate-hashes --no-header --allow-unsafe --upgrade --output-file requirements.txt requirements.in && \ - pip-compile --generate-hashes --no-header --allow-unsafe --upgrade --output-file dev-requirements.txt dev-requirements.in' - -.PHONY: pip-dev-upgrade -pip-dev-upgrade: ## Uses pip-compile to update all dev requirements that are not pinned -# in dev-requirements.in - docker run --rm -v "$(DIR):/code" -w /code -it python:3.14.6-slim-trixie \ - bash -c 'apt-get update && apt-get install gcc libpq-dev -y && \ - pip install pip-tools && \ - pip-compile --generate-hashes --no-header --allow-unsafe --upgrade --output-file dev-requirements.txt dev-requirements.in' - -.PHONY: pip-dev-update -pip-dev-update: ## Uses pip-compile to update dev-requirements.txt for upgrading a specific package -# It is critical that we run pip-compile via the same Python version -# that we're generating requirements for, otherwise the versions may -# be resolved differently. - docker run --rm -v "$(DIR):/code" -w /code -it python:3.14.6-slim-trixie \ - bash -c 'apt-get update && apt-get install gcc libpq-dev -y && \ - pip install pip-tools && \ - pip-compile --generate-hashes --no-header --allow-unsafe --upgrade-package $(PACKAGE) --output-file dev-requirements.txt dev-requirements.in' - - -.PHONY: upgrade-pip-tools -upgrade-pip-tools: ## Update the version of pip-tools used for other pip-related make commands - docker run --rm -v "$(DIR):/code" -w /code -it python:3.14.6-slim-trixie \ - bash -c 'pip install pip-tools && \ - pip-compile --generate-hashes --no-header --allow-unsafe --upgrade-package pip-tools --output-file pip-tools-requirements.txt pip-tools-requirements.in' - - -# Explanation of the below shell command should it ever break. -# 1. Set the field separator to ": ##" to parse lines for make targets. -# 2. Check for second field matching, skip otherwise. -# 3. Print fields 1 and 2 with colorized output. -# 4. Sort the list of make targets alphabetically -# 5. Format columns with colon as delimiter. -.PHONY: help -help: ## Prints this message and exits - @printf "Makefile for developing and testing Secure The News.\n" - @printf "Subcommands:\n\n" - @perl -F':\s+##\s+' -lanE '$$F[1] and say "\033[36m$$F[0]\033[0m : $$F[1]"' $(MAKEFILE_LIST) \ - | sort \ - | column -s ':' -t - -.PHONY: eslint -eslint: - docker compose exec node npm run js-lint - -.PHONY: stylelint -stylelint: - docker compose exec node npm run stylelint - -.PHONY: check-migrations -check-migrations: ## Check for ungenerated migrations - docker compose exec -T django /bin/bash -c "./manage.py makemigrations --dry-run --check" - -.PHONY: bandit -bandit: ## Runs bandit static code analysis in Python3 container. - @docker compose run --rm django ./scripts/bandit - -.PHONY: prod-push -prod-push: ## Publishes prod container image to registry - docker tag $(REMOTE_IMAGE):latest $(REMOTE_IMAGE):$(GIT_REV)-$(GIT_BR) - docker push $(REMOTE_IMAGE):latest - docker push $(REMOTE_IMAGE):$(GIT_REV)-$(GIT_BR) diff --git a/README.rst b/README.rst index fe25ee5af..7833dbd8a 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,12 @@ The installation instructions below assume you have the following software on yo * `docker `_ * `docker compose `_ +* `just `_, which runs the + project's developer commands. Run ``just`` on its own at any point to + list them. + +`Podman `_ works too, provided it has +"compose" support. Set ``CONTAINER_ENGINE=podman`` to select it. Local Development instructions ------------------------------ @@ -48,14 +54,12 @@ environment, run the following your first run: .. code:: bash - # One-time command to run and forget - make dev-init - - # Starts up the environment - docker compose up + # Starts up the environment (records your host UID in .env on the way, + # so a bare `docker compose up` works afterwards too) + just dev # Inject development data (also only needs to be run once) - docker compose exec django ./manage.py createdevdata + just createdevdata # install pre-commit and set up hooks pip install pre-commit @@ -65,7 +69,7 @@ environment, run the following your first run: # Add wagtail inventory to search wagtail pages by block type docker compose exec django ./manage.py block_inventory -You should be able to hit the web server interface by running ``make open-browser`` +You should be able to hit the web server interface by running ``just open-browser`` If you run into any issues starting the application locally, check the `troubleshooting doc `_ for solutions to common problems. @@ -88,7 +92,7 @@ To test your frontend code with jest, you can run the following command: .. code:: bash - docker compose exec node npm test + just test-js If tests need to be updated, you can run the following command: @@ -161,11 +165,11 @@ Add the desired dependency to the appropriate ``.in`` file, then run: .. code:: bash - make compile-pip-dependencies + just pip-compile All requirements files will be regenerated based on compatible versions. Multiple ``.in`` -files can be merged into a single ``.txt`` file, for use with ``pip``. The Makefile -target handles the merging of multiple files. +files can be merged into a single ``.txt`` file, for use with ``pip``. The just +recipe handles the merging of multiple files. This process is the same if a requirement needs to be changed (i.e. its version number restricted) or removed. Make the appropriate change in the correct ``requirements.in`` file, then run the above command to compile the dependencies. @@ -176,7 +180,7 @@ There are separate commands to upgrade a package without changing the ``requirem .. code:: bash - make pip-update PACKAGE=package-name + just pip-compile --upgrade-package=package-name will update the package named ``package-name`` to the latest version allowed by the constraints in ``requirements.in`` and compile a new ``dev-requirements.txt`` and ``requirements.txt`` based on that version. @@ -184,7 +188,7 @@ If the package appears only in ``dev-requirements.in``, then you must use this c .. code:: bash - make pip-dev-update PACKAGE=package-name + just pip-compile-dev --upgrade-package=package-name which will update the package named ``package-name`` to the latest version allowed by the constraints in ``requirements.in`` and compile a new ``dev-requirements.txt``. @@ -196,15 +200,15 @@ Database import +++++++++++++++ Drop a postgres database dump into the root of the repo and rename it to -``import.db``. To import it into a running dev session (ensure ``make dev-go`` has -already been started) run ``make dev-import-db``. Note that this will not pull in +``import.db``. To import it into a running dev session (ensure ``just dev`` has +already been started) run ``just import-db``. Note that this will not pull in images that are referenced from an external site backup. Connect to postgresql service from host +++++++++++++++++++++++++++++++++++++++ -The postgresql service is exposed to your host on port ``15432``. If you have a GUI +The postgresql service is exposed to your host on ``127.0.0.1:5432``. If you have a GUI database manipulation application you'd like to utilize, your settings will be: * username - ``tracker`` @@ -222,7 +226,7 @@ reverse nginx proxy, and debug mode off using the following command: docker compose -f prod-docker-compose.yaml up -All subsequent docker compose files will need that explicit ``-f`` flag pointing +All subsequent docker compose commands will need that explicit ``-f`` flag pointing to the production-like compose file. Database snapshots @@ -235,13 +239,13 @@ therefore helpful to be able to easily restore the database to a known-good state when making experimental changes. There are two commands provided to assist in this. -``make dev-save-db``: Saves a snapshot of the current state of the +``just save-db``: Saves a snapshot of the current state of the database to a file in the ``db-snapshots`` folder. This file is named for the currently checked-out git branch. -``make dev-restore-db``: Restores the most recent snapshot for the +``just restore-db``: Restores the most recent snapshot for the currently checked-out git branch. If none can be found, that is, -``make dev-save-db`` has never been run for the current branch, this +``just save-db`` has never been run for the current branch, this command will do nothing. If a saved database is found, all data in database will be replaced with that from the file. Note that this command will terminate all connections to the database and delete all @@ -250,7 +254,7 @@ data there, so care is encouraged. Workflow suggestions. I find it helpful to have one snapshot for each active branch I'm working on or reviewing, as well as for develop. Checking out a new branch and running its migrations should be -followed by running ``make dev-save-db`` to give you a baseline to +followed by running ``just save-db`` to give you a baseline to return to when needed. When checking out a new branch after working on another, it can be @@ -286,7 +290,13 @@ for production use, run: .. code:: bash - docker build --build-arg USERID=1000 -t TAG -f devops/docker/ProdDjangoDockerfile . + docker build -t TAG -f ci/containers/Containerfile --target prod . + +and for the chart pregenerator service: + +.. code:: bash + + docker build -t TAG -f ci/containers/Containerfile --target chartgen . Running ------------- diff --git a/ci/containers/Containerfile b/ci/containers/Containerfile new file mode 100644 index 000000000..73f606536 --- /dev/null +++ b/ci/containers/Containerfile @@ -0,0 +1,237 @@ +# Multi-stage container build for the pressfreedomtracker.us website. +# Build targets: +# +# node-dev dev webpack watcher (docker-compose.yaml `node` service) +# chartgen-dev dev chart pregenerator, sources bind-mounted +# (docker-compose.yaml `node-chart-pregenerator` service) +# dev dev Django, code bind-mounted (docker-compose.yaml `django` service) +# prod prod Django, code + built assets baked in (prod-docker-compose.yaml) +# and the published ghcr.io/freedomofpress/pressfreedomtracker-us image +# chartgen prod chart pregenerator, server bundle baked in (prod-docker-compose.yaml) +# and the published ghcr.io/freedomofpress/pressfreedomtracker-us-chartgen image +# +# `assets`, `prod-deps` and `version` are internal build stages feeding `prod`, +# and `chartgen-base` feeds both chartgen targets; none are targeted directly. + +# ---- pinned base image digests ---- +# Bump the tag/digest here; every stage below references these, so this is +# the only place that needs editing on update -- except PYTHON_IMAGE, which is +# also mirrored by the justfile's `python_builder` and must be bumped with it. +# Names are fully qualified: an unqualified `node:` or `python:` resolves against +# the builder's unqualified-search-registries, which under podman can reach a +# non-Docker-Hub registry first. +# sha256 for node:26.5.0-trixie as of 2026-07-15 +ARG NODE_IMAGE=docker.io/library/node:26.5.0-trixie@sha256:0473e7dc433a1310f436edee02aa79737ec78a4b345433ab0963d4a256f9ad85 +# sha256 for python:3.14.6-slim-trixie on linux/amd64 as of 2026-06-17 +ARG PYTHON_IMAGE=docker.io/library/python:3.14.6-slim-trixie@sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061 + +# ---- node runtime base (dev watch service) ---- +FROM ${NODE_IMAGE} AS node-base + +# jq backs the `node` service healthcheck, which polls the `status` field of +# build/static/bundles/webpack-stats.json. +RUN apt-get update && apt-get install -y --no-install-recommends \ + jq \ + && rm -rf /var/lib/apt/lists/* + +COPY ci/containers/node-start.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/node-start.sh + +# node:*-trixie ships a `node` account at uid 1000, so a developer whose host uid +# is 1000 collides with it; reuse whichever account holds the uid rather than +# failing. Only the numeric uid matters here -- nothing references the account +# name -- so the unconditional useradd used in python-base below would buy nothing. +ARG USERID +RUN getent passwd "${USERID?USERID must be supplied}" || \ + useradd --create-home --no-log-init --uid "${USERID}" pft_node +USER ${USERID} + +FROM node-base AS node-dev +# No WORKDIR: compose sets `working_dir: /django` over the bind-mounted checkout, +# and node-start.sh uses only relative paths. +CMD [ "node-start.sh" ] + +# ---- chart pregenerator base (shared by chartgen-dev + chartgen) ---- +FROM ${NODE_IMAGE} AS chartgen-base + +# Not derived from node-base: that stage ends with a USER switch, and the font +# install below needs root. ttf-mscorefonts-installer lives in Debian's non-free +# component and fetches its payload from SourceForge at build time, so this +# stage depends on more than the registry being reachable. +# The pipefail SHELL satisfies hadolint DL4006 for the debconf pipe; both +# chartgen leaf stages inherit it. +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN echo "deb http://deb.debian.org/debian trixie main contrib non-free non-free-firmware" > /etc/apt/sources.list && \ + echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true" | debconf-set-selections && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + fontconfig \ + ttf-mscorefonts-installer && \ + fc-cache -f && \ + rm -rf /var/lib/apt/lists/* + +# Defaults to 1001, the uid publish.yaml ships, so a bare build of the `chartgen` +# target is the deployed artifact; the dev compose file overrides it with the +# host uid. Same getent-or-useradd idiom as node-base, for the same reason. +ARG USERID=1001 +RUN getent passwd "${USERID}" || \ + useradd --create-home --no-log-init --uid "${USERID}" pft_node + +# ---- dev chart pregenerator (src + client bind-mounted by compose) ---- +FROM chartgen-base AS chartgen-dev + +COPY ci/containers/chartgen-start.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/chartgen-start.sh + +# Dependencies are installed into the image at /app; compose mounts only +# chart_pregenerator/src and client/ over it, so node_modules never touches the +# host checkout and `npm ci` runs once per image build rather than per start. +ARG USERID=1001 +WORKDIR /app +RUN chown "${USERID}" /app +USER ${USERID} +COPY --chown=${USERID} \ + chart_pregenerator/package.json chart_pregenerator/package-lock.json \ + chart_pregenerator/dev.js chart_pregenerator/jest.config.js \ + chart_pregenerator/babel.config.json ./ +RUN npm ci +CMD [ "chartgen-start.sh" ] + +# ---- prod chart pregenerator (server bundle baked in) ---- +FROM chartgen-base AS chartgen + +LABEL org.opencontainers.image.vendor="Freedom of the Press Foundation" +LABEL org.opencontainers.image.title="pressfreedomtracker.us chart pregenerator" + +# The service imports its chart components straight out of client/, so both +# trees are copied in; the esbuild bundle in build/ is what CMD runs. +ARG USERID=1001 +COPY --chown=${USERID} chart_pregenerator /opt/chart-pregenerator +COPY --chown=${USERID} client /opt/chart-pregenerator/client +WORKDIR /opt/chart-pregenerator +USER ${USERID} +RUN npm install && npm run build +CMD [ "node", "./build/server.js" ] + +# ---- prod asset build (built once, copied into `prod`) ---- +FROM ${NODE_IMAGE} AS assets + +COPY ./ /src-files +WORKDIR /src-files + +# node_modules and .git are build-time only; stripping them lets `prod` copy this +# tree wholesale rather than filtering during chown, and keeps the packages and +# the git history out of the published image. Both still exist in the COPY layer +# above, but `assets` is a discarded stage that is never pushed -- `prod` only +# ever sees the result. +RUN npm install && npm run build && \ + rm -rf node_modules .git + +# ---- python base (shared by dev + prod) ---- +FROM ${PYTHON_IMAGE} AS python-base + +# The previous Dev and Prod images installed the same set; Dev merely split it +# across two stages. `gcc` is omitted as redundant -- build-essential depends on +# it. The -dev headers are retained for any dependency that lacks a cp314 wheel +# and falls back to an sdist. +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + git \ + libpq-dev \ + libssl-dev \ + libxml2-dev \ + libxslt-dev \ + libz-dev \ + netcat-traditional \ + python3-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY ci/containers/django-start.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/django-start.sh + +# Created unconditionally, because `prod` addresses this account by name (COPY +# --chown, chown, USER) -- a conditional useradd would defer the failure to those +# lines instead of this one. python:*-slim-trixie has nothing between uid 100 and +# `nobody`, so the uid is free both for the 1000 default and for any host uid dev +# passes in; a genuine collision fails loudly, right here. +ARG USERID=1000 +RUN useradd --create-home --no-log-init --uid "${USERID}" --user-group gcorn + +# Directories for named-volume permission handling; see +# https://github.com/docker/compose/issues/3270 +# /django is pre-created and app-owned before any stage sets it as WORKDIR -- the +# prod build's collectstatic writes ./static into it as an unprivileged user, and +# production never re-runs collectstatic (its root filesystem is read-only), so +# that baked output is what actually gets served. +RUN mkdir -p /django /django-media /django-static /django-logs /deploy && \ + chown -R gcorn:gcorn /django /django-media /django-static /django-logs /deploy + +# Set here rather than only in `prod` so one-shot invocations +# (`compose run --rm --no-deps django ...`, `docker run`) land in the app root +# whether or not the caller passes -w. +WORKDIR /django + +EXPOSE 8000 + +# ---- dev django (code bind-mounted by compose) ---- +FROM python-base AS dev + +# The checked-in lockfile, not a freshly recompiled one. dev-requirements.in is +# `-r requirements.txt` plus dev extras, so this superset covers both. Lockfile +# drift is caught explicitly by `just pip-check` rather than as a build artifact. +COPY dev-requirements.txt /requirements.txt +RUN pip install --no-deps --no-cache-dir --require-hashes -r /requirements.txt + +# Re-declared for portability, not because buildah needs it: ARG is documented as +# going out of scope at the end of the declaring stage, so `USER ${USERID}` below +# is only guaranteed to resolve where the ARG is in scope. buildah does carry the +# parent's value through (verified: this target builds to uid 1000 with no +# --build-arg and no re-declaration), but Docker/BuildKit is not obliged to. +ARG USERID=1000 +USER ${USERID} +CMD [ "django-start.sh" ] + +# ---- prod python dependencies ---- +# Kept apart from the app source so editing code does not reinstall the wheel set. +FROM python-base AS prod-deps + +COPY requirements.txt ./ +RUN pip install --no-deps --no-cache-dir --require-hashes -r requirements.txt + +# ---- deployment version metadata ---- +# The report covers git facts plus the python and OS versions, all of which +# come from python-base and so match `prod` exactly. Only the two generated +# text files are copied into `prod`. +FROM python-base AS version + +# .git is bind-mounted rather than COPYed so it is readable for the length of this +# one command and never lands in an image layer. Requires BuildKit or buildah; +# only the long-deprecated DOCKER_BUILDKIT=0 builder would choke. +# Built in /version rather than /django, which python-base hands to the app +# user -- git refuses to operate on a repo it does not own. +WORKDIR /version +COPY ci/scripts/version-file.sh /usr/local/bin/ +# safe.directory: the mounted .git carries the host checkout's ownership, which +# need not match the build user. Scoped to this discarded stage. +RUN --mount=type=bind,source=.git,target=/version/.git,ro \ + git config --global --add safe.directory /version && \ + version-file.sh + +# ---- prod django (code + built assets baked in) ---- +FROM prod-deps AS prod + +# Previously attached to the node asset stage, i.e. to an image that is discarded +# and never published, so the released image carried no labels at all. +LABEL org.opencontainers.image.vendor="Freedom of the Press Foundation" +LABEL org.opencontainers.image.title="pressfreedomtracker.us" + +COPY --from=assets --chown=gcorn:gcorn /src-files/ /django/ +COPY --from=version --chown=gcorn:gcorn /deploy/ /deploy/ + +RUN mkdir -p /etc/gunicorn && chown -R gcorn: /etc/gunicorn +COPY ci/containers/gunicorn/gunicorn.py /etc/gunicorn/gunicorn.py + +USER gcorn +RUN env DJANGO_WHITENOISE=1 DJANGO_DISABLE_DEBUG=1 DJANGO_NO_DB=1 ./manage.py collectstatic -c --noinput +CMD [ "django-start.sh" ] diff --git a/ci/containers/chartgen-start.sh b/ci/containers/chartgen-start.sh new file mode 100755 index 000000000..7ec49cfb5 --- /dev/null +++ b/ci/containers/chartgen-start.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Container entrypoint for the dev chart pregenerator (esbuild watch + nodemon). +set -e + +# exec so npm is PID 1 and receives SIGTERM directly; without it bash holds +# PID 1, swallows the signal, and `compose down` waits out the grace period. +exec npm run dev diff --git a/ci/containers/django-start.sh b/ci/containers/django-start.sh new file mode 100755 index 000000000..0e2707cd5 --- /dev/null +++ b/ci/containers/django-start.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Container entrypoint script for Django applications. +set -e + + +# Wait for node-start.sh to report that node_modules is populated. Bounded, and +# the marker is left in place on purpose: compose's `depends_on: service_healthy` +# is the real cold-start gate, so a `compose restart django` on its own must not +# block forever waiting for a marker no running node service will re-create. +wait_for_node() { + if [ "${DEPLOY_ENV}" == "dev" ]; then + echo "Waiting for node to populate node_modules..." + for _ in $(seq 30); do + [ -f .node_complete ] && return 0 + sleep 2 + done + echo "WARNING: .node_complete not seen after 60s; continuing anyway" + fi +} + +wait_for_postgres() { + echo "Waiting for postgres to start..." + until nc -z "${DJANGO_DB_HOST?}" "${DJANGO_DB_PORT?}" + do + sleep 2 + done +} + +django_start() { + ./manage.py migrate + if [ "${DJANGO_COLLECT_STATIC}" == "yes" ]; then + ./manage.py collectstatic -c --noinput + fi + if [ "${DJANGO_CREATEDEVDATA:-no}" == "yes" ]; then + ./manage.py createdevdata + fi + # exec so the server is PID 1 and receives SIGTERM directly; without it bash + # holds PID 1, swallows the signal, and shutdown waits for SIGKILL. + if [ "${DEPLOY_ENV}" == "dev" ]; then + ./ci/scripts/version-file.sh || echo "WARNING: version file creation failed" + exec ./manage.py runserver 0.0.0.0:8000 + else + exec gunicorn -c /etc/gunicorn/gunicorn.py "${DJANGO_APP_NAME?}.wsgi" + fi +} + +wait_for_postgres +wait_for_node +django_start diff --git a/devops/docker/gunicorn/gunicorn.py b/ci/containers/gunicorn/gunicorn.py similarity index 100% rename from devops/docker/gunicorn/gunicorn.py rename to ci/containers/gunicorn/gunicorn.py diff --git a/ci/containers/node-start.sh b/ci/containers/node-start.sh new file mode 100755 index 000000000..008aa9f0c --- /dev/null +++ b/ci/containers/node-start.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Container entrypoint for the dev webpack watcher. +set -e + +# node_modules lives in the bind-mounted checkout, not the image, so it has to be +# populated at runtime. +npm install + +# `.node_complete` is the handshake consumed by django-start.sh's wait_for_node(): +# it signals that node_modules is populated, so Django's own tooling can rely on +# the tree being complete. Compose's `depends_on: node: service_healthy` is the +# stronger gate for the bundles themselves. +touch .node_complete + +# exec so webpack is PID 1 and receives SIGTERM directly; without it bash holds +# PID 1, swallows the signal, and `compose down` waits out the grace period. +exec npm run start diff --git a/ci/scripts/browser-open.sh b/ci/scripts/browser-open.sh new file mode 100755 index 000000000..2879061d3 --- /dev/null +++ b/ci/scripts/browser-open.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Open a web-browser pointing at the dev django web-app, +# using the CLI tooling native to Mac/Linux. + +PLATFORM="$(uname -o)" + +# Dev only. There was once a `prod` branch here that looked for an nginx +# container on port 8080, but no compose file has defined an nginx service for +# some time, so it could never fire. +COMPOSE="${COMPOSE:-docker compose}" +DJANGO_URL="http://$($COMPOSE -f docker-compose.yaml port django 8000)" +export DJANGO_URL + +# Are we on Linux? +if [[ "${PLATFORM}" == *"linux"* ]]; then + xdg-open "${DJANGO_URL}" & +# I guess we are on Mac :shrug: +else + open "${DJANGO_URL}" & +fi diff --git a/ci/scripts/restoredb.sh b/ci/scripts/restoredb.sh new file mode 100755 index 000000000..f38eb841b --- /dev/null +++ b/ci/scripts/restoredb.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Restore a database dump for the current git branch. + +set -euo pipefail + +# Honour the engine chosen by the justfile; fall back to docker standalone. +COMPOSE="${COMPOSE:-docker compose}" + +# `/` in a branch name would otherwise become a directory in the dump path. +BRANCH="$(git rev-parse --abbrev-ref HEAD | tr / -)" +FOLDER="db-snapshots" +OWNER="tracker" +DBNAME="trackerdb" +CONTAINER="postgresql" + +# `pft-` is the current prefix; `pfi-` is matched too so snapshots taken before +# the rename still restore. +FILE="" +for prefix in "pft-$BRANCH" "pfi-$BRANCH"; do + FILE="$(find "$FOLDER" -iname "$prefix*.dump" 2>/dev/null | sort -r | head -n 1)" + [ -n "$FILE" ] && break +done +if [ -z "$FILE" ]; then + echo "no snapshots found for branch $BRANCH" >&2 + exit 1 +fi +echo "Restoring from: $FILE" + +# Terminate all other connections. The maintenance commands connect over the +# container's unix socket, which the postgres image trusts; `$OWNER` is the +# superuser this compose stack creates. +$COMPOSE exec -T "$CONTAINER" psql -o /dev/null -U "$OWNER" postgres \ + -c "ALTER DATABASE $DBNAME CONNECTION LIMIT 1;" +$COMPOSE exec -T "$CONTAINER" psql -o /dev/null -U "$OWNER" postgres \ + -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$DBNAME';" + +$COMPOSE exec -T "$CONTAINER" dropdb -U "$OWNER" "$DBNAME" +$COMPOSE exec -T "$CONTAINER" createdb -U "$OWNER" --encoding UTF8 \ + --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 --template=template0 \ + --owner "$OWNER" "$DBNAME" +# No `-n public`: the schema's text-search configuration depends on the +# `unaccent` extension, and extensions carry no schema in the dump's TOC, so a +# schema filter would drop the CREATE EXTENSION the restore then trips over. +$COMPOSE exec -T "$CONTAINER" pg_restore -U "$OWNER" -1 --no-owner \ + --role="$OWNER" --dbname="$DBNAME" < "$FILE" diff --git a/ci/scripts/savedb.sh b/ci/scripts/savedb.sh new file mode 100755 index 000000000..55fc09c51 --- /dev/null +++ b/ci/scripts/savedb.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Save a database snapshot for the current git branch. + +set -euo pipefail + +# Honour the engine chosen by the justfile; fall back to docker standalone. +COMPOSE="${COMPOSE:-docker compose}" + +# `/` in a branch name would otherwise become a directory in the dump path. +BRANCH="$(git rev-parse --abbrev-ref HEAD | tr / -)" +DATE="$(date +%Y-%m-%d-%H-%M-%S)" +DUMPFILE="pft-$BRANCH.$DATE.dump" +DBNAME="trackerdb" +FOLDER="db-snapshots" +OWNER="tracker" + +mkdir -p "$FOLDER" + +$COMPOSE exec -T postgresql pg_dump -U "$OWNER" --format=custom "$DBNAME" \ + > "$FOLDER/$DUMPFILE" + +echo "Saved snapshot: $FOLDER/$DUMPFILE" diff --git a/devops/scripts/version-file.sh b/ci/scripts/version-file.sh similarity index 78% rename from devops/scripts/version-file.sh rename to ci/scripts/version-file.sh index fab7c0004..56dfee366 100755 --- a/devops/scripts/version-file.sh +++ b/ci/scripts/version-file.sh @@ -35,6 +35,10 @@ esac echo "$commit" >"$short_version_out" +# No python dependency section: this report is generated in a build stage that +# has none of the app's dependencies installed, so a "pip freeze" here would +# describe a bare interpreter. The authoritative pinned set is requirements.txt, +# shipped in the image and installed with --no-deps --require-hashes. cat >"$full_version_out" < /etc/apt/sources.list && \ - echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true" | debconf-set-selections && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - fontconfig \ - ttf-mscorefonts-installer && \ - fc-cache -f && \ - rm -rf /var/lib/apt/lists/* - - -COPY devops/docker/node-pregenerator-start.sh /usr/bin/node-pregenerator-start.sh -RUN getent passwd "${USERID}" || useradd --create-home --no-log-init --uid "${USERID}" pft_node - -WORKDIR /app - -RUN chown -R "${USERID}" /app -USER ${USERID} - -COPY chart_pregenerator/package*.json ./ -COPY chart_pregenerator/dev.js ./ -COPY chart_pregenerator/jest.config.js ./ -COPY chart_pregenerator/babel.config.json ./ - -RUN npm ci -CMD [ "/usr/bin/node-pregenerator-start.sh" ] diff --git a/devops/docker/NodeChartPregeneratorDockerfile b/devops/docker/NodeChartPregeneratorDockerfile deleted file mode 100644 index 2a23ebd17..000000000 --- a/devops/docker/NodeChartPregeneratorDockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# sha256 as of 2026-07-15 -FROM node:26.5.0-trixie@sha256:0473e7dc433a1310f436edee02aa79737ec78a4b345433ab0963d4a256f9ad85 AS node-assets - -ARG USERID - -RUN echo "deb http://deb.debian.org/debian trixie main contrib non-free non-free-firmware" > /etc/apt/sources.list && \ - echo "ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true" | debconf-set-selections && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - fontconfig \ - ttf-mscorefonts-installer && \ - fc-cache -f && \ - rm -rf /var/lib/apt/lists/* - -RUN getent passwd "${USERID}" || useradd --create-home --no-log-init --uid "${USERID}" pft_node -COPY chart_pregenerator /opt/chart-pregenerator -COPY ./client /opt/chart-pregenerator/client -RUN chown -R "${USERID}" /opt/chart-pregenerator -WORKDIR /opt/chart-pregenerator - -USER ${USERID} - -RUN npm install -RUN npm run build - -CMD ["node", "./build/server.js"] diff --git a/devops/docker/NodeDockerfile b/devops/docker/NodeDockerfile deleted file mode 100644 index 0a0eddf0f..000000000 --- a/devops/docker/NodeDockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# sha256 as of 2026-07-15 -FROM node:26.5.0-trixie@sha256:0473e7dc433a1310f436edee02aa79737ec78a4b345433ab0963d4a256f9ad85 AS node-assets - -ARG USERID - -COPY devops/docker/node-start.sh /usr/bin/node-start.sh - -RUN apt-get update && apt-get install -y --no-install-recommends \ - jq \ - && rm -rf /var/lib/apt/lists/* - -RUN getent passwd "${USERID}" || useradd --create-home --no-log-init --uid "${USERID}" pft_node - -USER ${USERID} -CMD [ "/usr/bin/node-start.sh" ] diff --git a/devops/docker/ProdDjangoDockerfile b/devops/docker/ProdDjangoDockerfile deleted file mode 100644 index 604a78456..000000000 --- a/devops/docker/ProdDjangoDockerfile +++ /dev/null @@ -1,59 +0,0 @@ -# sha256 as of 2026-07-15 -FROM node:26.5.0-trixie@sha256:0473e7dc433a1310f436edee02aa79737ec78a4b345433ab0963d4a256f9ad85 AS node-assets - -# Install gulp globally so it can be used during builds -RUN npm install --global gulp-cli - -COPY ./ /src-files -RUN cd /src-files && ( npm install && npm run build ) - -# sha256 for python:3.14.6-slim-trixie on linux/amd64 as of 2026-06-17 -FROM python:3.14.6-slim-trixie@sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061 AS python-base - -LABEL MAINTAINER="Freedom of the Press Foundation" -LABEL APP="pressfreedomtracker.us" - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential libc6-dev \ - curl \ - git \ - libpq-dev \ - libssl-dev \ - netcat-traditional \ - && rm -rf /var/lib/apt/lists/* - - -COPY devops/docker/django-start.sh /usr/local/bin -RUN chmod +x /usr/local/bin/django-start.sh - -ARG USERID -RUN getent passwd "${USERID?USERID must be supplied}" || \ - useradd --create-home --no-log-init --uid "${USERID}" --user-group gcorn - -COPY --from=node-assets /src-files/ /django/ -# Unfortunately the chown flag in COPY is not -# available in my docker system version :( -RUN find /django -path /django/node_modules -prune -o -print -exec chown gcorn: '{}' \; - -WORKDIR /django -RUN pip install --no-deps --require-hashes -r /django/requirements.txt - - -# Really not used in production. Needed for mapped named volume -# permission handling https://github.com/docker/compose/issues/3270 -RUN mkdir /django-media /django-static /django-logs && \ - chown -R gcorn: /django-media && \ - chown -R gcorn: /django-static && \ - chown -R gcorn: /django-logs - -RUN mkdir -p /etc/gunicorn && chown -R gcorn: /etc/gunicorn -COPY devops/docker/gunicorn/gunicorn.py /etc/gunicorn/gunicorn.py - -RUN mkdir /deploy && chown -R gcorn: /deploy - -EXPOSE 8000 -USER gcorn -RUN /django/devops/scripts/version-file.sh - -RUN env DJANGO_WHITENOISE=1 DJANGO_DISABLE_DEBUG=1 DJANGO_NO_DB=1 ./manage.py collectstatic -c --noinput -CMD django-start.sh diff --git a/devops/docker/django-start.sh b/devops/docker/django-start.sh deleted file mode 100644 index c853bb1c0..000000000 --- a/devops/docker/django-start.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Container entrypoint script for Django applications. -set -e - -django_start() { - ./manage.py migrate - if [ "${DJANGO_COLLECT_STATIC}" == "yes" ]; then - ./manage.py collectstatic -c --noinput - fi - if [ "${DJANGO_CREATEDEVDATA:-no}" == "yes" ]; then - ./manage.py createdevdata - fi - if [ "${DEPLOY_ENV}" == "dev" ]; then - ./devops/scripts/version-file.sh || echo "WARNING: version file creation failed" - exec ./manage.py runserver 0.0.0.0:8000 - else - gunicorn -c /etc/gunicorn/gunicorn.py "${DJANGO_APP_NAME}.wsgi" - fi -} - -django_start diff --git a/devops/docker/node-pregenerator-start.sh b/devops/docker/node-pregenerator-start.sh deleted file mode 100755 index 2081a6460..000000000 --- a/devops/docker/node-pregenerator-start.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -# -# Start node pregeneration service - -set -x - -npm run dev diff --git a/devops/docker/node-start.sh b/devops/docker/node-start.sh deleted file mode 100755 index 92c386386..000000000 --- a/devops/docker/node-start.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# -# Start installing node dependencies - -set -x - -npm install && \ - npm run start diff --git a/devops/scripts/browser-open.sh b/devops/scripts/browser-open.sh deleted file mode 100755 index 4380e1000..000000000 --- a/devops/scripts/browser-open.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# -# -# Open a web-browser pointing to the django web-app -# using the CLI tooling native to Mac/Linux. Attempt -# basic environment detection so we dont have to prompt - -PLATFORM="$(uname -o)" - -COMPOSE_ENV=dev -# If nginx is running, we are surely in prod-land, this obviously -# is not error free logic. Its possible for a user to clone this repo -# into a folder with a unique name which would cause a full grep -# to fail. :shrug: -( docker ps | grep -q _nginx_1 ) && export COMPOSE_ENV=prod - -if [[ "${COMPOSE_ENV}" == "prod" ]]; then - DOCKER_COMPOSE_FILE="prod-docker-compose.yaml" - export CONTAINER="nginx" - export PORT=8080 -else - DOCKER_COMPOSE_FILE="docker-compose.yaml" - export CONTAINER="django" - export PORT=8000 -fi - -export DJANGO_URL="http://$(docker compose -f ${DOCKER_COMPOSE_FILE} port ${CONTAINER} ${PORT})" - -# Are we on Linux? -if [[ "${PLATFORM}" == *"linux"* ]]; then - xdg-open "${DJANGO_URL}" & -# I guess we are on Mac :shrug: -else - open "${DJANGO_URL}" & -fi diff --git a/devops/scripts/dev.sh b/devops/scripts/dev.sh deleted file mode 100755 index 33e8176a5..000000000 --- a/devops/scripts/dev.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# -# - -if [ $# -eq 0 ]; then echo "ERR - Missing positional molecule action argument"; exit 1; fi - -# Allow setting Django port to random values by env variable -if [ "${RAND_PORT-false}" != "false" ]; then - export RAND_PORT=true -fi - -# If the user already has a virtualenv activated. Lets not interfere with that -if [ -z "$VIRTUAL_ENV" ]; then - if [ ! -f devops/.venv/bin/activate ]; then virtualenv -p $(which python2) --no-site-packages devops/.venv; fi - source devops/.venv/bin/activate - - pip install -U -r devops/requirements.txt > /dev/null -fi - -molecule $1 -s dev diff --git a/devops/scripts/savedb.sh b/devops/scripts/savedb.sh deleted file mode 100755 index 1ebdb7971..000000000 --- a/devops/scripts/savedb.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# -# Save a database snapshot for the current git branch. - -BRANCH=`git rev-parse --abbrev-ref HEAD` -DATE=`date +%Y-%m-%d-%H-%M-%S` -DUMPFILE="pfi-$BRANCH.$DATE.dump" -DBNAME="trackerdb" -FOLDER="db-snapshots" -OWNER="tracker" - -if [ ! -d "$FOLDER" ]; then - mkdir $FOLDER -fi - -docker compose exec postgresql pg_dump -U $OWNER --format=custom $DBNAME > $FOLDER/$DUMPFILE diff --git a/devops/tests/test_basic.py b/devops/tests/test_basic.py deleted file mode 100644 index 76bf017d1..000000000 --- a/devops/tests/test_basic.py +++ /dev/null @@ -1,12 +0,0 @@ -def test_mainpage(host): - """ - Basic test to make sure home-page is coming up. - """ - SITE_STRING = "Press Freedom Tracker" - - URL = "http://localhost:8000" - - content = host.check_output(f"curl -k {URL}") - head = host.check_output(f"curl -I -k {URL}") - assert SITE_STRING in content - assert "HTTP/1.1 200 OK" in head diff --git a/docker-compose.yaml b/docker-compose.yaml index 22a9cb90e..ab14bb7af 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,26 +2,41 @@ networks: app: +x-variables: + db-name: &db-name trackerdb + db-username: &db-username tracker + db-password: &db-password trackerpassword + x-chart-pregenerator: host: &chart-host ${NODE_CHART_PREGENERATOR_HOST:-chartgenerator} port: &chart-port ${NODE_CHART_PREGENERATOR_PORT:-3000} services: postgresql: - image: postgres:17 + image: docker.io/library/postgres:17 init: true ports: - - '5432:5432' + - '127.0.0.1:5432:5432' volumes: - - ./:/django:ro + - ./:/django:ro,z environment: - POSTGRES_PASSWORD: trackerpassword - POSTGRES_USER: tracker - POSTGRES_DB: trackerdb + POSTGRES_PASSWORD: *db-password + POSTGRES_USER: *db-username + POSTGRES_DB: *db-name user: postgres healthcheck: test: - ['CMD', 'pg_isready', '-q', '-U', 'tracker', '--dbname', 'trackerdb'] + [ + 'CMD', + 'pg_isready', + '-q', + '-h', + 'localhost', + '-U', + *db-username, + '--dbname', + *db-name, + ] timeout: 45s interval: 10s retries: 10 @@ -31,7 +46,8 @@ services: - db selenium: - image: selenium/standalone-firefox-debug:latest + image: docker.io/selenium/standalone-firefox-debug:latest + init: true ports: - '127.0.0.1::5900' networks: @@ -40,41 +56,44 @@ services: - selenium node: - init: true + image: localhost/pressfreedomtracker-us-node build: context: . - dockerfile: devops/docker/NodeDockerfile + dockerfile: ci/containers/Containerfile + target: node-dev args: USERID: ${UID:?err} - image: localhost/pressfreedomtracker-us-node + init: true volumes: - - ./:/django + - ./:/django:z working_dir: /django user: ${UID:?err} healthcheck: test: jq '.status' build/static/bundles/webpack-stats.json |grep -q -e compiling -e error -e done interval: 10s start_period: 1m30s + userns_mode: keep-id networks: - app node-chart-pregenerator: - init: true + image: localhost/pressfreedomtracker-us-node-chart-pregenerator build: context: . - dockerfile: devops/docker/NodeChartPregeneratorDevDockerfile + dockerfile: ci/containers/Containerfile + target: chartgen-dev args: USERID: ${UID:?err} - image: localhost/pressfreedomtracker-us-node-chart-pregenerator + init: true volumes: - - ./:/django - - ./client:/app/client - - ./chart_pregenerator/src:/app/src + - ./:/django:z + - ./client:/app/client:z + - ./chart_pregenerator/src:/app/src:z environment: + DEPLOY_ENV: dev DJANGO_HOST: app DJANGO_PORT: 8000 PORT: *chart-port - DEPLOY_ENV: dev ports: - target: *chart-port published: *chart-port @@ -85,6 +104,7 @@ services: timeout: 5s retries: 10 start_period: 45s + userns_mode: keep-id networks: app: aliases: @@ -94,53 +114,55 @@ services: init: true stdin_open: true tty: true + image: localhost/pressfreedomtracker-us-django build: context: . - dockerfile: devops/docker/DevDjangoDockerfile + dockerfile: ci/containers/Containerfile + target: dev args: USERID: ${UID:?err} - image: localhost/pressfreedomtracker-us-django depends_on: - node: - condition: service_healthy postgresql: condition: service_healthy + node: + condition: service_healthy node-chart-pregenerator: condition: service_healthy - healthcheck: - test: - ['CMD', 'curl', '--fail', 'http://localhost:8000/health/ok/?monitor'] - interval: 10s - timeout: 5s - retries: 10 - start_period: 45s environment: + DEPLOY_ENV: dev DJANGO_ALLOWED_HOSTS: app localhost + DJANGO_CHART_HOST: node-chart-pregenerator + DJANGO_CHART_PORT: *chart-port DJANGO_CREATEDEVDATA: '${DJANGO_CREATEDEVDATA:-no}' - DJANGO_DB_PASSWORD: trackerpassword - DJANGO_DB_USER: tracker - DJANGO_DB_NAME: trackerdb - DJANGO_DB_PORT: 5432 DJANGO_DB_HOST: db - DJANGO_XMLTEST_OUTPUT: 'yes' - DEPLOY_ENV: dev + DJANGO_DB_NAME: *db-name + DJANGO_DB_PASSWORD: *db-password + DJANGO_DB_PORT: 5432 + DJANGO_DB_USER: *db-username DJANGO_PROFILE: '${DJANGO_PROFILE:-no}' - PYINSTRUMENT: '${PYINSTRUMENT:-no}' - SELENIUM_HOST: selenium - DJANGO_CHART_HOST: node-chart-pregenerator - DJANGO_CHART_PORT: *chart-port + DJANGO_XMLTEST_OUTPUT: 'yes' MAILCHIMP_API_KEY: - NEWSLETTER_REPLY_TO: NEWSLETTER_FROM_NAME: + NEWSLETTER_REPLY_TO: + PREPUB_DOCUMENT_ID: PREPUB_GOOGLE_SHEETS_CREDS: PREPUB_HEADER_ROW_INDEX: PREPUB_SHEET_NAME: - PREPUB_DOCUMENT_ID: + PYINSTRUMENT: '${PYINSTRUMENT:-no}' + SELENIUM_HOST: selenium working_dir: /django + userns_mode: keep-id volumes: - - ./:/django + - ./:/django:z ports: - '127.0.0.1:8000:8000' + healthcheck: + test: + ['CMD', 'curl', '--fail', 'http://localhost:8000/health/ok/?monitor'] + interval: 10s + timeout: 5s + retries: 10 + start_period: 45s networks: app: aliases: diff --git a/justfile b/justfile new file mode 100644 index 000000000..92d45612b --- /dev/null +++ b/justfile @@ -0,0 +1,155 @@ +# Recipes for developing and testing the pressfreedomtracker.us website. +# Run `just` (or `just --list`) to see everything available. +# +# Container engine: prefers docker for backwards compat. +# Override explicitly with `CONTAINER_ENGINE=podman just `. + +engine := env_var_or_default("CONTAINER_ENGINE", "docker") +compose := engine + " compose" +# Must stay in step with PYTHON_IMAGE in ci/containers/Containerfile: pip-compile +# resolves hashes against this interpreter, and the app installs the result. The +# tag alone is not enough -- Docker Hub rebuilds it in place for patches, so +# without the digest the two can silently drift apart. +python_builder := "docker.io/library/python:3.14.6-slim-trixie@sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061" + +# pinning a specific, recent version of pip-tools, so that the dev-env +# reuses the same tooling predictably. +# TODO: drop use of pip-tools in favor of more modern python package management. +pip_tools_version := "7.6.1" + +# Show available recipes. +default: + @just --list + +# Write the host UID into .env so containers build/run as your user (see README). +dev-init: + echo "UID=$(id -u)" > .env + +# Same, but non-destructive: every compose invocation interpolates ${UID:?err}, +# so recipes depend on this to work from a cold checkout without clobbering an +# .env the developer has added their own variables to. +[private] +env-check: + [ -f .env ] || echo "UID=$(id -u)" > .env + +# Run the webapp locally, via containers (--build keeps images in sync with the Containerfile). +dev: env-check + {{compose}} up --build + +alias compose := dev + +# Build all containers locally. +build: env-check + {{compose}} build + +# The static checks below run with `--no-deps`: their tooling is baked into the +# dev images, so they need neither postgres, selenium, the webpack watcher nor +# the chart pregenerator. That keeps them usable without `just dev` running, +# locally and in CI alike. + +# Check Python lint and formatting with ruff, without writing changes. +ruff: env-check + # TODO: move `ruff` execution to host context; it shouldn't be running in the container + {{compose}} run --rm -T --no-deps django bash -c "ruff check && ruff format --check" + +# Apply ruff's fixes and formatting in place. +ruff-fix: env-check + {{compose}} run --rm -T --no-deps django bash -c "ruff check --fix && ruff format" + +# Static security analysis with bandit. +bandit: env-check + {{compose}} run --rm -T --no-deps django ./scripts/bandit + +# --skip-checks: Django's system checks import tracker.urls, and +# incident/api/views.py builds its OpenAPI parameters from the database at +# import time. Skipping them keeps this stack-free; `just test` still runs +# the full check framework. The consistent-history warning is expected. + +# Fail if a model changed without a matching migration. +check-migrations: env-check + {{compose}} run --rm -T --no-deps django bash -c "./manage.py makemigrations --dry-run --check --skip-checks" + +# Jest, eslint and stylelint read sources directly rather than webpack's output, +# so no build is needed -- but node_modules lives in the bind-mounted tree, +# populated by the `node` service, so install it if absent. The guard is +# deliberately host-side, and skipping the install when the tree is already +# populated keeps a running `just dev` watcher undisturbed. +[private] +node-modules: env-check + [ -d node_modules ] || {{compose}} run --rm --no-deps node npm ci + +# Lint JavaScript with eslint. +eslint: node-modules + {{compose}} run --rm --no-deps node npm run js-lint + +# Lint SASS with stylelint. +stylelint: node-modules + {{compose}} run --rm --no-deps node npm run stylelint + +# Run all project linters. +lint: ruff bandit check-migrations eslint stylelint + +# Run the Django test suite with coverage (CI enforces 100% on changed lines via diff-cover). +test: + {{compose}} exec django bash -ec "\ + coverage run ./manage.py test --noinput; \ + coverage html; \ + coverage xml; \ + coverage report" + +# The pregenerator image carries its own node_modules and bind-mounts only +# src/ and client/, so it needs no guard and no stack either. + +# Run the jest suites for the frontend and the chart pregenerator. +test-js: node-modules + {{compose}} run --rm --no-deps node npm test + {{compose}} run --rm --no-deps node-chart-pregenerator npm run test + +# Inject development data into the postgresql database. +createdevdata: + {{compose}} exec django bash -c "./manage.py createdevdata" + +# Import a postgres export file located at ./import.db. +import-db: + {{compose}} exec -T postgresql bash -c "sed 's/OWNER TO [a-z]*/OWNER TO tracker/g' /django/import.db | psql trackerdb -U tracker > /dev/null" + +# Save a snapshot of the database for the current git branch. +save-db: + COMPOSE="{{compose}}" ./ci/scripts/savedb.sh + +# Restore the most recent database snapshot for the current git branch. +restore-db: + COMPOSE="{{compose}}" ./ci/scripts/restoredb.sh + +# Open a browser pointed at the running dev site. +open-browser: + COMPOSE="{{compose}}" ./ci/scripts/browser-open.sh + +alias browser := open-browser + +# Recompile prod + ci + dev lockfiles (forward flags, e.g. --upgrade or --upgrade-package=NAME). +pip-compile *FLAGS: (_pip-lock "requirements.txt" "requirements.in" FLAGS) (_pip-lock "ci-requirements.txt" "ci-requirements.in" FLAGS) (_pip-lock "dev-requirements.txt" "dev-requirements.in" FLAGS) + +# Recompile only the dev lockfile (same flags as pip-compile). +pip-compile-dev *FLAGS: (_pip-lock "dev-requirements.txt" "dev-requirements.in" FLAGS) + +# Recompile one lockfile in a clean builder matching the app's Python, so +# hashes resolve identically to production. +# The final chown hands the regenerated lockfile back to whoever owns the input; +# the builder runs as root, so without it a developer is left with root-owned +# requirements files in their checkout. +_pip-lock outfile infile *FLAGS: + {{engine}} run --rm -v "{{justfile_directory()}}:/code:z" -w /code {{python_builder}} \ + bash -c 'apt-get update && apt-get install -y --no-install-recommends gcc libpq-dev && \ + pip install pip-tools=={{pip_tools_version}} && \ + pip-compile --generate-hashes --no-header --allow-unsafe {{FLAGS}} \ + --output-file {{outfile}} {{infile}} && \ + chown "$(stat -c "%u:%g" {{infile}})" {{outfile}}' + +# Fail if the lockfiles are out of sync with the .in files. +pip-check: pip-compile + git diff --exit-code -- requirements.txt ci-requirements.txt dev-requirements.txt + +# Clean out local developer assets. +clean: + rm -rvf ./node_modules diff --git a/prod-docker-compose.yaml b/prod-docker-compose.yaml index 3108dbe9c..faefd5ef6 100644 --- a/prod-docker-compose.yaml +++ b/prod-docker-compose.yaml @@ -8,7 +8,8 @@ x-chart-pregenerator: services: postgresql: - image: postgres:17 + image: docker.io/library/postgres:17 + init: true ports: - '127.0.0.1::5432' environment: @@ -25,9 +26,9 @@ services: '-h', 'localhost', '-U', - 'freedompress', + 'tracker', '--dbname', - 'freedompressdb', + 'trackerdb', ] timeout: 45s interval: 10s @@ -38,42 +39,44 @@ services: - db django: + init: true build: context: . - dockerfile: devops/docker/ProdDjangoDockerfile + dockerfile: ci/containers/Containerfile + target: prod args: - USERID: 1234 - image: quay.io/freedomofpress/pressfreedomtrackerus + USERID: 1000 + image: ghcr.io/freedomofpress/pressfreedomtracker-us depends_on: postgresql: condition: service_healthy working_dir: /django volumes: - - ${HOST_STATIC_DIR:-fpf-django-media}:/django-media - - ${HOST_MEDIA_DIR:-fpf-django-static}:/django-static + - ${HOST_MEDIA_DIR:-pft-django-media}:/django-media + - ${HOST_STATIC_DIR:-pft-django-static}:/django-static environment: + DEPLOY_ENV: prod DJANGO_ALLOWED_HOSTS: app localhost + DJANGO_APP_NAME: tracker + DJANGO_CHART_HOST: node-chart-pregenerator + DJANGO_CHART_PORT: *chart-port DJANGO_COLLECT_STATIC: 'yes' DJANGO_CREATEDEVDATA: '${DJANGO_CREATEDEVDATA:-no}' - DJANGO_SETTINGS_MODULE: tracker.settings.production - DJANGO_SILENCED_SYSTEM_CHECKS: django_recaptcha.recaptcha_test_key_error - # Arbitrary key for this "prod-like" testing setup. Do not use in real life. - DJANGO_SECRET_KEY: 64a235ef65834d8b42fa2802dd4290cf61c773c3f31332039ec5c48165f744da - DJANGO_STATIC_ROOT: /django-static - DJANGO_MEDIA_ROOT: /django-media - DEPLOY_ENV: prod - DJANGO_DB_PASSWORD: trackerpassword - DJANGO_DB_USER: tracker + DJANGO_DB_HOST: db DJANGO_DB_NAME: trackerdb + DJANGO_DB_PASSWORD: trackerpassword DJANGO_DB_PORT: 5432 - DJANGO_DB_HOST: db - DJANGO_XMLTEST_OUTPUT: 'yes' - DJANGO_APP_NAME: tracker - DJANGO_ONION_HOSTNAME: pfts5wyphb5x.onion + DJANGO_DB_USER: tracker DJANGO_JSON_LOG: 'yes' + DJANGO_MEDIA_ROOT: /django-media + DJANGO_ONION_HOSTNAME: pfts5wyphb5x.onion + # Arbitrary key for this "prod-like" testing setup. Do not use in real life. + DJANGO_SECRET_KEY: 64a235ef65834d8b42fa2802dd4290cf61c773c3f31332039ec5c48165f744da + DJANGO_SETTINGS_MODULE: tracker.settings.production + DJANGO_SILENCED_SYSTEM_CHECKS: django_recaptcha.recaptcha_test_key_error + DJANGO_STATIC_ROOT: /django-static DJANGO_WHITENOISE: 'yes' - DJANGO_CHART_HOST: node-chart-pregenerator - DJANGO_CHART_PORT: *chart-port + DJANGO_XMLTEST_OUTPUT: 'yes' GOOGLE_SHEETS_CREDS: # More fake testing keys (see above); generate your own for deployment. RECAPTCHA_PUBLIC_KEY: 6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI @@ -84,26 +87,26 @@ services: timeout: 5s retries: 10 start_period: 45s - ports: - - '8000:8000' networks: app: aliases: - app + ports: + - '127.0.0.1:8000:8000' node-chart-pregenerator: read_only: true init: true build: context: . - dockerfile: devops/docker/NodeChartPregeneratorDockerfile + dockerfile: ci/containers/Containerfile + target: chartgen args: - USERID: 1234 - image: quay.io/freedomofpress/pft-chart-pregenerator + USERID: 1001 + image: ghcr.io/freedomofpress/pressfreedomtracker-us-chartgen environment: DJANGO_HOST: app PORT: *chart-port - DEPLOY_ENV: dev ports: - target: *chart-port published: *chart-port @@ -119,5 +122,5 @@ services: - *chart-host volumes: - fpf-django-static: - fpf-django-media: + pft-django-static: + pft-django-media: diff --git a/pyproject.toml b/pyproject.toml index 9360cc056..779a54819 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,6 @@ requires-python = ">=3.14" [tool.ruff] exclude = [ "node_modules", - "molecule", ] # exclude RUF012 because it results in a lot of false positives in Django code, diff --git a/scripts/bandit b/scripts/bandit index 0d3e4d47b..24912b3e0 100755 --- a/scripts/bandit +++ b/scripts/bandit @@ -1,4 +1,4 @@ #!/bin/bash set -e -bandit --recursive . --exclude node_modules,devops,.venv -ll +bandit --recursive . --exclude node_modules,ci,.venv -ll diff --git a/scripts/pytest b/scripts/pytest deleted file mode 100755 index 7a3820cfd..000000000 --- a/scripts/pytest +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -# Install dependencies -pip install --require-hashes -r ci-requirements.txt - -python -m pytest devops/tests/test_basic.py