Repository navigation
release: bump 20 packages and raise the floors their new code needs #477
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python Packages (test + publish) | |
| on: | |
| push: | |
| branches: [master] | |
| paths: | |
| - 'packages/**' | |
| - '.github/workflows/python-packages.yml' | |
| - '.github/scripts/changed_packages.py' | |
| pull_request: | |
| branches: [master] | |
| paths: | |
| - 'packages/**' | |
| - '.github/workflows/python-packages.yml' | |
| - '.github/scripts/changed_packages.py' | |
| release: | |
| types: [published] | |
| # Cancel stale CI runs from earlier pushes on the same branch. Skip on | |
| # release events — those are one-shot and must complete. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name != 'release' }} | |
| jobs: | |
| # ─── Discover affected packages from the diff ──────────────────────────── | |
| # Only the packages that changed (or that depend on a changed package) get | |
| # tested. Workflow-file changes and release events expand to "all packages". | |
| discover: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| packages: ${{ steps.affected.outputs.packages }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # need history for git diff | |
| - id: affected | |
| name: Compute affected package list | |
| env: | |
| GHA_EVENT: ${{ github.event_name }} | |
| PR_BASE: ${{ github.base_ref }} | |
| PUSH_BEFORE: ${{ github.event.before }} | |
| RELEASE_TAG: ${{ github.event.release.tag_name }} | |
| run: | | |
| set -euo pipefail | |
| if [ "$GHA_EVENT" = "release" ]; then | |
| # The tag identifies exactly one package. Test that package + | |
| # its transitive dependents — not the full 30+ matrix, which | |
| # is wasted work since nothing else in the tree changed for | |
| # this release event. | |
| pkgs=$(python3 .github/scripts/changed_packages.py \ | |
| --from-tag "$RELEASE_TAG") | |
| else | |
| if [ "$GHA_EVENT" = "pull_request" ]; then | |
| BASE_REF="origin/${PR_BASE}" | |
| elif [ "$GHA_EVENT" = "push" ]; then | |
| BASE_REF="${PUSH_BEFORE}" | |
| else | |
| BASE_REF="" | |
| fi | |
| if [ -z "${BASE_REF}" ] \ | |
| || [ "${BASE_REF}" = "0000000000000000000000000000000000000000" ]; then | |
| pkgs=$(python3 .github/scripts/changed_packages.py --all) | |
| else | |
| export BASE_REF | |
| pkgs=$(python3 .github/scripts/changed_packages.py) | |
| fi | |
| fi | |
| echo "Affected packages: $pkgs" | |
| echo "packages=$pkgs" >> "$GITHUB_OUTPUT" | |
| # ─── Tests run only for affected packages ──────────────────────────────── | |
| test: | |
| needs: discover | |
| if: needs.discover.outputs.packages != '[]' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.11', '3.12'] | |
| package: ${{ fromJSON(needs.discover.outputs.packages) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| # No `cache: pip` — we install via uv, so pip's cache dir | |
| # (~/.cache/pip) is never populated. setup-uv handles its own | |
| # cache below. | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: 'packages/**/pyproject.toml' | |
| # Install every in-repo MIDAS package as editable in a single uv | |
| # resolve. uv resolves the entire dep graph in one pass, replacing the | |
| # 18 sequential conditional pip-install steps the old workflow had. | |
| # ALL packages are installed (none hidden) so cross-package deps on the | |
| # newly-published packages (e.g. pf_odf -> grain_odf) resolve locally | |
| # rather than against PyPI, where 0.1.0 isn't published yet. | |
| - name: Install all in-repo MIDAS packages (editable, single uv resolve) | |
| run: | | |
| set -euo pipefail | |
| shopt -s nullglob | |
| args=() | |
| for d in packages/midas_*/; do | |
| args+=("-e" "$d") | |
| done | |
| uv pip install --system "${args[@]}" | |
| - name: Install dev extras for ${{ matrix.package }} | |
| run: uv pip install --system -e "packages/${{ matrix.package }}[dev]" | |
| - name: Run tests for ${{ matrix.package }} | |
| if: matrix.package != 'midas_suite' | |
| run: | | |
| cd packages/${{ matrix.package }} | |
| # `slow`-marked tests (midas-index regression vs C IndexerOMP) | |
| # depend on the C binaries which CI doesn't build; skipif handles | |
| # that, but we exclude them explicitly here for speed. | |
| python -m pytest tests/ -v --tb=short -m "not slow" | |
| # midas_suite is a meta-package with no test suite. Run an import + | |
| # installed() smoke test instead, which verifies (a) the package | |
| # imports, (b) every declared sub-package was resolved, and (c) the | |
| # SUBPACKAGES tuple matches what's actually importable. | |
| - name: Smoke test midas_suite | |
| if: matrix.package == 'midas_suite' | |
| run: | | |
| python -c " | |
| import midas_suite | |
| status = midas_suite.installed() | |
| print(f'midas-suite {midas_suite.__version__} declares {len(midas_suite.SUBPACKAGES)} sub-packages:') | |
| missing = [] | |
| for name, ver in status.items(): | |
| flag = 'OK ' if ver else 'MISS' | |
| print(f' {flag} {name:30s} {ver if ver else \"NOT INSTALLED\"}') | |
| if not ver: | |
| missing.append(name) | |
| if missing: | |
| raise SystemExit(f'FAIL: {len(missing)} sub-package(s) failed to import: {missing}') | |
| print('All sub-packages resolved.') | |
| " | |
| # ─── Build + publish: triggered by GitHub release only ──────────────────── | |
| # The tag name selects which package to publish: | |
| # midas-stress-v* → midas_stress | |
| # midas-params-v* → midas_params | |
| # midas-plotting-v* → midas_plotting | |
| # midas-peakfit-v* → midas_peakfit | |
| # midas-integrate-v2-v* → midas_integrate_v2 (matched BEFORE v1 — prefix collision) | |
| # midas-integrate-v* → midas_integrate | |
| # midas-hkls-v* → midas_hkls | |
| # midas-calibrate-v2-v* → midas_calibrate_v2 (matched BEFORE v1 — prefix collision) | |
| # midas-calibrate-v* → midas_calibrate | |
| # midas-diffract-v* → midas_diffract | |
| # midas-index-v* → midas_index | |
| # midas-transforms-v* → midas_transforms | |
| # midas-zipper-v* → midas_zipper | |
| # midas-snapshot-v* → midas_snapshot | |
| # midas-nf-preprocess-v* → midas_nf_preprocess | |
| # midas-process-grains-v* → midas_process_grains | |
| # midas-propagate-v* → midas_propagate | |
| # midas-invert-v* → midas_invert | |
| # midas-uq-v* → midas_uq | |
| # midas-joint-ff-calibrate-v* → midas_joint_ff_calibrate | |
| # midas-fit-grain-v* → midas_fit_grain | |
| # midas-tomo-v* → midas_tomo | |
| # midas-dt-v* → midas_dt | |
| # midas-nf-fitorientation-v* → midas_nf_fitorientation | |
| # midas-nf-pipeline-v* → midas_nf_pipeline | |
| # midas-parsl-configs-v* → midas_parsl_configs | |
| # midas-ff-pipeline-v* → midas_ff_pipeline | |
| # midas-pipeline-v* → midas_pipeline | |
| # midas-suite-v* → midas_suite (meta-package) | |
| # | |
| # Release tags without one of these prefixes are ignored (no-op build). | |
| build: | |
| needs: test | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'release' | |
| outputs: | |
| package: ${{ steps.which.outputs.package }} | |
| pypi_name: ${{ steps.which.outputs.pypi_name }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: which | |
| name: Identify package from tag | |
| run: | | |
| TAG="${GITHUB_REF##*/}" | |
| echo "Tag: $TAG" | |
| # midas-dct-tt is FIRST on purpose. `midas-dct-v*` would also glob-match | |
| # `midas-dct-tt-v0.1.0`, so if a midas-dct-* package is ever added it must | |
| # not be able to shadow this branch by being inserted above it. Same prefix | |
| # collision already handled below for midas-integrate-v2 / midas-calibrate-v2. | |
| if [[ "$TAG" == midas-dct-tt-v* ]]; then | |
| echo "package=midas_dct_tt" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-dct-tt" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-stress-v* ]]; then | |
| echo "package=midas_stress" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-stress" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-params-v* ]]; then | |
| echo "package=midas_params" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-params" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-plotting-v* ]]; then | |
| echo "package=midas_plotting" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-plotting" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-peakfit-v* ]]; then | |
| echo "package=midas_peakfit" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-peakfit" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-distortion-v* ]]; then | |
| echo "package=midas_distortion" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-distortion" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-integrate-v2-v* ]]; then | |
| # NOTE: this branch MUST come before `midas-integrate-v*` — | |
| # the v1 glob also matches `midas-integrate-v2-v0.0.0` etc. | |
| echo "package=midas_integrate_v2" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-integrate-v2" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-integrate-v* ]]; then | |
| echo "package=midas_integrate" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-integrate" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-hkls-v* ]]; then | |
| echo "package=midas_hkls" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-hkls" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-calibrate-v2-v* ]]; then | |
| # NOTE: this branch MUST come before `midas-calibrate-v*` — | |
| # the v1 glob also matches `midas-calibrate-v2-v0.0.0` etc. | |
| echo "package=midas_calibrate_v2" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-calibrate-v2" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-calibrate-v* ]]; then | |
| echo "package=midas_calibrate" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-calibrate" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-diffract-v* ]]; then | |
| echo "package=midas_diffract" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-diffract" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-index-v* ]]; then | |
| echo "package=midas_index" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-index" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-transforms-v* ]]; then | |
| echo "package=midas_transforms" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-transforms" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-zipper-v* ]]; then | |
| echo "package=midas_zipper" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-zipper" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-snapshot-v* ]]; then | |
| echo "package=midas_snapshot" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-snapshot" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-nf-preprocess-v* ]]; then | |
| echo "package=midas_nf_preprocess" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-nf-preprocess" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-process-grains-v* ]]; then | |
| echo "package=midas_process_grains" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-process-grains" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-propagate-v* ]]; then | |
| echo "package=midas_propagate" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-propagate" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-invert-v* ]]; then | |
| echo "package=midas_invert" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-invert" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-uq-v* ]]; then | |
| echo "package=midas_uq" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-uq" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-joint-ff-calibrate-v* ]]; then | |
| echo "package=midas_joint_ff_calibrate" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-joint-ff-calibrate" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-fit-grain-v* ]]; then | |
| echo "package=midas_fit_grain" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-fit-grain" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-tomo-v* ]]; then | |
| echo "package=midas_tomo" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-tomo" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-dt-v* ]]; then | |
| echo "package=midas_dt" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-dt" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-nf-fitorientation-v* ]]; then | |
| echo "package=midas_nf_fitorientation" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-nf-fitorientation" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-nf-pipeline-v* ]]; then | |
| echo "package=midas_nf_pipeline" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-nf-pipeline" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-parsl-configs-v* ]]; then | |
| echo "package=midas_parsl_configs" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-parsl-configs" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-ff-pipeline-v* ]]; then | |
| echo "package=midas_ff_pipeline" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-ff-pipeline" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-pipeline-v* ]]; then | |
| echo "package=midas_pipeline" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-pipeline" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-suite-v* ]]; then | |
| echo "package=midas_suite" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-suite" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-2d-v* ]]; then | |
| echo "package=midas_2d" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-2d" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-ckernel-v* ]]; then | |
| echo "package=midas_ckernel" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-ckernel" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-ddd-v* ]]; then | |
| echo "package=midas_ddd" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-ddd" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-saxs-v* ]]; then | |
| echo "package=midas_saxs" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-saxs" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-defect-v* ]]; then | |
| echo "package=midas_defect" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-defect" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-dfxm-v* ]]; then | |
| echo "package=midas_dfxm" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-dfxm" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-grain-odf-v* ]]; then | |
| echo "package=midas_grain_odf" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-grain-odf" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-pdf-v* ]]; then | |
| echo "package=midas_pdf" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-pdf" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-pf-odf-v* ]]; then | |
| echo "package=midas_pf_odf" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-pf-odf" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-pink-v* ]]; then | |
| echo "package=midas_pink" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-pink" >> "$GITHUB_OUTPUT" | |
| elif [[ "$TAG" == midas-xaf-v* ]]; then | |
| echo "package=midas_xaf" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=midas-xaf" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "Tag $TAG does not match any known package prefix; skipping." | |
| echo "package=" >> "$GITHUB_OUTPUT" | |
| echo "pypi_name=" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Set up Python | |
| if: steps.which.outputs.package != '' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install build tools | |
| if: steps.which.outputs.package != '' | |
| run: pip install build | |
| - name: Build sdist and wheel | |
| if: steps.which.outputs.package != '' | |
| run: | | |
| cd packages/${{ steps.which.outputs.package }} | |
| # scikit-build-core packages (e.g. midas-index) compile a native | |
| # binary, so `python -m build` emits a platform-tagged wheel | |
| # (linux_x86_64) that PyPI rejects — it only accepts manylinux*. | |
| # Per project_no_c_pypi_packages.md the C binary is build-from-source, | |
| # so we publish the sdist only; pip compiles it at install time | |
| # (or falls back to the Python-only path). Pure-Python packages | |
| # keep the sdist + universal wheel. | |
| if grep -q 'scikit_build_core.build' pyproject.toml; then | |
| echo "scikit-build-core backend detected — building sdist only." | |
| python -m build --sdist | |
| else | |
| python -m build | |
| fi | |
| - name: Upload artifacts | |
| if: steps.which.outputs.package != '' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.which.outputs.pypi_name }}-dist | |
| path: packages/${{ steps.which.outputs.package }}/dist/ | |
| publish: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'release' && needs.build.outputs.package != '' | |
| environment: pypi | |
| permissions: | |
| id-token: write # for trusted publishing (OIDC) | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.build.outputs.pypi_name }}-dist | |
| path: dist/ | |
| - name: Publish to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| # Uses trusted publishing (OIDC). One-time setup per package at pypi.org: | |
| # Settings > Publishing > Add a new pending publisher | |
| # Owner: marinerhemant | |
| # Repo: MIDAS | |
| # Workflow: python-packages.yml | |
| # Environment: pypi | |
| # The same GitHub environment (pypi) is shared across packages; | |
| # the PyPI side identifies which project to publish to by the | |
| # PyPI project name baked into the uploaded dist/*. |