diff --git a/.github/actions/build-plugin/action.yml b/.github/actions/build-plugin/action.yml index 5506596b84..e93a384669 100644 --- a/.github/actions/build-plugin/action.yml +++ b/.github/actions/build-plugin/action.yml @@ -20,6 +20,9 @@ inputs: build-type: description: "release or debug" required: true + cudart: + description: "static, dynamic, or none" + required: true artifact-name: description: "Name for the config.log artifact uploaded on failure" required: true @@ -61,6 +64,7 @@ runs: --enable-werror=yes \ --enable-picky-compiler=yes \ --enable-platform-aws \ + ${{ inputs.cudart == 'dynamic' && '--enable-cudart-dynamic' || '' }} \ ${{ inputs.sdk == 'neuron' && '--enable-neuron' || '--with-cuda=/usr/local/cuda' }} \ ${{ inputs.build-type == 'debug' && '--enable-debug --enable-trace' || '' }} \ ${{ contains(inputs.tracing, 'lttng') && '--with-lttng' || '' }} \ @@ -70,6 +74,29 @@ runs: shell: bash run: make V=1 + - name: Verify dynamic CUDART isolation + if: inputs.cudart == 'dynamic' + shell: bash + run: | + set -euo pipefail + + shopt -s nullglob + plugins=(src/.libs/libnccl*.so) + if [ ${#plugins[@]} -eq 0 ]; then + echo "No core plugin shared libraries were built" >&2 + exit 1 + fi + + for plugin in "${plugins[@]}"; do + echo "Checking ${plugin}" + + if readelf -d "${plugin}" | grep -Eq 'NEEDED.*\[libcudart\.so'; then + echo "${plugin} has a direct CUDART dependency" >&2 + readelf -d "${plugin}" >&2 + exit 1 + fi + done + - name: Call `make check` shell: bash run: make check V=1 || (cat tests/unit/test-suite.log && exit 1) diff --git a/.github/workflows/distcheck.yaml b/.github/workflows/distcheck.yaml index f3a71db070..308bfd691d 100644 --- a/.github/workflows/distcheck.yaml +++ b/.github/workflows/distcheck.yaml @@ -30,10 +30,32 @@ jobs: build-type: - release - debug + cudart: + - static + - dynamic + - none exclude: # Neuron requires platform-aws, which requires newer EFA installer - efainstaller: 1.25.0 sdk: neuron + # Neuron builds do not use CUDART + - sdk: neuron + cudart: static + - sdk: neuron + cudart: dynamic + # CUDA builds require a CUDART linkage mode + - sdk: cuda + cudart: none + # Keep one dynamic CUDA release build on the latest EFA installer + - sdk: cuda + cudart: dynamic + build-type: debug + - sdk: cuda + cudart: dynamic + efainstaller: 1.34.0 + - sdk: cuda + cudart: dynamic + efainstaller: 1.25.0 include: - efainstaller: latest platform-aws: enable @@ -48,7 +70,7 @@ jobs: runs-on: ubuntu-latest container: ghcr.io/${{ github.repository }}/aws-ofi-nccl-al2023:${{ matrix.sdk }}-efa${{ matrix.efainstaller }} - name: al2023/${{ matrix.sdk }}/efa@${{ matrix.efainstaller }}/${{ matrix.build-type }} + name: al2023/${{ matrix.sdk }}/efa@${{ matrix.efainstaller }}/${{ matrix.build-type }}${{ matrix.sdk == 'cuda' && format('/cudart-{0}', matrix.cudart) || '' }} steps: - uses: actions/checkout@v6 @@ -67,12 +89,35 @@ jobs: --enable-werror=yes \ --enable-picky-compiler=yes \ --${{ matrix.platform-aws }}-platform-aws \ + ${{ matrix.cudart == 'dynamic' && '--enable-cudart-dynamic' || '' }} \ ${{ matrix.sdk == 'neuron' && '--enable-neuron' || '--with-cuda=/usr/local/cuda' }} \ ${{ matrix.build-type == 'debug' && '--enable-debug --enable-trace' || '' }} - name: Call `make` run: make V=1 + - name: Verify dynamic CUDART isolation + if: matrix.cudart == 'dynamic' + run: | + set -euo pipefail + + shopt -s nullglob + plugins=(src/.libs/libnccl*.so) + if [ ${#plugins[@]} -eq 0 ]; then + echo "No core plugin shared libraries were built" >&2 + exit 1 + fi + + for plugin in "${plugins[@]}"; do + echo "Checking ${plugin}" + + if readelf -d "${plugin}" | grep -Eq 'NEEDED.*\[libcudart\.so'; then + echo "${plugin} has a direct CUDART dependency" >&2 + readelf -d "${plugin}" >&2 + exit 1 + fi + done + - name: Call `make check` run: make check V=1 || (cat tests/unit/test-suite.log && exit 1) @@ -103,12 +148,43 @@ jobs: build-type: - release - debug + cudart: + - static + - dynamic + - none exclude: # NVTX requires CUDA, so skip the nvtx variants on neuron. - tracing: nvtx sdk: neuron - tracing: lttng-nvtx sdk: neuron + # Neuron builds do not use CUDART + - sdk: neuron + cudart: static + - sdk: neuron + cudart: dynamic + # CUDA builds require a CUDART linkage mode + - sdk: cuda + cudart: none + # Keep one dynamic CUDA release/GCC/no-tracing build on this OS + - sdk: cuda + cudart: dynamic + build-type: debug + - sdk: cuda + cudart: dynamic + tracing: lttng + - sdk: cuda + cudart: dynamic + tracing: nvtx + - sdk: cuda + cudart: dynamic + tracing: lttng-nvtx + - sdk: cuda + cudart: dynamic + cc: clang + - sdk: cuda + cudart: dynamic + cc-variant: legacy include: - cc-variant: latest cc: clang @@ -117,7 +193,7 @@ jobs: cc: gcc cc-version: 13 - name: u2204/${{ matrix.sdk }}/${{matrix.cc}}-${{matrix.cc-variant}}-${{matrix.tracing}}/${{ matrix.build-type }} + name: u2204/${{ matrix.sdk }}/${{matrix.cc}}-${{matrix.cc-variant}}-${{matrix.tracing}}/${{ matrix.build-type }}${{ matrix.sdk == 'cuda' && format('/cudart-{0}', matrix.cudart) || '' }} # The nvtx and lttng-nvtx tracing variants reuse the lttng container # image, which already has cuda-nvtx installed (added in PR #1224). Map # any nvtx-containing variant to the lttng tag so the container reference @@ -132,6 +208,7 @@ jobs: sdk: ${{ matrix.sdk }} tracing: ${{ matrix.tracing }} build-type: ${{ matrix.build-type }} + cudart: ${{ matrix.cudart }} artifact-name: ${{ matrix.cc }}-${{ matrix.cc-variant }}-${{ matrix.sdk }}-config.log Ubuntu2404: @@ -154,14 +231,48 @@ jobs: build-type: - release - debug + cudart: + - static + - dynamic + - none exclude: # NVTX requires CUDA, so skip the nvtx variants on neuron. - tracing: nvtx sdk: neuron - tracing: lttng-nvtx sdk: neuron - - name: U2404/${{ matrix.sdk }}/${{matrix.cc}}/${{matrix.tracing}}/${{ matrix.build-type }} + # Neuron builds do not use CUDART + - sdk: neuron + cudart: static + - sdk: neuron + cudart: dynamic + # CUDA builds require a CUDART linkage mode + - sdk: cuda + cudart: none + # Keep one dynamic CUDA release/GCC/no-tracing build on this OS + - sdk: cuda + cudart: dynamic + build-type: debug + - sdk: cuda + cudart: dynamic + tracing: lttng + - sdk: cuda + cudart: dynamic + tracing: nvtx + - sdk: cuda + cudart: dynamic + tracing: lttng-nvtx + - sdk: cuda + cudart: dynamic + cc: gcc-13 + - sdk: cuda + cudart: dynamic + cc: clang-18 + - sdk: cuda + cudart: dynamic + cc: clang-19 + + name: U2404/${{ matrix.sdk }}/${{matrix.cc}}/${{matrix.tracing}}/${{ matrix.build-type }}${{ matrix.sdk == 'cuda' && format('/cudart-{0}', matrix.cudart) || '' }} # See comment on u2204 job: nvtx and lttng-nvtx reuse the lttng container. container: ghcr.io/${{ github.repository }}/aws-ofi-nccl-ubuntu2404:${{ matrix.sdk }}-${{ matrix.cc }}-${{ (contains(matrix.tracing, 'nvtx') && 'lttng') || matrix.tracing }}-efalattest steps: @@ -173,6 +284,7 @@ jobs: sdk: ${{ matrix.sdk }} tracing: ${{ matrix.tracing }} build-type: ${{ matrix.build-type }} + cudart: ${{ matrix.cudart }} artifact-name: ${{ matrix.cc }}-${{ matrix.sdk }}-config.log Ubuntu2604: @@ -195,14 +307,48 @@ jobs: build-type: - release - debug + cudart: + - static + - dynamic + - none exclude: # NVTX requires CUDA, so skip the nvtx variants on neuron. - tracing: nvtx sdk: neuron - tracing: lttng-nvtx sdk: neuron - - name: U2604/${{ matrix.sdk }}/${{matrix.cc}}/${{matrix.tracing}}/${{ matrix.build-type }} + # Neuron builds do not use CUDART + - sdk: neuron + cudart: static + - sdk: neuron + cudart: dynamic + # CUDA builds require a CUDART linkage mode + - sdk: cuda + cudart: none + # Keep one dynamic CUDA release/GCC/no-tracing build on this OS + - sdk: cuda + cudart: dynamic + build-type: debug + - sdk: cuda + cudart: dynamic + tracing: lttng + - sdk: cuda + cudart: dynamic + tracing: nvtx + - sdk: cuda + cudart: dynamic + tracing: lttng-nvtx + - sdk: cuda + cudart: dynamic + cc: gcc-14 + - sdk: cuda + cudart: dynamic + cc: clang-19 + - sdk: cuda + cudart: dynamic + cc: clang-20 + + name: U2604/${{ matrix.sdk }}/${{matrix.cc}}/${{matrix.tracing}}/${{ matrix.build-type }}${{ matrix.sdk == 'cuda' && format('/cudart-{0}', matrix.cudart) || '' }} # Host runner is ubuntu-24.04; the 26.04 environment comes from the container. # See comment on u2204 job: nvtx and lttng-nvtx reuse the lttng container. container: ghcr.io/${{ github.repository }}/aws-ofi-nccl-ubuntu2604:${{ matrix.sdk }}-${{ matrix.cc }}-${{ (contains(matrix.tracing, 'nvtx') && 'lttng') || matrix.tracing }}-efalattest @@ -215,6 +361,7 @@ jobs: sdk: ${{ matrix.sdk }} tracing: ${{ matrix.tracing }} build-type: ${{ matrix.build-type }} + cudart: ${{ matrix.cudart }} artifact-name: ${{ matrix.cc }}-${{ matrix.sdk }}-u2604-config.log thread-safety-check: