diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..4ecb75c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ + version: 2 + updates: + - package-ecosystem: "github-actions" + directory: "/.github/workflows" + schedule: + interval: "monthly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fda3d4e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,106 @@ +name: ci + +on: + push: + branches: + - devel + pull_request: + branches: + - devel + +permissions: + contents: read + +concurrency: + group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} + cancel-in-progress: true + +jobs: + spindle-serial-ubuntu: + name: Testsuite (Serial, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + + - name: Setup Docker Compose + uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746 + with: + version: latest + + - name: Build spindle-serial-ubuntu image + id: serial-ubuntu-build + run: | + cd containers/spindle-serial-ubuntu + docker compose --progress=plain build + + - name: Bring spindle-serial-ubuntu up + id: serial-ubuntu-up + run: | + cd containers/spindle-serial-ubuntu + docker compose up -d + + - name: Verify munge works in spindle-serial-ubuntu + id: serial-ubuntu-munge + run: | + docker exec spindlenode bash -c 'munge -n | unmunge' + + - name: Run spindle-serial-ubuntu testsuite + id: serial-ubuntu-testsuite + run: | + docker exec spindlenode bash -c 'cd Spindle-build/testsuite && ./runTests' + + - name: Bring spindle-serial-ubuntu down + id: serial-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-serial-ubuntu + docker compose down + + spindle-flux-ubuntu: + name: Testsuite (Flux, Ubuntu) + environment: Spindle CI + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out Spindle + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + + - name: Setup Docker Compose + uses: docker/setup-compose-action@364cc21a5de5b1ee4a7f5f9d3fa374ce0ccde746 + with: + version: latest + + - name: Build spindle-flux-ubuntu image + id: flux-ubuntu-build + run: | + cd containers/spindle-flux-ubuntu + docker compose --progress=plain build + + - name: Bring spindle-flux-ubuntu up + id: flux-ubuntu-up + run: | + cd containers/spindle-flux-ubuntu + docker compose up -d --wait --wait-timeout 60 + + - name: Verify munge works in spindle-flux-ubuntu + id: flux-ubuntu-munge + run: | + docker exec node-1 bash -c 'munge -n | unmunge' + + - name: Run spindle-flux-ubuntu testsuite + id: flux-ubuntu-testsuite + run: | + docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./runTests --nodes=${workers} --tasks-per-node=3' + + - name: Bring spindle-flux-ubuntu down + id: flux-ubuntu-down + if: ${{ always() }} + continue-on-error: true + run: | + cd containers/spindle-flux-ubuntu + docker compose down + diff --git a/containers/spindle-flux-ubuntu/Dockerfile b/containers/spindle-flux-ubuntu/Dockerfile new file mode 100644 index 0000000..3af607a --- /dev/null +++ b/containers/spindle-flux-ubuntu/Dockerfile @@ -0,0 +1,67 @@ +# This is based on the Flux Container Tutorial +# See https://flux-framework.readthedocs.io/en/latest/tutorials/containers +ARG flux_sched_version=noble +FROM fluxrm/flux-sched:${flux_sched_version} AS builder +ARG replicas=4 +ENV workers=${replicas} +USER root + +RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ + && apt-get -qq install -y --no-install-recommends \ + autotools-dev \ + autoconf \ + automake \ + cmake \ + git \ + python3 \ + openssh-server \ + openssh-client \ + libdb-dev \ + apt-utils \ + dnsutils \ + iputils-ping \ + python3-pip \ + libgcrypt20 \ + libgcrypt20-dev \ + gdb \ + software-properties-common + +ARG USER=fluxuser +ARG CONFIG_ROOT=containers/spindle-flux-ubuntu + +# Allow fluxuser to run as other users so it can start munged +RUN sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers" + +# Configure flux +ENV STATE_DIR=/var/lib/flux +RUN mkdir -p ${STATE_DIR} /etc/flux/system /etc/flux/system/cron.d /etc/flux/config /run/flux /etc/flux/imp/conf.d +COPY ${CONFIG_ROOT}/flux/imp.toml /etc/flux/imp/conf.d/ +COPY ${CONFIG_ROOT}/flux/broker.toml /etc/flux/config/ +RUN mkdir -p /etc/flux/system/cron.d && \ + mkdir -p /mnt/curve && \ + flux keygen /mnt/curve/curve.cert && \ + flux R encode --hosts="node-[1-${workers}]" > /etc/flux/system/R && \ + chown -R ${USER}:${USER} /run/flux ${STATE_DIR} /mnt/curve/curve.cert + +# Build Spindle +WORKDIR /home/${USER} +# Copy the whole git repo into the container. +COPY . /home/${USER}/Spindle +COPY ${CONFIG_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN ./build_spindle.sh + +RUN chown -R ${USER}:${USER} /home/fluxuser && \ + chown -R ${USER}:${USER} /run/flux + +USER ${USER} +COPY ${CONFIG_ROOT}/scripts/flux_healthcheck.sh ./ +COPY ${CONFIG_ROOT}/scripts/entrypoint.sh ./ +ENV PATH /home/${USER}/Spindle-inst/bin:${PATH} +# Make libfabric work with fork. +ENV RDMAV_FORK_SAFE 1 +# Silence warning from hwloc about unsupported PCI device +# on GitHub-hosted runners. +ENV HWLOC_HIDE_ERRORS 2 + +ENTRYPOINT /bin/bash ./entrypoint.sh + diff --git a/containers/spindle-flux-ubuntu/docker-compose.yml b/containers/spindle-flux-ubuntu/docker-compose.yml new file mode 100644 index 0000000..e587e1d --- /dev/null +++ b/containers/spindle-flux-ubuntu/docker-compose.yml @@ -0,0 +1,70 @@ +# 4-node Flux cluster +# For information on running Flux in containers, see +# https://flux-framework.readthedocs.io/en/latest/tutorials/containers + +# `replicas` must match the number of nodes defined in the services section +x-shared-workers: + &workers + replicas: 4 + +# Ubuntu version to use (noble = 24.04) +x-shared-build-args: &shared-build-args + flux_sched_version: noble + <<: *workers + +# Docker prohibits copying files from outside of the build context. +# In order to be able to copy the whole repo into the container, +# we have to set the context to be the root of the repo. +# We then have to specify the path from there to the Dockerfile. +x-shared-build-context: &shared-build-context + context: ../.. + dockerfile: containers/spindle-flux-ubuntu/Dockerfile + args: *shared-build-args + +# Name of the node that runs the Flux broker +x-shared-environment: &shared-environment + mainHost: node-1 + <<: *workers + +networks: + flux: + driver: bridge + +# Common parameters for all nodes. +x-shared-node-parameters: &shared-node-parameters + build: *shared-build-context + networks: + - flux + environment: *shared-environment + cap_add: + - SYS_NICE # Required for libnuma + +services: + node-1: + <<: *shared-node-parameters + hostname: node-1 + container_name: node-1 + # Check whether all the workers have registered + # with the broker on the head node. + healthcheck: + test: ["CMD", "./flux_healthcheck.sh"] + start_period: 15s + interval: 5s + timeout: 10s + retries: 5 + + node-2: + <<: *shared-node-parameters + hostname: node-2 + container_name: node-2 + + node-3: + <<: *shared-node-parameters + hostname: node-3 + container_name: node-3 + + node-4: + <<: *shared-node-parameters + hostname: node-4 + container_name: node-4 + diff --git a/containers/spindle-flux-ubuntu/flux/broker.toml b/containers/spindle-flux-ubuntu/flux/broker.toml new file mode 100644 index 0000000..ad94017 --- /dev/null +++ b/containers/spindle-flux-ubuntu/flux/broker.toml @@ -0,0 +1,19 @@ +[exec] +imp = "/usr/libexec/flux/flux-imp" + +[access] +allow-guest-user = true +allow-root-owner = true + +[resource] +path = "/etc/flux/system/R" + +[bootstrap] +curve_cert = "/mnt/curve/curve.cert" +default_port = 8050 +default_bind = "tcp://eth0:%%p" +default_connect = "tcp://%%h:%%p" +hosts = [ + { host="node-[1-4]"}, +] + diff --git a/containers/spindle-flux-ubuntu/flux/imp.toml b/containers/spindle-flux-ubuntu/flux/imp.toml new file mode 100644 index 0000000..156e3b5 --- /dev/null +++ b/containers/spindle-flux-ubuntu/flux/imp.toml @@ -0,0 +1,3 @@ +[exec] +allowed-users = [ "flux", "root" ] +allowed-shells = [ "/usr/libexec/flux/flux-shell" ] diff --git a/containers/spindle-flux-ubuntu/scripts/build_spindle.sh b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh new file mode 100755 index 0000000..9257f85 --- /dev/null +++ b/containers/spindle-flux-ubuntu/scripts/build_spindle.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -euxo pipefail + +mkdir -p /home/${USER}/Spindle-build +cd /home/${USER}/Spindle-build +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=flux --enable-flux-plugin --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +make -j$(nproc) +make install + diff --git a/containers/spindle-flux-ubuntu/scripts/entrypoint.sh b/containers/spindle-flux-ubuntu/scripts/entrypoint.sh new file mode 100755 index 0000000..8d3944e --- /dev/null +++ b/containers/spindle-flux-ubuntu/scripts/entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Starts munged and the flux broker. +# +# For documentation on running Flux in containers, see +# https://flux-framework.readthedocs.io/en/latest/tutorials/containers + +brokerOptions="-Scron.directory=/etc/flux/system/cron.d \ + -Stbon.fanout=256 \ + -Srundir=/run/flux \ + -Sstatedir=${STATE_DIRECTORY:-/var/lib/flux} \ + -Slocal-uri=local:///run/flux/local \ + -Slog-stderr-level=6 \ + -Slog-stderr-mode=local" + +# Get the hostname that will resolve for the Docker bridge network. +address=$(echo $( nslookup "$( hostname -i )" | head -n 1 )) +parts=(${address//=/ }) +hostName=${parts[2]} +thisHost=(${hostName//./ }) +thisHost=${thisHost[0]} +echo $thisHost +export FLUX_FAKE_HOSTNAME=$thisHost + +# Start munged +sudo -u munge /usr/sbin/munged + +if [ ${thisHost} != "${mainHost}" ]; then + # Worker node -- wait for head node before connecting + sleep 15 + FLUX_FAKE_HOSTNAME=$thisHost flux start -o --config /etc/flux/config ${brokerOptions} sleep inf +else + # Head node + FLUX_FAKE_HOSTNAME=$thisHost flux start -o --config /etc/flux/config ${brokerOptions} sleep inf +fi + diff --git a/containers/spindle-flux-ubuntu/scripts/flux_healthcheck.sh b/containers/spindle-flux-ubuntu/scripts/flux_healthcheck.sh new file mode 100755 index 0000000..c8816db --- /dev/null +++ b/containers/spindle-flux-ubuntu/scripts/flux_healthcheck.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +FLUX_FREE_NODES=$(flux resource list -s free -n | awk '{print $2}') + +if [[ ${FLUX_FREE_NODES} -ne ${replicas} ]] ; then + echo "FAILED: Incorrect number of Flux free nodes: expected ${replicas}, got ${FLUX_FREE_NODES}" + exit 1 +fi + +echo "PASSED: Found ${FLUX_FREE_NODES} nodes out of ${replicas}; all nodes registered." + diff --git a/containers/spindle-serial-ubuntu/Dockerfile b/containers/spindle-serial-ubuntu/Dockerfile new file mode 100644 index 0000000..3070596 --- /dev/null +++ b/containers/spindle-serial-ubuntu/Dockerfile @@ -0,0 +1,60 @@ +ARG ubuntu_version=noble +FROM ubuntu:${ubuntu_version} +USER root + +RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ +# install latest pkg utils: + && apt-get -qq install -y --no-install-recommends \ + apt-utils + +RUN DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + locales \ + ca-certificates \ + wget \ + git \ + ssh \ + sudo \ + build-essential \ + pkg-config \ + autotools-dev \ + libtool \ + autoconf \ + automake \ + make \ + gfortran-13 \ + gcc-13 \ + g++-13 \ + munge \ + libmunge-dev \ + libhwloc-dev \ + mpich \ + libmpich-dev + +# Prevent hwloc from trying to use graphics cards +# as this fails when X is not running. +ENV HWLOC_COMPONENTS=-gl + +# Set up munge +RUN mkdir -p /run/munge && \ + chown munge:munge /run/munge && \ + chmod 0755 /run/munge + +ARG USER=spindleuser +ARG UID=1001 +ARG BUILD_ROOT=./containers/spindle-serial-ubuntu +COPY ${BUILD_ROOT}/scripts/add_docker_user.sh /add_docker_user.sh +RUN /add_docker_user.sh + +USER ${USER} +WORKDIR /home/${USER} +RUN mkdir -p /home/${USER}/Spindle +# Copy the Spindle repo into the container +COPY . /home/${USER}/Spindle +COPY ${BUILD_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN ./build_spindle.sh + +COPY ${BUILD_ROOT}/scripts/entrypoint.sh /home/${USER}/entrypoint.sh +ENV PATH /home/${USER}/Spindle-inst/bin:$PATH + +ENTRYPOINT /bin/bash ./entrypoint.sh + diff --git a/containers/spindle-serial-ubuntu/docker-compose.yml b/containers/spindle-serial-ubuntu/docker-compose.yml new file mode 100644 index 0000000..f90f4e8 --- /dev/null +++ b/containers/spindle-serial-ubuntu/docker-compose.yml @@ -0,0 +1,16 @@ +services: + spindlenode: + # The build context is set to ../.. so that we can copy + # the Spindle directory into the created image. + # Because we set the context, we then have to specify + # the path from the context to the Dockerfile. + build: + context: ../.. + dockerfile: containers/spindle-serial-ubuntu/Dockerfile + hostname: spindlenode + container_name: spindlenode + tty: true + # The NUMA tests require CAP_SYS_NICE. + cap_add: + - SYS_NICE + diff --git a/containers/spindle-serial-ubuntu/scripts/add_docker_user.sh b/containers/spindle-serial-ubuntu/scripts/add_docker_user.sh new file mode 100755 index 0000000..9ecfbfd --- /dev/null +++ b/containers/spindle-serial-ubuntu/scripts/add_docker_user.sh @@ -0,0 +1,7 @@ +#!/bin/sh +sudo groupadd -g ${UID} ${USER} +sudo useradd -g ${USER} -u ${UID} -d /home/${USER} -m ${USER} +# Allow user to run as other users so that munge can be started as the munge user +sudo sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers" +sudo adduser ${USER} sudo + diff --git a/containers/spindle-serial-ubuntu/scripts/build_spindle.sh b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh new file mode 100755 index 0000000..37b6491 --- /dev/null +++ b/containers/spindle-serial-ubuntu/scripts/build_spindle.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -euxo pipefail + +mkdir -p /home/${USER}/Spindle-build +cd /home/${USER}/Spindle-build +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=serial --with-localstorage=/tmp CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +make -j$(nproc) +make install + diff --git a/containers/spindle-serial-ubuntu/scripts/entrypoint.sh b/containers/spindle-serial-ubuntu/scripts/entrypoint.sh new file mode 100755 index 0000000..1d1f236 --- /dev/null +++ b/containers/spindle-serial-ubuntu/scripts/entrypoint.sh @@ -0,0 +1,4 @@ +printf "\nStarting munged\n" + +sudo -u munge /usr/sbin/munged --foreground +