diff --git a/tests/scripts/.gitignore b/tests/scripts/.gitignore new file mode 100644 index 0000000000..795233fac6 --- /dev/null +++ b/tests/scripts/.gitignore @@ -0,0 +1,11 @@ +# User-specific environment (contains local paths. Copied from .example) +env.sh + +# User's local slurm script (copied from .example) +run-slurm.sh + +# Slurm output files +slurm-*.txt + +# Test output directories +outputs/ diff --git a/tests/scripts/AGENTS.md b/tests/scripts/AGENTS.md new file mode 100644 index 0000000000..2ad70916bd --- /dev/null +++ b/tests/scripts/AGENTS.md @@ -0,0 +1,282 @@ +# Testing Guide for AI Agents + +This directory contains test runners for the aws-ofi-nccl plugin. This +document explains how to use them and interpret results. + +## Two distinct test suites + +1. **Plugin functional tests** (`tests/functional/`): Plugin-level tests + that exercise the net plugin API directly. Binaries are built by the + plugin's own build system (`make check` or individual binaries in + `$PREFIX/bin/`). Run with `run-functional-tests.sh`. + +2. **nccl-tests** (https://github.com/NVIDIA/nccl-tests): NVIDIA's + standalone collective benchmarks that exercise NCCL end-to-end through + a real network backend. NOT bundled with this repo. Run with + `run-nccl-tests.sh`. + +These are different things. Do not conflate them. + +## Environment setup + +```bash +cd tests/scripts +cp env.sh.example env.sh +# Edit env.sh: set STACKDIR, PLUGIN_INSTALL, NCCL_DIR, NCCL_TESTS_DIR, MPI_DIR +``` + +Key variables: + +| Variable | Purpose | +|----------|---------| +| `STACKDIR` | Root workspace directory | +| `PLUGIN_INSTALL` | Plugin install prefix (has `lib/` and `bin/`) | +| `NCCL_DIR` | Path to NCCL library (`libnccl.so`) | +| `NCCL_TESTS_DIR` | Path to nccl-tests build dir (has `all_reduce_perf`, etc.) | +| `MPI_DIR` | MPI installation root | +| `OUTPUT_DIR` | Where test output goes | +| `ASAN_PLUGIN_DIR` | ASAN-instrumented plugin lib dir (for MODE=asan) | +| `LIBASAN_PATH` | Path to `libasan.so` | + +## Running functional tests + +```bash +# Basic +MODE=regular PROTOCOL=RDMA TEST=nccl_message_transfer ./run-functional-tests.sh + +# All available functional tests: +# nccl_message_transfer, inflight_close, nccl_connection, +# ring, gin, grouped_recv, reuse_listen_comm + +# With SENDRECV protocol +MODE=regular PROTOCOL=SENDRECV TEST=nccl_message_transfer ./run-functional-tests.sh + +# Multi-rank (some tests need exactly 2, gin needs 8+) +NRANKS=2 TEST=inflight_close ./run-functional-tests.sh +NRANKS=8 TEST=gin ./run-functional-tests.sh +``` + +### Functional test success criteria + +- Exit code 0 (the script propagates mpirun's exit code directly) +- Output contains `Results: N/N passed` +- No `NCCL_OFI_WARN` in output +- No segfaults or aborts + +### Functional test failure modes + +- Exit code non-zero: test assertion failed +- `NCCL_OFI_WARN`: plugin detected an error condition +- Hang (no output for >60s): likely a deadlock in completion handling + +## Running nccl-tests + +### Build-install-test workflow + +The plugin loads from `$PLUGIN_INSTALL/lib/` at runtime. You must +`make install` after building for changes to take effect: + +```bash +cd /path/to/aws-ofi-nccl +make -j && make install +``` + +### EXTRA_TEST_ARGS explained + +The default args are `-n 15 -w 10 -b 1K -e 16G -f 2 -c 1 -R 0`: + +| Flag | Meaning | +|------|---------| +| `-n 15` | 15 test iterations | +| `-w 10` | 10 warmup iterations | +| `-b 1K -e 16G` | Message sizes from 1KB to 16GB | +| `-f 2` | Size doubles each step | +| `-c 1` | **Correctness check enabled** (validates data integrity) | +| `-R 0` | No random seed (deterministic) | + +The `-c 1` flag is critical: without it, nccl-tests only measures +performance and does not validate data. Always use `-c 1` when +testing plugin changes. + +### Invocation examples + +```bash +# Single node, 8 GPUs +BENCHMARK=all_reduce_perf NUM_NODES=1 RANKS_PER_NODE=8 ./run-nccl-tests.sh + +# Two nodes +BENCHMARK=all_reduce_perf NUM_NODES=2 HOSTFILE=/path/to/hosts ./run-nccl-tests.sh + +# With topology-aware placement +USE_TOPO_SORT=1 HOSTFILE=/path/to/hosts BENCHMARK=all_reduce_perf NUM_NODES=2 ./run-nccl-tests.sh + +# With split mask (single-rank communicators, tests control path) +NCCL_TESTS_SPLIT_MASK=0x7 BENCHMARK=all_reduce_perf NUM_NODES=1 ./run-nccl-tests.sh + +# Smaller message range for quick validation +EXTRA_TEST_ARGS="-n 5 -w 5 -b 1K -e 2G -f 2 -c 1 -R 0" ./run-nccl-tests.sh +``` + +Available benchmarks: `all_reduce_perf`, `all_gather_perf`, +`reduce_scatter_perf`, `broadcast_perf`, `sendrecv_perf`, +`alltoall_perf`, `scatter_perf`, `gather_perf`, `reduce_perf`, +`hypercube_perf`. + +### nccl-tests success criteria + +Two lines in the output footer are the validation signal: + +``` +# Out of bounds values : 0 OK +# Avg bus bandwidth : +``` + +A run is successful when: +1. `# Out of bounds values : 0 OK` is present +2. `# Avg bus bandwidth` is reported + +A bandwidth of 0 is normal when `NCCL_TESTS_SPLIT_MASK` produces +single-rank communicators (e.g. mask 0x7 on 1 node with 8 ranks). +This does not indicate failure. + +### nccl-tests failure modes + +- Job aborts before printing footer (SIGSEGV, SIGABRT, hang) +- `Out of bounds values` reports non-zero count +- The `#wrong` column in per-row results is non-zero (per-size corruption) +- Output directory renamed to `*-err` (trap on non-zero exit) +- Plugin not loading: verify `NET/OFI Using network AWS Libfabric` appears + in the output. If missing, check `LD_LIBRARY_PATH` in env.sh. + +### Debugging nccl-tests failures + +Enable detailed NCCL logging: + +```bash +NCCL_DEBUG=INFO NCCL_DEBUG_SUBSYS=INIT,NET ./run-nccl-tests.sh +``` + +Verify the plugin loaded: + +```bash +grep "NET/OFI" $OUTPUT_DIR/*/output.txt +# Should show: NET/OFI Using network AWS Libfabric +``` + +### Checking results + +```bash +./check-test-results.sh $OUTPUT_DIR +``` + +This checks for: correctness lines, error directories, crashes, and +plugin warnings. + +## ASAN mode + +Build the plugin with `--enable-asan`, install to a separate prefix: + +```bash +./configure --prefix=$STACKDIR/install-asan --enable-asan --enable-debug ... +make -j && make install +``` + +Run: + +```bash +MODE=asan TEST=nccl_message_transfer ./run-functional-tests.sh +``` + +### ASAN output interpretation + +- The script exits 0 even when leaks are reported (due to `halt_on_error=0`) +- Check the ASAN summary at the end of output to determine pass/fail +- **Plugin leaks** (stack contains `nccl_ofi_*`): real bugs to fix +- **MPI/libfabric leaks**: suppressed by `sanitizers/lsan.supp` +- `ASAN_OPTIONS=protect_shadow_gap=0` is required for CUDA compatibility + +Look for: +``` +SUMMARY: AddressSanitizer: N byte(s) leaked in M allocation(s). +``` + +If all leaks are from suppressed third-party code, the test passes. + +## Valgrind mode + +Use a normal debug build (NOT ASAN). Do not combine ASAN and Valgrind. + +```bash +MODE=valgrind TEST=nccl_message_transfer ./run-functional-tests.sh +``` + +Per-rank output goes to `$OUTPUT_DIR/valgrind---/`. +The parser runs automatically and produces a summary classifying +plugin vs third-party errors. + +### Valgrind output interpretation + +- **Plugin errors/leaks**: real bugs (stack contains `nccl_ofi_*`) +- **Non-plugin**: suppressed or third-party noise +- The parser deduplicates across ranks and reports unique stacks + +## Slurm integration + +The runners are Slurm-agnostic. A batch template is provided: + +```bash +cp run-slurm.sh.example run-slurm.sh +# Edit: set --partition, --nodes, --ntasks-per-node, SCRIPT_DIR +# Set RUN_TYPE via env var when submitting +RUN_TYPE=functional sbatch run-slurm.sh +RUN_TYPE=nccl-tests sbatch run-slurm.sh +``` + +Or wrap manually: + +```bash +# Functional test +sbatch -N 1 --ntasks-per-node=2 --exclusive -p YOUR_PARTITION \ + --wrap="cd /path/to/tests/scripts && MODE=regular TEST=nccl_message_transfer ./run-functional-tests.sh" + +# nccl-tests (multi-node) +sbatch -N 2 --ntasks-per-node=8 --exclusive -p YOUR_PARTITION \ + --wrap="cd /path/to/tests/scripts && BENCHMARK=all_reduce_perf NUM_NODES=2 RANKS_PER_NODE=8 HOSTFILE=<(scontrol show hostnames) ./run-nccl-tests.sh" +``` + +## Test matrix for validating plugin changes + +### Workflow + +```bash +# 1. Build and install +cd /path/to/aws-ofi-nccl +make -j && make install + +# 2. Run tests +cd tests/scripts +# Functional +MODE=regular PROTOCOL=RDMA TEST=nccl_message_transfer ./run-functional-tests.sh +MODE=regular PROTOCOL=SENDRECV TEST=nccl_message_transfer ./run-functional-tests.sh +# nccl-tests (submit via Slurm) +RUN_TYPE=nccl-tests NCCL_TESTS_SPLIT_MASK=0x0 sbatch --ntasks-per-node=8 run-slurm.sh +RUN_TYPE=nccl-tests NCCL_TESTS_SPLIT_MASK=0x7 sbatch --ntasks-per-node=8 run-slurm.sh + +# 3. Check results +./check-test-results.sh $OUTPUT_DIR +``` + +### Minimum validation before submitting a PR + +| Test | Config | What it catches | +|------|--------|-----------------| +| `nccl_message_transfer` | RDMA + SENDRECV | Basic data path | +| `inflight_close` | RDMA | Completion handling races | +| `ring` | RDMA | Multi-device communication | +| `all_reduce_perf` 1N mask 0x0 | RDMA | Single-node collective correctness | +| `all_reduce_perf` 2N mask 0x0 | RDMA | Multi-node collective correctness | +| `all_reduce_perf` 1N mask 0x7 | RDMA | Control path (no data transfer) | +| `all_reduce_perf` 2N mask 0x7 | RDMA | Multi-node control path | + +For changes touching memory management, add ASAN and/or Valgrind runs +of `nccl_message_transfer` and `inflight_close`. diff --git a/tests/scripts/README.md b/tests/scripts/README.md new file mode 100644 index 0000000000..059b502e2e --- /dev/null +++ b/tests/scripts/README.md @@ -0,0 +1,57 @@ +# tests/scripts + +Test runners for aws-ofi-nccl. Supports functional tests, nccl-tests +(NVIDIA collective benchmarks), and sanitizer modes (ASAN, Valgrind). + +## Setup + +```bash +cp env.sh.example env.sh +# Edit env.sh with your paths +``` + +## Quick start + +```bash +# Build and install the plugin first +cd /path/to/aws-ofi-nccl && make -j && make install + +# Then run tests +cd tests/scripts + +# Functional test (regular) +MODE=regular TEST=nccl_message_transfer ./run-functional-tests.sh + +# nccl-tests +BENCHMARK=all_reduce_perf NUM_NODES=2 ./run-nccl-tests.sh + +# ASAN +MODE=asan TEST=inflight_close ./run-functional-tests.sh + +# Valgrind +MODE=valgrind TEST=ring ./run-functional-tests.sh + +# Check results +./check-test-results.sh /path/to/output-dir +``` + +## Files + +``` +tests/scripts/ +├── env.sh.example # Environment template (copy to env.sh) +├── run-functional-tests.sh # Functional test runner +├── run-nccl-tests.sh # nccl-tests collective runner +├── run-slurm.sh.example # Slurm batch template +├── check-test-results.sh # Output validator +├── sanitizers/ +│ ├── lsan.supp # LSAN suppressions +│ └── valgrind.supp # Valgrind suppressions +├── valgrind/ +│ ├── rank_wrapper.sh # Per-rank output redirector +│ └── parse_valgrind.sh # Valgrind output analyzer +├── README.md # This file +└── AGENTS.md # Agent-specific testing guide +``` + +See `AGENTS.md` for detailed usage instructions and output interpretation. diff --git a/tests/scripts/check-test-results.sh b/tests/scripts/check-test-results.sh new file mode 100755 index 0000000000..063bf89e44 --- /dev/null +++ b/tests/scripts/check-test-results.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# +# Validate nccl-tests output files for correctness. +# +# Usage: +# ./check-test-results.sh [OUTPUT_DIR] +# +# Checks: +# - "Out of bounds values : 0 OK" present in each output +# - No *-err directories (indicating failed runs) +# - No crashes (SIGSEGV, SIGABRT) +# - No NCCL_OFI_WARN lines +# +# If OUTPUT_DIR is not provided, uses $OUTPUT_DIR from env.sh. +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [[ -n "${1:-}" ]]; then + CHECK_DIR="$1" +elif [[ -f "${SCRIPT_DIR}/env.sh" ]]; then + source "${SCRIPT_DIR}/env.sh" + CHECK_DIR="${OUTPUT_DIR}" +else + echo "Usage: $0 [OUTPUT_DIR]" + exit 1 +fi + +if [[ ! -d "$CHECK_DIR" ]]; then + echo "Error: directory not found: $CHECK_DIR" + exit 1 +fi + +echo "=== Checking test results in: $CHECK_DIR ===" +echo "" + +FAILURES=0 + +# Check for error directories +ERR_DIRS=$(find "$CHECK_DIR" -type d -name "*-err" 2>/dev/null) +ERR_COUNT=$(echo "$ERR_DIRS" | grep -c . 2>/dev/null || echo 0) +if [[ -n "$ERR_DIRS" && "$ERR_COUNT" -gt 0 ]]; then + echo "FAIL: Found $ERR_COUNT error directories:" + echo "$ERR_DIRS" | sed 's/^/ /' + echo "" + FAILURES=$((FAILURES + ERR_COUNT)) +fi + +# Check correctness in output files +OUTPUTS=$(find "$CHECK_DIR" -name "output.txt" -o -name "slurmout_*.txt" 2>/dev/null) +if [[ -z "$OUTPUTS" ]]; then + echo "WARNING: No output files found in $CHECK_DIR" + exit 0 +fi + +echo "--- Correctness ---" +while IFS= read -r f; do + if grep -q "Out of bounds values : 0 OK" "$f"; then + echo " PASS: $(basename "$(dirname "$f")")/$(basename "$f")" + else + echo " FAIL: $(basename "$(dirname "$f")")/$(basename "$f") - correctness check missing or failed" + FAILURES=$((FAILURES + 1)) + fi +done <<< "$OUTPUTS" +echo "" + +# Check for crashes +echo "--- Crashes ---" +CRASH_FILES=$(grep -rl -i "segmentation\|segfault\|SIGABRT\|SIGSEGV" "$CHECK_DIR" 2>/dev/null || true) +if [[ -n "$CRASH_FILES" ]]; then + CRASH_COUNT=$(echo "$CRASH_FILES" | wc -l) + echo " FAIL: Found crashes in $CRASH_COUNT file(s):" + echo "$CRASH_FILES" | sed 's/^/ /' + FAILURES=$((FAILURES + CRASH_COUNT)) +else + echo " PASS: No crashes detected" +fi +echo "" + +# Check for plugin warnings +echo "--- Plugin Warnings ---" +WARN_FILES=$(grep -rl "NCCL_OFI_WARN" "$CHECK_DIR" 2>/dev/null || true) +if [[ -n "$WARN_FILES" ]]; then + WARN_COUNT=$(echo "$WARN_FILES" | wc -l) + echo " WARNING: NCCL_OFI_WARN found in $WARN_COUNT file(s):" + echo "$WARN_FILES" | sed 's/^/ /' + # Warnings are informational, not failures +else + echo " PASS: No NCCL_OFI_WARN lines" +fi +echo "" + +# Summary +echo "=== Summary ===" +if [[ "$FAILURES" -eq 0 ]]; then + echo " ALL TESTS PASSED" + exit 0 +else + echo " $FAILURES FAILURE(S) DETECTED" + exit 1 +fi diff --git a/tests/scripts/env.sh.example b/tests/scripts/env.sh.example new file mode 100644 index 0000000000..5de999c18e --- /dev/null +++ b/tests/scripts/env.sh.example @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Environment configuration for test scripts. +# Copy this file to env.sh and edit the paths for your setup. +# +# Usage: source env.sh (done automatically by the runners) +# + +# Root of the workspace (parent of plugin install, nccl, nccl-tests, etc.) +STACKDIR=${STACKDIR:-/path/to/workspace} + +# Plugin install directory (contains lib/libnccl-net-ofi.so) +PLUGIN_INSTALL=${PLUGIN_INSTALL:-${STACKDIR}/install} + +# NCCL library directory +NCCL_DIR=${NCCL_DIR:-${STACKDIR}/nccl/build/lib} + +# nccl-tests build directory (contains all_reduce_perf, etc.) +NCCL_TESTS_DIR=${NCCL_TESTS_DIR:-${STACKDIR}/nccl-tests/build} + +# MPI installation +MPI_DIR=${MPI_DIR:-/opt/amazon/openmpi} +MPIRUN=${MPIRUN:-${MPI_DIR}/bin/mpirun} + +# Output directory for test results (should be outside the repo) +OUTPUT_DIR=${OUTPUT_DIR:-${STACKDIR}/test-outputs} + +# ASAN-instrumented plugin install (only needed for MODE=asan) +ASAN_PLUGIN_DIR=${ASAN_PLUGIN_DIR:-${STACKDIR}/install-asan/lib} + +# Path to libasan.so (auto-detected if not set) +LIBASAN_PATH=${LIBASAN_PATH:-$(find /usr/lib/gcc -name "libasan.so" 2>/dev/null | head -1)} + +# Build LD_LIBRARY_PATH +unset LD_LIBRARY_PATH +unset LD_PRELOAD +LD_LIBRARY_PATH=${NCCL_DIR} +LD_LIBRARY_PATH=${PLUGIN_INSTALL}/lib:${LD_LIBRARY_PATH} +LD_LIBRARY_PATH=${MPI_DIR}/lib:${LD_LIBRARY_PATH} +LD_LIBRARY_PATH=/opt/amazon/efa/lib:${LD_LIBRARY_PATH} +LD_LIBRARY_PATH=/opt/pmix/lib:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH diff --git a/tests/scripts/run-functional-tests.sh b/tests/scripts/run-functional-tests.sh new file mode 100755 index 0000000000..883871a85d --- /dev/null +++ b/tests/scripts/run-functional-tests.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# +# Unified functional test runner for aws-ofi-nccl. +# +# Usage: +# ./run-functional-tests.sh [OPTIONS] +# +# Options (via environment variables): +# MODE regular|asan|valgrind (default: regular) +# PROTOCOL RDMA|SENDRECV (default: RDMA) +# TEST Test binary name (default: nccl_message_transfer) +# CLI_ARGS Extra args to pass to the test binary +# NRANKS Number of MPI ranks (default: 2) +# +# Examples: +# MODE=regular PROTOCOL=RDMA TEST=nccl_message_transfer ./run-functional-tests.sh +# MODE=asan PROTOCOL=SENDRECV TEST=inflight_close ./run-functional-tests.sh +# MODE=valgrind TEST=ring ./run-functional-tests.sh +# +# Prerequisites: +# - Copy env.sh.example to env.sh and edit paths for your setup +# - For ASAN: build plugin with --enable-asan, install to ASAN_PLUGIN_DIR +# - For Valgrind: use a normal (non-ASAN) debug build +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source environment +if [[ ! -f "${SCRIPT_DIR}/env.sh" ]]; then + echo "Error: ${SCRIPT_DIR}/env.sh not found." + echo "Copy env.sh.example to env.sh and edit paths for your setup." + exit 1 +fi +source "${SCRIPT_DIR}/env.sh" + +# Defaults +MODE=${MODE:-regular} +PROTOCOL=${PROTOCOL:-RDMA} +TEST=${TEST:-nccl_message_transfer} +CLI_ARGS=${CLI_ARGS:-} +NRANKS=${NRANKS:-2} + +# Validate mode +if [[ "$MODE" != "regular" && "$MODE" != "asan" && "$MODE" != "valgrind" ]]; then + echo "Error: MODE must be regular, asan, or valgrind (got: $MODE)" + exit 1 +fi + +# Locate binary +BINARY="${PLUGIN_INSTALL}/bin/${TEST}" +if [[ "$MODE" == "asan" ]]; then + ASAN_INSTALL=${ASAN_PLUGIN_DIR%/lib} + BINARY="${ASAN_INSTALL}/bin/${TEST}" +fi + +if [[ ! -x "$BINARY" ]]; then + echo "Error: binary not found or not executable: $BINARY" + exit 1 +fi + +echo "=== Functional Test: ${TEST} ===" +echo "Mode: $MODE" +echo "Protocol: $PROTOCOL" +echo "Binary: $BINARY" +echo "Args: ${CLI_ARGS:-}" +echo "Ranks: $NRANKS" +echo "==========================" + +# Common mpirun args +MPI_ARGS=( + -n "$NRANKS" + --bind-to none + -x LD_LIBRARY_PATH + -x NCCL_DEBUG=${NCCL_DEBUG:-WARN} + -x "OFI_NCCL_PROTOCOL=${PROTOCOL}" + --mca pml ^cm + --mca btl tcp,self + --mca btl_tcp_if_exclude lo,docker0 +) + +case "$MODE" in + regular) + "${MPIRUN}" "${MPI_ARGS[@]}" "$BINARY" ${CLI_ARGS} + ;; + + asan) + if [[ -z "${LIBASAN_PATH:-}" || ! -f "${LIBASAN_PATH:-}" ]]; then + echo "Error: libasan.so not found. Set LIBASAN_PATH in env.sh." + exit 1 + fi + if [[ ! -d "$ASAN_PLUGIN_DIR" ]]; then + echo "Error: ASAN plugin dir not found: $ASAN_PLUGIN_DIR" + exit 1 + fi + export LD_LIBRARY_PATH="${ASAN_PLUGIN_DIR}:${LD_LIBRARY_PATH}" + "${MPIRUN}" "${MPI_ARGS[@]}" \ + -x "LD_PRELOAD=${LIBASAN_PATH}" \ + -x "ASAN_OPTIONS=protect_shadow_gap=0:detect_leaks=1:halt_on_error=0" \ + -x "LSAN_OPTIONS=suppressions=${SCRIPT_DIR}/sanitizers/lsan.supp" \ + "$BINARY" ${CLI_ARGS} + ;; + + valgrind) + SUPP_FILE="${SCRIPT_DIR}/sanitizers/valgrind.supp" + OUTDIR="${OUTPUT_DIR}/valgrind-${TEST}-${PROTOCOL}-$(date +%Y%m%d_%H%M%S)" + mkdir -p "$OUTDIR" + export OUTDIR + + echo "Per-rank output: $OUTDIR/" + + rc=0 + "${MPIRUN}" "${MPI_ARGS[@]}" \ + -x OUTDIR \ + "${SCRIPT_DIR}/valgrind/rank_wrapper.sh" \ + valgrind --tool=memcheck \ + --leak-check=full \ + --track-origins=yes \ + --fair-sched=yes \ + --suppressions="$SUPP_FILE" \ + "$BINARY" ${CLI_ARGS} || rc=$? + + echo "" + echo "=== Per-rank output files ===" + ls -lh "$OUTDIR"/rank_*.txt 2>/dev/null || echo " (no rank files found)" + + for f in "$OUTDIR"/rank_*.txt; do + [[ -f "$f" ]] || continue + echo "" + echo "=== Parsing $(basename "$f") ===" + "${SCRIPT_DIR}/valgrind/parse_valgrind.sh" "$f" || true + done + + exit "$rc" + ;; +esac diff --git a/tests/scripts/run-nccl-tests.sh b/tests/scripts/run-nccl-tests.sh new file mode 100755 index 0000000000..11ecf849c3 --- /dev/null +++ b/tests/scripts/run-nccl-tests.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# +# Run NCCL collective tests (nccl-tests) with the plugin loaded. +# +# Usage: +# ./run-nccl-tests.sh [OPTIONS] +# +# Options (via environment variables): +# BENCHMARK Test binary name (default: all_reduce_perf) +# NCCL_TESTS_SPLIT_MASK Split mask (default: 0x0) +# NUM_NODES Number of nodes (default: 1) +# RANKS_PER_NODE Ranks per node (default: 8) +# NUM_GPUS_PER_RANK GPUs per rank (default: 1) +# EXTRA_TEST_ARGS Extra args for nccl-tests (default: "-n 15 -w 10 -b 1K -e 16G -f 2 -c 1 -R 0") +# HOSTFILE Path to hostfile (optional, for multi-node) +# USE_TOPO_SORT Run hostfile-topologify.py (default: 0) +# +# Examples: +# BENCHMARK=all_reduce_perf NUM_NODES=2 ./run-nccl-tests.sh +# BENCHMARK=broadcast_perf NCCL_TESTS_SPLIT_MASK=0x7 ./run-nccl-tests.sh +# NUM_NODES=1 RANKS_PER_NODE=8 NCCL_TESTS_SPLIT_MASK=0x7 ./run-nccl-tests.sh +# +# Prerequisites: +# - Copy env.sh.example to env.sh and edit paths +# - nccl-tests built and NCCL_TESTS_DIR set in env.sh +# - For multi-node: provide HOSTFILE or run inside a Slurm allocation +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source environment +if [[ ! -f "${SCRIPT_DIR}/env.sh" ]]; then + echo "Error: ${SCRIPT_DIR}/env.sh not found." + echo "Copy env.sh.example to env.sh and edit paths for your setup." + exit 1 +fi +source "${SCRIPT_DIR}/env.sh" + +# Defaults +BENCHMARK=${BENCHMARK:-all_reduce_perf} +NCCL_TESTS_SPLIT_MASK=${NCCL_TESTS_SPLIT_MASK:-0x0} +NUM_NODES=${NUM_NODES:-1} +RANKS_PER_NODE=${RANKS_PER_NODE:-8} +NUM_GPUS_PER_RANK=${NUM_GPUS_PER_RANK:-1} +EXTRA_TEST_ARGS=${EXTRA_TEST_ARGS:-"-n 15 -w 10 -b 1K -e 16G -f 2 -c 1 -R 0"} +USE_TOPO_SORT=${USE_TOPO_SORT:-0} + +TOTAL_RANKS=$((NUM_NODES * RANKS_PER_NODE)) + +# Validate benchmark binary +BENCH_BIN="${NCCL_TESTS_DIR}/${BENCHMARK}" +if [[ ! -x "$BENCH_BIN" ]]; then + echo "Error: benchmark binary not found: $BENCH_BIN" + echo "Set NCCL_TESTS_DIR in env.sh to point to your nccl-tests build directory." + exit 1 +fi + +# Output directory +OUTDIR="${OUTPUT_DIR}/nccl-tests-${BENCHMARK}-${NUM_NODES}x${RANKS_PER_NODE}-mask${NCCL_TESTS_SPLIT_MASK}-$(date +%Y%m%d_%H%M%S)" +mkdir -p "$OUTDIR" + +echo "=== nccl-tests: ${BENCHMARK} ===" +echo "Nodes: $NUM_NODES" +echo "Ranks/node: $RANKS_PER_NODE" +echo "Total: $TOTAL_RANKS" +echo "Mask: $NCCL_TESTS_SPLIT_MASK" +echo "GPUs/rank: $NUM_GPUS_PER_RANK" +echo "Args: $EXTRA_TEST_ARGS" +echo "Output: $OUTDIR" +echo "==========================" + +# Build hostfile args +HOSTFILE_ARGS=() +if [[ -n "${HOSTFILE:-}" ]]; then + if [[ "$USE_TOPO_SORT" == "1" ]]; then + TOPO_SCRIPT="${SCRIPT_DIR}/../../contrib/scripts/topology_aware/hostfile-topologify.py" + if [[ ! -x "$TOPO_SCRIPT" ]]; then + echo "Error: hostfile-topologify.py not found at $TOPO_SCRIPT" + exit 1 + fi + TOPO_HOSTFILE=$(mktemp) + TOPO_RMAP=$(mktemp) + "$TOPO_SCRIPT" --input "$HOSTFILE" --output "$TOPO_HOSTFILE" + for host in $(cat "$TOPO_HOSTFILE"); do + seq 1 "$RANKS_PER_NODE" | xargs -I{} echo "$host" + done > "$TOPO_RMAP" + HOSTFILE_ARGS=(--hostfile "$TOPO_RMAP" --mca rmaps seq) + else + HOSTFILE_ARGS=(--hostfile "$HOSTFILE") + fi +fi + +# NCCL environment +export NCCL_NET="AWS Libfabric" +export NCCL_TESTS_SPLIT_MASK +export NCCL_BUFFSIZE=${NCCL_BUFFSIZE:-8388608} +export NCCL_P2P_NET_CHUNKSIZE=${NCCL_P2P_NET_CHUNKSIZE:-524288} + +# Run +LOGFILE="${OUTDIR}/output.txt" + +function cleanup() { + if [[ $? -ne 0 ]]; then + mv "$OUTDIR" "${OUTDIR}-err" 2>/dev/null || true + echo "FAILED: output moved to ${OUTDIR}-err/" + fi +} +trap cleanup EXIT + +echo "STARTTIME: $(date)" | tee "$LOGFILE" + +"${MPIRUN}" \ + -n "$TOTAL_RANKS" \ + -N "$RANKS_PER_NODE" \ + "${HOSTFILE_ARGS[@]}" \ + -x LD_LIBRARY_PATH \ + -x NCCL_NET \ + -x NCCL_TESTS_SPLIT_MASK \ + -x NCCL_BUFFSIZE \ + -x NCCL_P2P_NET_CHUNKSIZE \ + -x NCCL_DEBUG=${NCCL_DEBUG:-INFO} \ + -x NCCL_DEBUG_SUBSYS=${NCCL_DEBUG_SUBSYS:-INIT,NET} \ + --mca pml ^cm \ + --mca btl tcp,self \ + --mca btl_tcp_if_exclude lo,docker0 \ + --bind-to none \ + "$BENCH_BIN" -g "$NUM_GPUS_PER_RANK" $EXTRA_TEST_ARGS \ + 2>&1 | tee -a "$LOGFILE" + +echo "ENDTIME: $(date)" | tee -a "$LOGFILE" + +# Quick validation +echo "" +echo "=== Validation ===" +if grep -q "Out of bounds values : 0 OK" "$LOGFILE"; then + echo "PASS: Out of bounds values : 0 OK" +else + echo "FAIL: correctness check not found or failed" + exit 1 +fi + +BW=$(grep "Avg bus bandwidth" "$LOGFILE" | awk '{print $NF}') +echo "Avg bus bandwidth: ${BW:-N/A}" + +# Reset trap on success +trap - EXIT +echo "Output: $OUTDIR/output.txt" diff --git a/tests/scripts/run-slurm.sh.example b/tests/scripts/run-slurm.sh.example new file mode 100755 index 0000000000..b0f9af3cba --- /dev/null +++ b/tests/scripts/run-slurm.sh.example @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Slurm batch template for running tests. +# Copy this file, adjust the SBATCH directives and SCRIPT_DIR below, +# then submit with: sbatch run-slurm.sh +# +# Control what runs via RUN_TYPE (default: functional): +# RUN_TYPE=functional sbatch run-slurm.sh +# RUN_TYPE=nccl-tests sbatch run-slurm.sh +# +#SBATCH --time=00:10:00 +#SBATCH --exclusive +#SBATCH --output=slurm-%j.txt +# +# Adjust these for your cluster: +#SBATCH --partition=CHANGEME +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=8 + +set -euo pipefail + +# Set this to the absolute path of tests/scripts/ in your checkout +SCRIPT_DIR=${SCRIPT_DIR:-/path/to/aws-ofi-nccl/tests/scripts} + +RUN_TYPE=${RUN_TYPE:-functional} + +case "$RUN_TYPE" in + functional) + export MODE=${MODE:-regular} + export PROTOCOL=${PROTOCOL:-RDMA} + export TEST=${TEST:-nccl_message_transfer} + export NRANKS=${NRANKS:-2} + exec "${SCRIPT_DIR}/run-functional-tests.sh" + ;; + nccl-tests) + export BENCHMARK=${BENCHMARK:-all_reduce_perf} + export NCCL_TESTS_SPLIT_MASK=${NCCL_TESTS_SPLIT_MASK:-0x0} + export NUM_NODES=${SLURM_JOB_NUM_NODES} + export RANKS_PER_NODE=${SLURM_NTASKS_PER_NODE} + HOSTFILE=$(mktemp) + scontrol show hostnames > "$HOSTFILE" + export HOSTFILE + exec "${SCRIPT_DIR}/run-nccl-tests.sh" + ;; + *) + echo "Error: RUN_TYPE must be 'functional' or 'nccl-tests' (got: $RUN_TYPE)" + exit 1 + ;; +esac diff --git a/tests/scripts/sanitizers/lsan.supp b/tests/scripts/sanitizers/lsan.supp new file mode 100644 index 0000000000..6dd1b2ed9b --- /dev/null +++ b/tests/scripts/sanitizers/lsan.supp @@ -0,0 +1,30 @@ +# LSAN (LeakSanitizer) suppression file for aws-ofi-nccl. +# Suppresses known leaks from third-party libraries (MPI, libfabric, hwloc). +# Plugin leaks (nccl_ofi_*) are intentionally NOT suppressed. + +# libfabric internals +leak:ofi_nic_dup +leak:fi_dupinfo +leak:fi_getinfo +leak:fi_allocinfo +leak:efa_post_send_validate + +# OpenMPI runtime +leak:orte_ess_base_proc_binding +leak:opal_argv_append_nosize +leak:ompi_op_base_op_select +leak:ompi_mpi_init +leak:mca_btl_ofi +leak:mca_mpool_hugepage +leak:libopen-pal + +# hwloc (MPI dependency) +leak:hwloc_bitmap_alloc +leak:hwloc_topology_init + +# glibc internals triggered by MPI +leak:__vasprintf_internal +leak:___asprintf_chk + +# Unsymbolized third-party allocations (EFA provider, nvidia driver) +leak: diff --git a/tests/scripts/sanitizers/valgrind.supp b/tests/scripts/sanitizers/valgrind.supp new file mode 100644 index 0000000000..1345f7ed0c --- /dev/null +++ b/tests/scripts/sanitizers/valgrind.supp @@ -0,0 +1,181 @@ +# Valgrind suppression file for aws-ofi-nccl. +# Suppresses known leaks/errors from third-party libraries. +# Plugin leaks (nccl_ofi_*) are intentionally NOT suppressed. + +# --- CUDA driver and runtime --- +{ + cuda_cuInit + Memcheck:Leak + ... + fun:cuInit +} +{ + cuda_cudaGetDriverEntryPoint + Memcheck:Leak + ... + fun:cudaGetDriverEntryPointByVersion +} +{ + cuda_driver_lib + Memcheck:Leak + ... + obj:*libcuda.so* +} +{ + cuda_driver_cond + Memcheck:Cond + ... + obj:*libcuda.so* +} + +# --- libfabric and EFA provider --- +{ + libfabric_fi_getinfo + Memcheck:Leak + ... + fun:fi_getinfo* +} +{ + libfabric_fi_ini + Memcheck:Leak + ... + fun:fi_ini +} +{ + efa_provider_init + Memcheck:Leak + ... + fun:fi_efa_ini +} +{ + efa_device_list + Memcheck:Leak + ... + fun:efa_device_list_initialize +} +{ + ibverbs_get_device_list + Memcheck:Leak + ... + fun:ibv_get_device_list* +} +{ + libfabric_cond + Memcheck:Cond + ... + obj:*libfabric* +} +{ + efa_cond + Memcheck:Cond + ... + obj:*libefa* +} + +# --- OpenMPI runtime --- +{ + openmpi_orte_finalize + Memcheck:Leak + ... + fun:orte_finalize +} +{ + openmpi_mpi_finalize + Memcheck:Leak + ... + fun:ompi_mpi_finalize +} +{ + openmpi_mpi_init + Memcheck:Leak + ... + fun:ompi_mpi_init +} +{ + openmpi_libopen_pal + Memcheck:Leak + ... + obj:*libopen-pal* +} +{ + openmpi_libopen_rte + Memcheck:Leak + ... + obj:*libopen-rte* +} +{ + openmpi_libmpi + Memcheck:Leak + ... + obj:*libmpi* +} +{ + openmpi_cond + Memcheck:Cond + ... + obj:*libopen-pal* +} + +# --- hwloc (MPI dependency) --- +{ + hwloc_topology_init + Memcheck:Leak + ... + fun:hwloc_topology_init +} +{ + hwloc_bitmap_alloc + Memcheck:Leak + ... + fun:hwloc_bitmap_alloc +} + +# --- pthread internals --- +{ + pthread_once + Memcheck:Leak + ... + fun:__pthread_once_slow +} + +# --- GDR API (gdrcopy) --- +{ + gdr_open_ioctl + Memcheck:Param + ioctl(generic) + ... + fun:gdr_open +} +{ + gdr_map_cond + Memcheck:Cond + ... + fun:gdr_map_v2 +} +{ + gdr_map_value + Memcheck:Value8 + ... + fun:gdr_map_v2 +} +{ + gdr_unmap_cond + Memcheck:Cond + ... + fun:gdr_unmap +} +{ + gdr_unmap_value + Memcheck:Value8 + ... + fun:gdr_unmap +} + +# --- OpenMPI TCP BTL uninitialized send --- +{ + openmpi_btl_tcp_send + Memcheck:Param + socketcall.sendto(msg) + ... + fun:mca_btl_tcp_send_blocking +} diff --git a/tests/scripts/valgrind/parse_valgrind.sh b/tests/scripts/valgrind/parse_valgrind.sh new file mode 100755 index 0000000000..57d4f46aad --- /dev/null +++ b/tests/scripts/valgrind/parse_valgrind.sh @@ -0,0 +1,204 @@ +#!/bin/bash +# Parse Valgrind memcheck reports from aws-ofi-nccl test output. +# Extracts errors and memory leaks, classifies plugin vs non-plugin, +# deduplicates across PIDs/ranks, and produces a concise summary. +# +# Handles: Invalid read/write, Conditional jump, Use of uninitialised value, +# Syscall param, memory leaks (definitely/indirectly/possibly lost). +# +# Usage: ./parse_valgrind.sh + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + exit 1 +fi + +INPUT="$1" +[[ -f "$INPUT" ]] || { echo "Error: $INPUT not found"; exit 1; } + +PARSE_TMPDIR=$(mktemp -d) +trap 'rm -rf "$PARSE_TMPDIR"' EXIT + +if ! command -v gawk &>/dev/null; then + echo "Error: gawk is required but not found" >&2 + exit 1 +fi + +plugin_blocks="$PARSE_TMPDIR/plugin_errors.txt" +other_blocks="$PARSE_TMPDIR/other_errors.txt" +dedup_plugin="$PARSE_TMPDIR/dedup_plugin.txt" +plugin_leaks="$PARSE_TMPDIR/plugin_leaks.txt" +other_leaks="$PARSE_TMPDIR/other_leaks.txt" + +# Single-pass awk: extract blocks, classify, and deduplicate all at once. +gawk ' +BEGIN { + plugin_re = "(nccl_ofi_[a-z_]+\\.(cpp|h)|platform-aws\\.cpp):[0-9]+" + fullpath_re = "aws-ofi-nccl/(src|include)/" +} + +function flush_block() { + if (buf == "") return + + total_blocks++ + + is_plugin = 0 + if (buf ~ plugin_re || buf ~ fullpath_re) is_plugin = 1 + + is_leak = (err_type ~ /definitely lost|indirectly lost|possibly lost/) + + if (is_plugin) { + plugin_count++ + print buf > "'"$plugin_blocks"'" + + if (is_leak) { + print buf > "'"$plugin_leaks"'" + plugin_leak_count++ + if (match(header_line, /== ([0-9,]+) (\([0-9,]+ direct, [0-9,]+ indirect\) )?bytes in ([0-9,]+) blocks are (definitely|indirectly|possibly) lost/, parts)) { + gsub(/,/, "", parts[1]) + gsub(/,/, "", parts[3]) + leak_bytes = parts[1] + 0 + leak_blocks_n = parts[3] + 0 + leak_kind = parts[4] + plugin_leak_bytes[leak_kind] += leak_bytes + plugin_leak_blocks[leak_kind] += leak_blocks_n + } + } else { + plugin_error_count++ + } + + first_src = "" + n_lines = split(buf, lines, "\n") + for (i = 1; i <= n_lines; i++) { + if (first_src == "" && match(lines[i], "\\(" plugin_re "\\)")) { + first_src = substr(lines[i], RSTART, RLENGTH) + } + if (first_src == "" && lines[i] ~ fullpath_re) { + match(lines[i], /aws-ofi-nccl\/(src|include)\/[^ )]+/) + if (RSTART > 0) first_src = substr(lines[i], RSTART, RLENGTH) + } + } + dk = err_type "|" first_src + if (!(dk in seen)) { + seen[dk] = 1 + dedup_order[++dedup_n] = dk + dedup_headers[dk] = header_line + dedup_bodies[dk] = buf + dedup_counts[dk] = 1 + dedup_is_leak[dk] = is_leak + } else { + dedup_counts[dk]++ + } + } else { + other_count++ + print buf > "'"$other_blocks"'" + if (is_leak) { + print buf > "'"$other_leaks"'" + other_leak_count++ + } + } + buf = "" + header_line = "" + err_type = "" +} + +/^==[0-9]+== (Syscall|Conditional|Use of|Invalid|Uninitialised)/ { + flush_block() + buf = $0 "\n" + header_line = $0 + match($0, /== (Syscall|Conditional|Use of uninitialised|Invalid read|Invalid write)/, m) + err_type = (m[1] != "") ? m[1] : "unknown" + in_block = 1 + next +} + +/^==[0-9]+== [0-9,]+ (\([0-9,]+ direct, [0-9,]+ indirect\) )?bytes in [0-9,]+ blocks are (definitely|indirectly|possibly) lost/ { + flush_block() + buf = $0 "\n" + header_line = $0 + match($0, /(definitely|indirectly|possibly) lost/, m) + err_type = m[0] + in_block = 1 + next +} + +in_block && /^==[0-9]+== $/ { buf = buf $0 "\n"; flush_block(); in_block = 0; next } +in_block && !/^==[0-9]+=/ { flush_block(); in_block = 0; next } +in_block { buf = buf $0 "\n" } +END { + flush_block() + + dedup_error_n = 0 + dedup_leak_n = 0 + for (i = 1; i <= dedup_n; i++) { + k = dedup_order[i] + if (!dedup_is_leak[k]) { + dedup_error_n++ + printf "[%d occurrences] %s\n%s\n", dedup_counts[k], dedup_headers[k], dedup_bodies[k] > "'"$dedup_plugin"'.errors" + } else { + dedup_leak_n++ + printf "[%d occurrences] %s\n%s\n", dedup_counts[k], dedup_headers[k], dedup_bodies[k] > "'"$dedup_plugin"'.leaks" + } + } + + printf "%d %d %d %d %d %d %d %d\n", \ + total_blocks, plugin_count, other_count, dedup_n, \ + plugin_error_count, plugin_leak_count, dedup_error_n, dedup_leak_n \ + > "'"$PARSE_TMPDIR"'/stats.txt" + + for (kind in plugin_leak_bytes) { + printf "%s %d %d\n", kind, plugin_leak_bytes[kind], plugin_leak_blocks[kind] \ + > "'"$PARSE_TMPDIR"'/leak_bytes.txt" + } +} +' "$INPUT" + +# Read stats +read raw_count plugin_count other_count unique_count \ + plugin_error_count plugin_leak_count dedup_error_n dedup_leak_n \ + < "$PARSE_TMPDIR/stats.txt" 2>/dev/null || { + echo "No valgrind errors or leaks found in $INPUT" + exit 0 +} +parsed_count=$((plugin_count + other_count)) +num_pids=$(grep -oP '==\K[0-9]+(?===)' "$INPUT" | sort -u | wc -l || true) + +# Output report +echo "============================================================" +echo " Valgrind Memcheck Analysis: NCCL OFI Plugin" +echo " Input: $INPUT" +echo " PIDs (ranks): $num_pids" +echo "============================================================" +echo "" +echo "--- PLUGIN ERRORS (non-leak) ---" +echo " Total: $plugin_error_count Unique: $dedup_error_n" +echo "" +echo "--- PLUGIN LEAKS ---" +echo " Total: $plugin_leak_count Unique: $dedup_leak_n" +if [[ -f "$PARSE_TMPDIR/leak_bytes.txt" ]]; then + echo " By kind:" + while read kind bytes blocks; do + printf " %-20s %'d bytes in %'d blocks\n" "$kind:" "$bytes" "$blocks" + done < "$PARSE_TMPDIR/leak_bytes.txt" | sort +fi +echo "" +echo "--- NON-PLUGIN (third-party) ---" +echo " Count: $other_count" +echo "" +echo "--- DEDUPLICATED PLUGIN ERRORS ---" +if [[ -f "$dedup_plugin.errors" ]]; then + cat "$dedup_plugin.errors" +else + echo " None." +fi +echo "" +echo "--- DEDUPLICATED PLUGIN LEAKS ---" +if [[ -f "$dedup_plugin.leaks" ]]; then + cat "$dedup_plugin.leaks" +else + echo " None." +fi +echo "" +echo "============================================================" diff --git a/tests/scripts/valgrind/rank_wrapper.sh b/tests/scripts/valgrind/rank_wrapper.sh new file mode 100755 index 0000000000..07387b089d --- /dev/null +++ b/tests/scripts/valgrind/rank_wrapper.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Per-rank wrapper: redirects each rank's stdout/stderr to a separate file. +# OUTDIR must be set by the caller. OMPI_COMM_WORLD_RANK is set by OpenMPI. +RANK="${OMPI_COMM_WORLD_RANK:-unknown}" +exec "$@" > "${OUTDIR}/rank_${RANK}.txt" 2>&1