diff --git a/bin/aomp_common_vars b/bin/aomp_common_vars index 4d1e5a5e00..b337c7f127 100644 --- a/bin/aomp_common_vars +++ b/bin/aomp_common_vars @@ -9,6 +9,10 @@ # # The bash syntax ENVNAME=${ENVNAME:-value} checks if ENVNAME is # already set. If not, it assigns it the value following the :- +# +# Most variables defined here are consumed by the sourcing scripts, +# so their uses are invisible to static analysis. +# shellcheck disable=SC2034 # --- Start standard header to set AOMP environment variables ---- realpath=$(realpath "$0") @@ -262,9 +266,11 @@ function cmquot() { # copy & pasted into the user's terminal. Output should not be further quoted. function shquot() { local -a output + local arg Escaped output=() for arg in "$@"; do - output+=(\'${arg//\'/\'\\\'\'}\') + Escaped=${arg//\'/\'\\\'\'} + output+=("'${Escaped}'") done printf '%s' "${output[*]}" } @@ -568,11 +574,11 @@ declare -a ASAN_FLAGS if [ "$AOMP_BUILD_SANITIZER" == 1 ]; then if [ -f "${LLVM_INSTALL_LOC}/bin/clang" ]; then - ASAN_LIB=$(${LLVM_INSTALL_LOC}/bin/clang --print-file-name=libclang_rt.asan-x86_64.so) + ASAN_LIB=$("${LLVM_INSTALL_LOC}"/bin/clang --print-file-name=libclang_rt.asan-x86_64.so) ASAN_LIB_PATH=$(dirname "$ASAN_LIB") if [ ! -d "$ASAN_LIB_PATH" ]; then #Suppose above search path fails then via 'LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON' the host asan library would lose the suffix - ASAN_LIB=$(${LLVM_INSTALL_LOC}/bin/clang --print-file-name=libclang_rt.asan.so) + ASAN_LIB=$("${LLVM_INSTALL_LOC}"/bin/clang --print-file-name=libclang_rt.asan.so) ASAN_LIB_PATH=$(dirname "$ASAN_LIB") fi else @@ -860,3 +866,146 @@ function help_build_aomp(){ EOF exit 0 } + +# Merge the contents of a variable with a list of input items. +# Usage: merge_variable_with_inputs --delimiter \ +# [--append|--prepend] [ ...] +# Default delimiter: ":" +# Default mode: --prepend (new inputs take priority / appear first). +# Empty/unset inputs are skipped; no dangling delimiters are produced. +# +# Example: +# merge_variable_with_inputs PATH "${AOMP:+${AOMP}/bin}" +# merge_variable_with_inputs CMAKE_MODULE_PATH --delimiter ";" \ +# --append "${AOMP:+${AOMP}/lib/cmake}" +function merge_variable_with_inputs() { + local VarName="$1"; shift + local Delimiter=":" + local Mode="prepend" + + while [ "$#" -gt 0 ]; do + case "$1" in + --delimiter) Delimiter="$2"; shift 2 ;; + --append) Mode="append"; shift ;; + --prepend) Mode="prepend"; shift ;; + --) shift; break ;; + *) break ;; + esac + done + + local -a InputItems=("$@") + local -n __MergeTargetVar="${VarName}" + + local Joined="" + local Item + for Item in "${InputItems[@]}"; do + [ -n "${Item}" ] && \ + Joined="${Joined:+${Joined}${Delimiter}}${Item}" + done + + # join2(a, b): a + Delimiter + b, but only if BOTH are non-empty; + # otherwise whichever side is non-empty (or "" if both are). + if [ "${Mode}" == "append" ]; then + __MergeTargetVar="${__MergeTargetVar}${__MergeTargetVar:+${Joined:+${Delimiter}}}${Joined}" + else + __MergeTargetVar="${Joined}${Joined:+${__MergeTargetVar:+${Delimiter}}}${__MergeTargetVar}" + fi +} + +# Return the path containing the requested CMake module directory. +# Usage: get_cmake_module_path [ ...] +# Searches each given Path, in order, for /lib/cmake/. +# Empty path arguments are skipped (safe to pass an unset/empty variable). +# If no Path arguments are given, defaults to searching AOMP, then ROCM_PATH +# (or a path derived from AOMP, if set), then /opt/rocm. +function get_cmake_module_path() { + local Module="$1" + shift + local -a SearchPaths=("$@") + + # No explicit candidates given: fall back to the default search order. + # Default: AOMP -> ROCM_PATH (if empty: AOMP/../..) -> /opt/rocm + if [ "$#" -eq 0 ]; then + SearchPaths=( + "${AOMP}" + "${ROCM_PATH:-${AOMP:+$(realpath -m "${AOMP}/../../")}}" + "/opt/rocm" + ) + fi + + # Assumptions: + # All CMake modules are located within ./lib/cmake/. + # All CMake module configuration files are named *onfig*.cmake. + local Path + for Path in "${SearchPaths[@]}"; do + if [ -n "${Path}" ] && [ -d "${Path}/lib/cmake/${Module}" ] && + compgen -G "${Path}/lib/cmake/${Module}/${Module}"'*onfig*.cmake' \ + > /dev/null; then + # Return the parent(!) directory. + echo "${Path}/lib/cmake" + return 0 + fi + done + return 1 +} + +# Keep the HIP headers of the ROCm under test ahead of every other ROCm's. +# Usage: add_cmake_rocm_header_priority_args [ ...] +# Appends one CMake argument per language (default: HIP); does nothing when +# ships no HIP headers of its own. +# +# A borrowed hipBLAS/rocBLAS exports its whole include directory, which CMake +# emits as -isystem, ahead of the compiler's own ROCm that clang searches last. +# CMAKE__COMPILER_ARG1 is the only slot in front of those includes; every +# *_FLAGS variable lands behind them. Keep CMAKE__COMPILER a plain path, +# CMake reassigns ARG1 when the compiler itself carries arguments. +function add_cmake_rocm_header_priority_args() { + local -n __HeaderPriorityArgs="$1"; shift + local RocmRoot="$1"; shift + local -a Languages=("$@") + [ "${#Languages[@]}" -gt 0 ] || Languages=("HIP") + + [ -n "${RocmRoot}" ] && [ -d "${RocmRoot}/include/hip" ] || return 0 + + local Language + for Language in "${Languages[@]}"; do + __HeaderPriorityArgs+=( + "-DCMAKE_${Language}_COMPILER_ARG1:STRING=-isystem ${RocmRoot}/include") + done +} + +# Verify that the ROCm header priority survived the CMake run. +# Usage: check_cmake_rocm_header_priority [ ...] +# A lost entry only breaks the build while the two ROCm header sets are +# incompatible, otherwise it silently builds against the wrong ones. +function check_cmake_rocm_header_priority() { + local BuildDir="$1"; shift + local RocmRoot="$1"; shift + local -a Languages=("$@") + [ "${#Languages[@]}" -gt 0 ] || Languages=("HIP") + + [ -n "${RocmRoot}" ] && [ -d "${RocmRoot}/include/hip" ] || return 0 + + local CacheFile="${BuildDir}/CMakeCache.txt" + if [ ! -f "${CacheFile}" ]; then + echo "ERROR: ${CacheFile} not found" + return 1 + fi + + local Language Entry Msg Status=0 + for Language in "${Languages[@]}"; do + Entry=$(grep "^CMAKE_${Language}_COMPILER_ARG1:" "${CacheFile}" || true) + case "${Entry}" in + *"-isystem ${RocmRoot}/include"*) ;; + *) + Msg="ERROR: CMAKE_${Language}_COMPILER_ARG1 lost" + Msg+=" ${RocmRoot}/include, its HIP headers can be shadowed" + Msg+=" by another ROCm" + echo "${Msg}" + Status=1 + ;; + esac + done + + return "${Status}" +} diff --git a/bin/run_llama.sh b/bin/run_llama.sh index ddb2f6cd2e..66105052cb 100755 --- a/bin/run_llama.sh +++ b/bin/run_llama.sh @@ -8,9 +8,12 @@ # Build script for LLaMA with HIP support using AOMP compiler +ScriptDir=$(dirname "$(realpath "$0")") + # shellcheck source=/dev/null -. aomp_common_vars +. "${ScriptDir}"/aomp_common_vars +: "${ROCM_PATH:=$(realpath -m "${AOMP}/../..")}" : "${AOMP_GPU:=gfx90a}" : "${LLAMA_GPU:=$AOMP_GPU}" @@ -20,10 +23,25 @@ : "${LLAMA_BUILD_MODE:=Release}" : "${LLAMA_TESTS_LOG_LOCATION:=$LLAMA_TLDIR/logs}" +# CTest 'test-backend-ops' exceeds the 1500s default timeout. +# Measured on MI350X: ~2500s +: "${LLAMA_CTEST_TIMEOUT:=3600}" + # Model to use in benchmarks (default is a smaller model) : "${LLAMA_BENCH_HF_ID:=ggml-org/gemma-3-1b-it-GGUF}" : "${LLAMA_CACHE:=$HOME/.cache/llama.cpp}" +# Add AOMP and ROCM_PATH to PATH and LD_LIBRARY_PATH and export them. +merge_variable_with_inputs PATH "${AOMP:+${AOMP}/bin}" "${ROCM_PATH:+${ROCM_PATH}/bin}" +merge_variable_with_inputs LD_LIBRARY_PATH \ + "${AOMP:+${AOMP}/lib}" \ + "${AOMP:+${AOMP}/lib/x86_64-unknown-linux-gnu}" \ + "${ROCM_PATH:+${ROCM_PATH}/lib}" + +export PATH +export LD_LIBRARY_PATH +export ROCM_PATH + pushd "${AOMP_REPOS_TEST}" || exit mkdir -p "${LLAMA_TLDIR}" && cd "${LLAMA_TLDIR}" || exit @@ -92,24 +110,72 @@ else cd .. fi -echo "Configuring build with CMake..." if [ "${DoConfigure}" == "yes" ]; then + echo "Configuring build with CMake..." rm -rf "${LLAMA_BUILD_DIR}" - cmake -B build \ - -S src \ - -DCMAKE_PREFIX_PATH="${AOMP}"/lib/cmake \ - -DGGML_HIP=On \ - -DCMAKE_BUILD_TYPE="${LLAMA_BUILD_MODE}" \ - -DGPU_TARGETS="${LLAMA_GPU}" \ - ${CmakeGenerator:+"${CmakeGenerator}"} \ - -DCMAKE_C_COMPILER="${AOMP}"/bin/clang \ - -DCMAKE_CXX_COMPILER="${AOMP}"/bin/clang++ \ - -DCMAKE_HIP_COMPILER="${AOMP}"/bin/clang++ + + CMakeArgs=() + if [ -n "${CmakeGenerator}" ]; then + CMakeArgs+=("${CmakeGenerator}") + fi + + # Determine the CMake module paths of required ROCm packages. + CMakePrefixPath="${ROCM_PATH}" + declare -A SeenPrefix=() + for Package in hip hipblas rocblas; do + if ! PackageCmakeDir=$(get_cmake_module_path "${Package}"); then + Msg="ERROR: no CMake package '${Package}' below ${AOMP}," + Msg+=" ${ROCM_PATH} or /opt/rocm" + echo "${Msg}" + exit 1 + fi + + # Anything found below the ROCm under test needs no extra prefix. + # Removing a prefix that is present shortens the path, so a path that + # comes back unchanged did not start with it. + if [ "${PackageCmakeDir#"${AOMP}"/}" != "${PackageCmakeDir}" ] || + [ "${PackageCmakeDir#"${ROCM_PATH}"/}" != "${PackageCmakeDir}" ]; then + continue + fi + + Msg="WARNING: ${ROCM_PATH} does not provide ${Package}," + Msg+=" using ${PackageCmakeDir}" + echo "${Msg}" + + # Two packages commonly resolve to the same place; append it only once. + if [ -z "${SeenPrefix[${PackageCmakeDir}]:-}" ]; then + SeenPrefix["${PackageCmakeDir}"]=1 + CMakePrefixPath+=";${PackageCmakeDir}" + fi + done + unset SeenPrefix + + CMakeArgs+=("-S" "src") + CMakeArgs+=("-B" "build") + CMakeArgs+=("-DCMAKE_PREFIX_PATH=${CMakePrefixPath}") + CMakeArgs+=("-DGGML_HIP=On") + CMakeArgs+=("-DCMAKE_BUILD_TYPE=${LLAMA_BUILD_MODE}") + CMakeArgs+=("-DGPU_TARGETS=${LLAMA_GPU}") + CMakeArgs+=("-DCMAKE_C_COMPILER=${AOMP}/bin/clang") + CMakeArgs+=("-DCMAKE_CXX_COMPILER=${AOMP}/bin/clang++") + CMakeArgs+=("-DCMAKE_HIP_COMPILER=${AOMP}/bin/clang++") + + # CMake modules export their whole include directory, HIP headers included, + # which would shadow the ROCm under test and e.g. its HIP headers. + add_cmake_rocm_header_priority_args CMakeArgs "${ROCM_PATH}" + + printf 'cmake'; printf ' %q' "${CMakeArgs[@]}"; printf '\n' + cmake "${CMakeArgs[@]}" 2>&1 | + tee "${LLAMA_TESTS_LOG_LOCATION}/cmake-configure.log" + + # Make sure the ROCm header priority is preserved. + check_cmake_rocm_header_priority "${LLAMA_BUILD_DIR}" "${ROCM_PATH}" || exit 1 fi if [ "${DoCompile}" == "yes" ]; then echo "Building LLaMA..." - cmake --build "${LLAMA_BUILD_DIR}" --parallel -j "${AOMP_BUILD_JOBS}" + cmake --build "${LLAMA_BUILD_DIR}" --parallel -j "${AOMP_BUILD_JOBS}" 2>&1 | + tee "${LLAMA_TESTS_LOG_LOCATION}/cmake-build.log" fi if [ "${DoCTest}" == "yes" ]; then @@ -118,7 +184,8 @@ if [ "${DoCTest}" == "yes" ]; then echo "Log in ${LLAMA_TESTS_LOG_LOCATION}/ctest.log" # Some model files are git-lfs and come from huggingface. They will auto-download during test - ctest --output-on-failure 2>&1 | tee "${LLAMA_TESTS_LOG_LOCATION}/ctest.log" + ctest --output-on-failure --timeout "${LLAMA_CTEST_TIMEOUT}" 2>&1 | + tee "${LLAMA_TESTS_LOG_LOCATION}/ctest.log" fi run_llama_bench() {