[llamacpp] Fix CMake config and EnVar setup

bin/aomp_common_vars

@@ -9,6 +9,10 @@
 #
 #     The bash syntax ENVNAME=${ENVNAME:-value} checks if ENVNAME is
 #     already set. If not, it assigns it the value following the :-
+#
+#     Most variables defined here are consumed by the sourcing scripts,
+#     so their uses are invisible to static analysis.
+# shellcheck disable=SC2034
 
 # --- Start standard header to set AOMP environment variables ----
 realpath=$(realpath "$0")
@@ -262,9 +266,11 @@ function cmquot() {
 # copy & pasted into the user's terminal.  Output should not be further quoted.
 function shquot() {
   local -a output
+  local arg Escaped
   output=()
   for arg in "$@"; do
-    output+=(\'${arg//\'/\'\\\'\'}\')
+    Escaped=${arg//\'/\'\\\'\'}
+    output+=("'${Escaped}'")
   done
   printf '%s' "${output[*]}"
 }
@@ -568,11 +574,11 @@ declare -a ASAN_FLAGS
 
 if [ "$AOMP_BUILD_SANITIZER" == 1 ]; then
    if [ -f "${LLVM_INSTALL_LOC}/bin/clang" ]; then
-      ASAN_LIB=$(${LLVM_INSTALL_LOC}/bin/clang --print-file-name=libclang_rt.asan-x86_64.so)
+      ASAN_LIB=$("${LLVM_INSTALL_LOC}"/bin/clang --print-file-name=libclang_rt.asan-x86_64.so)
       ASAN_LIB_PATH=$(dirname "$ASAN_LIB")
       if [ ! -d "$ASAN_LIB_PATH" ]; then
         #Suppose above search path fails then via 'LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=ON' the host asan library would lose the suffix
-        ASAN_LIB=$(${LLVM_INSTALL_LOC}/bin/clang --print-file-name=libclang_rt.asan.so)
+        ASAN_LIB=$("${LLVM_INSTALL_LOC}"/bin/clang --print-file-name=libclang_rt.asan.so)
         ASAN_LIB_PATH=$(dirname "$ASAN_LIB")
       fi
     else
@@ -860,3 +866,146 @@ function help_build_aomp(){
 EOF
    exit 0
 }
+
+# Merge the contents of a variable with a list of input items.
+# Usage: merge_variable_with_inputs <VarName> --delimiter <Delim> \
+#          [--append|--prepend] <Input1> [<Input2> ...]
+# Default delimiter: ":"
+# Default mode: --prepend (new inputs take priority / appear first).
+# Empty/unset inputs are skipped; no dangling delimiters are produced.
+#
+# Example:
+# merge_variable_with_inputs PATH "${AOMP:+${AOMP}/bin}"
+# merge_variable_with_inputs CMAKE_MODULE_PATH --delimiter ";" \
+#                            --append "${AOMP:+${AOMP}/lib/cmake}"
+function merge_variable_with_inputs() {
+  local VarName="$1"; shift
+  local Delimiter=":"
+  local Mode="prepend"
+
+  while [ "$#" -gt 0 ]; do
+    case "$1" in
+      --delimiter) Delimiter="$2"; shift 2 ;;
+      --append)    Mode="append";  shift   ;;
+      --prepend)   Mode="prepend"; shift   ;;
+      --)          shift; break ;;
+      *)           break ;;
+    esac
+  done
+
+  local -a InputItems=("$@")
+  local -n __MergeTargetVar="${VarName}"
+
+  local Joined=""
+  local Item
+  for Item in "${InputItems[@]}"; do
+    [ -n "${Item}" ] && \
+      Joined="${Joined:+${Joined}${Delimiter}}${Item}"
+  done
+
+  # join2(a, b): a + Delimiter + b, but only if BOTH are non-empty;
+  # otherwise whichever side is non-empty (or "" if both are).
+  if [ "${Mode}" == "append" ]; then
+    __MergeTargetVar="${__MergeTargetVar}${__MergeTargetVar:+${Joined:+${Delimiter}}}${Joined}"
+  else
+    __MergeTargetVar="${Joined}${Joined:+${__MergeTargetVar:+${Delimiter}}}${__MergeTargetVar}"
+  fi
+}
+
+# Return the path containing the requested CMake module directory.
+# Usage: get_cmake_module_path <Module> [<Path1> <Path2> ...]
+# Searches each given Path, in order, for <Path>/lib/cmake/<Module>.
+# Empty path arguments are skipped (safe to pass an unset/empty variable).
+# If no Path arguments are given, defaults to searching AOMP, then ROCM_PATH
+# (or a path derived from AOMP, if set), then /opt/rocm.
+function get_cmake_module_path() {
+  local Module="$1"
+  shift
+  local -a SearchPaths=("$@")
+
+  # No explicit candidates given: fall back to the default search order.
+  # Default: AOMP -> ROCM_PATH (if empty: AOMP/../..) -> /opt/rocm
+  if [ "$#" -eq 0 ]; then
+    SearchPaths=(
+      "${AOMP}"
+      "${ROCM_PATH:-${AOMP:+$(realpath -m "${AOMP}/../../")}}"
+      "/opt/rocm"
+    )
+  fi
+
+  # Assumptions:
+  # All CMake modules are located within ./lib/cmake/<Module>.
+  # All CMake module configuration files are named <Module>*onfig*.cmake.
+  local Path
+  for Path in "${SearchPaths[@]}"; do
+    if [ -n "${Path}" ] && [ -d "${Path}/lib/cmake/${Module}" ] &&
+       compgen -G "${Path}/lib/cmake/${Module}/${Module}"'*onfig*.cmake' \
+         > /dev/null; then
+      # Return the parent(!) directory.
+      echo "${Path}/lib/cmake"
+      return 0
+    fi
+  done
+  return 1
+}
+
+# Keep the HIP headers of the ROCm under test ahead of every other ROCm's.
+# Usage: add_cmake_rocm_header_priority_args <ArrayName> <RocmRoot> [<Lang> ...]
+# Appends one CMake argument per language (default: HIP); does nothing when
+# <RocmRoot> ships no HIP headers of its own.
+#
+# A borrowed hipBLAS/rocBLAS exports its whole include directory, which CMake
+# emits as -isystem, ahead of the compiler's own ROCm that clang searches last.
+# CMAKE_<LANG>_COMPILER_ARG1 is the only slot in front of those includes; every
+# *_FLAGS variable lands behind them.  Keep CMAKE_<LANG>_COMPILER a plain path,
+# CMake reassigns ARG1 when the compiler itself carries arguments.
+function add_cmake_rocm_header_priority_args() {
+  local -n __HeaderPriorityArgs="$1"; shift
+  local RocmRoot="$1"; shift
+  local -a Languages=("$@")
+  [ "${#Languages[@]}" -gt 0 ] || Languages=("HIP")
+
+  [ -n "${RocmRoot}" ] && [ -d "${RocmRoot}/include/hip" ] || return 0
+
+  local Language
+  for Language in "${Languages[@]}"; do
+    __HeaderPriorityArgs+=(
+      "-DCMAKE_${Language}_COMPILER_ARG1:STRING=-isystem ${RocmRoot}/include")
+  done
+}
+
+# Verify that the ROCm header priority survived the CMake run.
+# Usage: check_cmake_rocm_header_priority <BuildDir> <RocmRoot> [<Lang> ...]
+# A lost entry only breaks the build while the two ROCm header sets are
+# incompatible, otherwise it silently builds against the wrong ones.
+function check_cmake_rocm_header_priority() {
+  local BuildDir="$1"; shift
+  local RocmRoot="$1"; shift
+  local -a Languages=("$@")
+  [ "${#Languages[@]}" -gt 0 ] || Languages=("HIP")
+
+  [ -n "${RocmRoot}" ] && [ -d "${RocmRoot}/include/hip" ] || return 0
+
+  local CacheFile="${BuildDir}/CMakeCache.txt"
+  if [ ! -f "${CacheFile}" ]; then
+    echo "ERROR: ${CacheFile} not found"
+    return 1
+  fi
+
+  local Language Entry Msg Status=0
+  for Language in "${Languages[@]}"; do
+    Entry=$(grep "^CMAKE_${Language}_COMPILER_ARG1:" "${CacheFile}" || true)
+    case "${Entry}" in
+    *"-isystem ${RocmRoot}/include"*) ;;
+    *)
+      Msg="ERROR: CMAKE_${Language}_COMPILER_ARG1 lost"
+      Msg+=" ${RocmRoot}/include, its HIP headers can be shadowed"
+      Msg+=" by another ROCm"
+      echo "${Msg}"
+      Status=1
+      ;;
+    esac
+  done
+
+  return "${Status}"
+}

bin/run_llama.sh

@@ -8,9 +8,12 @@
 
 # Build script for LLaMA with HIP support using AOMP compiler
 
+ScriptDir=$(dirname "$(realpath "$0")")
+
 # shellcheck source=/dev/null
-. aomp_common_vars
+. "${ScriptDir}"/aomp_common_vars
 
+: "${ROCM_PATH:=$(realpath -m "${AOMP}/../..")}"
 : "${AOMP_GPU:=gfx90a}"
 : "${LLAMA_GPU:=$AOMP_GPU}"
 
@@ -20,10 +23,25 @@
 : "${LLAMA_BUILD_MODE:=Release}"
 : "${LLAMA_TESTS_LOG_LOCATION:=$LLAMA_TLDIR/logs}"
 
+# CTest 'test-backend-ops' exceeds the 1500s default timeout.
+# Measured on MI350X: ~2500s
+: "${LLAMA_CTEST_TIMEOUT:=3600}"
+
 # Model to use in benchmarks (default is a smaller model)
 : "${LLAMA_BENCH_HF_ID:=ggml-org/gemma-3-1b-it-GGUF}"
 : "${LLAMA_CACHE:=$HOME/.cache/llama.cpp}"
 
+# Add AOMP and ROCM_PATH to PATH and LD_LIBRARY_PATH and export them.
+merge_variable_with_inputs PATH "${AOMP:+${AOMP}/bin}" "${ROCM_PATH:+${ROCM_PATH}/bin}"
+merge_variable_with_inputs LD_LIBRARY_PATH \
+  "${AOMP:+${AOMP}/lib}" \
+  "${AOMP:+${AOMP}/lib/x86_64-unknown-linux-gnu}" \
+  "${ROCM_PATH:+${ROCM_PATH}/lib}"
+
+export PATH
+export LD_LIBRARY_PATH
+export ROCM_PATH
+
 pushd "${AOMP_REPOS_TEST}" || exit
 mkdir -p "${LLAMA_TLDIR}" && cd "${LLAMA_TLDIR}" || exit
 
@@ -92,24 +110,72 @@ else
   cd ..
 fi
 
-echo "Configuring build with CMake..."
 if [ "${DoConfigure}" == "yes" ]; then
+  echo "Configuring build with CMake..."
   rm -rf "${LLAMA_BUILD_DIR}"
-  cmake -B build \
-    -S src \
-    -DCMAKE_PREFIX_PATH="${AOMP}"/lib/cmake \
-    -DGGML_HIP=On \
-    -DCMAKE_BUILD_TYPE="${LLAMA_BUILD_MODE}" \
-    -DGPU_TARGETS="${LLAMA_GPU}" \
-    ${CmakeGenerator:+"${CmakeGenerator}"} \
-    -DCMAKE_C_COMPILER="${AOMP}"/bin/clang \
-    -DCMAKE_CXX_COMPILER="${AOMP}"/bin/clang++ \
-    -DCMAKE_HIP_COMPILER="${AOMP}"/bin/clang++
+
+  CMakeArgs=()
+  if [ -n "${CmakeGenerator}" ]; then
+    CMakeArgs+=("${CmakeGenerator}")
+  fi
+
+  # Determine the CMake module paths of required ROCm packages.
+  CMakePrefixPath="${ROCM_PATH}"
+  declare -A SeenPrefix=()
+  for Package in hip hipblas rocblas; do
+    if ! PackageCmakeDir=$(get_cmake_module_path "${Package}"); then
+      Msg="ERROR: no CMake package '${Package}' below ${AOMP},"
+      Msg+=" ${ROCM_PATH} or /opt/rocm"
+      echo "${Msg}"
+      exit 1
+    fi
+
+    # Anything found below the ROCm under test needs no extra prefix.
+    # Removing a prefix that is present shortens the path, so a path that
+    # comes back unchanged did not start with it.
+    if [ "${PackageCmakeDir#"${AOMP}"/}" != "${PackageCmakeDir}" ] ||
+       [ "${PackageCmakeDir#"${ROCM_PATH}"/}" != "${PackageCmakeDir}" ]; then
+      continue
+    fi
+
+    Msg="WARNING: ${ROCM_PATH} does not provide ${Package},"
+    Msg+=" using ${PackageCmakeDir}"
+    echo "${Msg}"
+
+    # Two packages commonly resolve to the same place; append it only once.
+    if [ -z "${SeenPrefix[${PackageCmakeDir}]:-}" ]; then
+      SeenPrefix["${PackageCmakeDir}"]=1
+      CMakePrefixPath+=";${PackageCmakeDir}"
+    fi
+  done
+  unset SeenPrefix
+
+  CMakeArgs+=("-S" "src")
+  CMakeArgs+=("-B" "build")
+  CMakeArgs+=("-DCMAKE_PREFIX_PATH=${CMakePrefixPath}")
+  CMakeArgs+=("-DGGML_HIP=On")
+  CMakeArgs+=("-DCMAKE_BUILD_TYPE=${LLAMA_BUILD_MODE}")
+  CMakeArgs+=("-DGPU_TARGETS=${LLAMA_GPU}")
+  CMakeArgs+=("-DCMAKE_C_COMPILER=${AOMP}/bin/clang")
+  CMakeArgs+=("-DCMAKE_CXX_COMPILER=${AOMP}/bin/clang++")
+  CMakeArgs+=("-DCMAKE_HIP_COMPILER=${AOMP}/bin/clang++")
+
+  # CMake modules export their whole include directory, HIP headers included,
+  # which would shadow the ROCm under test and e.g. its HIP headers.
+  add_cmake_rocm_header_priority_args CMakeArgs "${ROCM_PATH}"
+
+  printf 'cmake'; printf ' %q' "${CMakeArgs[@]}"; printf '\n'
+  cmake "${CMakeArgs[@]}" 2>&1 |
+    tee "${LLAMA_TESTS_LOG_LOCATION}/cmake-configure.log"
+
+  # Make sure the ROCm header priority is preserved.
+  check_cmake_rocm_header_priority "${LLAMA_BUILD_DIR}" "${ROCM_PATH}" || exit 1
 fi
 
 if [ "${DoCompile}" == "yes" ]; then
   echo "Building LLaMA..."
-  cmake --build "${LLAMA_BUILD_DIR}" --parallel -j "${AOMP_BUILD_JOBS}"
+  cmake --build "${LLAMA_BUILD_DIR}" --parallel -j "${AOMP_BUILD_JOBS}" 2>&1 |
+    tee "${LLAMA_TESTS_LOG_LOCATION}/cmake-build.log"
 fi
 
 if [ "${DoCTest}" == "yes" ]; then
@@ -118,7 +184,8 @@ if [ "${DoCTest}" == "yes" ]; then
   echo "Log in ${LLAMA_TESTS_LOG_LOCATION}/ctest.log"
 
   # Some model files are git-lfs and come from huggingface. They will auto-download during test
-  ctest --output-on-failure 2>&1 | tee "${LLAMA_TESTS_LOG_LOCATION}/ctest.log"
+  ctest --output-on-failure --timeout "${LLAMA_CTEST_TIMEOUT}" 2>&1 |
+    tee "${LLAMA_TESTS_LOG_LOCATION}/ctest.log"
 fi
 
 run_llama_bench() {