Provide debug environment and support for retrieving core dumps

.bazelrc

@@ -22,6 +22,8 @@ common --http_timeout_scaling=2.0
 # directory is often deleted while the ANDROID_HOME environment variable remains set.
 common --action_env=ANDROID_HOME=""
 
+common --cxxopt=-std=c++17
+build --cxxopt=-std=c++17
 build --java_language_version=17
 build --tool_java_language_version=17
 build --java_runtime_version=remotejdk_17
@@ -72,6 +74,8 @@ build:x86_64-linux --extra_toolchains=@score_toolchains_rust//toolchains/ferroce
 
 test:x86_64-linux --//config:integration_mode=docker
 test:x86_64-linux --//config:unit_mode=host
+# Show a failing test's log (incl. the crash-dump banner) in the console.
+test:x86_64-linux --test_output=errors
 
 # Target configuration for CPU:AArch64|OS:Linux build (do not use it in case of system toolchains!)
 build:arm64-linux --config=stub
@@ -181,3 +185,12 @@ test:tsan --test_tag_filters=-no-tsan
 test:tsan --build_tests_only
 test:tsan --cxxopt=-Wno-maybe-uninitialized
 test:tsan --cxxopt=-Wno-redundant-move
+
+# Build variant. --config=release is the default (slim image, no debug support) 
+# — equivalent to specifying no variant. --config=debug adds gdb to the
+# test image so cores can be analysed in-container and a backtrace is auto-captured. 
+# -c dbg is implied so the backtrace carries file/line symbols.
+build:release --//config:debug=False
+build:debug --compilation_mode=dbg
+test:debug --test_env=SCORE_ENABLE_DEBUG=1
+test:debug --//config:debug=True

.gitignore

@@ -71,6 +71,10 @@ target/
 tests/**/*.html
 tests/**/*.xml
 
+# Backup of the kernel core_pattern saved by the integration test core-dump
+# capture (--config=core_dump); auto-removed after restore
+/.original_core_pattern
+
 # IDE Code files
 *.orig
 .venv_docs

MODULE.bazel

@@ -19,6 +19,7 @@ bazel_dep(name = "rules_python", version = "1.8.5")
 bazel_dep(name = "rules_rust", version = "0.68.2-score")
 bazel_dep(name = "rules_cc", version = "0.2.17")
 bazel_dep(name = "rules_oci", version = "2.3.0")
+bazel_dep(name = "rules_distroless", version = "0.8.0")
 bazel_dep(name = "rules_shell", version = "0.6.1")
 bazel_dep(name = "aspect_rules_lint", version = "2.3.0")
 bazel_dep(name = "buildifier_prebuilt", version = "8.5.1")
@@ -175,6 +176,20 @@ oci.pull(
 )
 use_repo(oci, "debian-test-runtime", "debian-test-runtime_linux_amd64")
 
+# gdb (+ dependency closure) layered onto the test image; see
+# tests/utils/environments/x86_64-linux/gdb_apt.yaml. Regenerate the lockfile
+# with: bazel run @gdb_apt//:lock
+apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt")
+apt.install(
+    name = "gdb_apt",
+    lock = "//tests/utils/environments/x86_64-linux:gdb_apt.lock.json",
+    manifest = "//tests/utils/environments/x86_64-linux:gdb_apt.yaml",
+    # Normalize to a merged-usr layout so the layer does not clobber the base
+    # image's /bin, /lib, ... usr-merge symlinks (which would break /bin/sh).
+    mergedusr = True,
+)
+use_repo(apt, "gdb_apt")
+
 bazel_dep(name = "score_baselibs", version = "0.2.10")
 
 # Hedron's Compile Commands Extractor for Bazel

config/BUILD

@@ -50,6 +50,21 @@ config_setting(
     },
 )
 
+# Build variant. --config=release is the default (slim image); --config=debug adds
+# the gdb layer to the test image so core dumps can be analysed in-container (see
+# .bazelrc, tests/utils/environments/x86_64-linux/BUILD).
+bool_flag(
+    name = "debug",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "debug_enabled",
+    flag_values = {
+        ":debug": "True",
+    },
+)
+
 # How to run unit tests:
 #
 # - qemu: in a QEMU virtual machine

tests/integration/readme.md

@@ -23,3 +23,128 @@ Currently the following configs are supported:
 - `host`
 - `x86_64-linux`
 
+## Debug environment
+
+The debug environment is **opt-in**: add `--config=debug` to include it. 
+It adds gdb to the docker image, builds the code in debug variant (using "-c dbg") and if process is crashing, a core dump is created. Furthermore the backtrace of the thread which provoked the crash is printed to the console.
+
+The
+default (`--config=release`, equivalent to specifying no variant) builds the slim
+image without it. 
+
+**Attention: It is needed to influence the kernel `core_pattern` value of
+your host system to have core dump support!**
+
+
+How it works:
+- `--config=debug` forwards `SCORE_ENABLE_DEBUG=1` into the test
+  environment and sets the `//config:debug` build flag (see `.bazelrc`); the
+  shared pytest plugin keys off the env var, individual tests need no adaptions.
+- The build flag adds a `gdb` layer to the test image (`score_itf_examples`) so
+  cores can be analysed inside the container against matching libraries. Normal
+  runs build the image without `gdb`, unchanged.
+- The sandbox container runs privileged with an unlimited core-file `ulimit` and
+  a read-write bind-mount of the workspace root.
+- A shared fixture sets the kernel `core_pattern` to a sandbox-local path
+  (`/tmp/score_cores/core.%e.%p.%s.%t`). On teardown it symbolizes each core
+  **inside the container** (where the binary and matching libraries live) into a
+  `<core>.bt.txt` backtrace, copies the cores and backtraces into the Bazel test
+  outputs, and restores the original `core_pattern`.
+- Before changing `core_pattern`, the fixture mirrors the original value to
+  `.original_core_pattern` in the workspace root. The sandboxed test process sees
+  the source tree read-only, so this file is written from inside the privileged
+  container via the workspace bind-mount (hence it is root-owned). It is removed
+  again once the value is restored, so it exists only if a run is force-killed.
+
+Further technical limitations are described in [Important: the `core_pattern` is a global kernel setting](#important-the-core_pattern-is-a-global-kernel-setting)
+
+### Getting a crash dump
+
+Run the (crashing) test with `--config=debug`, disabling the cache so it
+actually executes:
+```
+bazel test //tests/integration/<test> --config=x86_64-linux --config=debug --nocache_test_results
+```
+
+If a crash dump was created, a `CRASH DUMP` section is printed right under the
+pytest `FAILURES` section at the end of the run (the `x86_64-linux` config
+enables `--test_output=errors`, so the failing log is shown automatically):
+```
+=================================== FAILURES ===================================
+...
+================================== CRASH DUMP ==================================
+CRASH DUMP HAS BEEN CREATED! See <.../test.outputs/cores> for details.
+
+core.launch_manager.42.6.1787209649:
+    Program terminated with signal SIGABRT, Aborted.
+    #0  0x... in ?? () from /lib/x86_64-linux-gnu/libc.so.6
+    #1  0x... in raise () from /lib/x86_64-linux-gnu/libc.so.6
+    #2  0x... in abort () from /lib/x86_64-linux-gnu/libc.so.6
+    #3  0x... in <your crashing frame> at <file>:<line>
+    ...
+    Full backtrace (all threads): <.../cores/core.launch_manager.*.bt.txt>
+    Reopen in gdb inside the test image:
+        docker run --rm -it -v <.../launch_manager>:/tmp/.../launch_manager:ro -v <.../cores>:/cores:ro score_itf_examples:latest gdb /tmp/.../launch_manager /cores/core.launch_manager.*
+=========================== short test summary info ============================
+```
+The crashing thread's stack is printed **inline** (symbolized inside the
+container, so libraries match). The printed paths are absolute and
+copy-pasteable. Core files are named `core.<exe>.<pid>.<signal>.<time>` (signal
+`11` = `SIGSEGV`, `6` = `SIGABRT`); they are only produced on an actual crash and
+can be large (hundreds of MB).
+
+### Analysing a crash dump
+
+The crashing thread is already in the banner above. For the **full all-threads**
+dump, open the auto-captured backtrace file the fixture wrote next to the core:
+```
+cat "$(bazel info bazel-testlogs)/tests/integration/<test>/<test>/test.outputs/cores/"*.bt.txt
+```
+`--config=debug` builds with `-c dbg` automatically, so the backtrace carries
+file/line information — no extra flag needed.
+
+The banner already prints a ready-to-run `docker run … gdb …` command per core.
+It must run **inside** the test image (tagged `score_itf_examples:latest`, which
+ships `gdb`), not on the host: the core references the container's libraries, so a
+host `gdb` unwind walks garbage. The command mounts the host binary at its
+original in-container path (so gdb matches the core without warnings) and the
+cores directory, then drops you into an interactive session:
+```
+CORES="$(bazel info bazel-testlogs)/tests/integration/<test>/<test>/test.outputs/cores"
+BIN="$(bazel info bazel-bin)/<path-to-crashing-binary>"
+docker run --rm -it -v "$BIN:/tmp/<remote>/<binary>:ro" -v "$CORES:/cores:ro" \
+    score_itf_examples:latest \
+    gdb /tmp/<remote>/<binary> /cores/core.*
+```
+Running host `gdb` against the core is a last resort; if you must, point it at an
+exported container rootfs via `set sysroot` / `solib-search-path`.
+
+## Important: the `core_pattern` is a global kernel setting
+
+`core_pattern` is **not** namespaced per container - it is shared with the
+host. The fixture therefore changes it globally at the start of a test and
+restores the original value on teardown. Two consequences:
+
+- **Run one crashing target at a time when investigating.** Concurrent tests
+  race on the shared `core_pattern`, so a parallel run can miss dumps or leave
+  a stale value.
+- **If a run is force-killed before teardown**, the restore may not run and
+  `core_pattern` is left pointing at `/tmp/score_cores/...`. The original value
+  was saved to `.original_core_pattern` in the workspace root before it was
+  changed, so you never lose it. **Simply re-running a core-dump test recovers
+  automatically**: the fixture sees the leftover backup, treats it (not the
+  current test value) as the original, and restores it on teardown. To fix it by
+  hand instead:
+  ```
+  cat /proc/sys/kernel/core_pattern          # current (likely /tmp/score_cores/...)
+  cat .original_core_pattern                 # the value to restore
+  # On WSL, write the saved value back from a privileged container:
+  docker run --rm --privileged -v "$PWD/.original_core_pattern:/orig:ro" \
+    debian:bookworm-slim \
+    bash -c 'cat /orig > /proc/sys/kernel/core_pattern'
+  rm .original_core_pattern
+  ```
+  If `.original_core_pattern` is absent, the restore already ran (the file is
+  removed on success). The default `core_pattern` differs per system (e.g.
+  `|/wsl-capture-crash ...` on WSL, `|/lib/systemd/systemd-coredump ...` on
+  systemd hosts, or plain `core`).

tests/utils/bazel/integration.bzl

@@ -101,11 +101,15 @@ def integration_test(
             "--local-dir=/tmp/score_itf_host/{}".format(name),
         ],
     })
-    final_plugins = ["//tests/utils/plugins:integration_plugin"] + select({
+
+    # integration_plugin is listed last so pytest registers it after the
+    # target plugin: its docker_configuration fixture then overrides the
+    # score_itf default (last-registered -p plugin wins fixture overrides).
+    final_plugins = select({
         "//config:integration_docker": ["@score_itf//score/itf/plugins:docker_plugin"],
         "//config:integration_qemu": ["@score_itf//score/itf/plugins:qemu_plugin"],
         "//config:integration_host": ["//tests/utils/plugins:localhost_plugin"],
-    })
+    }) + ["//tests/utils/plugins:integration_plugin"]
 
     # The QEMU plugin uses a hardcoded port so we can only run one test at a time.
     # See https://github.com/eclipse-score/itf/issues/125.

tests/utils/environments/x86_64-linux/BUILD

@@ -26,11 +26,19 @@ pkg_tar(
     srcs = [":ecu_logging_config_files"],
 )
 
+# Under --config=debug the image gains gdb (and its dependency closure) so
+# core dumps can be analysed inside the container against matching libraries;
+# otherwise it stays slim. The gdb layer is built by rules_distroless from
+# gdb_apt.yaml (regenerate its lockfile with:
+# bazel run @gdb_apt//:lock).
 oci_image(
     name = "image",
     base = "@debian-test-runtime",
     target_compatible_with = ["@platforms//os:linux"],
-    tars = [":ecu_logging_config_layer"],
+    tars = [":ecu_logging_config_layer"] + select({
+        "//config:debug_enabled": ["@gdb_apt//:flat"],
+        "//conditions:default": [],
+    }),
 )
 
 oci_load(