From 97bece219c4468dc456ea7822dd197701e7c5f7a Mon Sep 17 00:00:00 2001 From: "amelia@ivis.ai" Date: Thu, 3 Sep 2026 18:43:56 +0900 Subject: [PATCH 1/2] fix(launch_manager): enable ASSERT_DBG in integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #579, which wired SCORE_LANGUAGE_FUTURECPP_ASSERT_LEVEL_DEBUG into unit tests via lm_cc_test but left integration tests untouched. Integration tests link the production launch_manager daemon binary directly, so enabling the define there would mean enabling it on the target that ships in production. Instead, split the daemon's cc_binary into a launch_manager_binary macro (score/launch_manager/src/daemon/daemon.bzl) so a second target, launch_manager_debug_asserts, can share the same srcs/deps but add the debug-assert define, leaving the production launch_manager target untouched. Each tests/integration/*/BUILD now points its `binaries` list at the debug variant. integration_test() gains a binary_renames option (backed by pkg_files's renames) so the packaged binary still shows up as `launch_manager` in the test environment — the test scripts hardcode that filename, and the debug-variant target has a different Bazel target name. Test plan: - bazel build --lockfile_mode=error --config=x86_64-linux //score/launch_manager/src/daemon/... - bazel test --lockfile_mode=error --config=x86_64-linux //score/launch_manager/... (16/16 executed unit tests pass) - bazel run //:format.check_Starlark_with_buildifier - Ran the 19 integration tests locally in host mode with the debug-assert variant; failures observed in this environment reproduce identically against the unmodified production binary (signal-handling and a Rust-toolchain GLIBC mismatch, both pre-existing and unrelated to this change). --- score/launch_manager/BUILD | 7 +++ score/launch_manager/src/daemon/BUILD | 33 +++++-------- score/launch_manager/src/daemon/daemon.bzl | 46 +++++++++++++++++++ tests/integration/complex_monitoring/BUILD | 3 +- .../crash_ignores_dependents/BUILD | 3 +- tests/integration/crash_on_startup/BUILD | 3 +- .../fallback_to_same_target_restarts/BUILD | 3 +- .../incorrect_config_non_reporting/BUILD | 3 +- .../lm_shutdown_during_rt_switch/BUILD | 3 +- .../lm_shutdown_during_switch_to_off/BUILD | 3 +- tests/integration/parallel_launch/BUILD | 3 +- .../process_complex_rep_failure/BUILD | 3 +- .../process_crash_monitoring/BUILD | 3 +- tests/integration/process_fd_leak/BUILD | 3 +- tests/integration/process_launch_args/BUILD | 3 +- .../process_simple_rep_failure/BUILD | 3 +- .../process_wrong_binary_failure/BUILD | 3 +- .../rt_running_when_process_exits/BUILD | 3 +- tests/integration/sandbox_options/BUILD | 3 +- tests/integration/shutdown_signal/BUILD | 3 +- tests/integration/smoke/BUILD | 3 +- tests/integration/switch_run_target/BUILD | 3 +- tests/utils/bazel/integration.bzl | 6 +++ 23 files changed, 108 insertions(+), 41 deletions(-) create mode 100644 score/launch_manager/src/daemon/daemon.bzl diff --git a/score/launch_manager/BUILD b/score/launch_manager/BUILD index 55dd202efc..fc11e11494 100644 --- a/score/launch_manager/BUILD +++ b/score/launch_manager/BUILD @@ -46,6 +46,13 @@ alias( actual = "//score/launch_manager/src/daemon:launch_manager", ) +# Debug-assert build variant of the daemon, for integration tests only — +# see //score/launch_manager/src/daemon:launch_manager_debug_asserts. +alias( + name = "launch_manager_debug_asserts", + actual = "//score/launch_manager/src/daemon:launch_manager_debug_asserts", +) + cc_library( name = "error", hdrs = ["src/execution_error.h"], diff --git a/score/launch_manager/src/daemon/BUILD b/score/launch_manager/src/daemon/BUILD index 34705a5949..76984d2c5b 100644 --- a/score/launch_manager/src/daemon/BUILD +++ b/score/launch_manager/src/daemon/BUILD @@ -10,27 +10,16 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -load("@rules_cc//cc:defs.bzl", "cc_binary") +load(":daemon.bzl", "launch_manager_binary") -cc_binary( - name = "launch_manager", - srcs = ["src/main.cpp"], - linkopts = select({ - "@platforms//os:linux": ["-lpthread"], - "//conditions:default": [], - }), - visibility = ["//score/launch_manager:__subpackages__"], - deps = [ - "//score/launch_manager/src/daemon/src/alive_monitor", - "//score/launch_manager/src/daemon/src/common:assertion_handler", - "//score/launch_manager/src/daemon/src/common:log", - "//score/launch_manager/src/daemon/src/configuration:flatbuffer_config_loader", - "//score/launch_manager/src/daemon/src/osal:ipc_comms", - "//score/launch_manager/src/daemon/src/process_group_manager", - "//score/launch_manager/src/daemon/src/process_group_manager:alive_monitor_thread", - "//score/launch_manager/src/daemon/src/recovery_client", - "//score/launch_manager/src/daemon/src/supervision_control_client:supervision_control_notifier", - "//score/launch_manager/src/daemon/src/watchdog:watchdog_factory", - "@score_baselibs//score/language/futurecpp", - ], +# Production binary — ships as-is, debug-level asserts stay compiled out. +launch_manager_binary(name = "launch_manager") + +# Opt-in variant for integration tests: same srcs/deps as the production +# binary, but with SCORE_LANGUAGE_FUTURECPP_ASSERT_LEVEL_DEBUG enabled so the +# integration suite actually exercises DBG-level asserts (see lifecycle#579, +# which did the same for unit tests via lm_cc_test). +launch_manager_binary( + name = "launch_manager_debug_asserts", + defines = ["SCORE_LANGUAGE_FUTURECPP_ASSERT_LEVEL_DEBUG"], ) diff --git a/score/launch_manager/src/daemon/daemon.bzl b/score/launch_manager/src/daemon/daemon.bzl new file mode 100644 index 0000000000..2a403ebe42 --- /dev/null +++ b/score/launch_manager/src/daemon/daemon.bzl @@ -0,0 +1,46 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Bazel macros for the launch_manager daemon binary.""" + +load("@rules_cc//cc:defs.bzl", "cc_binary") + +def launch_manager_binary(name, defines = []): + """Defines a launch_manager daemon binary variant. + + Args: + name: Name of the binary target. + defines: Extra preprocessor defines for this variant. + """ + cc_binary( + name = name, + srcs = ["src/main.cpp"], + linkopts = select({ + "@platforms//os:linux": ["-lpthread"], + "//conditions:default": [], + }), + defines = defines, + visibility = ["//score/launch_manager:__subpackages__"], + deps = [ + "//score/launch_manager/src/daemon/src/alive_monitor", + "//score/launch_manager/src/daemon/src/common:assertion_handler", + "//score/launch_manager/src/daemon/src/common:log", + "//score/launch_manager/src/daemon/src/configuration:flatbuffer_config_loader", + "//score/launch_manager/src/daemon/src/osal:ipc_comms", + "//score/launch_manager/src/daemon/src/process_group_manager", + "//score/launch_manager/src/daemon/src/process_group_manager:alive_monitor_thread", + "//score/launch_manager/src/daemon/src/recovery_client", + "//score/launch_manager/src/daemon/src/supervision_control_client:supervision_control_notifier", + "//score/launch_manager/src/daemon/src/watchdog:watchdog_factory", + "@score_baselibs//score/language/futurecpp", + ], + ) diff --git a/tests/integration/complex_monitoring/BUILD b/tests/integration/complex_monitoring/BUILD index 15306f47ec..088084ae71 100644 --- a/tests/integration/complex_monitoring/BUILD +++ b/tests/integration/complex_monitoring/BUILD @@ -52,8 +52,9 @@ integration_test( binaries = [ ":component_complex_monitoring", ":control_client_test_driver", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", "//tests/utils/test_helper:verification_process", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":complex_monitoring.json", ) diff --git a/tests/integration/crash_ignores_dependents/BUILD b/tests/integration/crash_ignores_dependents/BUILD index 039993802c..14b5a2ffea 100644 --- a/tests/integration/crash_ignores_dependents/BUILD +++ b/tests/integration/crash_ignores_dependents/BUILD @@ -41,7 +41,8 @@ integration_test( ":config", ":test_process", ":process_crashing_once", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":crash_ignores_dependents.json", ) diff --git a/tests/integration/crash_on_startup/BUILD b/tests/integration/crash_on_startup/BUILD index 620abbc63d..040d57d90b 100644 --- a/tests/integration/crash_on_startup/BUILD +++ b/tests/integration/crash_on_startup/BUILD @@ -29,9 +29,10 @@ integration_test( srcs = ["crash_on_startup.py"], binaries = [ ":control_client_test_driver", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", "//tests/utils/test_helper:process_crashing_on_startup_n_times", "//tests/utils/test_helper:verification_process", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":crash_on_startup.json", ) diff --git a/tests/integration/fallback_to_same_target_restarts/BUILD b/tests/integration/fallback_to_same_target_restarts/BUILD index 9c39ec7713..5ce7a79862 100644 --- a/tests/integration/fallback_to_same_target_restarts/BUILD +++ b/tests/integration/fallback_to_same_target_restarts/BUILD @@ -41,7 +41,8 @@ integration_test( ":config", ":control_client_test_driver", ":process_crashing_once", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":fallback_to_same_target_restarts.json", ) diff --git a/tests/integration/incorrect_config_non_reporting/BUILD b/tests/integration/incorrect_config_non_reporting/BUILD index d84b30f28f..262a6069ef 100644 --- a/tests/integration/incorrect_config_non_reporting/BUILD +++ b/tests/integration/incorrect_config_non_reporting/BUILD @@ -29,7 +29,8 @@ integration_test( srcs = ["test_incorrect_config_non_reporting.py"], binaries = [ ":non_reporting_process", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":non_reporting_config.json", ) diff --git a/tests/integration/lm_shutdown_during_rt_switch/BUILD b/tests/integration/lm_shutdown_during_rt_switch/BUILD index d5b2c0d52c..4e5bdac283 100644 --- a/tests/integration/lm_shutdown_during_rt_switch/BUILD +++ b/tests/integration/lm_shutdown_during_rt_switch/BUILD @@ -50,7 +50,8 @@ integration_test( "//tests/utils/test_helper:process_hanging_on_sigterm", ":component_c", ":control_client_test_driver", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":lm_shutdown_during_rt_switch.json", ) diff --git a/tests/integration/lm_shutdown_during_switch_to_off/BUILD b/tests/integration/lm_shutdown_during_switch_to_off/BUILD index 88512b86c0..772c96b765 100644 --- a/tests/integration/lm_shutdown_during_switch_to_off/BUILD +++ b/tests/integration/lm_shutdown_during_switch_to_off/BUILD @@ -38,7 +38,8 @@ integration_test( binaries = [ "//tests/utils/test_helper:process_hanging_on_sigterm", ":control_client_test_driver", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":lm_shutdown_during_switch_to_off.json", ) diff --git a/tests/integration/parallel_launch/BUILD b/tests/integration/parallel_launch/BUILD index 2688898cb7..ab3479d0c4 100644 --- a/tests/integration/parallel_launch/BUILD +++ b/tests/integration/parallel_launch/BUILD @@ -40,7 +40,8 @@ integration_test( binaries = [ ":component_parallel_launch", ":control_client_test_driver", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":parallel_launch.json", ) diff --git a/tests/integration/process_complex_rep_failure/BUILD b/tests/integration/process_complex_rep_failure/BUILD index e7deefcbb0..fe90682975 100644 --- a/tests/integration/process_complex_rep_failure/BUILD +++ b/tests/integration/process_complex_rep_failure/BUILD @@ -41,8 +41,9 @@ integration_test( binaries = [ ":complex_reporting_process", ":control_client_test_driver", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", "//tests/utils/test_helper:verification_process", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":process_complex_rep_failure.json", ) diff --git a/tests/integration/process_crash_monitoring/BUILD b/tests/integration/process_crash_monitoring/BUILD index 43865f42b1..e5ca7fca4c 100644 --- a/tests/integration/process_crash_monitoring/BUILD +++ b/tests/integration/process_crash_monitoring/BUILD @@ -41,8 +41,9 @@ integration_test( ":config", ":control_client_test_driver", ":process_crashing_on_runtime", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", "//tests/utils/test_helper:verification_process", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":process_crash_monitoring.json", ) diff --git a/tests/integration/process_fd_leak/BUILD b/tests/integration/process_fd_leak/BUILD index 33e79c1606..ff6d6bfea1 100644 --- a/tests/integration/process_fd_leak/BUILD +++ b/tests/integration/process_fd_leak/BUILD @@ -67,7 +67,8 @@ integration_test( ":control_client_test_driver", ":native", ":reporting", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":process_fd_leak.json", ) diff --git a/tests/integration/process_launch_args/BUILD b/tests/integration/process_launch_args/BUILD index 279ba3a614..c3ea22824d 100644 --- a/tests/integration/process_launch_args/BUILD +++ b/tests/integration/process_launch_args/BUILD @@ -28,7 +28,8 @@ integration_test( srcs = ["process_launch_args.py"], binaries = [ ":process_initial", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":process_launch_args.json", ) diff --git a/tests/integration/process_simple_rep_failure/BUILD b/tests/integration/process_simple_rep_failure/BUILD index f0baf01713..a8ed8e71e2 100644 --- a/tests/integration/process_simple_rep_failure/BUILD +++ b/tests/integration/process_simple_rep_failure/BUILD @@ -40,8 +40,9 @@ integration_test( binaries = [ ":control_client_test_driver", ":process_simple_reporting", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", "//tests/utils/test_helper:verification_process", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":process_simple_rep_failure.json", ) diff --git a/tests/integration/process_wrong_binary_failure/BUILD b/tests/integration/process_wrong_binary_failure/BUILD index 8bc684a776..a937f656af 100644 --- a/tests/integration/process_wrong_binary_failure/BUILD +++ b/tests/integration/process_wrong_binary_failure/BUILD @@ -29,8 +29,9 @@ integration_test( srcs = ["process_wrong_binary_failure.py"], binaries = [ ":control_client_test_driver", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", "//tests/utils/test_helper:verification_process", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":process_wrong_binary_failure.json", ) diff --git a/tests/integration/rt_running_when_process_exits/BUILD b/tests/integration/rt_running_when_process_exits/BUILD index f82f94d835..f7991f897d 100644 --- a/tests/integration/rt_running_when_process_exits/BUILD +++ b/tests/integration/rt_running_when_process_exits/BUILD @@ -42,8 +42,9 @@ integration_test( binaries = [ ":control_client_test_driver", ":filesystem_reader", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", "//tests/utils/testing_utils:touch_file.sh", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":rt_running_when_process_exits.json", ) diff --git a/tests/integration/sandbox_options/BUILD b/tests/integration/sandbox_options/BUILD index 37029d1463..c02dce7566 100644 --- a/tests/integration/sandbox_options/BUILD +++ b/tests/integration/sandbox_options/BUILD @@ -41,8 +41,9 @@ integration_test( ":sandbox_options_process_a", ":sandbox_options_process_b", ":sandbox_options_process_c", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", "//tests/utils/test_helper:verification_process", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":sandbox_options.json", ) diff --git a/tests/integration/shutdown_signal/BUILD b/tests/integration/shutdown_signal/BUILD index 61650f6446..3c97e4f232 100644 --- a/tests/integration/shutdown_signal/BUILD +++ b/tests/integration/shutdown_signal/BUILD @@ -48,7 +48,8 @@ integration_test( binaries = [ ":control_client_test_driver", ":shutdown_signal_process", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":shutdown_signal.json", ) diff --git a/tests/integration/smoke/BUILD b/tests/integration/smoke/BUILD index 5bbf47291f..88fdf9ff17 100644 --- a/tests/integration/smoke/BUILD +++ b/tests/integration/smoke/BUILD @@ -42,7 +42,8 @@ integration_test( binaries = [ ":control_client_test_driver", ":gtest_process", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":lifecycle_smoketest.json", ) diff --git a/tests/integration/switch_run_target/BUILD b/tests/integration/switch_run_target/BUILD index f1bdb9301f..9ad3f02189 100644 --- a/tests/integration/switch_run_target/BUILD +++ b/tests/integration/switch_run_target/BUILD @@ -76,7 +76,8 @@ integration_test( ":component_d", ":component_e", ":control_client_test_driver", - "//score/launch_manager", + "//score/launch_manager:launch_manager_debug_asserts", ], + binary_renames = {"//score/launch_manager:launch_manager_debug_asserts": "launch_manager"}, config = ":switch_run_target.json", ) diff --git a/tests/utils/bazel/integration.bzl b/tests/utils/bazel/integration.bzl index be754b1947..d25466e9b8 100644 --- a/tests/utils/bazel/integration.bzl +++ b/tests/utils/bazel/integration.bzl @@ -24,6 +24,7 @@ def integration_test( files = [], config = None, install_prefix = SCORE_TEST_INSTALL_PREFIX, + binary_renames = {}, **kwargs): """Creates an integration test. @@ -41,6 +42,10 @@ def integration_test( files: Additional files config: Launch manager configuration file install_prefix: Installation prefix for the test environment + binary_renames: Optional {label: filename} overrides for entries in + `binaries` — lets a differently-named target (e.g. a debug-assert + build variant) be installed under the filename the test scripts + expect (e.g. "launch_manager"). **kwargs: Miscellaneous arguments passed through to `py_itf_test` """ @@ -49,6 +54,7 @@ def integration_test( srcs = binaries, attributes = pkg_attributes(mode = "0555"), prefix = "tests/{}".format(name), + renames = binary_renames, ) if config: From ce8b4ca7bca36dff73ae960d165e383b5dece5d8 Mon Sep 17 00:00:00 2001 From: "amelia@ivis.ai" Date: Fri, 4 Sep 2026 13:01:04 +0900 Subject: [PATCH 2/2] fix(tests): signal the wrapped binary, not the fakeroot shell LocalAsyncProcess.stop() sent SIGTERM to the whole process group so that the actual test binary (launched under fakeroot) would receive it and run its cleanup code. But fakeroot's /bin/sh wrapper traps EXIT/INT, not TERM, so the broadcast killed the wrapper itself before it could wait() for its child and relay the real exit code -- the tracked returncode then reported the wrapper's own signal death (-15) regardless of whether the wrapped binary actually shut down cleanly. Signal only the process(es) fakeroot launched instead, and let the wrapper's shell script finish normally and propagate the real exit status. Test plan: - Ran the launch_manager integration suite locally in host mode (--//config:integration_mode=host): all 17 tests unaffected by pre-existing local environment limitations (a missing SCHED_FIFO capability, and a GLIBC/rustc mismatch pulled in by an unrelated Rust dependency) now pass and correctly report the wrapped binary's own exit status. --- tests/utils/plugins/localhost.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/utils/plugins/localhost.py b/tests/utils/plugins/localhost.py index 09d4e83111..ae38fc0822 100644 --- a/tests/utils/plugins/localhost.py +++ b/tests/utils/plugins/localhost.py @@ -77,10 +77,24 @@ def wait(self, timeout_s: float = 15) -> int: def stop(self) -> int: if self.is_running(): - # Kill the entire process group so that children (e.g. the actual - # daemon binary launched under fakeroot) receive SIGTERM and can - # run their cleanup code before exiting. - os.killpg(os.getpgid(self._process.pid), signal.SIGTERM) + # Signal only the process(es) fakeroot launched, not the whole + # process group. fakeroot's /bin/sh wrapper traps EXIT/INT but not + # TERM, so killpg() would kill the wrapper itself before it can + # wait() for its child and relay the real exit code -- self._process + # would then report the wrapper's own signal death (-15) instead of + # the actual binary's exit status, regardless of whether that + # binary shut down cleanly. + children = subprocess.run( + ["pgrep", "-P", str(self._process.pid)], + capture_output=True, + text=True, + ).stdout.split() + targets = [int(pid) for pid in children] or [self._process.pid] + for pid in targets: + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass for _ in range(5): time.sleep(1) if not self.is_running():