From 4625d1ff0bf065b4b89b9ed14182704a2b432465 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Sat, 18 Jul 2026 17:34:11 -0400 Subject: [PATCH] Allow the watchdog to continue restarting an app indefinitely --- supervisor/apps/app.py | 37 +++++++++-- supervisor/apps/const.py | 1 + supervisor/apps/model.py | 7 ++ supervisor/apps/validate.py | 5 ++ supervisor/const.py | 8 +++ tests/apps/test_app.py | 124 +++++++++++++++++++++++++++++++++++- 6 files changed, 176 insertions(+), 6 deletions(-) diff --git a/supervisor/apps/app.py b/supervisor/apps/app.py index da779c590d8..54084777465 100644 --- a/supervisor/apps/app.py +++ b/supervisor/apps/app.py @@ -57,6 +57,7 @@ AppStartup, AppState, BusEvent, + WatchdogRestartPolicy, ) from ..coresys import CoreSys from ..docker.app import DockerApp @@ -101,6 +102,7 @@ from ..utils.sentry import async_capture_exception from .const import ( WATCHDOG_MAX_ATTEMPTS, + WATCHDOG_MAX_RETRY_SECONDS, WATCHDOG_RETRY_SECONDS, WATCHDOG_THROTTLE_MAX_CALLS, WATCHDOG_THROTTLE_PERIOD, @@ -1758,7 +1760,21 @@ def _restore_data(): async def _restart_after_problem( self, state: ContainerState, exit_code: int | None = None ): - """Restart unhealthy or failed app.""" + """Restart unhealthy or failed app, rate-limited to break crash loops.""" + await self._do_restart_after_problem(state, exit_code) + + async def _do_restart_after_problem( + self, + state: ContainerState, + exit_code: int | None = None, + *, + unlimited: bool = False, + ) -> None: + """Restart an unhealthy or failed app. + + When unlimited is True the per-invocation attempt cap is lifted so the + watchdog keeps retrying forever (WatchdogRestartPolicy.UNLIMITED). + """ attempts = 0 while await self.instance.current_state() == state: if not self.in_progress: @@ -1796,7 +1812,7 @@ async def _restart_after_problem( else: break - if attempts >= WATCHDOG_MAX_ATTEMPTS: + if not unlimited and attempts >= WATCHDOG_MAX_ATTEMPTS: _LOGGER.critical( "Watchdog cannot restart app %s, failed all %s attempts", self.name, @@ -1804,8 +1820,13 @@ async def _restart_after_problem( ) break - # Exponential backoff to spread retries over the throttle window - delay = WATCHDOG_RETRY_SECONDS * (1 << max(attempts - 1, 0)) + # Exponential backoff, capped so unlimited retries settle at a steady + # interval instead of growing without bound. The shift bound only keeps + # the intermediate value small; WATCHDOG_MAX_RETRY_SECONDS is the real cap. + delay = min( + WATCHDOG_RETRY_SECONDS * (1 << min(max(attempts - 1, 0), 8)), + WATCHDOG_MAX_RETRY_SECONDS, + ) _LOGGER.debug( "Watchdog will retry app %s in %s seconds (attempt %s)", self.name, @@ -1855,7 +1876,13 @@ async def watchdog_container(self, event: DockerContainerStateEvent) -> None: ContainerState.STOPPED, ContainerState.UNHEALTHY, ]: - await self._restart_after_problem(event.state, event.exit_code) + if self.watchdog_restart_policy == WatchdogRestartPolicy.UNLIMITED: + # Opted out of crash-loop protection: keep restarting forever. + await self._do_restart_after_problem( + event.state, event.exit_code, unlimited=True + ) + else: + await self._restart_after_problem(event.state, event.exit_code) async def refresh_path_cache(self) -> None: """Refresh cache of existing paths.""" diff --git a/supervisor/apps/const.py b/supervisor/apps/const.py index 21270ed8f06..a086ec40a59 100644 --- a/supervisor/apps/const.py +++ b/supervisor/apps/const.py @@ -39,6 +39,7 @@ class MappingType(StrEnum): ATTR_READ_ONLY = "read_only" ATTR_PATH = "path" WATCHDOG_RETRY_SECONDS = 10 +WATCHDOG_MAX_RETRY_SECONDS = 300 WATCHDOG_MAX_ATTEMPTS = 5 WATCHDOG_THROTTLE_PERIOD = timedelta(minutes=30) WATCHDOG_THROTTLE_MAX_CALLS = 10 diff --git a/supervisor/apps/model.py b/supervisor/apps/model.py index 3a1846ed051..7b7be8cc9f4 100644 --- a/supervisor/apps/model.py +++ b/supervisor/apps/model.py @@ -77,6 +77,7 @@ ATTR_VERSION_TIMESTAMP, ATTR_VIDEO, ATTR_WATCHDOG, + ATTR_WATCHDOG_RESTART_POLICY, ATTR_WEBUI, MACHINE_DEPRECATED, SECURITY_DEFAULT, @@ -87,6 +88,7 @@ AppStage, AppStartup, CpuArch, + WatchdogRestartPolicy, ) from ..coresys import CoreSys from ..docker.const import Capabilities @@ -308,6 +310,11 @@ def watchdog_url(self) -> str | None: """Return URL to for watchdog or None.""" return self.data.get(ATTR_WATCHDOG) + @property + def watchdog_restart_policy(self) -> WatchdogRestartPolicy: + """Return how the watchdog should restart the app after a failure.""" + return WatchdogRestartPolicy(self.data[ATTR_WATCHDOG_RESTART_POLICY]) + @property def ingress_port(self) -> int | None: """Return Ingress port.""" diff --git a/supervisor/apps/validate.py b/supervisor/apps/validate.py index 4d0a12e986d..69b042a51a8 100644 --- a/supervisor/apps/validate.py +++ b/supervisor/apps/validate.py @@ -96,6 +96,7 @@ ATTR_VERSION, ATTR_VIDEO, ATTR_WATCHDOG, + ATTR_WATCHDOG_RESTART_POLICY, ATTR_WEBUI, INGRESS_DYNAMIC_PORT_MAX, INGRESS_DYNAMIC_PORT_MIN, @@ -107,6 +108,7 @@ AppStage, AppStartup, AppState, + WatchdogRestartPolicy, ) from ..docker.const import Capabilities from ..validate import ( @@ -457,6 +459,9 @@ def _migrate(config: dict[str, Any]): vol.Optional(ATTR_WATCHDOG): vol.Match( r"^(?:https?|\[PROTO:\w+\]|tcp):\/\/\[HOST\]:(\[PORT:\d+\]|\d+).*$" ), + vol.Optional( + ATTR_WATCHDOG_RESTART_POLICY, default=WatchdogRestartPolicy.RATE_LIMITED + ): vol.Coerce(WatchdogRestartPolicy), vol.Optional(ATTR_WEBUI): vol.Match( r"^(?:https?|\[PROTO:\w+\]):\/\/\[HOST\]:\[PORT:\d+\].*$" ), diff --git a/supervisor/const.py b/supervisor/const.py index 4e5ffc6a3dd..2f11caa018d 100644 --- a/supervisor/const.py +++ b/supervisor/const.py @@ -398,6 +398,7 @@ ATTR_VPN = "vpn" ATTR_WAIT_BOOT = "wait_boot" ATTR_WATCHDOG = "watchdog" +ATTR_WATCHDOG_RESTART_POLICY = "watchdog_restart_policy" ATTR_WEBUI = "webui" ATTR_WIFI = "wifi" @@ -495,6 +496,13 @@ class AppStage(StrEnum): DEPRECATED = "deprecated" +class WatchdogRestartPolicy(StrEnum): + """How the watchdog restarts an app after it fails.""" + + RATE_LIMITED = "rate_limited" + UNLIMITED = "unlimited" + + class AppState(StrEnum): """State of app.""" diff --git a/tests/apps/test_app.py b/tests/apps/test_app.py index 2400f5c1c1e..4836121f36e 100644 --- a/tests/apps/test_app.py +++ b/tests/apps/test_app.py @@ -16,15 +16,21 @@ from securetar import SecureTarArchive, SecureTarFile from supervisor.apps.app import App -from supervisor.apps.const import AppBackupMode +from supervisor.apps.const import ( + WATCHDOG_MAX_ATTEMPTS, + WATCHDOG_THROTTLE_MAX_CALLS, + AppBackupMode, +) from supervisor.apps.model import AppModel from supervisor.const import ( ATTR_ADVANCED, ATTR_LOCATION, ATTR_PORTS, + ATTR_WATCHDOG_RESTART_POLICY, AppBoot, AppState, BusEvent, + WatchdogRestartPolicy, ) from supervisor.coresys import CoreSys from supervisor.docker.app import DockerApp @@ -35,6 +41,7 @@ AppFileReadError, AppPortConflict, AppPrePostBackupCommandReturnedError, + AppsError, AppsJobError, AppUnknownError, AudioUpdateError, @@ -418,6 +425,121 @@ async def test_watchdog_during_attach( assert restart.call_count == restart_count +async def test_watchdog_restart_policy_config(install_app_ssh: App) -> None: + """Test watchdog restart policy defaults to rate limited and reads config.""" + assert ( + install_app_ssh.watchdog_restart_policy == WatchdogRestartPolicy.RATE_LIMITED + ) + + install_app_ssh.data[ATTR_WATCHDOG_RESTART_POLICY] = WatchdogRestartPolicy.UNLIMITED + assert install_app_ssh.watchdog_restart_policy == WatchdogRestartPolicy.UNLIMITED + + +async def test_watchdog_restart_unlimited_ignores_rate_limit( + coresys: CoreSys, install_app_ssh: App +) -> None: + """Test unlimited restart policy keeps restarting past the crash-loop rate limit.""" + with patch.object(DockerApp, "attach"): + await install_app_ssh.load() + + install_app_ssh.watchdog = True + install_app_ssh._manual_stop = False # pylint: disable=protected-access + install_app_ssh.data[ATTR_WATCHDOG_RESTART_POLICY] = WatchdogRestartPolicy.UNLIMITED + + # Fire more failures than the rate limit would ever allow + events = WATCHDOG_THROTTLE_MAX_CALLS + 5 + + # Watchdog does ``await (await self.start())`` because App.start returns + # an asyncio.Task. The mock must mirror that shape. + done_task = asyncio.get_running_loop().create_future() + done_task.set_result(None) + with ( + patch.object(App, "start", AsyncMock(return_value=done_task)) as start, + patch.object(DockerApp, "current_state", return_value=ContainerState.FAILED), + patch.object(DockerApp, "stop"), + ): + for _ in range(events): + await _fire_test_event( + coresys, + f"addon_{TEST_ADDON_SLUG}", + ContainerState.FAILED, + exit_code=1, + ) + + # Every failure was reanimated; the rate limit never kicked in + assert start.call_count == events + + +async def test_watchdog_rate_limited_gives_up( + coresys: CoreSys, install_app_ssh: App +) -> None: + """Test the default rate-limited policy stops restarting once the crash loop trips.""" + with patch.object(DockerApp, "attach"): + await install_app_ssh.load() + + install_app_ssh.watchdog = True + install_app_ssh._manual_stop = False # pylint: disable=protected-access + assert ( + install_app_ssh.watchdog_restart_policy == WatchdogRestartPolicy.RATE_LIMITED + ) + + events = WATCHDOG_THROTTLE_MAX_CALLS + 5 + + done_task = asyncio.get_running_loop().create_future() + done_task.set_result(None) + with ( + patch.object(App, "start", AsyncMock(return_value=done_task)) as start, + patch.object(DockerApp, "current_state", return_value=ContainerState.FAILED), + patch.object(DockerApp, "stop"), + ): + for _ in range(events): + await _fire_test_event( + coresys, + f"addon_{TEST_ADDON_SLUG}", + ContainerState.FAILED, + exit_code=1, + ) + + # The watchdog gave up: it restarted at most up to the rate limit, not once per failure + assert start.call_count <= WATCHDOG_THROTTLE_MAX_CALLS + assert start.call_count < events + + +async def test_watchdog_restart_unlimited_ignores_attempt_limit( + coresys: CoreSys, install_app_ssh: App +) -> None: + """Test unlimited policy keeps retrying a failing start past the attempt cap.""" + with patch.object(DockerApp, "attach"): + await install_app_ssh.load() + + install_app_ssh.watchdog = True + install_app_ssh._manual_stop = False # pylint: disable=protected-access + install_app_ssh.data[ATTR_WATCHDOG_RESTART_POLICY] = WatchdogRestartPolicy.UNLIMITED + + # Fail more times than the capped attempt count allows, then finally succeed + fail_count = WATCHDOG_MAX_ATTEMPTS + 3 + + done_task = asyncio.get_running_loop().create_future() + done_task.set_result(None) + start_results = [AppsError() for _ in range(fail_count)] + [done_task] + with ( + patch.object(App, "start", AsyncMock(side_effect=start_results)) as start, + patch.object(DockerApp, "current_state", return_value=ContainerState.FAILED), + patch.object(DockerApp, "stop"), + patch("supervisor.apps.app.WATCHDOG_RETRY_SECONDS", 0), + patch("supervisor.apps.app.async_capture_exception"), + ): + await _fire_test_event( + coresys, + f"addon_{TEST_ADDON_SLUG}", + ContainerState.FAILED, + exit_code=1, + ) + + # It retried well past WATCHDOG_MAX_ATTEMPTS and only stopped once start succeeded + assert start.call_count == fail_count + 1 + + @pytest.mark.usefixtures("install_app_ssh") async def test_install_update_fails_if_out_of_date(coresys: CoreSys): """Test install or update of app fails when supervisor or plugin is out of date."""