Allow the watchdog to continue restarting an app indefinitely

supervisor/apps/app.py

@@ -57,6 +57,7 @@
     AppStartup,
     AppState,
     BusEvent,
+    WatchdogRestartPolicy,
 )
 from ..coresys import CoreSys
 from ..docker.app import DockerApp
@@ -101,6 +102,7 @@
 from ..utils.sentry import async_capture_exception
 from .const import (
     WATCHDOG_MAX_ATTEMPTS,
+    WATCHDOG_MAX_RETRY_SECONDS,
     WATCHDOG_RETRY_SECONDS,
     WATCHDOG_THROTTLE_MAX_CALLS,
     WATCHDOG_THROTTLE_PERIOD,
@@ -1758,7 +1760,21 @@ def _restore_data():
     async def _restart_after_problem(
         self, state: ContainerState, exit_code: int | None = None
     ):
-        """Restart unhealthy or failed app."""
+        """Restart unhealthy or failed app, rate-limited to break crash loops."""
+        await self._do_restart_after_problem(state, exit_code)
+
+    async def _do_restart_after_problem(
+        self,
+        state: ContainerState,
+        exit_code: int | None = None,
+        *,
+        unlimited: bool = False,
+    ) -> None:
+        """Restart an unhealthy or failed app.
+
+        When unlimited is True the per-invocation attempt cap is lifted so the
+        watchdog keeps retrying forever (WatchdogRestartPolicy.UNLIMITED).
+        """
         attempts = 0
         while await self.instance.current_state() == state:
             if not self.in_progress:
@@ -1796,16 +1812,21 @@ async def _restart_after_problem(
                 else:
                     break
 
-            if attempts >= WATCHDOG_MAX_ATTEMPTS:
+            if not unlimited and attempts >= WATCHDOG_MAX_ATTEMPTS:
                 _LOGGER.critical(
                     "Watchdog cannot restart app %s, failed all %s attempts",
                     self.name,
                     attempts,
                 )
                 break
 
-            # Exponential backoff to spread retries over the throttle window
-            delay = WATCHDOG_RETRY_SECONDS * (1 << max(attempts - 1, 0))
+            # Exponential backoff, capped so unlimited retries settle at a steady
+            # interval instead of growing without bound. The shift bound only keeps
+            # the intermediate value small; WATCHDOG_MAX_RETRY_SECONDS is the real cap.
+            delay = min(
+                WATCHDOG_RETRY_SECONDS * (1 << min(max(attempts - 1, 0), 8)),
+                WATCHDOG_MAX_RETRY_SECONDS,
+            )
             _LOGGER.debug(
                 "Watchdog will retry app %s in %s seconds (attempt %s)",
                 self.name,
@@ -1855,7 +1876,13 @@ async def watchdog_container(self, event: DockerContainerStateEvent) -> None:
             ContainerState.STOPPED,
             ContainerState.UNHEALTHY,
         ]:
-            await self._restart_after_problem(event.state, event.exit_code)
+            if self.watchdog_restart_policy == WatchdogRestartPolicy.UNLIMITED:
+                # Opted out of crash-loop protection: keep restarting forever.
+                await self._do_restart_after_problem(
+                    event.state, event.exit_code, unlimited=True
+                )
+            else:
+                await self._restart_after_problem(event.state, event.exit_code)
 
     async def refresh_path_cache(self) -> None:
         """Refresh cache of existing paths."""

supervisor/apps/const.py

@@ -39,6 +39,7 @@ class MappingType(StrEnum):
 ATTR_READ_ONLY = "read_only"
 ATTR_PATH = "path"
 WATCHDOG_RETRY_SECONDS = 10
+WATCHDOG_MAX_RETRY_SECONDS = 300
 WATCHDOG_MAX_ATTEMPTS = 5
 WATCHDOG_THROTTLE_PERIOD = timedelta(minutes=30)
 WATCHDOG_THROTTLE_MAX_CALLS = 10

supervisor/apps/model.py

@@ -77,6 +77,7 @@
     ATTR_VERSION_TIMESTAMP,
     ATTR_VIDEO,
     ATTR_WATCHDOG,
+    ATTR_WATCHDOG_RESTART_POLICY,
     ATTR_WEBUI,
     MACHINE_DEPRECATED,
     SECURITY_DEFAULT,
@@ -87,6 +88,7 @@
     AppStage,
     AppStartup,
     CpuArch,
+    WatchdogRestartPolicy,
 )
 from ..coresys import CoreSys
 from ..docker.const import Capabilities
@@ -308,6 +310,11 @@ def watchdog_url(self) -> str | None:
         """Return URL to for watchdog or None."""
         return self.data.get(ATTR_WATCHDOG)
 
+    @property
+    def watchdog_restart_policy(self) -> WatchdogRestartPolicy:
+        """Return how the watchdog should restart the app after a failure."""
+        return WatchdogRestartPolicy(self.data[ATTR_WATCHDOG_RESTART_POLICY])
+
     @property
     def ingress_port(self) -> int | None:
         """Return Ingress port."""

supervisor/apps/validate.py

@@ -96,6 +96,7 @@
     ATTR_VERSION,
     ATTR_VIDEO,
     ATTR_WATCHDOG,
+    ATTR_WATCHDOG_RESTART_POLICY,
     ATTR_WEBUI,
     INGRESS_DYNAMIC_PORT_MAX,
     INGRESS_DYNAMIC_PORT_MIN,
@@ -107,6 +108,7 @@
     AppStage,
     AppStartup,
     AppState,
+    WatchdogRestartPolicy,
 )
 from ..docker.const import Capabilities
 from ..validate import (
@@ -457,6 +459,9 @@ def _migrate(config: dict[str, Any]):
         vol.Optional(ATTR_WATCHDOG): vol.Match(
             r"^(?:https?|\[PROTO:\w+\]|tcp):\/\/\[HOST\]:(\[PORT:\d+\]|\d+).*$"
         ),
+        vol.Optional(
+            ATTR_WATCHDOG_RESTART_POLICY, default=WatchdogRestartPolicy.RATE_LIMITED
+        ): vol.Coerce(WatchdogRestartPolicy),
         vol.Optional(ATTR_WEBUI): vol.Match(
             r"^(?:https?|\[PROTO:\w+\]):\/\/\[HOST\]:\[PORT:\d+\].*$"
         ),

supervisor/const.py

@@ -398,6 +398,7 @@
 ATTR_VPN = "vpn"
 ATTR_WAIT_BOOT = "wait_boot"
 ATTR_WATCHDOG = "watchdog"
+ATTR_WATCHDOG_RESTART_POLICY = "watchdog_restart_policy"
 ATTR_WEBUI = "webui"
 ATTR_WIFI = "wifi"
 
@@ -495,6 +496,13 @@ class AppStage(StrEnum):
     DEPRECATED = "deprecated"
 
 
+class WatchdogRestartPolicy(StrEnum):
+    """How the watchdog restarts an app after it fails."""
+
+    RATE_LIMITED = "rate_limited"
+    UNLIMITED = "unlimited"
+
+
 class AppState(StrEnum):
     """State of app."""
 

tests/apps/test_app.py

@@ -16,15 +16,21 @@
 from securetar import SecureTarArchive, SecureTarFile
 
 from supervisor.apps.app import App
-from supervisor.apps.const import AppBackupMode
+from supervisor.apps.const import (
+    WATCHDOG_MAX_ATTEMPTS,
+    WATCHDOG_THROTTLE_MAX_CALLS,
+    AppBackupMode,
+)
 from supervisor.apps.model import AppModel
 from supervisor.const import (
     ATTR_ADVANCED,
     ATTR_LOCATION,
     ATTR_PORTS,
+    ATTR_WATCHDOG_RESTART_POLICY,
     AppBoot,
     AppState,
     BusEvent,
+    WatchdogRestartPolicy,
 )
 from supervisor.coresys import CoreSys
 from supervisor.docker.app import DockerApp
@@ -35,6 +41,7 @@
     AppFileReadError,
     AppPortConflict,
     AppPrePostBackupCommandReturnedError,
+    AppsError,
     AppsJobError,
     AppUnknownError,
     AudioUpdateError,
@@ -418,6 +425,121 @@
         assert restart.call_count == restart_count
 
 
+async def test_watchdog_restart_policy_config(install_app_ssh: App) -> None:
+    """Test watchdog restart policy defaults to rate limited and reads config."""
+    assert (
+        install_app_ssh.watchdog_restart_policy == WatchdogRestartPolicy.RATE_LIMITED
+    )
+
+    install_app_ssh.data[ATTR_WATCHDOG_RESTART_POLICY] = WatchdogRestartPolicy.UNLIMITED
+    assert install_app_ssh.watchdog_restart_policy == WatchdogRestartPolicy.UNLIMITED
+
+
+async def test_watchdog_restart_unlimited_ignores_rate_limit(
+    coresys: CoreSys, install_app_ssh: App
+) -> None:
+    """Test unlimited restart policy keeps restarting past the crash-loop rate limit."""
+    with patch.object(DockerApp, "attach"):
+        await install_app_ssh.load()
+
+    install_app_ssh.watchdog = True
+    install_app_ssh._manual_stop = False  # pylint: disable=protected-access
+    install_app_ssh.data[ATTR_WATCHDOG_RESTART_POLICY] = WatchdogRestartPolicy.UNLIMITED
+
+    # Fire more failures than the rate limit would ever allow
+    events = WATCHDOG_THROTTLE_MAX_CALLS + 5
+
+    # Watchdog does ``await (await self.start())`` because App.start returns
+    # an asyncio.Task. The mock must mirror that shape.
+    done_task = asyncio.get_running_loop().create_future()
+    done_task.set_result(None)
+    with (
+        patch.object(App, "start", AsyncMock(return_value=done_task)) as start,
+        patch.object(DockerApp, "current_state", return_value=ContainerState.FAILED),
+        patch.object(DockerApp, "stop"),
+    ):
+        for _ in range(events):
+            await _fire_test_event(
+                coresys,
+                f"addon_{TEST_ADDON_SLUG}",
+                ContainerState.FAILED,
+                exit_code=1,
+            )
+
+    # Every failure was reanimated; the rate limit never kicked in
+    assert start.call_count == events
+
+
+async def test_watchdog_rate_limited_gives_up(
+    coresys: CoreSys, install_app_ssh: App
+) -> None:
+    """Test the default rate-limited policy stops restarting once the crash loop trips."""
+    with patch.object(DockerApp, "attach"):
+        await install_app_ssh.load()
+
+    install_app_ssh.watchdog = True
+    install_app_ssh._manual_stop = False  # pylint: disable=protected-access
+    assert (
+        install_app_ssh.watchdog_restart_policy == WatchdogRestartPolicy.RATE_LIMITED
+    )
+
+    events = WATCHDOG_THROTTLE_MAX_CALLS + 5
+
+    done_task = asyncio.get_running_loop().create_future()
+    done_task.set_result(None)
+    with (
+        patch.object(App, "start", AsyncMock(return_value=done_task)) as start,
+        patch.object(DockerApp, "current_state", return_value=ContainerState.FAILED),
+        patch.object(DockerApp, "stop"),
+    ):
+        for _ in range(events):
+            await _fire_test_event(
+                coresys,
+                f"addon_{TEST_ADDON_SLUG}",
+                ContainerState.FAILED,
+                exit_code=1,
+            )
+
+    # The watchdog gave up: it restarted at most up to the rate limit, not once per failure
+    assert start.call_count <= WATCHDOG_THROTTLE_MAX_CALLS
+    assert start.call_count < events
+
+
+async def test_watchdog_restart_unlimited_ignores_attempt_limit(
+    coresys: CoreSys, install_app_ssh: App
+) -> None:
+    """Test unlimited policy keeps retrying a failing start past the attempt cap."""
+    with patch.object(DockerApp, "attach"):
+        await install_app_ssh.load()
+
+    install_app_ssh.watchdog = True
+    install_app_ssh._manual_stop = False  # pylint: disable=protected-access
+    install_app_ssh.data[ATTR_WATCHDOG_RESTART_POLICY] = WatchdogRestartPolicy.UNLIMITED
+
+    # Fail more times than the capped attempt count allows, then finally succeed
+    fail_count = WATCHDOG_MAX_ATTEMPTS + 3
+
+    done_task = asyncio.get_running_loop().create_future()
+    done_task.set_result(None)
+    start_results = [AppsError() for _ in range(fail_count)] + [done_task]
+    with (
+        patch.object(App, "start", AsyncMock(side_effect=start_results)) as start,
+        patch.object(DockerApp, "current_state", return_value=ContainerState.FAILED),
+        patch.object(DockerApp, "stop"),
+        patch("supervisor.apps.app.WATCHDOG_RETRY_SECONDS", 0),
+        patch("supervisor.apps.app.async_capture_exception"),
+    ):
+        await _fire_test_event(
+            coresys,
+            f"addon_{TEST_ADDON_SLUG}",
+            ContainerState.FAILED,
+            exit_code=1,
+        )
+
+    # It retried well past WATCHDOG_MAX_ATTEMPTS and only stopped once start succeeded
+    assert start.call_count == fail_count + 1
+
+
 @pytest.mark.usefixtures("install_app_ssh")
 async def test_install_update_fails_if_out_of_date(coresys: CoreSys):
     """Test install or update of app fails when supervisor or plugin is out of date."""