fix(deployer): decouple the restart backoff from the crash restart count

apps/deployer/lib/deployer/monitor.ex

@@ -18,6 +18,7 @@ defmodule Deployer.Monitor do
           language: String.t() | nil,
           status: :idle | :running | :starting,
           crash_restart_count: integer(),
+          consecutive_crash_count: integer(),
           force_restart_count: integer(),
           start_time: nil | integer(),
           timeout_app_ready: integer(),
@@ -33,6 +34,7 @@ defmodule Deployer.Monitor do
             language: nil,
             status: :idle,
             crash_restart_count: 0,
+            consecutive_crash_count: 0,
             force_restart_count: 0,
             start_time: nil,
             timeout_app_ready: nil,

apps/deployer/lib/deployer/monitor/application.ex

@@ -16,6 +16,7 @@ defmodule Deployer.Monitor.Application do
 
   @monitor_table "monitor-table"
   @new_deploy_topic "deployex::new_deploy"
+  @max_backoff_ms :timer.minutes(5)
 
   ### ==========================================================================
   ### Callback functions
@@ -137,7 +138,12 @@ defmodule Deployer.Monitor.Application do
 
     Engine.notify_application_running(sname)
 
-    {:noreply, update_non_blocking_state(%{state | status: :running})}
+    # NOTE: The application reached a stable state, so the backoff sequence starts
+    #       over on the next crash. The crash_restart_count is a lifetime total and
+    #       is never reset here.
+    state = %{state | status: :running, consecutive_crash_count: 0}
+
+    {:noreply, update_non_blocking_state(state)}
   end
 
   def handle_info({:check_running, _pid, _sname}, state) do
@@ -162,6 +168,9 @@ defmodule Deployer.Monitor.Application do
     # Update the number of crash restarts
     crash_restart_count = state.crash_restart_count + 1
 
+    # Crashes since the application was last seen running, it drives the backoff only
+    consecutive_crash_count = state.consecutive_crash_count + 1
+
     Foundation.Notifications.notify("crash_restart", %{
       node: Node.self(),
       sname: state.sname,
@@ -170,14 +179,16 @@ defmodule Deployer.Monitor.Application do
       crash_restart_count: crash_restart_count
     })
 
-    # Retry with backoff pattern
-    trigger_run_service(state.sname, 2 * crash_restart_count * 1000)
+    # Retry with backoff pattern, capped to avoid unbounded growth
+    backoff = min(2 * consecutive_crash_count * 1000, @max_backoff_ms)
+    trigger_run_service(state.sname, backoff)
 
     {:noreply,
      update_non_blocking_state(%{
        state
        | current_pid: nil,
-         crash_restart_count: crash_restart_count
+         crash_restart_count: crash_restart_count,
+         consecutive_crash_count: consecutive_crash_count
      })}
   end
 

apps/deployer/test/monitor_test.exs

@@ -680,6 +680,58 @@ defmodule Deployer.MonitorTest do
       assert :ok = MonitorApp.stop_service(name, sname)
     end
 
+    @tag :capture_log
+    test "consecutive_crash_count resets to 0 after application reports running", %{
+      elixir_name: name,
+      elixir_sname: sname,
+      ports: ports
+    } do
+      test_event_ref = make_ref()
+      test_pid_process = self()
+      os_pid = 123_456
+      FixtureFiles.create_bin_files(sname)
+
+      Deployer.StatusMock
+      |> stub(:current_version_map, fn ^sname ->
+        %Catalog.Version{version: "1.0.0"}
+      end)
+
+      Host.CommanderMock
+      |> stub(:run_link, fn _command, _options ->
+        Process.send_after(test_pid_process, {:handle_ref_event, test_event_ref}, 100)
+        {:ok, test_pid_process, os_pid}
+      end)
+      |> stub(:run, fn _commands, _options -> {:ok, test_pid_process} end)
+      |> stub(:stop, fn ^test_pid_process -> :ok end)
+
+      assert {:ok, pid} =
+               MonitorApp.start_service(%Service{
+                 name: name,
+                 sname: sname,
+                 language: "elixir",
+                 ports: ports,
+                 timeout_app_ready: 10
+               })
+
+      assert_receive {:handle_ref_event, ^test_event_ref}, 1_000
+
+      assert %{status: :running, crash_restart_count: 0, consecutive_crash_count: 0} =
+               MonitorApp.state(sname)
+
+      # Crash the application
+      send(pid, {:EXIT, test_pid_process, :forcing_restart})
+
+      # Wait for the backoff (2 * 1 * 1000 = 2000ms) plus margin for check_running
+      :timer.sleep(2_500)
+
+      # After the restart and check_running fires, the backoff counter resets while
+      # the lifetime crash_restart_count is preserved
+      assert %{status: :running, crash_restart_count: 1, consecutive_crash_count: 0} =
+               MonitorApp.state(sname)
+
+      assert :ok = MonitorApp.stop_service(name, sname)
+    end
+
     @tag :capture_log
     test "Don't restart Application if EXIT message is not valid", %{
       elixir_name: name,