From 43965607c9078db8bfe5a73f8826a9e2e9b76d07 Mon Sep 17 00:00:00 2001 From: Matthias Zirnstein Date: Sun, 12 Jul 2026 19:49:34 +0200 Subject: [PATCH 1/5] feat: Show health check result on failure Before kamal-proxy switches traffic to a new release, it performs a health check. If the health check fails, Kamal automatically rolls back the newly deployed container. Once the rollback has completed, it is no longer possible to manually run a curl command against the failed container to inspect the health check response and understand what went wrong. To improve debugging, we execute a curl health check while the failed container still exists and log both the HTTP status code and the response body in the Kamal deployment logs. This has been extremely helpful when diagnosing deployment issues caused by misconfigurations, especially in GitHub Actions, where reproducing the failed state can be difficult. --- lib/kamal/cli/app/boot.rb | 13 ++++++++++++- lib/kamal/commands/app/proxy.rb | 10 ++++++++++ lib/kamal/configuration/docs/proxy.yml | 4 ++++ lib/kamal/configuration/proxy.rb | 4 ++++ test/commands/app_test.rb | 12 ++++++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) diff --git a/lib/kamal/cli/app/boot.rb b/lib/kamal/cli/app/boot.rb index 5516761a8..6a5b89d10 100644 --- a/lib/kamal/cli/app/boot.rb +++ b/lib/kamal/cli/app/boot.rb @@ -54,7 +54,12 @@ def start_new_version if running_proxy? endpoint = capture_with_info(*app.container_id_for_version(version)).strip raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty? - execute *app.deploy(target: endpoint) + begin + execute *app.deploy(target: endpoint) + rescue SSHKit::Command::Failed + show_health_check_response(endpoint) if role.proxy.healthcheck_debug? + raise + end else Kamal::Cli::Healthcheck::Poller.wait_for_healthy { capture_with_info(*app.status(version: version)) } end @@ -63,6 +68,12 @@ def start_new_version raise e end + def show_health_check_response(endpoint) + response = capture_with_info(*app.health_check_response(target: endpoint), raise_on_non_zero_exit: false) + error "Health check response:\n#{response}" if response.present? + rescue StandardError + end + def stop_new_version execute *app.stop(version: version), raise_on_non_zero_exit: false end diff --git a/lib/kamal/commands/app/proxy.rb b/lib/kamal/commands/app/proxy.rb index ea5da2c00..ab785f73f 100644 --- a/lib/kamal/commands/app/proxy.rb +++ b/lib/kamal/commands/app/proxy.rb @@ -21,6 +21,16 @@ def remove_proxy_app_directory remove_directory config.proxy_boot.app_directory end + def health_check_response(target:) + health_check_path = role.proxy.proxy_config.dig("healthcheck", "path") || "/up" + app_port = role.proxy.app_port + + pipe \ + docker(:inspect, "--format", "'{{.NetworkSettings.Networks.kamal.IPAddress}}'", target), + [ :xargs, "-I{}", :curl, "-s", "-o", "-", "-w", "'\\n%{http_code}'", "--max-time", "5", + "http://{}:#{app_port}#{health_check_path}" ] + end + def create_ssl_directory make_directory(File.join(config.proxy_boot.tls_directory, role.name)) end diff --git a/lib/kamal/configuration/docs/proxy.yml b/lib/kamal/configuration/docs/proxy.yml index 87e428e3c..9042f88bc 100644 --- a/lib/kamal/configuration/docs/proxy.yml +++ b/lib/kamal/configuration/docs/proxy.yml @@ -114,10 +114,14 @@ proxy: # the deploy timeout, with a 5-second timeout for each request. # # Once the app is up, the proxy will stop hitting the healthcheck endpoint. + # + # Set `debug` to true to log the body of the last failed healthcheck response + # when a deploy fails, which can help diagnose why the container is unhealthy. healthcheck: interval: 3 path: /health timeout: 3 + debug: false # Buffering # diff --git a/lib/kamal/configuration/proxy.rb b/lib/kamal/configuration/proxy.rb index 3f2d83fad..719542542 100644 --- a/lib/kamal/configuration/proxy.rb +++ b/lib/kamal/configuration/proxy.rb @@ -67,6 +67,10 @@ def path_prefixes proxy_config["path_prefixes"] || proxy_config["path_prefix"]&.split(",") || [] end + def healthcheck_debug? + !!proxy_config.dig("healthcheck", "debug") + end + def deploy_options { host: hosts, diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index dac14ef50..344022ed9 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -599,6 +599,18 @@ class CommandsAppTest < ActiveSupport::TestCase new_command.remove_proxy_app_directory.join(" ") end + test "health_check_response" do + assert_equal \ + "docker inspect --format '{{.NetworkSettings.Networks.kamal.IPAddress}}' abc123 | xargs -I{} curl -s -o - -w '\\n%{http_code}' --max-time 5 http://{}:80/up", + new_command.health_check_response(target: "abc123").join(" ") + end + + test "health_check_response with custom path" do + assert_equal \ + "docker inspect --format '{{.NetworkSettings.Networks.kamal.IPAddress}}' abc123 | xargs -I{} curl -s -o - -w '\\n%{http_code}' --max-time 5 http://{}:80/healthz", + new_command(proxy: { "healthcheck" => { "path" => "/healthz" } }).health_check_response(target: "abc123").join(" ") + end + private def new_command(role: "web", host: "1.1.1.1", **additional_config) config = Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999") From d26b121d3b7edec05911a56848a4b628fb6da1de Mon Sep 17 00:00:00 2001 From: Matthias Zirnstein Date: Mon, 27 Jul 2026 07:24:25 +0200 Subject: [PATCH 2/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- lib/kamal/cli/app/boot.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/kamal/cli/app/boot.rb b/lib/kamal/cli/app/boot.rb index 6a5b89d10..bb74ec79c 100644 --- a/lib/kamal/cli/app/boot.rb +++ b/lib/kamal/cli/app/boot.rb @@ -71,7 +71,8 @@ def start_new_version def show_health_check_response(endpoint) response = capture_with_info(*app.health_check_response(target: endpoint), raise_on_non_zero_exit: false) error "Health check response:\n#{response}" if response.present? - rescue StandardError + rescue StandardError => e + info "Could not fetch health check response: #{e.class}: #{e.message}" end def stop_new_version From 9d88c38adce4d7fbcc9336eb61a9b449f7f4d1d4 Mon Sep 17 00:00:00 2001 From: Matthias Zirnstein Date: Mon, 27 Jul 2026 14:35:16 +0200 Subject: [PATCH 3/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- lib/kamal/configuration/docs/proxy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/kamal/configuration/docs/proxy.yml b/lib/kamal/configuration/docs/proxy.yml index 9042f88bc..a68ee8975 100644 --- a/lib/kamal/configuration/docs/proxy.yml +++ b/lib/kamal/configuration/docs/proxy.yml @@ -115,8 +115,8 @@ proxy: # # Once the app is up, the proxy will stop hitting the healthcheck endpoint. # - # Set `debug` to true to log the body of the last failed healthcheck response - # when a deploy fails, which can help diagnose why the container is unhealthy. + # Set `debug` to true to log the HTTP status code and response body of the last failed healthcheck + # when a deploy fails (this may include sensitive data), which can help diagnose why the container is unhealthy. healthcheck: interval: 3 path: /health From adaaa2ebb5297cd64965b662150561f52ef29145 Mon Sep 17 00:00:00 2001 From: Matthias Zirnstein Date: Fri, 31 Jul 2026 20:04:57 +0200 Subject: [PATCH 4/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- lib/kamal/configuration/docs/proxy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/kamal/configuration/docs/proxy.yml b/lib/kamal/configuration/docs/proxy.yml index a68ee8975..22b6b0633 100644 --- a/lib/kamal/configuration/docs/proxy.yml +++ b/lib/kamal/configuration/docs/proxy.yml @@ -115,9 +115,9 @@ proxy: # # Once the app is up, the proxy will stop hitting the healthcheck endpoint. # - # Set `debug` to true to log the HTTP status code and response body of the last failed healthcheck - # when a deploy fails (this may include sensitive data), which can help diagnose why the container is unhealthy. - healthcheck: + # Set `debug` to true to log the HTTP status code and response body from an additional + # debug healthcheck request made after a deploy fails (this may include sensitive data), + # which can help diagnose why the container is unhealthy. interval: 3 path: /health timeout: 3 From d34a88c1545b53493b6ad14bf9d3c4cd6e218cd2 Mon Sep 17 00:00:00 2001 From: Matthias Zirnstein Date: Sat, 1 Aug 2026 22:01:02 +0200 Subject: [PATCH 5/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- lib/kamal/configuration/docs/proxy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/kamal/configuration/docs/proxy.yml b/lib/kamal/configuration/docs/proxy.yml index 22b6b0633..32d45810d 100644 --- a/lib/kamal/configuration/docs/proxy.yml +++ b/lib/kamal/configuration/docs/proxy.yml @@ -118,6 +118,7 @@ proxy: # Set `debug` to true to log the HTTP status code and response body from an additional # debug healthcheck request made after a deploy fails (this may include sensitive data), # which can help diagnose why the container is unhealthy. + healthcheck: interval: 3 path: /health timeout: 3