diff --git a/lib/kamal/cli/app/boot.rb b/lib/kamal/cli/app/boot.rb index 5516761a8..bb74ec79c 100644 --- a/lib/kamal/cli/app/boot.rb +++ b/lib/kamal/cli/app/boot.rb @@ -54,7 +54,12 @@ def start_new_version if running_proxy? endpoint = capture_with_info(*app.container_id_for_version(version)).strip raise Kamal::Cli::BootError, "Failed to get endpoint for #{role} on #{host}, did the container boot?" if endpoint.empty? - execute *app.deploy(target: endpoint) + begin + execute *app.deploy(target: endpoint) + rescue SSHKit::Command::Failed + show_health_check_response(endpoint) if role.proxy.healthcheck_debug? + raise + end else Kamal::Cli::Healthcheck::Poller.wait_for_healthy { capture_with_info(*app.status(version: version)) } end @@ -63,6 +68,13 @@ def start_new_version raise e end + def show_health_check_response(endpoint) + response = capture_with_info(*app.health_check_response(target: endpoint), raise_on_non_zero_exit: false) + error "Health check response:\n#{response}" if response.present? + rescue StandardError => e + info "Could not fetch health check response: #{e.class}: #{e.message}" + end + def stop_new_version execute *app.stop(version: version), raise_on_non_zero_exit: false end diff --git a/lib/kamal/commands/app/proxy.rb b/lib/kamal/commands/app/proxy.rb index ea5da2c00..ab785f73f 100644 --- a/lib/kamal/commands/app/proxy.rb +++ b/lib/kamal/commands/app/proxy.rb @@ -21,6 +21,16 @@ def remove_proxy_app_directory remove_directory config.proxy_boot.app_directory end + def health_check_response(target:) + health_check_path = role.proxy.proxy_config.dig("healthcheck", "path") || "/up" + app_port = role.proxy.app_port + + pipe \ + docker(:inspect, "--format", "'{{.NetworkSettings.Networks.kamal.IPAddress}}'", target), + [ :xargs, "-I{}", :curl, "-s", "-o", "-", "-w", "'\\n%{http_code}'", "--max-time", "5", + "http://{}:#{app_port}#{health_check_path}" ] + end + def create_ssl_directory make_directory(File.join(config.proxy_boot.tls_directory, role.name)) end diff --git a/lib/kamal/configuration/docs/proxy.yml b/lib/kamal/configuration/docs/proxy.yml index 87e428e3c..32d45810d 100644 --- a/lib/kamal/configuration/docs/proxy.yml +++ b/lib/kamal/configuration/docs/proxy.yml @@ -114,10 +114,15 @@ proxy: # the deploy timeout, with a 5-second timeout for each request. # # Once the app is up, the proxy will stop hitting the healthcheck endpoint. + # + # Set `debug` to true to log the HTTP status code and response body from an additional + # debug healthcheck request made after a deploy fails (this may include sensitive data), + # which can help diagnose why the container is unhealthy. healthcheck: interval: 3 path: /health timeout: 3 + debug: false # Buffering # diff --git a/lib/kamal/configuration/proxy.rb b/lib/kamal/configuration/proxy.rb index 3f2d83fad..719542542 100644 --- a/lib/kamal/configuration/proxy.rb +++ b/lib/kamal/configuration/proxy.rb @@ -67,6 +67,10 @@ def path_prefixes proxy_config["path_prefixes"] || proxy_config["path_prefix"]&.split(",") || [] end + def healthcheck_debug? + !!proxy_config.dig("healthcheck", "debug") + end + def deploy_options { host: hosts, diff --git a/test/commands/app_test.rb b/test/commands/app_test.rb index dac14ef50..344022ed9 100644 --- a/test/commands/app_test.rb +++ b/test/commands/app_test.rb @@ -599,6 +599,18 @@ class CommandsAppTest < ActiveSupport::TestCase new_command.remove_proxy_app_directory.join(" ") end + test "health_check_response" do + assert_equal \ + "docker inspect --format '{{.NetworkSettings.Networks.kamal.IPAddress}}' abc123 | xargs -I{} curl -s -o - -w '\\n%{http_code}' --max-time 5 http://{}:80/up", + new_command.health_check_response(target: "abc123").join(" ") + end + + test "health_check_response with custom path" do + assert_equal \ + "docker inspect --format '{{.NetworkSettings.Networks.kamal.IPAddress}}' abc123 | xargs -I{} curl -s -o - -w '\\n%{http_code}' --max-time 5 http://{}:80/healthz", + new_command(proxy: { "healthcheck" => { "path" => "/healthz" } }).health_check_response(target: "abc123").join(" ") + end + private def new_command(role: "web", host: "1.1.1.1", **additional_config) config = Kamal::Configuration.new(@config.merge(additional_config), destination: @destination, version: "999")