diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index 1ed2773b6..faeaa9311 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -22,6 +22,8 @@ on: - 'tests/check-windows-appservice-normalization.ps1' - 'tests/check-apply-wrapper-flags.sh' - 'tests/check-helm-agentteams.sh' + - 'manager/scripts/init/start-manager-agent.sh' + - 'manager/tests/test-manager-startup-recovery.sh' workflow_dispatch: ~ jobs: @@ -53,6 +55,9 @@ jobs: - name: Check declarative apply wrapper flags run: bash tests/check-apply-wrapper-flags.sh + - name: Check Manager startup Worker recovery ownership + run: bash manager/tests/test-manager-startup-recovery.sh + - name: Set up Helm uses: azure/setup-helm@v4 with: diff --git a/agentteams-controller/test/integration/controller/worker_test.go b/agentteams-controller/test/integration/controller/worker_test.go index 5e3a04677..9bc48a3bc 100644 --- a/agentteams-controller/test/integration/controller/worker_test.go +++ b/agentteams-controller/test/integration/controller/worker_test.go @@ -359,6 +359,9 @@ func TestWorkerPodDeleted_Recreates(t *testing.T) { workerName := fixtures.UniqueName("test-pod-deleted") worker := fixtures.NewTestWorker(workerName) + worker.Spec.Env = map[string]string{ + "WORKER_SETTING": "preserve-me", + } if err := k8sClient.Create(ctx, worker); err != nil { t.Fatalf("failed to create Worker CR: %v", err) @@ -383,6 +386,14 @@ func TestWorkerPodDeleted_Recreates(t *testing.T) { // Phase should converge back to Running after recreation. waitForRunning(t, worker) + + recreated, ok := mockBackend.FindCreateReq(workerName) + if !ok { + t.Fatalf("no backend.Create request recorded for recreated worker %q", workerName) + } + if got := recreated.Env["WORKER_SETTING"]; got != "preserve-me" { + t.Errorf("recreated worker env WORKER_SETTING=%q, want %q", got, "preserve-me") + } } // --------------------------------------------------------------------------- diff --git a/manager/scripts/init/start-manager-agent.sh b/manager/scripts/init/start-manager-agent.sh index ae0df1bf7..f9e6ef9ec 100755 --- a/manager/scripts/init/start-manager-agent.sh +++ b/manager/scripts/init/start-manager-agent.sh @@ -914,11 +914,13 @@ if [ "${CMS_TRACES_ENABLED}" = "true" ]; then fi # ============================================================ -# Detect container runtime (for Worker creation) +# Detect container runtime for explicit Worker management commands. +# Worker recovery is owned by the controller reconciliation loop; this +# startup script must not create Worker resources. # ============================================================ source /opt/agentteams/scripts/lib/container-api.sh if container_api_available; then - log "Container runtime socket detected at ${CONTAINER_SOCKET} — direct Worker creation enabled" + log "Container runtime socket detected at ${CONTAINER_SOCKET} — Worker management enabled" export AGENTTEAMS_CONTAINER_RUNTIME="socket" elif [ "${AGENTTEAMS_RUNTIME}" = "aliyun" ] || [ "${AGENTTEAMS_RUNTIME}" = "k8s" ]; then log "Cloud/K8s mode — Workers created via controller API" @@ -928,61 +930,6 @@ else export AGENTTEAMS_CONTAINER_RUNTIME="none" fi -# ============================================================ -# Recreate Worker containers as needed after Manager restart. -# Workers are on agentteams-net; Docker DNS resolves *-local.agentteams.io via -# the Manager's network aliases, so IP changes don't require worker recreation. -# Only recreate stopped/missing workers. -# ============================================================ -if container_api_available; then - _workers_json=$(agt get workers -o json 2>/dev/null || echo '{"workers":[]}') - for _worker_name in $(echo "${_workers_json}" | jq -r '.workers[].name'); do - [ -z "${_worker_name}" ] && continue - - _status=$(container_status_worker "${_worker_name}") - if [ "${_status}" = "running" ]; then - log "Worker running: ${_worker_name}, skipping" - continue - fi - # Container missing or stopped — recreate. - log "Worker container ${_status}: ${_worker_name}, recreating..." - _creds_file="/data/worker-creds/${_worker_name}.env" - if [ -f "${_creds_file}" ]; then - source "${_creds_file}" - _runtime=$(echo "${_workers_json}" | jq -r --arg w "${_worker_name}" '.workers[] | select(.name == $w) | .runtime // "openclaw"') - _recreated=false - for _attempt in 1 2 3; do - _env_map="" - _create_body="" - _env_map=$(jq -cn \ - --arg name "${_worker_name}" \ - --arg fak "${_worker_name}" \ - --arg fsk "${WORKER_MINIO_PASSWORD:-}" \ - --arg fs_domain "${AGENTTEAMS_FS_DOMAIN:-fs-local.agentteams.io}" \ - --arg controller_url "${AGENTTEAMS_CONTROLLER_URL:-}" \ - '{ - "AGENTTEAMS_WORKER_NAME": $name, - "AGENTTEAMS_FS_ENDPOINT": ("http://" + ($fs_domain | split(":")[0]) + ":9000"), - "AGENTTEAMS_FS_ACCESS_KEY": $fak, - "AGENTTEAMS_FS_SECRET_KEY": $fsk - } - | if $controller_url != "" then . + {"AGENTTEAMS_CONTROLLER_URL": $controller_url} else . end') - _create_body=$(jq -cn --arg name "${_worker_name}" --arg runtime "${_runtime}" --argjson env "${_env_map}" '{name: $name, runtime: $runtime, env: $env}') - worker_backend_create "${_create_body}" > /dev/null 2>&1 && _recreated=true && break - log " Attempt ${_attempt}/3 failed for ${_worker_name}, retrying in $((5 * _attempt))s..." - sleep $((5 * _attempt)) - done - if [ "${_recreated}" = true ]; then - log " Recreated ${_runtime} worker: ${_worker_name}" - else - log " ERROR: Failed to recreate ${_worker_name} after 3 attempts" - fi - else - log " WARNING: No credentials found for ${_worker_name} (${_creds_file} missing), skipping" - fi - done -fi - # ============================================================ # Notify workers of builtin updates if upgrade happened # Builtin files (AGENTS.md, skills) are already synced by upgrade-builtins.sh diff --git a/manager/tests/test-manager-startup-recovery.sh b/manager/tests/test-manager-startup-recovery.sh new file mode 100644 index 000000000..de77f39bf --- /dev/null +++ b/manager/tests/test-manager-startup-recovery.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Regression tests for Manager startup Worker recovery ownership. + +set -uo pipefail + +PASS=0 +FAIL=0 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +STARTUP_SCRIPT="${PROJECT_ROOT}/manager/scripts/init/start-manager-agent.sh" + +pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } +fail() { echo " FAIL: $1"; echo " expected: $2"; echo " got: $3"; FAIL=$((FAIL + 1)); } + +assert_contains() { + local desc="$1" needle="$2" haystack="$3" + if printf '%s\n' "${haystack}" | grep -qF -- "${needle}"; then + pass "${desc}" + else + fail "${desc}" "contains '${needle}'" "not found" + fi +} + +assert_not_contains() { + local desc="$1" needle="$2" haystack="$3" + if ! printf '%s\n' "${haystack}" | grep -qF -- "${needle}"; then + pass "${desc}" + else + fail "${desc}" "does not contain '${needle}'" "found '${needle}'" + fi +} + +startup_source="$(<"${STARTUP_SCRIPT}")" + +echo "=== Manager startup Worker recovery ownership ===" +assert_contains "startup still detects the configured container runtime" \ + 'export AGENTTEAMS_CONTAINER_RUNTIME=' "${startup_source}" +assert_contains "startup still loads the Worker management API helpers" \ + 'source /opt/agentteams/scripts/lib/container-api.sh' "${startup_source}" +assert_not_contains "startup no longer creates Worker CRs during recovery" \ + 'worker_backend_create' "${startup_source}" +assert_not_contains "startup no longer enumerates Workers for recovery" \ + 'Recreate Worker containers as needed after Manager restart.' "${startup_source}" + +if bash -n "${STARTUP_SCRIPT}"; then + pass "startup script has valid Bash syntax" +else + fail "startup script has valid Bash syntax" "bash -n succeeds" "bash -n failed" +fi + +if [ "${FAIL}" -eq 0 ]; then + echo "All ${PASS} tests passed." + exit 0 +fi + +echo "${FAIL} of $((PASS + FAIL)) tests failed." +exit 1