Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/helm-lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ on:
- 'tests/check-windows-appservice-normalization.ps1'
- 'tests/check-apply-wrapper-flags.sh'
- 'tests/check-helm-agentteams.sh'
- 'manager/scripts/init/start-manager-agent.sh'
- 'manager/tests/test-manager-startup-recovery.sh'
workflow_dispatch: ~

jobs:
Expand Down Expand Up @@ -53,6 +55,9 @@ jobs:
- name: Check declarative apply wrapper flags
run: bash tests/check-apply-wrapper-flags.sh

- name: Check Manager startup Worker recovery ownership
run: bash manager/tests/test-manager-startup-recovery.sh

- name: Set up Helm
uses: azure/setup-helm@v4
with:
Expand Down
11 changes: 11 additions & 0 deletions agentteams-controller/test/integration/controller/worker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,9 @@ func TestWorkerPodDeleted_Recreates(t *testing.T) {

workerName := fixtures.UniqueName("test-pod-deleted")
worker := fixtures.NewTestWorker(workerName)
worker.Spec.Env = map[string]string{
"WORKER_SETTING": "preserve-me",
}

if err := k8sClient.Create(ctx, worker); err != nil {
t.Fatalf("failed to create Worker CR: %v", err)
Expand All @@ -383,6 +386,14 @@ func TestWorkerPodDeleted_Recreates(t *testing.T) {

// Phase should converge back to Running after recreation.
waitForRunning(t, worker)

recreated, ok := mockBackend.FindCreateReq(workerName)
if !ok {
t.Fatalf("no backend.Create request recorded for recreated worker %q", workerName)
}
if got := recreated.Env["WORKER_SETTING"]; got != "preserve-me" {
t.Errorf("recreated worker env WORKER_SETTING=%q, want %q", got, "preserve-me")
}
}

// ---------------------------------------------------------------------------
Expand Down
61 changes: 4 additions & 57 deletions manager/scripts/init/start-manager-agent.sh
Original file line number Diff line number Diff line change
Expand Up @@ -914,11 +914,13 @@ if [ "${CMS_TRACES_ENABLED}" = "true" ]; then
fi

# ============================================================
# Detect container runtime (for Worker creation)
# Detect container runtime for explicit Worker management commands.
# Worker recovery is owned by the controller reconciliation loop; this
# startup script must not create Worker resources.
# ============================================================
source /opt/agentteams/scripts/lib/container-api.sh
if container_api_available; then
log "Container runtime socket detected at ${CONTAINER_SOCKET} — direct Worker creation enabled"
log "Container runtime socket detected at ${CONTAINER_SOCKET} — Worker management enabled"
export AGENTTEAMS_CONTAINER_RUNTIME="socket"
elif [ "${AGENTTEAMS_RUNTIME}" = "aliyun" ] || [ "${AGENTTEAMS_RUNTIME}" = "k8s" ]; then
log "Cloud/K8s mode — Workers created via controller API"
Expand All @@ -928,61 +930,6 @@ else
export AGENTTEAMS_CONTAINER_RUNTIME="none"
fi

# ============================================================
# Recreate Worker containers as needed after Manager restart.
# Workers are on agentteams-net; Docker DNS resolves *-local.agentteams.io via
# the Manager's network aliases, so IP changes don't require worker recreation.
# Only recreate stopped/missing workers.
# ============================================================
if container_api_available; then
_workers_json=$(agt get workers -o json 2>/dev/null || echo '{"workers":[]}')
for _worker_name in $(echo "${_workers_json}" | jq -r '.workers[].name'); do
[ -z "${_worker_name}" ] && continue

_status=$(container_status_worker "${_worker_name}")
if [ "${_status}" = "running" ]; then
log "Worker running: ${_worker_name}, skipping"
continue
fi
# Container missing or stopped — recreate.
log "Worker container ${_status}: ${_worker_name}, recreating..."
_creds_file="/data/worker-creds/${_worker_name}.env"
if [ -f "${_creds_file}" ]; then
source "${_creds_file}"
_runtime=$(echo "${_workers_json}" | jq -r --arg w "${_worker_name}" '.workers[] | select(.name == $w) | .runtime // "openclaw"')
_recreated=false
for _attempt in 1 2 3; do
_env_map=""
_create_body=""
_env_map=$(jq -cn \
--arg name "${_worker_name}" \
--arg fak "${_worker_name}" \
--arg fsk "${WORKER_MINIO_PASSWORD:-}" \
--arg fs_domain "${AGENTTEAMS_FS_DOMAIN:-fs-local.agentteams.io}" \
--arg controller_url "${AGENTTEAMS_CONTROLLER_URL:-}" \
'{
"AGENTTEAMS_WORKER_NAME": $name,
"AGENTTEAMS_FS_ENDPOINT": ("http://" + ($fs_domain | split(":")[0]) + ":9000"),
"AGENTTEAMS_FS_ACCESS_KEY": $fak,
"AGENTTEAMS_FS_SECRET_KEY": $fsk
}
| if $controller_url != "" then . + {"AGENTTEAMS_CONTROLLER_URL": $controller_url} else . end')
_create_body=$(jq -cn --arg name "${_worker_name}" --arg runtime "${_runtime}" --argjson env "${_env_map}" '{name: $name, runtime: $runtime, env: $env}')
worker_backend_create "${_create_body}" > /dev/null 2>&1 && _recreated=true && break
log " Attempt ${_attempt}/3 failed for ${_worker_name}, retrying in $((5 * _attempt))s..."
sleep $((5 * _attempt))
done
if [ "${_recreated}" = true ]; then
log " Recreated ${_runtime} worker: ${_worker_name}"
else
log " ERROR: Failed to recreate ${_worker_name} after 3 attempts"
fi
else
log " WARNING: No credentials found for ${_worker_name} (${_creds_file} missing), skipping"
fi
done
fi

# ============================================================
# Notify workers of builtin updates if upgrade happened
# Builtin files (AGENTS.md, skills) are already synced by upgrade-builtins.sh
Expand Down
57 changes: 57 additions & 0 deletions manager/tests/test-manager-startup-recovery.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/bin/bash
Comment thread
014-code marked this conversation as resolved.
# Regression tests for Manager startup Worker recovery ownership.

set -uo pipefail

PASS=0
FAIL=0
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
STARTUP_SCRIPT="${PROJECT_ROOT}/manager/scripts/init/start-manager-agent.sh"

pass() { echo " PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo " FAIL: $1"; echo " expected: $2"; echo " got: $3"; FAIL=$((FAIL + 1)); }

assert_contains() {
local desc="$1" needle="$2" haystack="$3"
if printf '%s\n' "${haystack}" | grep -qF -- "${needle}"; then
pass "${desc}"
else
fail "${desc}" "contains '${needle}'" "not found"
fi
}

assert_not_contains() {
local desc="$1" needle="$2" haystack="$3"
if ! printf '%s\n' "${haystack}" | grep -qF -- "${needle}"; then
pass "${desc}"
else
fail "${desc}" "does not contain '${needle}'" "found '${needle}'"
fi
}

startup_source="$(<"${STARTUP_SCRIPT}")"

echo "=== Manager startup Worker recovery ownership ==="
assert_contains "startup still detects the configured container runtime" \
'export AGENTTEAMS_CONTAINER_RUNTIME=' "${startup_source}"
assert_contains "startup still loads the Worker management API helpers" \
'source /opt/agentteams/scripts/lib/container-api.sh' "${startup_source}"
assert_not_contains "startup no longer creates Worker CRs during recovery" \
'worker_backend_create' "${startup_source}"
assert_not_contains "startup no longer enumerates Workers for recovery" \
'Recreate Worker containers as needed after Manager restart.' "${startup_source}"

if bash -n "${STARTUP_SCRIPT}"; then
pass "startup script has valid Bash syntax"
else
fail "startup script has valid Bash syntax" "bash -n succeeds" "bash -n failed"
fi

if [ "${FAIL}" -eq 0 ]; then
echo "All ${PASS} tests passed."
exit 0
fi

echo "${FAIL} of $((PASS + FAIL)) tests failed."
exit 1