From 2887c10f541f2ca1030d574c1d0b25d62efe5bcd Mon Sep 17 00:00:00 2001 From: Doug Martin Date: Fri, 28 Aug 2026 06:06:31 -0400 Subject: [PATCH 1/3] chore(devops): add teardown script for v2-v3 redirect monitoring Reverses deploy-monitoring.sh, removing the cutover soak dashboard, its five alarms, the error-fallthrough metric filter, both Synthetics canaries with their generated Lambdas and log groups, the canary artifact bucket, and the canary execution role. Sources config.env for the function name, artifact bucket, and role ARN so those values are not duplicated from the deploy script. Discovers the canary Lambda log groups by prefix, which catches the orphans left by earlier canary generations. Defaults to a dry run; deletion requires an explicit --apply. The redirect itself is out of scope: the CloudFront function, clone distribution, temp subdomain, and Route 53 records are untouched, and the function's own log group is kept since the live function still writes to it. --- .../v2-v3-redirect/teardown-monitoring.sh | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100755 devops/cloudfront-functions/v2-v3-redirect/teardown-monitoring.sh diff --git a/devops/cloudfront-functions/v2-v3-redirect/teardown-monitoring.sh b/devops/cloudfront-functions/v2-v3-redirect/teardown-monitoring.sh new file mode 100755 index 0000000000..264e06f369 --- /dev/null +++ b/devops/cloudfront-functions/v2-v3-redirect/teardown-monitoring.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# Reverse deploy-monitoring.sh: remove the seven monitoring checks created for the +# V2->V3 redirect cutover soak, plus the supporting resources that exist only to +# feed them. +# +# 1. CloudWatch dashboard +# 2. the five alarms +# 3. the error-fallthrough log metric filter +# 4. both Synthetics canaries, with their generated cwsyn-* Lambdas +# 5. the canary Lambda log groups (Synthetics does not remove these itself) +# 6. the canary artifact bucket +# 7. the canary execution role +# +# Scope boundary: the redirect itself stays up. This never touches the CloudFront +# function, the clone distribution, the temp subdomain, or Route 53. The function's +# own log group is kept because the live function still writes to it; only the +# metric filter on it is removed. +# +# Runs as a dry run unless passed --apply. + +set -euo pipefail +cd "$(dirname "$0")" +source ./config.env + +REGION_US_E1="us-east-1" +LOG_GROUP="/aws/cloudfront/function/$FUNCTION_NAME" + +# These literals must match the names deploy-monitoring.sh creates. +DASHBOARD_NAME="codap-v2-v3-redirect" +CANARY_PREFIX="codap-v2-v3" +CANARY_LOG_PREFIX="/aws/lambda/cwsyn-$CANARY_PREFIX" +ALARMS=( + "codap-v2-v3-redirect-FunctionExecutionErrors" + "codap-v2-v3-redirect-error-fallthrough" + "codap-v2-v3-redirect-FunctionThrottles" + "codap-v2-v3-redirect-5xxErrorRate" + "codap-v2-v3-redirect-4xxErrorRate" +) + +APPLY=false +[ "${1:-}" = "--apply" ] && APPLY=true + +run() { + if $APPLY; then + echo " + $*" + "$@" + else + echo " [dry-run] $*" + fi +} + +$APPLY || echo "DRY RUN. Re-run with --apply to actually delete." +echo + +echo "1) dashboard $DASHBOARD_NAME" +run aws cloudwatch delete-dashboards --dashboard-names "$DASHBOARD_NAME" \ + --region "$REGION_US_E1" + +echo "2) alarms" +run aws cloudwatch delete-alarms --alarm-names "${ALARMS[@]}" --region "$REGION_US_E1" + +echo "3) error-fallthrough metric filter (the log group itself is kept)" +run aws logs delete-metric-filter \ + --log-group-name "$LOG_GROUP" \ + --filter-name "codap-v2-v3-redirect-error-fallthrough" \ + --region "$REGION_US_E1" + +# A RUNNING canary cannot be deleted, so stop it and wait for the state to settle +# first. --delete-lambda removes the generated cwsyn-* Lambda and its layers, which +# are otherwise left orphaned. +echo "4) Synthetics canaries" +for canary in v3-reachability redirect-correctness; do + name="$CANARY_PREFIX-$canary" + if ! aws synthetics get-canary --name "$name" --region "$REGION_US_E1" >/dev/null 2>&1; then + echo " canary $name not present, skipping" + continue + fi + state=$(aws synthetics get-canary --name "$name" --region "$REGION_US_E1" \ + --query "Canary.Status.State" --output text) + if [ "$state" = "RUNNING" ]; then + run aws synthetics stop-canary --name "$name" --region "$REGION_US_E1" + if $APPLY; then + echo -n " waiting for $name to stop" + for _ in $(seq 1 60); do + state=$(aws synthetics get-canary --name "$name" --region "$REGION_US_E1" \ + --query "Canary.Status.State" --output text) + [ "$state" != "RUNNING" ] && [ "$state" != "STOPPING" ] && break + echo -n "." + sleep 5 + done + echo " $state" + fi + fi + run aws synthetics delete-canary --name "$name" --delete-lambda --region "$REGION_US_E1" +done + +# Deleting a Lambda leaves its log group behind, and a canary re-pointed by +# delete+recreate leaves a log group per generation. Discover by prefix so the +# orphans from earlier generations are caught too. +echo "5) canary Lambda log groups" +mapfile -t CANARY_LOG_GROUPS < <( + aws logs describe-log-groups \ + --log-group-name-prefix "$CANARY_LOG_PREFIX" \ + --region "$REGION_US_E1" \ + --query "logGroups[].logGroupName" --output text | tr '\t' '\n' +) +for lg in "${CANARY_LOG_GROUPS[@]}"; do + [ -z "$lg" ] && continue + run aws logs delete-log-group --log-group-name "$lg" --region "$REGION_US_E1" +done + +# The bucket is unversioned, so a recursive delete empties it completely. Deleting +# the bucket also releases the globally unique name. +echo "6) canary artifact bucket" +if [ -z "${SYNTHETICS_ARTIFACT_BUCKET:-}" ]; then + echo " SYNTHETICS_ARTIFACT_BUCKET not set in config.env, skipping" +else + run aws s3 rm "s3://$SYNTHETICS_ARTIFACT_BUCKET" --recursive + run aws s3api delete-bucket --bucket "$SYNTHETICS_ARTIFACT_BUCKET" \ + --region "$REGION_US_E1" +fi + +echo "7) canary execution role" +if [ -z "${SYNTHETICS_ROLE_ARN:-}" ]; then + echo " SYNTHETICS_ROLE_ARN not set in config.env, skipping" +else + role_name="${SYNTHETICS_ROLE_ARN##*/}" + mapfile -t INLINE_POLICIES < <( + aws iam list-role-policies --role-name "$role_name" \ + --query "PolicyNames" --output text | tr '\t' '\n' + ) + for policy in "${INLINE_POLICIES[@]}"; do + [ -z "$policy" ] && continue + run aws iam delete-role-policy --role-name "$role_name" --policy-name "$policy" + done + mapfile -t ATTACHED_POLICIES < <( + aws iam list-attached-role-policies --role-name "$role_name" \ + --query "AttachedPolicies[].PolicyArn" --output text | tr '\t' '\n' + ) + for arn in "${ATTACHED_POLICIES[@]}"; do + [ -z "$arn" ] && continue + run aws iam detach-role-policy --role-name "$role_name" --policy-arn "$arn" + done + run aws iam delete-role --role-name "$role_name" +fi + +echo +if $APPLY; then + echo "teardown-monitoring.sh complete." +else + echo "Dry run complete. Nothing was deleted." +fi From 88140d7efeeebf0341d6ed5903079b48f8fdabba Mon Sep 17 00:00:00 2001 From: Doug Martin Date: Sat, 5 Sep 2026 06:46:14 -0400 Subject: [PATCH 2/3] fix(devops): surface AWS failures in teardown-monitoring.sh mapfile returns success regardless of what the command feeding it did, so a failed AWS listing was indistinguishable from an empty one. A denied or throttled describe-log-groups or IAM list call left its resources in place while the script ran to completion and exited 0. Capture each listing in a command substitution first so set -e aborts on the original error. Wait for a canary to reach a state delete-canary accepts rather than only polling a RUNNING one, and fail with an explicit message when it has not settled before the deadline. Previously the poll could expire with the canary still running and fall through to a delete that AWS rejects, aborting the teardown partway with a bare ConflictException. A canary found already STOPPING skipped the wait entirely and hit the same rejection. --- .../v2-v3-redirect/teardown-monitoring.sh | 78 ++++++++++++------- 1 file changed, 51 insertions(+), 27 deletions(-) diff --git a/devops/cloudfront-functions/v2-v3-redirect/teardown-monitoring.sh b/devops/cloudfront-functions/v2-v3-redirect/teardown-monitoring.sh index 264e06f369..b31beb7583 100755 --- a/devops/cloudfront-functions/v2-v3-redirect/teardown-monitoring.sh +++ b/devops/cloudfront-functions/v2-v3-redirect/teardown-monitoring.sh @@ -37,6 +37,9 @@ ALARMS=( "codap-v2-v3-redirect-4xxErrorRate" ) +# How long to wait for a canary to reach a state delete-canary accepts. +CANARY_SETTLE_TIMEOUT_SECS=300 + APPLY=false [ "${1:-}" = "--apply" ] && APPLY=true @@ -49,6 +52,21 @@ run() { fi } +canary_state() { + aws synthetics get-canary --name "$1" --region "$REGION_US_E1" \ + --query "Canary.Status.State" --output text +} + +# CanaryState is one of CREATING, READY, STARTING, RUNNING, UPDATING, STOPPING, +# STOPPED, ERROR, DELETING. delete-canary is only accepted once the canary has +# come to rest; every other state is either running or mid-transition. +canary_settled() { + case "$1" in + READY|STOPPED|ERROR) return 0 ;; + *) return 1 ;; + esac +} + $APPLY || echo "DRY RUN. Re-run with --apply to actually delete." echo @@ -75,21 +93,27 @@ for canary in v3-reachability redirect-correctness; do echo " canary $name not present, skipping" continue fi - state=$(aws synthetics get-canary --name "$name" --region "$REGION_US_E1" \ - --query "Canary.Status.State" --output text) + state=$(canary_state "$name") if [ "$state" = "RUNNING" ]; then run aws synthetics stop-canary --name "$name" --region "$REGION_US_E1" - if $APPLY; then - echo -n " waiting for $name to stop" - for _ in $(seq 1 60); do - state=$(aws synthetics get-canary --name "$name" --region "$REGION_US_E1" \ - --query "Canary.Status.State" --output text) - [ "$state" != "RUNNING" ] && [ "$state" != "STOPPING" ] && break - echo -n "." - sleep 5 - done - echo " $state" - fi + fi + # Any unsettled state has to be waited out, including a canary that was already + # STOPPING or mid-transition before this script ran. + if $APPLY && ! canary_settled "$state"; then + echo -n " waiting for $name to settle (was $state)" + deadline=$((SECONDS + CANARY_SETTLE_TIMEOUT_SECS)) + while ! canary_settled "$state"; do + if [ "$SECONDS" -ge "$deadline" ]; then + echo + echo "FAIL: canary $name is still $state after ${CANARY_SETTLE_TIMEOUT_SECS}s." + echo " delete-canary would be rejected. Re-run once it has settled." + exit 1 + fi + echo -n "." + sleep 5 + state=$(canary_state "$name") + done + echo " $state" fi run aws synthetics delete-canary --name "$name" --delete-lambda --region "$REGION_US_E1" done @@ -98,12 +122,14 @@ done # delete+recreate leaves a log group per generation. Discover by prefix so the # orphans from earlier generations are caught too. echo "5) canary Lambda log groups" -mapfile -t CANARY_LOG_GROUPS < <( - aws logs describe-log-groups \ - --log-group-name-prefix "$CANARY_LOG_PREFIX" \ - --region "$REGION_US_E1" \ - --query "logGroups[].logGroupName" --output text | tr '\t' '\n' -) +# mapfile succeeds whatever the command feeding it did, so a failed AWS listing +# would arrive here as "nothing to delete" and the teardown would report success +# with the resources still in place. Capture first so set -e sees the failure. +canary_log_groups=$(aws logs describe-log-groups \ + --log-group-name-prefix "$CANARY_LOG_PREFIX" \ + --region "$REGION_US_E1" \ + --query "logGroups[].logGroupName" --output text) +mapfile -t CANARY_LOG_GROUPS < <(printf '%s\n' "$canary_log_groups" | tr '\t' '\n') for lg in "${CANARY_LOG_GROUPS[@]}"; do [ -z "$lg" ] && continue run aws logs delete-log-group --log-group-name "$lg" --region "$REGION_US_E1" @@ -125,18 +151,16 @@ if [ -z "${SYNTHETICS_ROLE_ARN:-}" ]; then echo " SYNTHETICS_ROLE_ARN not set in config.env, skipping" else role_name="${SYNTHETICS_ROLE_ARN##*/}" - mapfile -t INLINE_POLICIES < <( - aws iam list-role-policies --role-name "$role_name" \ - --query "PolicyNames" --output text | tr '\t' '\n' - ) + inline_policies=$(aws iam list-role-policies --role-name "$role_name" \ + --query "PolicyNames" --output text) + mapfile -t INLINE_POLICIES < <(printf '%s\n' "$inline_policies" | tr '\t' '\n') for policy in "${INLINE_POLICIES[@]}"; do [ -z "$policy" ] && continue run aws iam delete-role-policy --role-name "$role_name" --policy-name "$policy" done - mapfile -t ATTACHED_POLICIES < <( - aws iam list-attached-role-policies --role-name "$role_name" \ - --query "AttachedPolicies[].PolicyArn" --output text | tr '\t' '\n' - ) + attached_policies=$(aws iam list-attached-role-policies --role-name "$role_name" \ + --query "AttachedPolicies[].PolicyArn" --output text) + mapfile -t ATTACHED_POLICIES < <(printf '%s\n' "$attached_policies" | tr '\t' '\n') for arn in "${ATTACHED_POLICIES[@]}"; do [ -z "$arn" ] && continue run aws iam detach-role-policy --role-name "$role_name" --policy-arn "$arn" From abf51a5af6b8a40526f27108d92530528f70a1c7 Mon Sep 17 00:00:00 2001 From: Doug Martin Date: Tue, 8 Sep 2026 16:29:26 -0400 Subject: [PATCH 3/3] fix(devops): address review on teardown-monitoring.sh Replaces the bash 4 mapfile calls, which abort mid-teardown on the macOS system bash (3.2.57), with a read loop over the captured listing. The other two call sites go away with the IAM role section below. Makes the teardown re-runnable after a partial failure. The dashboard, metric filter, log group, and S3 deletes now tolerate an already-absent resource and re-raise every other error, so a run stopped by the canary settle timeout or a transient AWS error can be re-run to finish the job. Stops deleting the Synthetics artifact bucket and execution role. deploy-monitoring.sh does not create either, PREFLIGHT.md has the operator supply pre-existing ones, and the config.env.example names are generic enough to be shared with other CODAP canaries. Only the two code packages and two run-artifact prefixes the deploy script writes are removed now. Also waits for delete-canary to complete before emptying the artifact prefixes, re-issues stop-canary for a canary caught in STARTING, rejects an unrecognized argument instead of silently dry-running, and adds the script to the README table. --- .../v2-v3-redirect/README.md | 1 + .../v2-v3-redirect/teardown-monitoring.sh | 147 +++++++++++------- 2 files changed, 95 insertions(+), 53 deletions(-) diff --git a/devops/cloudfront-functions/v2-v3-redirect/README.md b/devops/cloudfront-functions/v2-v3-redirect/README.md index b4a8a25acf..c09631cbb2 100644 --- a/devops/cloudfront-functions/v2-v3-redirect/README.md +++ b/devops/cloudfront-functions/v2-v3-redirect/README.md @@ -58,6 +58,7 @@ correct execution order, and each row points back at its requirement. | `dns-audit.sh` | Audit `*.codap.concord.org` records | R26c | | `deploy-monitoring.sh` | CloudWatch alarms, log metric filter, synthetic canaries | R26b | | `verify-alarms.sh` | Induce a synthetic error against each R26b check; confirm ALARM | G5 / DO-I3 | +| `teardown-monitoring.sh` | Remove the R26b monitoring after the soak (dry run unless `--apply`) | R26b (reverse) | | `flip.sh` / `rollback.sh` | Flip-day forward / reverse | R24/R24a/R25 | | `route53-change.sh` | Shared helper: UPSERT a Route 53 ALIAS A record | shared | | `PREFLIGHT.md` | Pre-flip pipeline (run this first) | this folder | diff --git a/devops/cloudfront-functions/v2-v3-redirect/teardown-monitoring.sh b/devops/cloudfront-functions/v2-v3-redirect/teardown-monitoring.sh index b31beb7583..a19c7e8ff5 100755 --- a/devops/cloudfront-functions/v2-v3-redirect/teardown-monitoring.sh +++ b/devops/cloudfront-functions/v2-v3-redirect/teardown-monitoring.sh @@ -8,15 +8,20 @@ # 3. the error-fallthrough log metric filter # 4. both Synthetics canaries, with their generated cwsyn-* Lambdas # 5. the canary Lambda log groups (Synthetics does not remove these itself) -# 6. the canary artifact bucket -# 7. the canary execution role +# 6. the canary code packages and run artifacts in the Synthetics bucket # # Scope boundary: the redirect itself stays up. This never touches the CloudFront # function, the clone distribution, the temp subdomain, or Route 53. The function's # own log group is kept because the live function still writes to it; only the # metric filter on it is removed. # -# Runs as a dry run unless passed --apply. +# The Synthetics artifact bucket and execution role are also kept. deploy-monitoring.sh +# does not create them (PREFLIGHT.md has the operator supply pre-existing ones, which may +# be shared with other CODAP canaries), so only the objects the deploy script writes are +# removed. Delete the bucket and role by hand if they were dedicated to this soak. +# +# Runs as a dry run unless passed --apply. Every delete tolerates an already-absent +# resource, so a run interrupted partway can be re-run to finish the job. set -euo pipefail cd "$(dirname "$0")" @@ -29,6 +34,7 @@ LOG_GROUP="/aws/cloudfront/function/$FUNCTION_NAME" DASHBOARD_NAME="codap-v2-v3-redirect" CANARY_PREFIX="codap-v2-v3" CANARY_LOG_PREFIX="/aws/lambda/cwsyn-$CANARY_PREFIX" +CANARIES=(v3-reachability redirect-correctness) ALARMS=( "codap-v2-v3-redirect-FunctionExecutionErrors" "codap-v2-v3-redirect-error-fallthrough" @@ -37,26 +43,58 @@ ALARMS=( "codap-v2-v3-redirect-4xxErrorRate" ) -# How long to wait for a canary to reach a state delete-canary accepts. -CANARY_SETTLE_TIMEOUT_SECS=300 +# How long to wait for a canary to reach a state delete-canary accepts, and then for +# the delete itself to complete. +CANARY_WAIT_TIMEOUT_SECS=300 -APPLY=false -[ "${1:-}" = "--apply" ] && APPLY=true +usage() { + echo "usage: $(basename "$0") [--apply]" >&2 + exit 2 +} -run() { - if $APPLY; then - echo " + $*" - "$@" - else +APPLY=false +case "${1:-}" in + --apply) APPLY=true ;; + "") ;; + *) usage ;; +esac +[ "$#" -le 1 ] || usage + +# Each service spells "it is not there" its own way. A delete that reports one of +# these has nothing left to do, which is what makes the script re-runnable. +NOT_FOUND_RE='ResourceNotFound|DashboardNotFound|NoSuchBucket|NoSuchKey|NoSuchEntity|NotFoundException' + +_run() { + local tolerate="$1"; shift + if ! $APPLY; then echo " [dry-run] $*" + return 0 + fi + echo " + $*" + local output status=0 + output=$("$@" 2>&1) || status=$? + if [ "$status" -ne 0 ] && $tolerate && printf '%s' "$output" | grep -Eq "$NOT_FOUND_RE"; then + echo " already absent" + return 0 + fi + if [ -n "$output" ]; then + printf '%s\n' "$output" fi + return "$status" } +run() { _run false "$@"; } +run_optional() { _run true "$@"; } + canary_state() { aws synthetics get-canary --name "$1" --region "$REGION_US_E1" \ --query "Canary.Status.State" --output text } +canary_exists() { + aws synthetics get-canary --name "$1" --region "$REGION_US_E1" >/dev/null 2>&1 +} + # CanaryState is one of CREATING, READY, STARTING, RUNNING, UPDATING, STOPPING, # STOPPED, ERROR, DELETING. delete-canary is only accepted once the canary has # come to rest; every other state is either running or mid-transition. @@ -71,14 +109,15 @@ $APPLY || echo "DRY RUN. Re-run with --apply to actually delete." echo echo "1) dashboard $DASHBOARD_NAME" -run aws cloudwatch delete-dashboards --dashboard-names "$DASHBOARD_NAME" \ +run_optional aws cloudwatch delete-dashboards --dashboard-names "$DASHBOARD_NAME" \ --region "$REGION_US_E1" +# delete-alarms is silent about names that do not exist. echo "2) alarms" run aws cloudwatch delete-alarms --alarm-names "${ALARMS[@]}" --region "$REGION_US_E1" echo "3) error-fallthrough metric filter (the log group itself is kept)" -run aws logs delete-metric-filter \ +run_optional aws logs delete-metric-filter \ --log-group-name "$LOG_GROUP" \ --filter-name "codap-v2-v3-redirect-error-fallthrough" \ --region "$REGION_US_E1" @@ -87,9 +126,9 @@ run aws logs delete-metric-filter \ # first. --delete-lambda removes the generated cwsyn-* Lambda and its layers, which # are otherwise left orphaned. echo "4) Synthetics canaries" -for canary in v3-reachability redirect-correctness; do +for canary in "${CANARIES[@]}"; do name="$CANARY_PREFIX-$canary" - if ! aws synthetics get-canary --name "$name" --region "$REGION_US_E1" >/dev/null 2>&1; then + if ! canary_exists "$name"; then echo " canary $name not present, skipping" continue fi @@ -101,76 +140,78 @@ for canary in v3-reachability redirect-correctness; do # STOPPING or mid-transition before this script ran. if $APPLY && ! canary_settled "$state"; then echo -n " waiting for $name to settle (was $state)" - deadline=$((SECONDS + CANARY_SETTLE_TIMEOUT_SECS)) + deadline=$((SECONDS + CANARY_WAIT_TIMEOUT_SECS)) while ! canary_settled "$state"; do if [ "$SECONDS" -ge "$deadline" ]; then echo - echo "FAIL: canary $name is still $state after ${CANARY_SETTLE_TIMEOUT_SECS}s." + echo "FAIL: canary $name is still $state after ${CANARY_WAIT_TIMEOUT_SECS}s." echo " delete-canary would be rejected. Re-run once it has settled." exit 1 fi echo -n "." sleep 5 state=$(canary_state "$name") + # A canary that was STARTING when this ran was never sent a stop: it reaches + # RUNNING on its own and would sit there until the timeout. Re-issuing the stop + # is harmless once it is already STOPPING. + if [ "$state" = "RUNNING" ]; then + aws synthetics stop-canary --name "$name" --region "$REGION_US_E1" >/dev/null 2>&1 || true + fi done echo " $state" fi run aws synthetics delete-canary --name "$name" --delete-lambda --region "$REGION_US_E1" + # delete-canary is asynchronous. Wait for the canary to go before step 6 empties its + # artifact prefix, so a run still in flight cannot write objects after the delete. + if $APPLY; then + echo -n " waiting for $name to disappear" + deadline=$((SECONDS + CANARY_WAIT_TIMEOUT_SECS)) + while canary_exists "$name"; do + if [ "$SECONDS" -ge "$deadline" ]; then + echo + echo "FAIL: canary $name still exists ${CANARY_WAIT_TIMEOUT_SECS}s after delete-canary." + echo " Re-run once it has finished deleting." + exit 1 + fi + echo -n "." + sleep 5 + done + echo " gone" + fi done # Deleting a Lambda leaves its log group behind, and a canary re-pointed by # delete+recreate leaves a log group per generation. Discover by prefix so the # orphans from earlier generations are caught too. echo "5) canary Lambda log groups" -# mapfile succeeds whatever the command feeding it did, so a failed AWS listing -# would arrive here as "nothing to delete" and the teardown would report success -# with the resources still in place. Capture first so set -e sees the failure. +# Captured rather than piped into the loop: pipefail makes a failed listing abort the +# script here, instead of reaching the loop as "nothing to delete" and letting the +# teardown report success with the log groups still standing. canary_log_groups=$(aws logs describe-log-groups \ --log-group-name-prefix "$CANARY_LOG_PREFIX" \ --region "$REGION_US_E1" \ - --query "logGroups[].logGroupName" --output text) -mapfile -t CANARY_LOG_GROUPS < <(printf '%s\n' "$canary_log_groups" | tr '\t' '\n') -for lg in "${CANARY_LOG_GROUPS[@]}"; do + --query "logGroups[].logGroupName" --output text | tr '\t' '\n') +while IFS= read -r lg; do [ -z "$lg" ] && continue - run aws logs delete-log-group --log-group-name "$lg" --region "$REGION_US_E1" -done + run_optional aws logs delete-log-group --log-group-name "$lg" --region "$REGION_US_E1" +done <<< "$canary_log_groups" -# The bucket is unversioned, so a recursive delete empties it completely. Deleting -# the bucket also releases the globally unique name. -echo "6) canary artifact bucket" +# The prefixes deploy-monitoring.sh writes: one code package per canary under code/, +# and one run-artifact prefix per canary. Nothing else in the bucket is touched. +echo "6) canary code packages and run artifacts" if [ -z "${SYNTHETICS_ARTIFACT_BUCKET:-}" ]; then echo " SYNTHETICS_ARTIFACT_BUCKET not set in config.env, skipping" else - run aws s3 rm "s3://$SYNTHETICS_ARTIFACT_BUCKET" --recursive - run aws s3api delete-bucket --bucket "$SYNTHETICS_ARTIFACT_BUCKET" \ - --region "$REGION_US_E1" -fi - -echo "7) canary execution role" -if [ -z "${SYNTHETICS_ROLE_ARN:-}" ]; then - echo " SYNTHETICS_ROLE_ARN not set in config.env, skipping" -else - role_name="${SYNTHETICS_ROLE_ARN##*/}" - inline_policies=$(aws iam list-role-policies --role-name "$role_name" \ - --query "PolicyNames" --output text) - mapfile -t INLINE_POLICIES < <(printf '%s\n' "$inline_policies" | tr '\t' '\n') - for policy in "${INLINE_POLICIES[@]}"; do - [ -z "$policy" ] && continue - run aws iam delete-role-policy --role-name "$role_name" --policy-name "$policy" - done - attached_policies=$(aws iam list-attached-role-policies --role-name "$role_name" \ - --query "AttachedPolicies[].PolicyArn" --output text) - mapfile -t ATTACHED_POLICIES < <(printf '%s\n' "$attached_policies" | tr '\t' '\n') - for arn in "${ATTACHED_POLICIES[@]}"; do - [ -z "$arn" ] && continue - run aws iam detach-role-policy --role-name "$role_name" --policy-arn "$arn" + for canary in "${CANARIES[@]}"; do + run_optional aws s3 rm "s3://$SYNTHETICS_ARTIFACT_BUCKET/$CANARY_PREFIX-$canary/" --recursive + run_optional aws s3 rm "s3://$SYNTHETICS_ARTIFACT_BUCKET/code/$CANARY_PREFIX-$canary.zip" done - run aws iam delete-role --role-name "$role_name" fi echo if $APPLY; then echo "teardown-monitoring.sh complete." + echo "The Synthetics artifact bucket and execution role were left in place." else echo "Dry run complete. Nothing was deleted." fi