diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 07595a26..281ac71d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -465,7 +465,10 @@ jobs: run: | set -uex lxc exec node-wrk0 -- sh -c "microceph.ceph osd crush rule ls" | grep -F microceph_auto_host - lxc exec node-wrk0 -- sh -c "microceph.ceph osd pool ls detail" | grep -F "crush_rule 2" + # Wait for pool migration to crush_rule 2 (host failure domain). + # The mgr-created .mgr pool may not yet exist when this step runs, + # so a bare `grep crush_rule 2` races against pool creation/migration. + lxc exec node-wrk0 -- /mnt/actionutils.sh wait_for_pool_crush_rule 2 - name: Add another OSD run: | diff --git a/tests/scripts/actionutils.sh b/tests/scripts/actionutils.sh index dec3435c..2414d6a3 100755 --- a/tests/scripts/actionutils.sh +++ b/tests/scripts/actionutils.sh @@ -1264,6 +1264,31 @@ function add_osd_to_node() { sleep 1 } +function wait_for_pool_crush_rule() { + # Wait until at least one pool exists on the cluster with the given + # crush_rule id. After the failure-domain auto-switch flips the default + # crush rule (e.g. osd -> host), the mgr-created .mgr pool may not yet + # exist, or pool migration may still be in flight, so an immediate + # `ceph osd pool ls detail | grep crush_rule N` race-fails. + local rule_id="${1?missing rule id}" + local tries="${2:-30}" + local out="" + + for ((i=0; i/dev/null || true) + if echo "$out" | grep -qF "crush_rule ${rule_id}"; then + echo "Found pool with crush_rule ${rule_id}" + return 0 + fi + sleep 2 + done + + echo "No pool reached crush_rule ${rule_id} after ${tries} tries" + echo "--- pool ls detail ---" + echo "$out" + return 1 +} + function wait_for_osds() { local expect="${1?missing}" local res=0