Skip to content

Commit 2d2351f

Browse files
danielhanchendanielhanchen
authored
Reclaim artifact storage after each prebuilt run (#10)
1 parent ff6b225 commit 2d2351f

1 file changed

Lines changed: 149 additions & 0 deletions

File tree

‎.github/workflows/unsloth-sd-prebuilt.yml‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,7 @@ jobs:
331331
name: sd-${{ needs.resolve.outputs.tag }}-bin-${{ matrix.label }}
332332
path: dist/sd-${{ needs.resolve.outputs.tag }}-bin-${{ matrix.label }}.zip
333333
if-no-files-found: error
334+
retention-days: 7
334335

335336
# Linux CUDA. The rest of this pipeline is CPU and Apple on the premise that GPU hosts use
336337
# diffusers/torch, and that premise fails for MiniMax-H3: its Diffusers path needs about
@@ -485,6 +486,7 @@ jobs:
485486
name: sd-${{ needs.resolve.outputs.tag }}-bin-Linux-Ubuntu-22.04-x86_64-cuda12
486487
path: dist/sd-${{ needs.resolve.outputs.tag }}-bin-Linux-Ubuntu-22.04-x86_64-cuda12.zip
487488
if-no-files-found: error
489+
retention-days: 7
488490

489491
- name: Evict stale ccache files
490492
# !cancelled(), unlike the save below: a cancelled job gets one short
@@ -572,10 +574,15 @@ jobs:
572574
name: sd-${{ needs.resolve.outputs.tag }}-bin-win-cpu-x64
573575
path: dist/sd-${{ needs.resolve.outputs.tag }}-bin-win-cpu-x64.zip
574576
if-no-files-found: error
577+
retention-days: 7
575578

576579
assemble:
577580
name: Assemble + publish
578581
needs: [resolve, build-unix, build-windows, build-linux-cuda]
582+
# Consumed by `reclaim` to tell "these bundles are now release assets" from
583+
# "nothing will ever read these". Set only after draft=false lands.
584+
outputs:
585+
published: ${{ steps.publish.outputs.published }}
579586
if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }}
580587
runs-on: ubuntu-22.04
581588
permissions:
@@ -649,6 +656,7 @@ jobs:
649656
retention-days: 7
650657

651658
- name: Publish GitHub release
659+
id: publish
652660
if: ${{ (github.event_name == 'schedule' || inputs.publish) && needs.resolve.outputs.exists != 'true' }}
653661
run: |
654662
set -eux
@@ -673,3 +681,144 @@ jobs:
673681
--notes "$NOTES" \
674682
dist/*
675683
gh release edit "$TAG" --repo "$REPO" --draft=false
684+
echo "published=true" >> "$GITHUB_OUTPUT"
685+
686+
687+
# Ported from unslothai/llama.cpp's unsloth-prebuilt.yml `reclaim` job, which
688+
# this pipeline was copied from without it. Nothing here ever deleted its own
689+
# artifacts, so every run's bundles sat until GitHub's retention expired them:
690+
# measured 2026-08-11, 168 live artifacts / 13.49 GiB, spread over 27 runs
691+
# whose binaries were already published as release assets.
692+
reclaim:
693+
name: Reclaim artifact storage
694+
needs: [resolve, assemble]
695+
# always(), so a run that publishes NOTHING still cleans up after itself.
696+
# Gating on `published` is what leaks: a workflow_dispatch defaults to
697+
# publish:false, and a cancelled run never reaches publish either.
698+
if: ${{ always() }}
699+
runs-on: ubuntu-24.04
700+
timeout-minutes: 20
701+
permissions:
702+
actions: write # delete this run's artifacts
703+
contents: read # read the release asset list
704+
steps:
705+
- name: Delete artifacts already published as release assets
706+
if: ${{ needs.assemble.outputs.published == 'true' }}
707+
# Never fail a published release over cleanup.
708+
continue-on-error: true
709+
env:
710+
GH_TOKEN: ${{ github.token }}
711+
TAG: ${{ needs.resolve.outputs.tag }}
712+
run: |
713+
set -euo pipefail
714+
repo="$GITHUB_REPOSITORY"
715+
716+
if [ -z "${TAG:-}" ]; then
717+
echo "no tag resolved; leaving artifacts untouched"
718+
exit 0
719+
fi
720+
721+
# Gate 1: the release must exist and be published, not a draft.
722+
draft="$(gh release view "$TAG" --repo "$repo" --json isDraft -q .isDraft 2>/dev/null || echo missing)"
723+
if [ "$draft" != "false" ]; then
724+
echo "release $TAG is '$draft', not a published release; leaving artifacts untouched"
725+
exit 0
726+
fi
727+
assets="$RUNNER_TEMP/reclaim-assets.txt"
728+
arts="$RUNNER_TEMP/reclaim-arts.tsv"
729+
gh release view "$TAG" --repo "$repo" --json assets -q '.assets[].name' | sort > "$assets"
730+
echo "release $TAG has $(wc -l < "$assets") assets"
731+
732+
# Gate 2: only THIS run's artifacts are even considered, so the step
733+
# cannot reach another run's -- including a concurrent build's.
734+
gh api "repos/$repo/actions/runs/$GITHUB_RUN_ID/artifacts" --paginate \
735+
-q '.artifacts[] | select(.expired==false) | "\(.id)\t\(.size_in_bytes)\t\(.name)"' > "$arts" || true
736+
echo "this run has $(grep -c . "$arts" || true) live artifacts"
737+
738+
freed=0; deleted=0; kept=0; failed=0
739+
while IFS="$(printf '\t')" read -r id size name; do
740+
[ -z "${id:-}" ] && continue
741+
# Gate 3: delete only what is provably already on the release.
742+
# Build children upload `sd-<tag>-bin-<label>`; assemble publishes it
743+
# as `<name>.zip`. Anything that does not match is KEPT -- that is
744+
# what protects a partial publish, and it also keeps the
745+
# `unsloth-sd-prebuilt-<tag>` full-set fallback artifact.
746+
# -F: fixed string. Without it every `.` in the name is a regex
747+
# wildcard, and the tag carries dots (Ubuntu-22.04).
748+
if grep -qxF -- "${name}.zip" "$assets" || grep -qxF -- "${name}.tar.gz" "$assets"; then
749+
# < /dev/null so the command can never consume the loop's stdin
750+
# and silently truncate the sweep to one artifact.
751+
if err="$(gh api -X DELETE "repos/$repo/actions/artifacts/$id" --silent < /dev/null 2>&1)"; then
752+
freed=$(( freed + size )); deleted=$(( deleted + 1 ))
753+
else
754+
printf ' could not delete %s: %s\n' "$name" "$err"
755+
failed=$(( failed + 1 ))
756+
fi
757+
else
758+
printf ' KEEP %s (no matching release asset)\n' "$name"
759+
kept=$(( kept + 1 ))
760+
fi
761+
done < "$arts"
762+
763+
echo "deleted $deleted artifacts, freed $(( freed / 1048576 )) MiB, kept $kept, failed $failed"
764+
if [ "$failed" -gt 0 ]; then
765+
echo "::warning::$failed artifact(s) could not be deleted; storage will be reclaimed by retention instead"
766+
fi
767+
{
768+
echo "### Artifact storage reclaimed"
769+
echo ""
770+
echo "| metric | value |"
771+
echo "| --- | --- |"
772+
echo "| release | \`$TAG\` |"
773+
echo "| artifacts deleted | $deleted |"
774+
echo "| storage freed | $(( freed / 1048576 )) MiB |"
775+
echo "| kept (no release asset) | $kept |"
776+
echo "| delete failures | $failed |"
777+
} >> "$GITHUB_STEP_SUMMARY"
778+
779+
- name: Delete artifacts of a run that published nothing
780+
# The other step's name-match against release assets is meaningless
781+
# here: either no release was written, or the tag belongs to a DIFFERENT
782+
# run's release. So the rule is simply that nothing will ever consume
783+
# these -- a publish:false dispatch is a test, and a cancelled or failed
784+
# run is not resumable past the missing legs -- and they are deleted.
785+
if: ${{ needs.assemble.outputs.published != 'true' }}
786+
continue-on-error: true
787+
env:
788+
GH_TOKEN: ${{ github.token }}
789+
run: |
790+
set -euo pipefail
791+
repo="$GITHUB_REPOSITORY"
792+
arts="$RUNNER_TEMP/reclaim-unpublished.tsv"
793+
794+
# Same containment as the published path: only THIS run's artifacts
795+
# are listed, so the step cannot reach a concurrent build's.
796+
gh api "repos/$repo/actions/runs/$GITHUB_RUN_ID/artifacts" --paginate \
797+
-q '.artifacts[] | select(.expired==false) | "\(.id)\t\(.size_in_bytes)\t\(.name)"' > "$arts" || true
798+
n="$(grep -c . "$arts" || true)"
799+
echo "run published nothing; deleting its $n live artifact(s)"
800+
801+
freed=0; deleted=0; failed=0
802+
while IFS="$(printf '\t')" read -r id size name; do
803+
[ -z "${id:-}" ] && continue
804+
if err="$(gh api -X DELETE "repos/$repo/actions/artifacts/$id" --silent < /dev/null 2>&1)"; then
805+
freed=$(( freed + size )); deleted=$(( deleted + 1 ))
806+
else
807+
printf ' could not delete %s: %s\n' "$name" "$err"
808+
failed=$(( failed + 1 ))
809+
fi
810+
done < "$arts"
811+
812+
echo "deleted $deleted artifacts, freed $(( freed / 1048576 )) MiB, failed $failed"
813+
if [ "$failed" -gt 0 ]; then
814+
echo "::warning::$failed artifact(s) could not be deleted; storage will be reclaimed by retention instead"
815+
fi
816+
{
817+
echo "### Artifact storage reclaimed (unpublished run)"
818+
echo ""
819+
echo "| metric | value |"
820+
echo "| --- | --- |"
821+
echo "| artifacts deleted | $deleted |"
822+
echo "| storage freed | $(( freed / 1048576 )) MiB |"
823+
echo "| delete failures | $failed |"
824+
} >> "$GITHUB_STEP_SUMMARY"

0 commit comments

Comments
 (0)