diff --git a/.gitea/workflows/docker-publish.yml b/.gitea/workflows/docker-publish.yml index a9bee3e..1ca1622 100644 --- a/.gitea/workflows/docker-publish.yml +++ b/.gitea/workflows/docker-publish.yml @@ -658,13 +658,18 @@ jobs: EXPECTED_NODE_MAJOR=$(sed -n 's/^ARG NODE_VERSION=\([0-9][0-9]*\).*/\1/p' Dockerfile.base) export EXPECTED_NODE_MAJOR bash scripts/ci-reclaim-docker.sh assert-protected "$SMOKE_IMAGE" + # Pin the image with a running container for the whole test, so a + # raw prune from a job we do not control (other repos, the other + # Gitea instance's runner on the same VM) cannot remove it either. + bash scripts/ci-reclaim-docker.sh hold "$SMOKE_IMAGE" bash scripts/smoke-test.sh "$SMOKE_IMAGE" - # The label keeps this image from every job's reclaim, so it is removed - # here instead. A job killed before this step leaves an orphan, which the - # reclaim's age sweep deletes once it is older than SMOKE_IMAGE_MAX_AGE. + # The label keeps this image from every job's reclaim and the holder pins + # it, so both are removed here. A job killed before this step leaves an + # orphan: the holder exits after 1 h, and the reclaim's age sweep deletes + # the image once it is older than SMOKE_IMAGE_MAX_AGE. - name: Remove smoke image if: always() - run: docker image rm "$SMOKE_IMAGE" || true + run: bash scripts/ci-reclaim-docker.sh release "$SMOKE_IMAGE" # ── Phase 3b: amd64 smoke for the studio variant ──────────────────── # Additive + independent of the core `smoke` job: gates ONLY @@ -737,13 +742,18 @@ jobs: EXPECTED_NODE_MAJOR=$(sed -n 's/^ARG NODE_VERSION=\([0-9][0-9]*\).*/\1/p' Dockerfile.base) export EXPECTED_NODE_MAJOR bash scripts/ci-reclaim-docker.sh assert-protected "$SMOKE_IMAGE" + # Pin the image with a running container for the whole test, so a + # raw prune from a job we do not control (other repos, the other + # Gitea instance's runner on the same VM) cannot remove it either. + bash scripts/ci-reclaim-docker.sh hold "$SMOKE_IMAGE" bash scripts/smoke-test.sh "$SMOKE_IMAGE" - # The label keeps this image from every job's reclaim, so it is removed - # here instead. A job killed before this step leaves an orphan, which the - # reclaim's age sweep deletes once it is older than SMOKE_IMAGE_MAX_AGE. + # The label keeps this image from every job's reclaim and the holder pins + # it, so both are removed here. A job killed before this step leaves an + # orphan: the holder exits after 1 h, and the reclaim's age sweep deletes + # the image once it is older than SMOKE_IMAGE_MAX_AGE. - name: Remove smoke image if: always() - run: docker image rm "$SMOKE_IMAGE" || true + run: bash scripts/ci-reclaim-docker.sh release "$SMOKE_IMAGE" # ── Phase 4: multi-arch publish ───────────────────────────────────── build-variant: diff --git a/.gitea/workflows/lint.yml b/.gitea/workflows/lint.yml index 672ced5..054daf5 100644 --- a/.gitea/workflows/lint.yml +++ b/.gitea/workflows/lint.yml @@ -91,8 +91,8 @@ jobs: run: bash scripts/check-workflow-shell.sh .gitea/workflows - name: No raw docker prune in workflows (shared runner daemon) - # Jobs on the same runner share its Docker daemon (runner-a1 runs - # several docker-publish.yml jobs at once), and + # Everything on one runner host shares its Docker daemon (runner-a1 + # runs several jobs at once; runner-2 also serves gitea.egl.lan), and # a bare `docker system prune -a` in one job deletes the smoke image # another job is testing (v1.12.2 run 740 shipped without its -studio # tags that way). Reclaim goes through scripts/ci-reclaim-docker.sh, diff --git a/AGENTS.md b/AGENTS.md index f91bd18..2cbb095 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,12 +45,15 @@ re-brand of opencode-devbox's `pi-only` variant. update-description). The `-studio` variant adds independent `smoke-studio` + `build-variant-studio` jobs that gate only the `-studio` tags (never the core `:latest` release). -- `scripts/ci-reclaim-docker.sh` — the ONLY way a CI job frees disk. Jobs on - the same runner share its Docker daemon (runner-a1 runs several at once), so a bare `docker system prune -a` in one +- `scripts/ci-reclaim-docker.sh` — the ONLY way a CI job frees disk. Everything + on one runner host shares its Docker daemon (runner-a1 runs several jobs at + once; runner-2 also hosts a runner for gitea.egl.lan), so a bare `docker system prune -a` in one job deletes the smoke image another job is mid-test on (v1.12.2 run 740 lost its `-studio` tags that way). The script spares images labelled `se.jordbo.pi-devbox.ci-smoke-image`; the smoke jobs apply that label, - assert it (`assert-protected`) and remove their own image at the end. + assert it (`assert-protected`), pin the image with a running container for + the test (`hold`, against raw prunes from jobs we do not control) and + `release` it at the end. `lint.yml` fails on any raw `docker system|image prune` in a workflow. ## Versioning scheme diff --git a/CHANGELOG.md b/CHANGELOG.md index de1a1c2..779c4d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,9 +17,11 @@ Pre-v1.0.0 tags followed the pi npm version (`v{pi_version}[letter]`). - **CI: a smoke job's image could be deleted mid-test by another job's disk reclaim.** This is why v1.12.2's `-studio` tags were not published by its - tag run. Jobs that land on the same runner share its Docker daemon (in run - 740, runner-a1 ran `smoke-studio` and `build-variant` at the same time), and - each job began with `docker system prune -af --volumes`, which removes every + tag run. Everything on one runner host shares its Docker daemon: concurrent + jobs of a runner with capacity > 1 (in run 740, runner-a1 ran `smoke-studio` + and `build-variant` at the same time), and the jobs of other act_runner + containers on the same VM (runner-2 has one for gitea.jordbo.se and one for + gitea.egl.lan). Each pi-devbox job began with `docker system prune -af --volumes`, which removes every image that no container is using at that moment. A smoke image is unused between `smoke-test.sh`'s one-shot `docker run`s. In run 740, `smoke` passed and `build-variant` started. Its prune logged @@ -33,8 +35,14 @@ Pre-v1.0.0 tags followed the pi npm version (`v{pi_version}[letter]`). volumes, build cache) but never prunes an image labelled `se.jordbo.pi-devbox.ci-smoke-image`. The smoke builds apply that label, a preflight fails if it is missing, and each smoke job removes its own image at - the end. Smoke images left by a killed job are swept once they are older - than 6 h. Smoke tags carry the run id (`pi-devbox:smoke-`), so two + the end. Jobs we do not control can still run a raw prune on the same + daemon: opencode-devbox's workflows do, in 7 places, and so may whatever + gitea.egl.lan runs. So the smoke step also pins its image with a running + container (`ci-reclaim-docker.sh hold`) for the whole test, because no prune + removes an image a running container uses. Only the few seconds between the + build step loading the image and the pin starting are left unprotected. The + holder exits by itself after 1 h, and smoke images left by a killed job are + swept once they are older than 6 h. Smoke tags carry the run id (`pi-devbox:smoke-`), so two runs on different refs cannot test or delete each other's image. `lint.yml` fails on any raw `docker system|image prune` in a workflow. Job order is unchanged: the studio jobs still gate only the `-studio` tags. Reordering @@ -45,7 +53,10 @@ Pre-v1.0.0 tags followed the pi npm version (`v{pi_version}[letter]`). fresh labelled image, deletes an unlabelled one and a labelled one created in 1970, keeps an image a running container uses, and prints no errors. `assert-protected` passes the labelled image and fails both an unlabelled - and a missing one. + and a missing one. With `hold`, an unlabelled image survives both a raw + `docker system prune -af --volumes` and `docker image prune -af` (control: + without the holder, the same prune deletes it). `hold` fails on a missing + image, and `release` removes the holder and the image. ## v1.12.2 — 2026-10-22 diff --git a/scripts/ci-reclaim-docker.sh b/scripts/ci-reclaim-docker.sh index 9c81415..12a4daf 100755 --- a/scripts/ci-reclaim-docker.sh +++ b/scripts/ci-reclaim-docker.sh @@ -7,13 +7,17 @@ # # bash scripts/ci-reclaim-docker.sh # reclaim (default) # bash scripts/ci-reclaim-docker.sh assert-protected IMG # smoke preflight +# bash scripts/ci-reclaim-docker.sh hold IMG # pin IMG while testing +# bash scripts/ci-reclaim-docker.sh release IMG # drop the pin + IMG # bash scripts/ci-reclaim-docker.sh lint WORKFLOW... # lint.yml guard # -# WHY THIS EXISTS. Jobs that land on the same runner share its Docker daemon -# (runner-a1 runs several at once: in run 740 it had smoke-studio and -# build-variant going together), and each job used to start with -# `docker system prune -af --volumes`. That removes -# every image no container is using at that instant, and a smoke image is +# WHY THIS EXISTS. Everything that runs on one runner host shares its Docker +# daemon: concurrent jobs of a runner with capacity > 1 (runner-a1 had +# smoke-studio and build-variant going together in run 740), and the jobs of +# other act_runner containers on the same VM (runner-2 has one registered with +# gitea.jordbo.se and one with gitea.egl.lan). Each pi-devbox job used to start +# with `docker system prune -af --volumes`. That removes every image no +# container is using at that instant, and a smoke image is # exactly that for most of its life: smoke-test.sh runs one `docker run --rm` # per assertion, so between assertions nothing holds the image. v1.12.2 run 740 # (2026-10-22): smoke passed, build-variant started and pruned, and its log @@ -23,8 +27,14 @@ # locally", build-variant-studio was skipped, and the release went out without # its -studio tags. Job ORDER cannot close this: smoke and smoke-studio prune # each other too whenever their start times drift apart, and other runs share -# the daemon. So the smoke images carry a label, and this script never prunes -# an image that has it. +# the daemon. Two defences, because we only control our own jobs: +# - the smoke images carry a label, and this script never prunes an image +# that has it (covers pi-devbox's own jobs, before and during the test); +# - `hold` starts a container from the image for the whole smoke step. No +# prune removes an image a running container uses, so this also covers +# jobs that never call this script (opencode-devbox's raw prunes, +# anything gitea.egl.lan runs). What is left is the few seconds between +# the build step loading the image and `hold` starting. # # WHAT IT DOES. The same reclaim as `docker system prune -af --volumes` + # `docker builder prune -af`, done per object type so the image step can take @@ -86,6 +96,29 @@ assert_protected() { echo "OK: ${img} is present and carries ${SMOKE_IMAGE_LABEL}=true" } +holder_name() { printf 'ci-hold-%s' "$(printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '_')"; } + +# The holder exits on its own after an hour (--rm), so a job killed before +# `release` does not pin its image forever; the age sweep then removes it. +hold() { + local img="${1:?usage: $0 hold }" name + name=$(holder_name "$img") + docker rm -f "$name" >/dev/null 2>&1 || true + if ! docker run -d --rm --name "$name" --network none --entrypoint "" \ + "$img" sleep 3600 >/dev/null; then + echo "::error::could not start holder ${name} for ${img}; is the image still in the daemon?" + return 1 + fi + echo "OK: ${img} pinned by running container ${name} (exits by itself after 1 h)" +} + +release() { + local img="${1:?usage: $0 release }" + docker rm -f "$(holder_name "$img")" >/dev/null 2>&1 || true + docker image rm "$img" >/dev/null 2>&1 || true + echo "released ${img}" +} + lint_workflows() { [ "$#" -gt 0 ] || { echo "usage: $0 lint ..." >&2; return 2; } local hits @@ -101,6 +134,8 @@ lint_workflows() { case "${1:-reclaim}" in reclaim) reclaim ;; assert-protected) shift; assert_protected "$@" ;; + hold) shift; hold "$@" ;; + release) shift; release "$@" ;; lint) shift; lint_workflows "$@" ;; - *) echo "usage: $0 [reclaim | assert-protected | lint ...]" >&2; exit 2 ;; + *) echo "usage: $0 [reclaim | assert-protected | hold | release | lint ...]" >&2; exit 2 ;; esac