diff --git a/.gitea/workflows/docker-publish.yml b/.gitea/workflows/docker-publish.yml index ba06b0a..a9bee3e 100644 --- a/.gitea/workflows/docker-publish.yml +++ b/.gitea/workflows/docker-publish.yml @@ -90,6 +90,12 @@ env: IMAGE: ${{ vars.DOCKERHUB_USERNAME }}/pi-devbox RELEASE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || inputs.release_tag }} PROMOTE_LATEST: ${{ github.ref_type == 'tag' && 'true' || inputs.promote_latest }} + # Label that protects a smoke job's loaded image from the disk reclaim every + # job runs on the shared runner daemon (scripts/ci-reclaim-docker.sh says + # why: v1.12.2 run 740 lost pi-devbox:smoke-studio to build-variant's prune + # 30 ms after loading it). Set once here; the smoke builds apply it, the + # reclaim script honours it, the smoke preflight asserts it. + SMOKE_IMAGE_LABEL: se.jordbo.pi-devbox.ci-smoke-image jobs: # ── Phase 1: decide whether base needs rebuilding ────────────────── @@ -538,8 +544,7 @@ jobs: /usr/lib/jvm 2>/dev/null || true apt-get clean || true rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* || true - docker system prune -af --volumes || true - docker builder prune -af || true + bash scripts/ci-reclaim-docker.sh df -h / || true - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -597,6 +602,10 @@ jobs: needs.resolve-versions.result == 'success' && (needs.build-base.result == 'success' || needs.build-base.result == 'skipped') runs-on: ubuntu-latest + env: + # Per run, so two runs on different refs cannot test or delete each + # other's image on the shared daemon. run_id is stable across re-runs. + SMOKE_IMAGE: pi-devbox:smoke-${{ github.run_id }} container: image: catthehacker/ubuntu:act-latest steps: @@ -610,8 +619,7 @@ jobs: /usr/local/lib/android /usr/local/share/powershell \ /usr/local/share/chromium /usr/local/share/boost \ /usr/lib/jvm 2>/dev/null || true - docker system prune -af --volumes || true - docker builder prune -af || true + bash scripts/ci-reclaim-docker.sh - uses: docker/setup-buildx-action@v4 with: {driver-opts: network=host} - uses: docker/login-action@v3 @@ -626,7 +634,8 @@ jobs: platforms: linux/amd64 push: false load: true - tags: pi-devbox:smoke + tags: ${{ env.SMOKE_IMAGE }} + labels: ${{ env.SMOKE_IMAGE_LABEL }}=true build-args: | BASE_IMAGE=${{ env.IMAGE }}:${{ needs.base-decide.outputs.base_tag }} PI_VERSION=${{ needs.resolve-versions.outputs.pi_version }} @@ -648,7 +657,14 @@ jobs: # Asserting the BUILT image matches it also catches a stale cached layer. EXPECTED_NODE_MAJOR=$(sed -n 's/^ARG NODE_VERSION=\([0-9][0-9]*\).*/\1/p' Dockerfile.base) export EXPECTED_NODE_MAJOR - bash scripts/smoke-test.sh pi-devbox:smoke + bash scripts/ci-reclaim-docker.sh assert-protected "$SMOKE_IMAGE" + bash scripts/smoke-test.sh "$SMOKE_IMAGE" + # The label keeps this image from every job's reclaim, so it is removed + # here instead. A job killed before this step leaves an orphan, which the + # reclaim's age sweep deletes once it is older than SMOKE_IMAGE_MAX_AGE. + - name: Remove smoke image + if: always() + run: docker image rm "$SMOKE_IMAGE" || true # ── Phase 3b: amd64 smoke for the studio variant ──────────────────── # Additive + independent of the core `smoke` job: gates ONLY @@ -662,6 +678,10 @@ jobs: needs.resolve-versions.result == 'success' && (needs.build-base.result == 'success' || needs.build-base.result == 'skipped') runs-on: ubuntu-latest + env: + # Per run, so two runs on different refs cannot test or delete each + # other's image on the shared daemon. run_id is stable across re-runs. + SMOKE_IMAGE: pi-devbox:smoke-studio-${{ github.run_id }} container: image: catthehacker/ubuntu:act-latest steps: @@ -675,8 +695,7 @@ jobs: /usr/local/lib/android /usr/local/share/powershell \ /usr/local/share/chromium /usr/local/share/boost \ /usr/lib/jvm 2>/dev/null || true - docker system prune -af --volumes || true - docker builder prune -af || true + bash scripts/ci-reclaim-docker.sh - uses: docker/setup-buildx-action@v4 with: {driver-opts: network=host} - uses: docker/login-action@v3 @@ -691,7 +710,8 @@ jobs: platforms: linux/amd64 push: false load: true - tags: pi-devbox:smoke-studio + tags: ${{ env.SMOKE_IMAGE }} + labels: ${{ env.SMOKE_IMAGE_LABEL }}=true build-args: | BASE_IMAGE=${{ env.IMAGE }}:${{ needs.base-decide.outputs.base_tag }} PI_VERSION=${{ needs.resolve-versions.outputs.pi_version }} @@ -716,7 +736,14 @@ jobs: # Asserting the BUILT image matches it also catches a stale cached layer. EXPECTED_NODE_MAJOR=$(sed -n 's/^ARG NODE_VERSION=\([0-9][0-9]*\).*/\1/p' Dockerfile.base) export EXPECTED_NODE_MAJOR - bash scripts/smoke-test.sh pi-devbox:smoke-studio + bash scripts/ci-reclaim-docker.sh assert-protected "$SMOKE_IMAGE" + bash scripts/smoke-test.sh "$SMOKE_IMAGE" + # The label keeps this image from every job's reclaim, so it is removed + # here instead. A job killed before this step leaves an orphan, which the + # reclaim's age sweep deletes once it is older than SMOKE_IMAGE_MAX_AGE. + - name: Remove smoke image + if: always() + run: docker image rm "$SMOKE_IMAGE" || true # ── Phase 4: multi-arch publish ───────────────────────────────────── build-variant: @@ -741,8 +768,7 @@ jobs: /usr/local/lib/android /usr/local/share/powershell \ /usr/local/share/chromium /usr/local/share/boost \ /usr/lib/jvm 2>/dev/null || true - docker system prune -af --volumes || true - docker builder prune -af || true + bash scripts/ci-reclaim-docker.sh - uses: docker/setup-qemu-action@v3 with: {platforms: arm64} - uses: docker/setup-buildx-action@v4 @@ -834,8 +860,7 @@ jobs: /usr/local/lib/android /usr/local/share/powershell \ /usr/local/share/chromium /usr/local/share/boost \ /usr/lib/jvm 2>/dev/null || true - docker system prune -af --volumes || true - docker builder prune -af || true + bash scripts/ci-reclaim-docker.sh - uses: docker/setup-qemu-action@v3 with: {platforms: arm64} - uses: docker/setup-buildx-action@v4 diff --git a/.gitea/workflows/lint.yml b/.gitea/workflows/lint.yml index 8084bab..4552991 100644 --- a/.gitea/workflows/lint.yml +++ b/.gitea/workflows/lint.yml @@ -90,6 +90,14 @@ jobs: # actionlint so the more precise diagnostic surfaces first. run: bash scripts/check-workflow-shell.sh .gitea/workflows + - name: No raw docker prune in workflows (shared runner daemon) + # Every docker-publish.yml job shares the runner's Docker daemon, and + # a bare `docker system prune -a` in one job deletes the smoke image + # another job is testing (v1.12.2 run 740 shipped without its -studio + # tags that way). Reclaim goes through scripts/ci-reclaim-docker.sh, + # which spares labelled smoke images; this keeps it that way. + run: bash scripts/ci-reclaim-docker.sh lint .gitea/workflows/*.yml + - name: link-pi-models.sh contract (two-sided, no docker needed) # The entrypoint links ~/.pi/agent/models.json to a mounted shared file # and must never replace a real file or a user's own link. Tested here diff --git a/AGENTS.md b/AGENTS.md index e07de43..b1e1841 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,13 @@ re-brand of opencode-devbox's `pi-only` variant. update-description). The `-studio` variant adds independent `smoke-studio` + `build-variant-studio` jobs that gate only the `-studio` tags (never the core `:latest` release). +- `scripts/ci-reclaim-docker.sh` — the ONLY way a CI job frees disk. All jobs + share the runner's Docker daemon, so a bare `docker system prune -a` in one + job deletes the smoke image another job is mid-test on (v1.12.2 run 740 lost + its `-studio` tags that way). The script spares images labelled + `se.jordbo.pi-devbox.ci-smoke-image`; the smoke jobs apply that label, + assert it (`assert-protected`) and remove their own image at the end. + `lint.yml` fails on any raw `docker system|image prune` in a workflow. ## Versioning scheme diff --git a/CHANGELOG.md b/CHANGELOG.md index c9cb6bb..3db6ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,41 @@ Pre-v1.0.0 tags followed the pi npm version (`v{pi_version}[letter]`). --- +## Unreleased + +### Fixed + +- **CI: a smoke job's image could be deleted mid-test by another job's disk + reclaim.** This is why v1.12.2's `-studio` tags were not published by its + tag run. All `docker-publish.yml` jobs share the runner's Docker daemon, and + each began with `docker system prune -af --volumes`, which removes every + image that no container is using at that moment. A smoke image is unused + between `smoke-test.sh`'s one-shot `docker run`s. In run 740, `smoke` + passed and `build-variant` started. Its prune logged + `untagged: pi-devbox:smoke-studio` at 22:58:49.11Z, 30 ms after + `smoke-studio` had loaded that image. Every studio assertion then failed with + "Unable to find image … locally", and `build-variant-studio` was skipped. + `v1.12.2` and `latest` were published; `v1.12.2-studio` and + `latest-studio` were not, until the failed jobs were re-run. Now every job + reclaims disk through the new `scripts/ci-reclaim-docker.sh`. It does the + same work as before (stopped containers, unused images, networks, anonymous + volumes, build cache) but never prunes an image labelled + `se.jordbo.pi-devbox.ci-smoke-image`. The smoke builds apply that label, a + preflight fails if it is missing, and each smoke job removes its own image at + the end. Smoke images left by a killed job are swept once they are older + than 6 h. Smoke tags carry the run id (`pi-devbox:smoke-`), so two + runs on different refs cannot test or delete each other's image. `lint.yml` + fails on any raw `docker system|image prune` in a workflow. Job order is + unchanged: the studio jobs still gate only the `-studio` tags. Reordering + could not have closed the race anyway, since the two smoke jobs also prune + each other whenever their start times drift. Measured 2026-10-23 on an + isolated `docker:29-dind` (engine 29.9.2; the runner has 29.7.2). The old + prune deletes an unused labelled image (the bug). The new reclaim keeps a + fresh labelled image, deletes an unlabelled one and a labelled one created + in 1970, keeps an image a running container uses, and prints no errors. + `assert-protected` passes the labelled image and fails both an unlabelled + and a missing one. + ## v1.12.2 — 2026-10-22 Patch release. Headline: **pi `1.4.0`** (audited, see *Changed* below), plus diff --git a/scripts/ci-reclaim-docker.sh b/scripts/ci-reclaim-docker.sh new file mode 100755 index 0000000..4dbaa96 --- /dev/null +++ b/scripts/ci-reclaim-docker.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# ci-reclaim-docker.sh — free disk on the CI runner's Docker daemon WITHOUT +# deleting a smoke image that another job is still testing. +# +# Used by every docker-publish.yml job that reclaims disk (build-base, smoke, +# smoke-studio, build-variant, build-variant-studio). Not shipped in the image. +# +# bash scripts/ci-reclaim-docker.sh # reclaim (default) +# bash scripts/ci-reclaim-docker.sh assert-protected IMG # smoke preflight +# bash scripts/ci-reclaim-docker.sh lint WORKFLOW... # lint.yml guard +# +# WHY THIS EXISTS. The jobs share ONE Docker daemon (the runner's socket), and +# each used to start with `docker system prune -af --volumes`. That removes +# every image no container is using at that instant, and a smoke image is +# exactly that for most of its life: smoke-test.sh runs one `docker run --rm` +# per assertion, so between assertions nothing holds the image. v1.12.2 run 740 +# (2026-10-22): smoke passed, build-variant started and pruned, and its log +# shows `untagged: pi-devbox:smoke-studio` at 22:58:49.11Z, 30 ms after +# smoke-studio had imported that image (22:58:49.08Z). Every smoke-studio +# assertion then failed with "Unable to find image 'pi-devbox:smoke-studio' +# locally", build-variant-studio was skipped, and the release went out without +# its -studio tags. Job ORDER cannot close this: smoke and smoke-studio prune +# each other too whenever their start times drift apart, and other runs share +# the daemon. So the smoke images carry a label, and this script never prunes +# an image that has it. +# +# WHAT IT DOES. The same reclaim as `docker system prune -af --volumes` + +# `docker builder prune -af`, done per object type so the image step can take +# a filter: +# 1. stopped containers (running ones are never touched) +# 2. unused images WITHOUT the label (the old `system prune -a` image step) +# 3. unused images WITH the label, older than SMOKE_IMAGE_MAX_AGE +# — orphans of a smoke job that died before its own cleanup step +# 4. unused networks, anonymous volumes (what `system prune --volumes` did) +# 5. the build cache +# Each step is `|| true`, as before: failing to reclaim disk must not fail the +# job. A filter the daemon rejected would show up as an error line in the log. +# +# `assert-protected IMG` is the smoke jobs' preflight: it fails unless IMG is in +# the daemon AND carries the label. Without it, a build that silently lost the +# label would look fine until the next time a sibling job's prune landed +# mid-test; and a missing image gets one clear error line instead of every +# assertion failing with "Unable to find image ... locally". +# +# `lint WORKFLOW...` fails if any workflow runs `docker system prune` or +# `docker image prune` itself. One such line in any job reopens the race, so +# lint.yml runs this on every push rather than relying on review. +# +# Environment: +# SMOKE_IMAGE_LABEL label key that protects an image (default below; the +# workflow sets it once in its top-level env) +# SMOKE_IMAGE_MAX_AGE `until=` value for the orphan sweep (default 6h; a +# smoke job takes ~10 min). Compared against the image's +# Created time, which is the build time: nothing here +# sets SOURCE_DATE_EPOCH. +set -u + +SMOKE_IMAGE_LABEL="${SMOKE_IMAGE_LABEL:-se.jordbo.pi-devbox.ci-smoke-image}" +SMOKE_IMAGE_MAX_AGE="${SMOKE_IMAGE_MAX_AGE:-6h}" + +step() { printf '+ %s\n' "$*"; "$@" || true; } + +reclaim() { + step docker container prune -f + step docker image prune -af --filter "label!=${SMOKE_IMAGE_LABEL}" + step docker image prune -af --filter "label=${SMOKE_IMAGE_LABEL}" \ + --filter "until=${SMOKE_IMAGE_MAX_AGE}" + step docker network prune -f + step docker volume prune -f + step docker builder prune -af +} + +assert_protected() { + local img="${1:?usage: $0 assert-protected }" val + if ! val=$(docker image inspect \ + --format "{{ index .Config.Labels \"${SMOKE_IMAGE_LABEL}\" }}" "$img" 2>&1); then + echo "::error::${img} is not in the runner's Docker daemon (${val}). Was it removed by a concurrent prune that does not honour ${SMOKE_IMAGE_LABEL}?" + return 1 + fi + if [ "$val" != "true" ]; then + echo "::error::${img} lacks the label ${SMOKE_IMAGE_LABEL}=true (got '${val}'), so a sibling job's reclaim could delete it mid-test. Check the build step's labels: input." + return 1 + fi + echo "OK: ${img} is present and carries ${SMOKE_IMAGE_LABEL}=true" +} + +lint_workflows() { + [ "$#" -gt 0 ] || { echo "usage: $0 lint ..." >&2; return 2; } + local hits + hits=$(grep -nE '^[^#]*docker[[:space:]]+(system|image)[[:space:]]+prune' "$@" || true) + if [ -n "$hits" ]; then + echo "::error::raw docker prune in a workflow. It deletes smoke images other jobs are testing on the shared runner daemon; call 'bash scripts/ci-reclaim-docker.sh' instead." + printf '%s\n' "$hits" + return 1 + fi + echo "OK: no raw 'docker system|image prune' in $# workflow file(s)" +} + +case "${1:-reclaim}" in + reclaim) reclaim ;; + assert-protected) shift; assert_protected "$@" ;; + lint) shift; lint_workflows "$@" ;; + *) echo "usage: $0 [reclaim | assert-protected | lint ...]" >&2; exit 2 ;; +esac