ci: workflow-level 3-attempt retry around buildx build --push

Belt-and-braces against transient registry-1.docker.io blips (rate limits, brief 5xx, CDN flap). Replaces all five push docker/build-push- action@v7 invocations (1 base + 4 variants) with shell: bash steps that run docker buildx build --push in a for-loop with backoff (15s, 30s). Smoke build steps (load: true, no push) are untouched. Does NOT mask deterministic failures: a true regression (e.g. the cache-export 400 we hit 2026-05-23..28) fails all 3 attempts identically and the job still fails by design. Orthogonal layer to both cache-export disablement and the ci-release-watcher skill's transient-rerun heuristic. - AGENTS.md: new Critical conventions bullet documenting the retry pattern, the consistency rule across push steps, and why it's duplicated rather than factored (Gitea Actions doesn't support reusable composite shell steps cleanly). - CHANGELOG.md: Unreleased section addendum, no image-side change. No image-side change.
2026-05-28 16:32:41 +02:00
parent 6cc2670a93
commit 1fe5b5df91
3 changed files with 178 additions and 60 deletions
@@ -184,14 +184,17 @@ jobs:
          username: ${{ vars.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

-      - name: Build and push base (multi-arch)
-        uses: docker/build-push-action@v7
-        with:
-          context: .
-          file: Dockerfile.base
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ env.IMAGE }}:${{ needs.base-decide.outputs.base_tag }}
+      - name: Build and push base (multi-arch) — with retry
+        shell: bash
+        env:
+          BASE_TAG_FULL: ${{ env.IMAGE }}:${{ needs.base-decide.outputs.base_tag }}
+        run: |
+          set -euo pipefail
+          # 3-attempt retry around `docker buildx build --push` for transient
+          # registry-1.docker.io blips. Does NOT mask deterministic failures:
+          # a true regression (e.g. cache-export 400 hit 2026-05-23..28) will
+          # fail all 3 attempts identically and the job still fails — by
+          # design.
          # Registry cache disabled: buildkit's cache-export (mode=max) hits a
          # reproducible HTTP 400 from registry-1.docker.io on the resumable-
          # upload PUT (state-token format mismatch on Hub CDN, suspected to
@@ -200,6 +203,25 @@ jobs:
          # tag itself is content-addressed (base-<hash>) so unchanged bases
          # short-circuit at the probe step and never re-build anyway. Re-
          # enable when upstream resolves; tracked in CHANGELOG v1.15.12.
+          for attempt in 1 2 3; do
+            echo "==> Build+push attempt ${attempt}/3"
+            if docker buildx build \
+                --platform linux/amd64,linux/arm64 \
+                --file Dockerfile.base \
+                --push \
+                --tag "${BASE_TAG_FULL}" \
+                .; then
+              echo "==> Attempt ${attempt} succeeded"
+              exit 0
+            fi
+            if [[ "${attempt}" -lt 3 ]]; then
+              backoff=$(( attempt * 15 ))
+              echo "==> Attempt ${attempt} failed, sleeping ${backoff}s before retry"
+              sleep "${backoff}"
+            fi
+          done
+          echo "==> All 3 build+push attempts failed"
+          exit 1

  # ── Phase 3: amd64 smoke per variant (gates the multi-arch publish) ─
  # Each smoke job builds amd64-only against the base tag and runs
@@ -425,18 +447,40 @@ jobs:
            fi
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
-      - uses: docker/build-push-action@v7
-        with:
-          context: .
-          file: Dockerfile.variant
-          platforms: linux/amd64,linux/arm64
-          push: true
-          build-args: |
-            BASE_IMAGE=${{ env.IMAGE }}:${{ needs.base-decide.outputs.base_tag }}
-            INSTALL_OPENCODE=true
-            INSTALL_OMOS=false
-            INSTALL_PI=false
-          tags: ${{ steps.tags.outputs.tags }}
+      - name: Build and push variant (with retry)
+        shell: bash
+        env:
+          TAGS: ${{ steps.tags.outputs.tags }}
+          BASE_IMAGE_FULL: ${{ env.IMAGE }}:${{ needs.base-decide.outputs.base_tag }}
+        run: |
+          set -euo pipefail
+          TAG_FLAGS=()
+          while IFS= read -r t; do [[ -n "$t" ]] && TAG_FLAGS+=( -t "$t" ); done <<< "${TAGS}"
+          # 3-attempt retry around `docker buildx build --push` (see build-base
+          # step for full rationale). Variant: base (opencode only).
+          for attempt in 1 2 3; do
+            echo "==> Build+push attempt ${attempt}/3"
+            if docker buildx build \
+                --platform linux/amd64,linux/arm64 \
+                --file Dockerfile.variant \
+                --push \
+                --build-arg "BASE_IMAGE=${BASE_IMAGE_FULL}" \
+                --build-arg "INSTALL_OPENCODE=true" \
+                --build-arg "INSTALL_OMOS=false" \
+                --build-arg "INSTALL_PI=false" \
+                "${TAG_FLAGS[@]}" \
+                .; then
+              echo "==> Attempt ${attempt} succeeded"
+              exit 0
+            fi
+            if [[ "${attempt}" -lt 3 ]]; then
+              backoff=$(( attempt * 15 ))
+              echo "==> Attempt ${attempt} failed, sleeping ${backoff}s before retry"
+              sleep "${backoff}"
+            fi
+          done
+          echo "==> All 3 build+push attempts failed"
+          exit 1

  build-variant-omos:
    needs: [base-decide, smoke-omos, resolve-versions]
@@ -473,19 +517,41 @@ jobs:
            fi
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
-      - uses: docker/build-push-action@v7
-        with:
-          context: .
-          file: Dockerfile.variant
-          platforms: linux/amd64,linux/arm64
-          push: true
-          build-args: |
-            BASE_IMAGE=${{ env.IMAGE }}:${{ needs.base-decide.outputs.base_tag }}
-            INSTALL_OPENCODE=true
-            INSTALL_OMOS=true
-            INSTALL_PI=false
-            OMOS_VERSION=${{ needs.resolve-versions.outputs.omos_version }}
-          tags: ${{ steps.tags.outputs.tags }}
+      - name: Build and push variant (with retry)
+        shell: bash
+        env:
+          TAGS: ${{ steps.tags.outputs.tags }}
+          BASE_IMAGE_FULL: ${{ env.IMAGE }}:${{ needs.base-decide.outputs.base_tag }}
+          OMOS_VERSION: ${{ needs.resolve-versions.outputs.omos_version }}
+        run: |
+          set -euo pipefail
+          TAG_FLAGS=()
+          while IFS= read -r t; do [[ -n "$t" ]] && TAG_FLAGS+=( -t "$t" ); done <<< "${TAGS}"
+          # 3-attempt retry (see build-base step for rationale). Variant: omos.
+          for attempt in 1 2 3; do
+            echo "==> Build+push attempt ${attempt}/3"
+            if docker buildx build \
+                --platform linux/amd64,linux/arm64 \
+                --file Dockerfile.variant \
+                --push \
+                --build-arg "BASE_IMAGE=${BASE_IMAGE_FULL}" \
+                --build-arg "INSTALL_OPENCODE=true" \
+                --build-arg "INSTALL_OMOS=true" \
+                --build-arg "INSTALL_PI=false" \
+                --build-arg "OMOS_VERSION=${OMOS_VERSION}" \
+                "${TAG_FLAGS[@]}" \
+                .; then
+              echo "==> Attempt ${attempt} succeeded"
+              exit 0
+            fi
+            if [[ "${attempt}" -lt 3 ]]; then
+              backoff=$(( attempt * 15 ))
+              echo "==> Attempt ${attempt} failed, sleeping ${backoff}s before retry"
+              sleep "${backoff}"
+            fi
+          done
+          echo "==> All 3 build+push attempts failed"
+          exit 1

  build-variant-with-pi:
    needs: [base-decide, smoke-with-pi, resolve-versions]
@@ -522,19 +588,41 @@ jobs:
            fi
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
-      - uses: docker/build-push-action@v7
-        with:
-          context: .
-          file: Dockerfile.variant
-          platforms: linux/amd64,linux/arm64
-          push: true
-          build-args: |
-            BASE_IMAGE=${{ env.IMAGE }}:${{ needs.base-decide.outputs.base_tag }}
-            INSTALL_OPENCODE=true
-            INSTALL_OMOS=false
-            INSTALL_PI=true
-            PI_VERSION=${{ needs.resolve-versions.outputs.pi_version }}
-          tags: ${{ steps.tags.outputs.tags }}
+      - name: Build and push variant (with retry)
+        shell: bash
+        env:
+          TAGS: ${{ steps.tags.outputs.tags }}
+          BASE_IMAGE_FULL: ${{ env.IMAGE }}:${{ needs.base-decide.outputs.base_tag }}
+          PI_VERSION: ${{ needs.resolve-versions.outputs.pi_version }}
+        run: |
+          set -euo pipefail
+          TAG_FLAGS=()
+          while IFS= read -r t; do [[ -n "$t" ]] && TAG_FLAGS+=( -t "$t" ); done <<< "${TAGS}"
+          # 3-attempt retry (see build-base step for rationale). Variant: with-pi.
+          for attempt in 1 2 3; do
+            echo "==> Build+push attempt ${attempt}/3"
+            if docker buildx build \
+                --platform linux/amd64,linux/arm64 \
+                --file Dockerfile.variant \
+                --push \
+                --build-arg "BASE_IMAGE=${BASE_IMAGE_FULL}" \
+                --build-arg "INSTALL_OPENCODE=true" \
+                --build-arg "INSTALL_OMOS=false" \
+                --build-arg "INSTALL_PI=true" \
+                --build-arg "PI_VERSION=${PI_VERSION}" \
+                "${TAG_FLAGS[@]}" \
+                .; then
+              echo "==> Attempt ${attempt} succeeded"
+              exit 0
+            fi
+            if [[ "${attempt}" -lt 3 ]]; then
+              backoff=$(( attempt * 15 ))
+              echo "==> Attempt ${attempt} failed, sleeping ${backoff}s before retry"
+              sleep "${backoff}"
+            fi
+          done
+          echo "==> All 3 build+push attempts failed"
+          exit 1

  build-variant-omos-with-pi:
    needs: [base-decide, smoke-omos-with-pi, resolve-versions]
@@ -571,20 +659,43 @@ jobs:
            fi
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
-      - uses: docker/build-push-action@v7
-        with:
-          context: .
-          file: Dockerfile.variant
-          platforms: linux/amd64,linux/arm64
-          push: true
-          build-args: |
-            BASE_IMAGE=${{ env.IMAGE }}:${{ needs.base-decide.outputs.base_tag }}
-            INSTALL_OPENCODE=true
-            INSTALL_OMOS=true
-            INSTALL_PI=true
-            PI_VERSION=${{ needs.resolve-versions.outputs.pi_version }}
-            OMOS_VERSION=${{ needs.resolve-versions.outputs.omos_version }}
-          tags: ${{ steps.tags.outputs.tags }}
+      - name: Build and push variant (with retry)
+        shell: bash
+        env:
+          TAGS: ${{ steps.tags.outputs.tags }}
+          BASE_IMAGE_FULL: ${{ env.IMAGE }}:${{ needs.base-decide.outputs.base_tag }}
+          PI_VERSION: ${{ needs.resolve-versions.outputs.pi_version }}
+          OMOS_VERSION: ${{ needs.resolve-versions.outputs.omos_version }}
+        run: |
+          set -euo pipefail
+          TAG_FLAGS=()
+          while IFS= read -r t; do [[ -n "$t" ]] && TAG_FLAGS+=( -t "$t" ); done <<< "${TAGS}"
+          # 3-attempt retry (see build-base step for rationale). Variant: omos-with-pi.
+          for attempt in 1 2 3; do
+            echo "==> Build+push attempt ${attempt}/3"
+            if docker buildx build \
+                --platform linux/amd64,linux/arm64 \
+                --file Dockerfile.variant \
+                --push \
+                --build-arg "BASE_IMAGE=${BASE_IMAGE_FULL}" \
+                --build-arg "INSTALL_OPENCODE=true" \
+                --build-arg "INSTALL_OMOS=true" \
+                --build-arg "INSTALL_PI=true" \
+                --build-arg "PI_VERSION=${PI_VERSION}" \
+                --build-arg "OMOS_VERSION=${OMOS_VERSION}" \
+                "${TAG_FLAGS[@]}" \
+                .; then
+              echo "==> Attempt ${attempt} succeeded"
+              exit 0
+            fi
+            if [[ "${attempt}" -lt 3 ]]; then
+              backoff=$(( attempt * 15 ))
+              echo "==> Attempt ${attempt} failed, sleeping ${backoff}s before retry"
+              sleep "${backoff}"
+            fi
+          done
+          echo "==> All 3 build+push attempts failed"
+          exit 1

  # ── Phase 5: promote base-<hash> → base-latest (manifest copy only) ─
  promote-base-latest: