pi-devbox/scripts/vendor-mempalace-skill.sh

#!/usr/bin/env bash
# vendor-mempalace-skill.sh — refresh the vendored mempalace skill snapshot
# AND its recorded provenance, together, so the two cannot drift apart.
#
# WHY THIS EXISTS
# ---------------
# rootfs/usr/local/share/pi-devbox/skills/mempalace/SKILL.md is a snapshot of a
# file owned by the PRIVATE skillset repo (see VENDORED.md). Because the image
# cannot clone that repo, refreshing the snapshot was a manual `cp` — and the
# result was anonymous: nothing recorded WHICH skillset commit the bytes came
# from. The only staleness check available was a hand-maintained phrase canary
# in scripts/smoke-test.sh, which by construction detects "older than the phrase
# I remembered to pin", never "older than skillset main".
#
# Two facts now travel with the snapshot: the skillset commit it was taken from
# (ARG SKILLSET_SNAPSHOT_REF in Dockerfile.variant) and the sha256 of the bytes
# themselves (measured at build time into build-manifest.json). This script is
# the only thing that should ever write the first one, because a `cp` without a
# matching ARG bump produces a manifest that CONFIDENTLY LIES — worse than the
# anonymous snapshot it replaced.
#
# HARDENED after peer review (pi@emb-7kj4vr4g, logstream correlation
# skills-provenance-review, 2026-08-26) proved the original --check could print
# OK and exit 0 without actually verifying anything: `git show <ref>:<path>`
# emits NOTHING when the ref/path doesn't resolve, and `sha256sum` still hashes
# that empty stdin, so "ref not found" silently collided with "the file really
# is 0 bytes". Depending on which side of the comparison hit the collision this
# fell through as either a false MISMATCH (blaming provenance for what was
# really an incomplete clone) or, worse, a false OK. See EXIT STATUS below —
# "cannot determine" is now its own outcome, distinct from "confirmed wrong",
# which is the same distinction the phrase canary this script replaced lacked.
#
# USAGE
#   scripts/vendor-mempalace-skill.sh [skillset-root] [--force]
#       refresh: rewrite the snapshot and the ARG together.
#   scripts/vendor-mempalace-skill.sh --check [skillset-root]
#       verify only, writes nothing. The root path and any flag may appear in
#       either order — a positional-only parser previously made `<root>
#       --check` silently run a refresh instead of the verification asked for.
#
#   skillset-root defaults to /workspace/skillset, then $HOME/skillset.
#
#   --force  (refresh mode only) proceed even when the recorded ref cannot be
#            proven to be an ancestor of the skillset's current HEAD — i.e.
#            skip the guard against silently REWINDING provenance, which a
#            detached HEAD, an older checkout, or a shallow clone lacking the
#            recorded commit can all trigger. Meant to be used deliberately,
#            not habitually: each use is a human deciding a rewind is fine.
#
# --check answers "is the committed snapshot really skillset@<recorded ref>?"
# — the question CI cannot answer without a credential for the private repo,
# and which anyone with the skillset checked out can answer for free.
#
# EXIT STATUS (same three codes in both modes)
#   0  the operation succeeded, or (--check) the record is verified truthful.
#      This INCLUDES a truthful record that is merely stale — upstream has
#      moved on since the recorded ref, or the local working tree has since
#      diverged. A NOTICE is printed to stderr, but the snapshot is not being
#      accused of lying, so this is not a release-blocking failure. Skipping a
#      refresh is a legitimate release-day choice (see AGENTS.md); this exit
#      code is what makes that choice checkable rather than merely asserted.
#   1  refused: a CONFIRMED problem. Dirty upstream file; a refresh that would
#      rewind past the recorded ref; or (--check) the vendored bytes provably
#      do NOT match the file at the recorded ref — a lying record.
#   2  cannot determine: the recorded ref, or the path at that ref, is not
#      resolvable in this clone. Commonly a shallow clone missing history, or
#      a ref that was rewritten or never pushed. Deliberately NOT the same as
#      1 — "I can't tell" must never be reported as "it's wrong".
set -euo pipefail

cd "$(dirname "$0")/.."

DOCKERFILE="Dockerfile.variant"
VENDORED="rootfs/usr/local/share/pi-devbox/skills/mempalace/SKILL.md"
ARG_NAME="SKILLSET_SNAPSHOT_REF"
REL_PATH="skills/mempalace/SKILL.md"

die() { printf '%s: %s\n' "$(basename "$0")" "$1" >&2; exit 1; }

# Parse flags and the optional root path in either order, and reject anything
# unrecognised rather than silently absorbing it.
MODE="refresh"
FORCE=0
ROOT=""
for arg in "$@"; do
  case "$arg" in
    --check) MODE="check" ;;
    --force) FORCE=1 ;;
    # This is the one script whose argument ORDER was itself a landmine, so the
    # path that documents the trap must not be the path that errors.
    -h|--help)
      awk 'NR>1 && /^#/ { sub(/^# ?/, ""); print; next } NR>1 { exit }' "$0"
      exit 0
      ;;
    --*) die "unknown option: $arg (try --help)" ;;
    *)
      [ -z "$ROOT" ] || die "unexpected extra argument: $arg (root already set to $ROOT)"
      ROOT="$arg"
      ;;
  esac
done
if [ "$MODE" = "check" ] && [ "$FORCE" = 1 ]; then
  die "--force has no effect with --check (nothing is written); remove it"
fi

if [ -z "$ROOT" ]; then
  for candidate in /workspace/skillset "$HOME/skillset"; do
    if [ -d "$candidate/.git" ]; then
      ROOT="$candidate"
      break
    fi
  done
fi

[ -n "$ROOT" ] || die "no skillset clone found (pass one: $(basename "$0") /path/to/skillset)"
[ -d "$ROOT/.git" ] || die "not a git clone: $ROOT"
[ -f "$ROOT/$REL_PATH" ] || die "no $REL_PATH in $ROOT"
[ -f "$VENDORED" ] || die "vendored snapshot missing: $VENDORED"
# LOAD-BEARING, DO NOT DELETE AS "REDUNDANT WITH THE EXISTENCE PROBES": -f
# accepts an empty file, and sha256 of an empty file equals sha256 of a failed
# pipeline's empty stdin. Guarding it HERE, before mode dispatch, makes that
# collision unreachable by construction rather than by a probe further down --
# which also means no test below exercises the collision any more. Remove this
# line and the false "OK" for a nonexistent ref returns with nothing failing.
[ -s "$VENDORED" ] || die "vendored snapshot is empty: $VENDORED"

head_sha=$(git -C "$ROOT" rev-parse HEAD 2>/dev/null) || die "cannot read HEAD of $ROOT"
recorded=$(grep -oE "^ARG ${ARG_NAME}=[0-9a-f]{40}$" "$DOCKERFILE" | cut -d= -f2 || true)
[ -n "$recorded" ] || die "no 'ARG ${ARG_NAME}=<40-hex>' line in $DOCKERFILE"

sha_of() { sha256sum "$1" | cut -d' ' -f1; }
vendored_sha=$(sha_of "$VENDORED")
upstream_sha=$(sha_of "$ROOT/$REL_PATH")

# Does $REL_PATH exist at HEAD at all? Proven with `cat-file -e` BEFORE
# hashing anything. Piping a failed `git show` straight into sha256sum, as
# this script used to, hashes an EMPTY stream and produces sha256(""): a real,
# collidable value — not a representation of absence. That collapsed "doesn't
# exist" and "exists and happens to be empty" into the same signal, which is
# exactly the defect class the peer review found in --check's at_ref, below.
blob_sha=""
if git -C "$ROOT" cat-file -e "HEAD:$REL_PATH" 2>/dev/null; then
  blob_sha=$(git -C "$ROOT" show "HEAD:$REL_PATH" | sha256sum | cut -d' ' -f1)
fi

upstream_dirty=""
if [ -z "$blob_sha" ]; then
  upstream_dirty="not present at HEAD (untracked, or absent at this commit)"
elif [ "$blob_sha" != "$upstream_sha" ]; then
  if ! git -C "$ROOT" diff --quiet -- "$REL_PATH" 2>/dev/null; then
    upstream_dirty="modified but not committed"
  elif ! git -C "$ROOT" diff --cached --quiet -- "$REL_PATH" 2>/dev/null; then
    upstream_dirty="staged but not committed"
  else
    upstream_dirty="different at HEAD than in the working tree"
  fi
fi

if [ "$MODE" = "check" ]; then
  # Resolve the recorded ref the same careful way: existence is proven with
  # `cat-file -e` before anything is hashed, and "the ref itself is missing"
  # is reported distinctly from "the ref resolves but the path isn't there
  # at it" — both used to be silently swallowed into a plausible sha256("").
  ref_exists=0
  path_at_ref_exists=0
  at_ref=""
  if git -C "$ROOT" cat-file -e "${recorded}^{commit}" 2>/dev/null; then
    ref_exists=1
    if git -C "$ROOT" cat-file -e "${recorded}:${REL_PATH}" 2>/dev/null; then
      path_at_ref_exists=1
      at_ref=$(git -C "$ROOT" show "${recorded}:${REL_PATH}" | sha256sum | cut -d' ' -f1)
    fi
  fi

  printf 'recorded ref:      %s\n' "$recorded"
  printf 'vendored sha256:   %s\n' "$vendored_sha"
  if [ "$path_at_ref_exists" = 1 ]; then
    printf 'sha256 at ref:     %s\n' "$at_ref"
  elif [ "$ref_exists" = 1 ]; then
    printf 'sha256 at ref:     <%s not present at %s>\n' "$REL_PATH" "${recorded:0:7}"
  else
    printf 'sha256 at ref:     <%s not present in this clone>\n' "${recorded:0:7}"
  fi
  printf 'skillset HEAD:     %s (%s)\n' "$head_sha" "$upstream_sha"
  if [ -n "$upstream_dirty" ]; then
    printf 'live working tree: %s\n' "$upstream_dirty"
  fi

  rc=0
  if [ "$ref_exists" != 1 ]; then
    printf 'CANNOT-DETERMINE: %s is not present in %s — fetch, or check against a complete clone\n' "$recorded" "$ROOT" >&2
    rc=2
  elif [ "$path_at_ref_exists" != 1 ]; then
    printf 'MISMATCH: %s does not exist at %s in this clone — the recorded ref cannot be describing these bytes\n' "$REL_PATH" "$recorded" >&2
    rc=1
  elif [ "$at_ref" != "$vendored_sha" ]; then
    printf 'MISMATCH: the vendored snapshot is NOT the file at the recorded ref\n' >&2
    rc=1
  else
    printf 'OK: the vendored snapshot is exactly skillset@%s:%s\n' "${recorded:0:7}" "$REL_PATH"
  fi

  # Staleness is orthogonal to truthfulness: a record can correctly describe
  # an old commit even after upstream has moved on, and a dirty local working
  # tree in $ROOT doesn't rewrite git history either — it says nothing about
  # whether the RECORDED, committed ref describes the RECORDED, committed
  # bytes. Only worth reporting once we already know rc=0 (truthful) — a
  # MISMATCH or CANNOT-DETERMINE is the dominant fact and a staleness note
  # would only muddy it.
  if [ "$rc" = 0 ] && [ "$vendored_sha" != "$upstream_sha" ]; then
    # Name the ACTUAL cause. "working tree differs" is wrong when the tree is
    # clean and the ref simply moved on — a message that names the wrong cause
    # is the same defect class as a canary pinned to a deleted phrase.
    if [ "$recorded" != "$head_sha" ] && [ "$blob_sha" = "$upstream_sha" ]; then
      # Do not ASSERT which side is newer — test it. Asserting that HEAD is the
      # newer side points the operator at a refresh (which costs a ~67-minute
      # base rebuild) when the real remedy may be `git pull` in this clone. The
      # refresh path below already uses this primitive; reuse it here.
      if git -C "$ROOT" merge-base --is-ancestor "$recorded" "$head_sha" 2>/dev/null; then
        printf 'NOTICE: %s has moved on to %s; the snapshot describes the older %s (stale, not untruthful — refresh to catch up)\n' \
          "$ROOT" "${head_sha:0:7}" "${recorded:0:7}" >&2
      elif git -C "$ROOT" merge-base --is-ancestor "$head_sha" "$recorded" 2>/dev/null; then
        printf 'NOTICE: %s is BEHIND at %s; the snapshot describes the newer %s — pull this clone, do NOT refresh the snapshot\n' \
          "$ROOT" "${head_sha:0:7}" "${recorded:0:7}" >&2
      else
        printf 'NOTICE: %s (HEAD %s) and the recorded %s have DIVERGED — neither is an ancestor of the other; reconcile the clone before refreshing\n' \
          "$ROOT" "${head_sha:0:7}" "${recorded:0:7}" >&2
      fi
    else
      printf 'NOTICE: the working tree of %s differs from the snapshot (HEAD %s)\n' \
        "$ROOT" "${head_sha:0:7}" >&2
    fi
  fi

  exit "$rc"
fi

[ -z "$upstream_dirty" ] || die "$ROOT/$REL_PATH is $upstream_dirty — commit it first, or the recorded ref would not describe these bytes"

if [ "$vendored_sha" = "$upstream_sha" ] && [ "$recorded" = "$head_sha" ]; then
  printf 'already current: snapshot == skillset@%s\n' "${head_sha:0:7}"
  exit 0
fi

# Refuse to silently REWIND provenance. `git checkout <tag>`, a detached HEAD,
# or an older checkout can all leave $ROOT's HEAD behind the already-recorded
# ref; without this guard a refresh there would happily rewrite both the ARG
# and the bytes backwards and report it as an ordinary update.
if [ "$recorded" != "$head_sha" ]; then
  if git -C "$ROOT" cat-file -e "${recorded}^{commit}" 2>/dev/null; then
    if ! git -C "$ROOT" merge-base --is-ancestor "$recorded" "$head_sha" 2>/dev/null; then
      if [ "$FORCE" != 1 ]; then
        die "refusing: $ROOT's HEAD ($head_sha) is not a descendant of the recorded ref ($recorded) — this looks like a rewind. Pass --force if this is intentional."
      fi
      printf 'WARNING: --force set; %s is not an ancestor of HEAD %s — proceeding anyway\n' "${recorded:0:7}" "${head_sha:0:7}" >&2
    fi
  else
    if [ "$FORCE" != 1 ]; then
      printf 'CANNOT-DETERMINE: %s is not present in %s (shallow clone?) — fetch full history to verify this refresh moves forward, or pass --force to proceed without that guarantee\n' "$recorded" "$ROOT" >&2
      exit 2
    fi
    printf 'WARNING: --force set; %s could not be resolved in %s — proceeding without verifying forward motion\n' "${recorded:0:7}" "$ROOT" >&2
  fi
fi

# Written FROM THE REF, not copied from the working tree, so the pair cannot
# be a lie by construction. Via a temp file so a failed write cannot leave a
# half-vendored snapshot behind.
snap_tmp=$(mktemp)
if ! git -C "$ROOT" show "HEAD:$REL_PATH" > "$snap_tmp" 2>/dev/null; then
  rm -f -- "$snap_tmp"
  die "cannot read HEAD:$REL_PATH from $ROOT"
fi
chmod 0644 -- "$snap_tmp"
mv -- "$snap_tmp" "$VENDORED"
[ "$(sha_of "$VENDORED")" = "$blob_sha" ] \
  || die "internal: written snapshot does not match HEAD:$REL_PATH"

# In-place, and only the exact pinned line: a broad sed on this Dockerfile
# could rewrite one of the other *_REF ARGs.
tmp=$(mktemp)
sed "s|^ARG ${ARG_NAME}=.*\$|ARG ${ARG_NAME}=${head_sha}|" "$DOCKERFILE" > "$tmp"
chmod 0644 -- "$tmp"
mv -- "$tmp" "$DOCKERFILE"

new_recorded=$(grep -oE "^ARG ${ARG_NAME}=[0-9a-f]{40}$" "$DOCKERFILE" | cut -d= -f2 || true)
[ "$new_recorded" = "$head_sha" ] || die "failed to rewrite ${ARG_NAME} in $DOCKERFILE"

printf 'snapshot:  %s -> %s\n' "${vendored_sha:0:12}" "$(sha_of "$VENDORED" | cut -c1-12)"
printf 'ref:       %s -> %s\n' "${recorded:0:7}" "${head_sha:0:7}"
printf '\nNOTE: %s is hashed into base_tag, so this costs a base rebuild\n' "$VENDORED"
printf 'on the next tag (~67 min). Also re-pin the phrase canary in\n'
printf 'scripts/smoke-test.sh if the section it names changed.\n'