pi-devbox/scripts/smoke-test.sh

#!/usr/bin/env bash
# smoke-test.sh — sanity checks for the pi-devbox image
#
# Usage: ./scripts/smoke-test.sh <image>
#
# Verifies:
#   - pi binary present and (if EXPECTED_PI_VERSION set) matches CI's resolved version
#   - new v1.0.0 base additions (pandoc, graphviz, imagemagick, yq, tealdeer)
#   - typst PDF engine for pandoc (Unreleased) — `pandoc --pdf-engine=typst`
#   - non-modal editors nano + micro (alongside nvim)
#   - terminfo for modern emulators: xterm-kitty, xterm-ghostty, wezterm,
#     alacritty, foot (kitty-terminfo + ncurses-term + compiled ghostty alias)
#   - tmux 0-indexing baked in /etc/tmux.conf (required for pi-studio variants)
#   - pi-toolkit cloned at /opt/pi-toolkit
#   - pi-extensions cloned at /opt/pi-extensions
#   - pi-atelier vendored at /opt/pi-atelier, registered from /opt (not npm:),
#     and >= the version floor pi's TUI requires (see the floor test)
#   - pi-fork + pi-observational-memory cloned with node_modules baked
#   - entrypoint deploys pi-toolkit keybindings symlink
#   - entrypoint deploys ≥4 extensions
#   - mempalace bridge symlink present
#   - settings.json bootstrapped
#   - pi-fork + pi-observational-memory registered in settings.json packages[]
#     via `pi install`
#   - pi-devbox-version command present + wraps the build manifest correctly
#     (human, --json, --quiet)
#   - (studio variant only, auto-detected) pi-studio cloned + prebuilt
#     client bundle present + registered via `pi install`
#   - image size within threshold

set -euo pipefail

IMAGE="${1:?usage: $0 <image>}"
PASS=0; FAIL=0
# pi-devbox v1.0.0 (decoupled from opencode-devbox) added pandoc, graphviz,
# imagemagick, yq, tealdeer, a baked /etc/tmux.conf, and the non-modal
# editors nano + micro (~15 MB combined). v1.6.0 baked in agent-browser +
# Playwright Chromium (~291 MB net after dropping the unused headless-shell
# build), which lifted the baseline. CI amd64 actuals observed on run 512
# (v1.6.1): 3411 MB non-studio, 3574 MB studio. Threshold below carries
# ~225 MB margin above the studio number to absorb minor arch/build-cache
# differences and small future growth without false reds, while still
# catching an unexpected +GB regression.
SIZE_THRESHOLD_MB=3800

run() {
  local label="$1"; local cmd="$2"
  if docker run --rm --entrypoint="" "$IMAGE" sh -c "$cmd" >/dev/null 2>&1; then
    printf "  ✅  %s\n" "$label"; PASS=$((PASS+1))
  else
    printf "  ❌  %s\n" "$label"; FAIL=$((FAIL+1))
  fi
}

# Stricter version of `run` that asserts an expected substring in stdout.
# Catches the "image bytes silently identical to previous release" class of
# regression — Docker layer cache hit on `npm install -g <pkg>` because the
# bare command string is identical across builds, even when `latest` would
# resolve differently. Discovered 2026-05-23 — every pi-devbox release
# v0.74.0..v0.75.5 had been shipping the same image bytes.
run_expect() {
  local label="$1"; local cmd="$2"; local expect="$3"
  local out
  out=$(docker run --rm --entrypoint="" "$IMAGE" sh -c "$cmd" 2>&1) || true
  if echo "$out" | grep -Fq "$expect"; then
    printf "  ✅  %s (got %s)\n" "$label" "$expect"; PASS=$((PASS+1))
  else
    printf "  ❌  %s — expected substring %q, got: %s\n" "$label" "$expect" "$out"; FAIL=$((FAIL+1))
  fi
}

echo "=== pi-devbox smoke test: $IMAGE ==="
echo ""

# ── Binaries ─────────────────────────────────────────────────────────
echo "── Binaries ──"
if [ -n "${EXPECTED_PI_VERSION:-}" ]; then
  run_expect "pi version matches build arg" "pi --version" "$EXPECTED_PI_VERSION"
else
  run "pi"              "pi --version"
fi
run "node"            "node --version"
run "git"             "git --version"
run "aws"             "aws --version"
run "uv"              "uv --version"
run "nvim"            "nvim --version"
run "nano"            "nano --version"
run "micro"           "micro --version"
run "kitty-terminfo"  "infocmp -x xterm-kitty >/dev/null 2>&1"
run "terminfo: modern emulators (ncurses-term)" 'for t in wezterm alacritty foot ghostty st-256color; do infocmp -x "$t" >/dev/null 2>&1 || exit 1; done'
run "terminfo: xterm-ghostty alias (tic)" "infocmp -x xterm-ghostty >/dev/null 2>&1"
run "nvim true-colour default (sysinit.vim)" "nvim --headless -c 'lua os.exit(vim.o.termguicolors and 0 or 1)'"
run "mempalace-mcp"   "mempalace-mcp --help"
# v1.0.0 base additions — verify presence and basic functionality.
run "pandoc"          "pandoc --version"
run "typst"           "typst --version"
run "pandoc+typst PDF engine" "printf '# hi\n' | pandoc --pdf-engine=typst -o /tmp/_smoke.pdf - && test -s /tmp/_smoke.pdf; rm -f /tmp/_smoke.pdf"
run "graphviz (dot)"  "dot -V"
run "imagemagick"     "magick --version"
run "yq (mikefarah v4)" "yq --version | grep -qE 'mikefarah.*version v4'"
run "tldr (tealdeer)" "tldr --version"
run "socat"           "socat -V"
run "studio-expose helper" "test -x /usr/local/bin/studio-expose"
run "image-baked pi-devbox-environment skill" \
    "test -f /usr/local/share/pi-devbox/skills/pi-devbox-environment/SKILL.md"
run "global-AGENTS append snippet present" \
    "test -f /usr/local/share/pi-devbox/pi-global-AGENTS.append.md"
run "pi-devbox block merged into pi-global-AGENTS.md" \
    "grep -q 'pi-devbox:managed-block' /opt/pi-toolkit/pi-global-AGENTS.md"
run "mempalace session-start pointer merged into global AGENTS.md" \
    "grep -q 'load the mempalace skill' /opt/pi-toolkit/pi-global-AGENTS.md"
# Vendored fallback skills (so a no-skillset container still resolves the
# AGENTS.md 'read the pi-extensions skill' pointer).
run "image-baked pi-extensions fallback skill" \
    "test -f /usr/local/share/pi-devbox/skills/pi-extensions/SKILL.md"
run "pi-extensions skill ships its helper" \
    "test -f /usr/local/share/pi-devbox/skills/pi-extensions/evaluate-extension-usage.py"
run "image-baked mempalace fallback skill" \
    "test -f /usr/local/share/pi-devbox/skills/mempalace/SKILL.md"
# Layered freshness: when the pinned pi-extensions clone carries the skill, the
# baked copy must be the fresh package copy (Option 1), not the stale snapshot.
run "pi-extensions skill refreshed from package when present" \
    "if [ -f /opt/pi-extensions/skill/SKILL.md ]; then cmp -s /opt/pi-extensions/skill/SKILL.md /usr/local/share/pi-devbox/skills/pi-extensions/SKILL.md; else true; fi"

# ── tmux 0-indexing (required for pi-studio variants) ─────────────────
echo ""
echo "── tmux config ──"
run_expect "/etc/tmux.conf has base-index 0" \
  "cat /etc/tmux.conf" "set -g base-index 0"
run_expect "/etc/tmux.conf has pane-base-index 0" \
  "cat /etc/tmux.conf" "set -g pane-base-index 0"

# ── Repo clones ───────────────────────────────────────────────────────
echo ""
echo "── Repo clones ──"
run "pi-toolkit clone"    "test -d /opt/pi-toolkit && git -C /opt/pi-toolkit rev-parse --short HEAD"
run "pi-extensions clone" "test -d /opt/pi-extensions && git -C /opt/pi-extensions rev-parse --short HEAD"
run "pi-fork clone + node_modules" \
    "test -f /opt/pi-fork/package.json && test -d /opt/pi-fork/node_modules"
run "pi-observational-memory clone + node_modules" \
    "test -f /opt/pi-observational-memory/package.json && test -d /opt/pi-observational-memory/node_modules"
# pi-atelier: deliberately NO node_modules assertion, unlike its siblings —
# it declares zero runtime dependencies (only peerDeps, satisfied by the baked
# pi) and has no build step, so Dockerfile.variant skips `npm install` for it.
# Assert what pi actually loads instead: the entry point named by its
# package.json `pi.extensions` key.
run "pi-atelier clone + entry point" \
    "test -f /opt/pi-atelier/package.json && test -f /opt/pi-atelier/extensions/index.ts"

# ── pi <-> pi-atelier compatibility floor (executable, not a comment) ──
# pi-atelier < 0.7.1 wraps pi's PRIVATE TUI renderer in a way that recurses
# under pi >= 0.84: pi hangs at startup burning CPU, with no error. Upstream
# fixed it in 0.7.1/0.7.2, but atelier's peerDependencies still say
# `>=0.80.7`, so neither npm nor pi can warn about the real floor. Both
# versions are pinned in Dockerfile.variant; this makes a bad PAIRING fail the
# build instead of publishing an image whose TUI never starts.
run_expect "pi-atelier >= 0.7.1 floor for pi >= 0.84 (startup-hang guard)" \
    'ge() { [ "$(printf "%s\n%s\n" "$1" "$2" | sort -V | head -n1)" = "$2" ]; }; AV=$(jq -r ".version // empty" /opt/pi-atelier/package.json 2>/dev/null); PV=$(pi --version 2>/dev/null | grep -oE "[0-9]+\.[0-9]+\.[0-9]+" | head -n1); if [ -z "$AV" ] || [ -z "$PV" ]; then echo "unreadable versions (atelier=$AV pi=$PV)"; elif ge "$PV" 0.84.0 && ! ge "$AV" 0.7.1; then echo "VIOLATION: pi $PV with pi-atelier $AV"; else echo "compatible: pi $PV + pi-atelier $AV"; fi' \
    "compatible:"

# pi-studio is present only in the :latest-studio variant. Auto-detect by
# probing /opt/pi-studio so this one script covers both variants.
if docker run --rm --entrypoint="" "$IMAGE" sh -c 'test -d /opt/pi-studio' >/dev/null 2>&1; then
  STUDIO_VARIANT=1
  echo "  ℹ️   pi-studio detected — running studio assertions"
  run "pi-studio clone + node_modules" \
      "test -f /opt/pi-studio/package.json && test -d /opt/pi-studio/node_modules"
  run "pi-studio prebuilt client bundle" \
      "test -f /opt/pi-studio/client/studio-client.js"
else
  STUDIO_VARIANT=0
  echo "  ℹ️   pi-studio not present (non-studio variant) — skipping studio clone checks"
fi

# ── Build provenance (manifest + OCI labels) ─────────────────────────
echo ""
echo "── Build provenance ──"
run "/etc/pi-devbox/build-manifest.json present" \
    "test -f /etc/pi-devbox/build-manifest.json"
run_expect "manifest records pi-extensions component" \
    "cat /etc/pi-devbox/build-manifest.json" '"pi-extensions"'
run_expect "manifest records pi-atelier" \
    "cat /etc/pi-devbox/build-manifest.json" '"pi-atelier"'
run_expect "manifest records pi_version" \
    "cat /etc/pi-devbox/build-manifest.json" '"pi_version"'
# Every component must be a resolved commit (or null for pi-studio in the
# non-studio variant) — 'unknown' means a clone silently failed to resolve.
run "manifest has no unresolved ('unknown') components" \
    "! grep -q '\"unknown\"' /etc/pi-devbox/build-manifest.json"
# pi-devbox-version wraps the manifest into a human-first command (this
# PR); verify the binary is present, executable, and both output modes work.
run "pi-devbox-version binary present + executable" \
    "test -x /usr/local/bin/pi-devbox-version"
run_expect "pi-devbox-version human output shows release tag" \
    "pi-devbox-version" "pi-devbox "
run_expect "pi-devbox-version --json round-trips the manifest" \
    "pi-devbox-version --json" '"release_tag"'
run_expect "pi-devbox-version --quiet is a compact one-liner" \
    "pi-devbox-version --quiet | wc -l" "1"
# OCI labels live in the image config, not the container fs — inspect them
# from the host docker rather than via `docker run`.
LBL=$(docker inspect --format '{{ index .Config.Labels "se.jordbo.pi-devbox.pi-extensions-ref" }}' "$IMAGE" 2>/dev/null || true)
if [ -n "$LBL" ] && [ "$LBL" != "<no value>" ]; then
  printf "  ✅  OCI label se.jordbo.pi-devbox.pi-extensions-ref=%s\n" "$LBL"; PASS=$((PASS+1))
else
  printf "  ❌  OCI label se.jordbo.pi-devbox.pi-extensions-ref missing or empty\n"; FAIL=$((FAIL+1))
fi

# ── Runtime deployment (needs entrypoint to run) ──────────────────────
echo ""
echo "── Runtime deployment ──"
# Spin up a long-running container WITHOUT overriding the entrypoint, so
# the baked entrypoint chain (entrypoint.sh → entrypoint-user.sh) runs and
# deploys pi-toolkit + pi-extensions to ~/.pi/agent/. Override CMD to
# tail -f /dev/null so the container stays alive while we docker-exec.
CID=$(docker run -d --rm "$IMAGE" tail -f /dev/null)
cleanup() { docker rm -f "$CID" >/dev/null 2>&1 || true; }
trap cleanup EXIT

# Wait for entrypoint-user.sh to finish deploying pi-toolkit + extensions.
# Gate on BOTH the keybindings symlink (deployed by pi-toolkit) AND the
# mempalace.ts bridge (deployed last by entrypoint-user.sh) AND ≥4 *.ts
# extensions present. Parallel build load can otherwise sample the *.ts
# count mid-deploy and produce a flake. See opencode-devbox c6f9d11
# (2026-06-08) — same fix transplanted.
for i in $(seq 1 45); do
  if docker exec "$CID" sh -c '
       test -L /home/developer/.pi/agent/keybindings.json && \
       test -L /home/developer/.pi/agent/extensions/mempalace.ts && \
       test -L /home/developer/.agents/skills/pi-devbox-environment && \
       test -L /home/developer/.agents/skills/pi-extensions && \
       test -L /home/developer/.agents/skills/mempalace && \
       count=$(ls -1 /home/developer/.pi/agent/extensions/*.ts 2>/dev/null | wc -l) && \
       [ "$count" -ge 4 ]
     ' >/dev/null 2>&1; then
    break
  fi
  sleep 1
done

exec_test() {
  local label="$1"; local cmd="$2"
  if docker exec -u developer "$CID" sh -c "$cmd" >/dev/null 2>&1; then
    printf "  ✅  %s\n" "$label"; PASS=$((PASS+1))
  else
    printf "  ❌  %s\n" "$label"; FAIL=$((FAIL+1))
  fi
}

exec_test "keybindings.json (pi-toolkit)"         'test -L $HOME/.pi/agent/keybindings.json && echo ok'
exec_test "extensions ≥ 4 (pi-extensions)"        'count=$(ls -1 $HOME/.pi/agent/extensions/*.ts 2>/dev/null | wc -l); [ $count -ge 4 ] && echo "$count extensions"'
exec_test "mempalace.ts bridge"                    'test -L $HOME/.pi/agent/extensions/mempalace.ts && echo ok'
exec_test "settings.json bootstrapped"             'test -f $HOME/.pi/agent/settings.json && echo ok'
exec_test "pi-devbox-environment skill linked"     'test -L $HOME/.agents/skills/pi-devbox-environment && test -f $HOME/.agents/skills/pi-devbox-environment/SKILL.md && echo ok'
exec_test "pi-extensions skill linked (fallback)"  'test -L $HOME/.agents/skills/pi-extensions && test -f $HOME/.agents/skills/pi-extensions/SKILL.md && echo ok'
exec_test "mempalace skill linked (fallback)"      'test -L $HOME/.agents/skills/mempalace && test -f $HOME/.agents/skills/mempalace/SKILL.md && echo ok'

# pi-fork + pi-observational-memory are registered by entrypoint-user.sh via
# `pi install /opt/<pkg>`, which runs slightly after the keybindings marker.
#
# Assert against the `packages` ARRAY, never a whole-file grep: the settings
# template ships a top-level "pi-fork" CONFIG block, so `grep -q pi-fork
# settings.json` passes even when `pi install /opt/pi-fork` never ran. That
# false green is exactly why the missing `fork` tool shipped unnoticed from
# v1.0.0 through v1.6.3.
pkg_registered_cmd() {
  printf "jq -e --arg n %s '(.packages // []) | any((type == \"string\") and (. == \"npm:\" + \$n or endswith(\"/\" + \$n)))' \$HOME/.pi/agent/settings.json" "$1"
}

for i in $(seq 1 15); do
  if docker exec -u developer "$CID" sh -c "$(pkg_registered_cmd pi-observational-memory)" \
       >/dev/null 2>&1; then
    break
  fi
  sleep 1
done
exec_test "pi-fork registered in packages[] (fork tool)" \
  "$(pkg_registered_cmd pi-fork)"
exec_test "pi-observational-memory registered in packages[] (recall tool)" \
  "$(pkg_registered_cmd pi-observational-memory)"

# pi-studio registration (studio variant only) — registered by the same
# entrypoint-user.sh local-path install loop as fork/obsmem.
if [ "${STUDIO_VARIANT:-0}" = "1" ]; then
  for i in $(seq 1 15); do
    if docker exec -u developer "$CID" sh -c "$(pkg_registered_cmd pi-studio)" \
         >/dev/null 2>&1; then
      break
    fi
    sleep 1
  done
  exec_test "pi-studio registered in packages[] (/studio command + studio_* tools)" \
    "$(pkg_registered_cmd pi-studio)"
fi

# pi-atelier registration. It is LAST in the entrypoint's install loop, so a
# pass here also means that loop ran to completion rather than dying midway.
for i in $(seq 1 15); do
  if docker exec -u developer "$CID" sh -c "$(pkg_registered_cmd pi-atelier)" \
       >/dev/null 2>&1; then
    break
  fi
  sleep 1
done
exec_test "pi-atelier registered in packages[] (TUI sidebar)" \
  "$(pkg_registered_cmd pi-atelier)"
# ...and registered from the vendored /opt copy, NOT as `npm:pi-atelier`: an
# npm: entry resolves through ~/.pi/npm-global on the config VOLUME, which
# outlives image upgrades and would silently keep an old, unaudited atelier —
# exactly the shape that pairs a stale 0.6.x with a new pi and hangs at startup.
exec_test "pi-atelier registered from /opt, not npm: (volume-shadowing guard)" \
  'jq -e "((.packages // []) | any((type == \"string\") and endswith(\"/pi-atelier\"))) and (((.packages // []) | any(. == \"npm:pi-atelier\")) | not)" $HOME/.pi/agent/settings.json'

# ── /tmp/sshcm directory created by entrypoint ────────────────────────
exec_test "/tmp/sshcm dir mode 700 (ssh ControlMaster)" \
  'test -d /tmp/sshcm && [ "$(stat -c %a /tmp/sshcm)" = "700" ] && echo ok'

# ── Image size ────────────────────────────────────────────────────────
echo ""
echo "── Image size ──"
# Sum all layers via `docker history`. Docker's `image inspect --format='{{.Size}}'`
# returns ONLY the variant-unique layer when the base is content-addressed and
# shared (the case in this repo's two-phase build), which understates the
# user-facing image size by 2+ GB. Summing layer sizes from history is the
# metric Hub displays to users and the one we actually want to gate on.
SIZE_MB=$(docker history --format '{{.Size}}' "$IMAGE" | python3 -c '
import sys, re
total=0.0
for line in sys.stdin:
    s=line.strip()
    if s in ("0B", ""): continue
    m=re.match(r"^([0-9.]+)(B|kB|MB|GB)$", s)
    if not m: continue
    v=float(m.group(1)); u=m.group(2)
    mult={"B":1/1048576,"kB":1/1024,"MB":1,"GB":1024}[u]
    total+=v*mult
print(int(total))
')
if [ -z "$SIZE_MB" ] || [ "$SIZE_MB" = "0" ]; then
  printf "  ⚠️   image size: could not parse — skipping check\n"
elif [ "$SIZE_MB" -le "$SIZE_THRESHOLD_MB" ]; then
  printf "  ✅  size: %d MB (threshold %d MB)\n" "$SIZE_MB" "$SIZE_THRESHOLD_MB"; PASS=$((PASS+1))
else
  printf "  ❌  size: %d MB exceeds threshold %d MB\n" "$SIZE_MB" "$SIZE_THRESHOLD_MB"; FAIL=$((FAIL+1))
fi

# ── Summary ───────────────────────────────────────────────────────────
echo ""
echo "=== Results: ${PASS} passed, ${FAIL} failed ==="
[ "$FAIL" -eq 0 ]