diff --git a/extensions/pi/README.md b/extensions/pi/README.md index 1c7898a..21c078b 100644 --- a/extensions/pi/README.md +++ b/extensions/pi/README.md @@ -378,6 +378,57 @@ the one failure this derivation exists to prevent. protocol says a broadcast owes nobody a reply — which also means broadcasting an ask demonstrably reaches no owed set, giving that documented anti-pattern teeth. +**Dormant asks — open, owed, and deliberately not announced.** Some asks are +planted *unanswerable on purpose*: a first-boot acceptance describes work that +becomes possible only when the device is next recreated, and it must stay owed +until then, because closing it early to tidy the mailbox is exactly how the work +gets lost. Derivation alone cannot tell that apart from neglected work, so such +an ask was announced at every session start and every poll for as long as it was +correctly waiting — measured at three days running on `emb-7kj4vr4g` +(2026-09-28 → 2026-10-01), and across three consecutive releases before that. +The ask was right; announcing it was wrong, and the cost landed on the human +reading the window, the one reader who cannot filter it. + +An ask may therefore declare, in its own `metadata`, the condition under which it +is merely waiting: + +```json +"dormant_unless": [ + { "kind": "json_field", "path": "/etc/pi-devbox/build-manifest.json", + "field": "release_tag", "baseline": "v1.9.4" }, + { "kind": "file_mtime", "path": "/etc/hostname", + "baseline": "2026-09-22T18:12:49Z" } +] +``` + +It is dormant while **every** condition still matches its baseline, and goes live +the moment **any** of them differs — the trigger such asks already stated in +prose ("act when EITHER differs"), now in a form the bridge can check. Two kinds, +both local-file-only: `file_mtime` (compared at whole seconds in UTC, because a +filesystem mtime carries sub-second residue that a reported ISO baseline never +will — comparing raw milliseconds would mark every such predicate permanently +"changed" and silently disable the feature) and `json_field` (dotted paths +allowed, compared as strings so a manifest holding `3` matches a baseline of +`"3"`). There is no expression language, no shell and no network: a general +evaluator in the path that decides whether work is *visible* is a far worse trade +than a clumsy schema. + +Dormancy is **withheld from the announcement, never from the mailbox**: the +wake-up injection lists dormant asks once per session with their ids, and a +mid-session poll mentions only a count, and only when the window is already open +for something else. They are not added to the resurface map, so one becomes +announceable the instant its baseline moves. + +**Dormancy must be proven, never assumed** — every unevaluable predicate shows +the ask as owed. A missing file, an unreadable one, a baseline that will not +parse, an unknown `kind`, a vanished field, a relative path, more than eight +conditions: each announces. The dangerous failure here is not a spurious nag but +work that disappears because a predicate could not be evaluated, which would be +indistinguishable from the ask being lost and would not surface until a release +needed it. An ask with no `dormant_unless` behaves exactly as it did before the +feature existed. `scripts/test-dormancy.sh` exercises all of it, positive and +negative arms both. + Gated on `MEMPALACE_PI_DEVICE` **and** `MEMPALACE_REMOTE_URL` (the same pair as the stamper, since an unstamped client has no address to be reached at), and disabled outright with `MEMPALACE_MAILBOX=0` (notifications alone with diff --git a/extensions/pi/mempalace.ts b/extensions/pi/mempalace.ts index f30869f..f8ff35d 100644 --- a/extensions/pi/mempalace.ts +++ b/extensions/pi/mempalace.ts @@ -92,6 +92,7 @@ */ import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { readFileSync, statSync } from "node:fs"; import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { Type } from "typebox"; @@ -157,6 +158,165 @@ const sleep = (ms: number): Promise => if (typeof t.unref === "function") t.unref(); }); +// ── Dormancy ────────────────────────────────────────────────────────────────── +/** + * Is a standing ask DORMANT — correct to leave open, wrong to announce? + * + * The problem this solves, measured rather than imagined. A first-boot + * acceptance ask is planted deliberately unanswerable: it describes work that + * becomes possible only when the device is next recreated, and it must STAY + * owed until then, because closing it early to tidy the mailbox is exactly how + * that work gets lost. But deriveOwed could only see "directed, open, not + * answered, not withdrawn", so such an ask is announced at every session start + * and every poll for as long as it is correctly waiting — three days running on + * emb-7kj4vr4g (2026-09-28 → 2026-10-01), and three consecutive releases before + * that. The ask was right; announcing it was wrong. The cost lands on the human + * reading the window, who is the one reader that cannot filter it. + * + * So an ask may now declare, in its own metadata, the condition under which it + * is merely waiting: + * + * "dormant_unless": [ + * { "kind": "json_field", "path": "/etc/pi-devbox/build-manifest.json", + * "field": "release_tag", "baseline": "v1.9.4" }, + * { "kind": "file_mtime", "path": "/etc/hostname", + * "baseline": "2026-09-22T18:12:49Z" } + * ] + * + * It is dormant while EVERY condition still matches its baseline, and goes live + * the moment ANY of them differs — which is the trigger those asks already + * state in prose ("act when EITHER differs"), now in a form a machine can + * check. Dormant asks are withheld from the ANNOUNCED owed set, never from the + * mailbox: the wake-up injection still lists them once per session, so they + * stay discoverable and cannot be silently dropped. + * + * THE SAFETY RULE, and the reason every branch below reports `unchanged: false` + * on doubt: dormancy must be PROVEN, never assumed. A missing file, an + * unreadable one, a baseline that will not parse, an unknown `kind`, a field + * that has vanished — each is a reason to SHOW the ask, not to hide it. The + * dangerous failure here is not a spurious nag; it is work that vanishes + * because a predicate could not be evaluated, which would be indistinguishable + * from the ask being lost and would not surface until a release needed it. + * + * No eval, no shell, no network: the predicate language is deliberately two + * fixed checks against local files. A general expression evaluator sitting in + * the path that decides whether work is VISIBLE is a far worse trade than a + * slightly clumsy schema. + */ +export type DormancyVerdict = { dormant: boolean; reason: string }; + +/** Split of the derived owed-set: what to announce, and what is merely waiting. */ +export type OwedSplit = { owed: LogEvent[]; dormant: LogEvent[] }; + +// Bounds, so a malformed or hostile event cannot turn a mailbox read into real +// work. Both are deliberately small: these predicates describe boot state, and +// anything needing more than a handful of local checks is not a dormancy rule. +const MAX_DORMANCY_CONDITIONS = 8; +const MAX_DORMANCY_JSON_BYTES = 256 * 1024; + +const dormancyCondition = (c: unknown): { unchanged: boolean; reason: string } => { + if (typeof c !== "object" || c === null || Array.isArray(c)) + return { unchanged: false, reason: "condition is not an object" }; + const { kind, path, baseline, field } = c as Record; + // Absolute paths only: a relative one would resolve against whatever cwd the + // pi process happens to hold, which is not a property of the device whose + // state the baseline describes. + if (typeof path !== "string" || !path.startsWith("/") || path.includes("\0")) + return { + unchanged: false, + reason: `path must be an absolute string, got ${JSON.stringify(path)}`, + }; + if (typeof baseline !== "string" && typeof baseline !== "number") + return { unchanged: false, reason: "baseline must be a string or a number" }; + + if (kind === "file_mtime") { + // Compared at WHOLE SECONDS in UTC, because the two routes that produce + // these baselines disagree below that: a filesystem mtime carries + // sub-second residue (measured: /etc/hostname at .773761009) that a + // hand-written or reported ISO baseline never will. Comparing raw + // milliseconds would make every such predicate permanently "changed", + // i.e. would silently disable the feature while appearing to work. + const expected = Date.parse(String(baseline)); + if (!Number.isFinite(expected)) + return { + unchanged: false, + reason: `baseline is not a parseable date: ${JSON.stringify(baseline)}`, + }; + let mtimeMs: number; + try { + mtimeMs = statSync(path).mtimeMs; + } catch { + return { unchanged: false, reason: `cannot stat ${path}` }; + } + return Math.floor(mtimeMs / 1000) === Math.floor(expected / 1000) + ? { unchanged: true, reason: `${path} mtime still at baseline` } + : { unchanged: false, reason: `${path} mtime moved from baseline` }; + } + + if (kind === "json_field") { + if (typeof field !== "string" || field.length === 0) + return { unchanged: false, reason: "json_field needs a non-empty field name" }; + let text: string; + try { + const buf = readFileSync(path); + if (buf.byteLength > MAX_DORMANCY_JSON_BYTES) + return { + unchanged: false, + reason: `${path} exceeds the ${MAX_DORMANCY_JSON_BYTES}-byte cap`, + }; + text = buf.toString("utf8"); + } catch { + return { unchanged: false, reason: `cannot read ${path}` }; + } + let doc: unknown; + try { + doc = JSON.parse(text); + } catch { + return { unchanged: false, reason: `${path} is not valid JSON` }; + } + let cur: unknown = doc; + for (const seg of field.split(".")) { + if ( + typeof cur !== "object" || + cur === null || + !Object.prototype.hasOwnProperty.call(cur, seg) + ) + return { unchanged: false, reason: `${path} has no field ${field}` }; + cur = (cur as Record)[seg]; + } + if (cur !== null && typeof cur === "object") + return { unchanged: false, reason: `field ${field} is not a scalar` }; + // Compared as strings on purpose: the baseline arrives as JSON metadata, so + // a manifest holding 3 and a baseline saying "3" are the same fact. + return String(cur) === String(baseline) + ? { unchanged: true, reason: `${field} still at baseline` } + : { unchanged: false, reason: `${field} moved from baseline` }; + } + + return { unchanged: false, reason: `unknown dormancy kind ${JSON.stringify(kind)}` }; +}; + +export function evaluateDormancy( + metadata: Record | null | undefined, +): DormancyVerdict { + const raw = metadata?.dormant_unless; + // Every one of these returns NOT dormant, i.e. "announce it". An ask without + // a predicate behaves exactly as it did before this feature existed. + if (raw === undefined || raw === null) return { dormant: false, reason: "no dormant_unless" }; + if (!Array.isArray(raw)) return { dormant: false, reason: "dormant_unless is not an array" }; + if (raw.length === 0) return { dormant: false, reason: "dormant_unless is empty" }; + if (raw.length > MAX_DORMANCY_CONDITIONS) + return { + dormant: false, + reason: `too many conditions (${raw.length} > ${MAX_DORMANCY_CONDITIONS})`, + }; + for (const c of raw) { + const v = dormancyCondition(c); + if (!v.unchanged) return { dormant: false, reason: v.reason }; + } + return { dormant: true, reason: `all ${raw.length} condition(s) still at baseline` }; +} + class StdioMcpClient implements IMcpClient { private proc: ChildProcessWithoutNullStreams | null = null; private nextId = 1; @@ -1205,8 +1365,8 @@ export default async function mempalaceExtension(pi: ExtensionAPI) { * Directed asks addressed to this device with no terminal reply from it, and * not explicitly withdrawn by whoever sent them (see isWithdrawn). */ - async function deriveOwed(): Promise { - if (!mailboxEnabled || !available) return []; + async function deriveOwed(): Promise { + if (!mailboxEnabled || !available) return { owed: [], dormant: [] }; try { const [candidatesRaw, mineRaw, inboundRaw] = await Promise.all([ // Every cursor-less event_list call in this file passes `order` explicitly. @@ -1254,13 +1414,21 @@ export default async function mempalaceExtension(pi: ExtensionAPI) { // You cannot owe yourself. return c.from_agent !== mailboxAddress; }); - if (candidates.length === 0) return []; + if (candidates.length === 0) return { owed: [], dormant: [] }; const mine = parseEvents(mineRaw); const inbound = parseEvents(inboundRaw); - return candidates.filter((c) => !isAnswered(c, mine) && !isWithdrawn(c, inbound)); + const live = candidates.filter((c) => !isAnswered(c, mine) && !isWithdrawn(c, inbound)); + // PARTITION, not filter. A dormant ask is still owed in the protocol + // sense — it has no terminal reply and must not be closed — so it stays in + // the return value where the mailbox can still account for it, and only + // the ANNOUNCING paths below treat the two halves differently. + const owed: LogEvent[] = []; + const dormant: LogEvent[] = []; + for (const c of live) (evaluateDormancy(c.metadata).dormant ? dormant : owed).push(c); + return { owed, dormant }; } catch { // Fail silent and open: a mailbox read must never break a session. - return []; + return { owed: [], dormant: [] }; } } @@ -1356,6 +1524,15 @@ export default async function mempalaceExtension(pi: ExtensionAPI) { }) .join("\n"); + const DORMANT_NOTE = + "Each of these declares a `dormant_unless` predicate in its metadata that STILL " + + "matches this device's current state, so the work it describes is not yet " + + "possible. They are NOT closed and NOT answered — leave them open. They return to " + + "the announced owed-set by themselves the moment a baseline stops matching. If a " + + "predicate cannot be evaluated at all (missing file, unparseable baseline, unknown " + + "kind) the ask is announced as owed instead, deliberately: dormancy has to be " + + "proven, never assumed."; + const OWED_HOWTO = "To close one, append an event on the SAME correlation_id with a terminal status " + "(applied/superseded/failed/blocked). An ack alone does NOT clear it, and neither " + @@ -1463,7 +1640,8 @@ export default async function mempalaceExtension(pi: ExtensionAPI) { lastMailboxPollAt = Date.now(); void (async () => { try { - const [owed, closed] = await Promise.all([deriveOwed(), deriveClosed()]); + const [owedSplit, closed] = await Promise.all([deriveOwed(), deriveClosed()]); + const { owed, dormant } = owedSplit; const now = Date.now(); const unseen = (list: LogEvent[]): LogEvent[] => list.filter((e) => { @@ -1491,6 +1669,13 @@ export default async function mempalaceExtension(pi: ExtensionAPI) { `${CLOSED_NOTE}\n\n${formatOwed(newsRaw)}`, ); } + // A dormant ask NEVER opens this window on its own — the early return + // above is what actually stops the nag. But once the window is open for + // something else, one line of count is nearly free and keeps a waiting + // ask from feeling lost between session starts. + if (dormant.length > 0) { + blocks.push(`${dormant.length} dormant ask(s), not listed here. ${DORMANT_NOTE}`); + } pi.sendMessage( { customType: "mempalace-mailbox", @@ -1555,10 +1740,11 @@ export default async function mempalaceExtension(pi: ExtensionAPI) { sections.push(`## mempalace_diary_read\n\n(error: ${(err as Error).message})`); } // Tier 1 mailbox: what this device owes a reply to, derived rather than read - // off a filter. deriveOwed() swallows its own failures and returns [], so a - // broken palace costs a missing section, never a broken wake-up. + // off a filter. deriveOwed() swallows its own failures and returns an empty + // split, so a broken palace costs a missing section, never a broken wake-up. if (mailboxEnabled) { - const [owed, closed] = await Promise.all([deriveOwed(), deriveClosed()]); + const [owedSplit, closed] = await Promise.all([deriveOwed(), deriveClosed()]); + const { owed, dormant } = owedSplit; if (owed.length > 0) { // Record what this injection showed, so the first mid-session poll does // not re-announce the identical list minutes later. Without this the @@ -1572,6 +1758,18 @@ export default async function mempalaceExtension(pi: ExtensionAPI) { `this device. ${OWED_HOWTO}\n\n${formatOwed(owed)}`, ); } + // Once per session, WITH ids. The wake-up injection is the one place a + // dormant ask should be visible: "what is this device still carrying?" is a + // session-start question, and answering it here is what keeps the mid-session + // silence honest rather than concealing. Deliberately NOT added to + // `surfaced`: that map exists to suppress repeats of ANNOUNCED work, and a + // dormant ask must stay announceable the instant its baseline moves. + if (dormant.length > 0) { + sections.push( + `## logstream mailbox (${dormant.length} dormant — waiting, nothing owed yet)\n\n` + + `${DORMANT_NOTE}\n\n${formatOwed(dormant)}`, + ); + } const news = closed.filter((e) => e.id && !surfaced.has(e.id)); if (news.length > 0) { const now = Date.now(); diff --git a/scripts/test-dormancy.sh b/scripts/test-dormancy.sh new file mode 100755 index 0000000..39f5174 --- /dev/null +++ b/scripts/test-dormancy.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# test-dormancy.sh — exercise the mailbox dormancy predicate (dormant_unless). +# +# Why this exists as a script and not a note: `evaluateDormancy` decides whether +# an ask is SHOWN to the agent, so a silent regression there does not look like a +# bug — it looks like an empty mailbox. The positive arms matter as much as the +# negative ones: a predicate that never fires makes the feature a no-op, and a +# predicate that fires too eagerly hides real work. Both arms run here. +# +# It cannot import extensions/pi/mempalace.ts in place, because that file imports +# `typebox`, which pi provides at RUNTIME and this repo has no node_modules for. +# So it copies the file into a temp tree with the resolved deps symlinked beside +# it. The copy is made BY this script on every run, so it cannot drift from the +# source the way a vendored duplicate would. +# +# Fixtures are created here rather than read from the host: the first draft used +# /etc/hostname and /etc/pi-devbox/build-manifest.json, which made the positive +# arms pass only on a pi-devbox container and silently flip to "not dormant" +# anywhere else — a device-dependent test that reports success by doing nothing. +# +# Exit codes, deliberately distinct: +# 0 every case behaved as specified +# 1 at least one case FAILED (a real regression) +# 2 INCONCLUSIVE — deps could not be resolved, so nothing was proven +# (never 0: a test that skips quietly is the failure mode it should catch) + +set -uo pipefail + +# ── Args ────────────────────────────────────────────────────────────────────── +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + sed -n '2,25p' "$0" | sed 's/^# \?//' + exit 0 +fi + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SRC="$REPO_ROOT/extensions/pi/mempalace.ts" +[[ -r "$SRC" ]] || { + echo "INCONCLUSIVE: cannot read $SRC" >&2 + exit 2 +} + +# ── Resolve pi's runtime deps (discovered, never hardcoded) ─────────────────── +GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)" +PI_PKG="" +for cand in \ + "${GLOBAL_ROOT:+$GLOBAL_ROOT/@earendil-works/pi-coding-agent}" \ + "/usr/lib/node_modules/@earendil-works/pi-coding-agent" \ + "/usr/local/lib/node_modules/@earendil-works/pi-coding-agent"; do + [[ -n "$cand" && -d "$cand" ]] && { + PI_PKG="$cand" + break + } +done +[[ -n "$PI_PKG" && -d "$PI_PKG/node_modules/typebox" ]] || { + echo "INCONCLUSIVE: pi-coding-agent / typebox not resolvable (looked under 'npm root -g')." >&2 + echo " Nothing was proven. Install pi, or run this where pi is installed." >&2 + exit 2 +} + +# Node must be able to strip types from a .ts entry point (Node >= 22.6). +node -e 'process.exit(0)' 2>/dev/null || { + echo "INCONCLUSIVE: no usable node" >&2 + exit 2 +} + +# ── Build the throwaway tree ────────────────────────────────────────────────── +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/extensions/pi" "$WORK/node_modules/@earendil-works" +cp "$SRC" "$WORK/extensions/pi/mempalace.ts" +ln -s "$PI_PKG/node_modules/typebox" "$WORK/node_modules/typebox" +ln -s "$PI_PKG" "$WORK/node_modules/@earendil-works/pi-coding-agent" +printf '{"type":"module"}\n' > "$WORK/package.json" + +# Prove the copy is the source, so a PASS cannot be about a stale file. +if ! cmp -s "$SRC" "$WORK/extensions/pi/mempalace.ts"; then + echo "INCONCLUSIVE: the copy differs from the source" >&2 + exit 2 +fi + +# ── Fixtures, owned by this test ────────────────────────────────────────────── +printf '{"release_tag":"v1.9.4","nested":{"deep":{"leaf":"found"}},"count":3}\n' > "$WORK/manifest.json" +printf 'not json at all\n' > "$WORK/notjson.txt" +: > "$WORK/stamp" + +# ── The truth table ─────────────────────────────────────────────────────────── +cat > "$WORK/run.mjs" <<'EOF' +import { statSync } from "node:fs"; +import { evaluateDormancy } from "./extensions/pi/mempalace.ts"; + +const W = process.env.WORK; +const STAMP = `${W}/stamp`; +const MAN = `${W}/manifest.json`; +// Floor to whole seconds: that is the contract, and the residue below proves +// why the implementation must do the same. +const atBaseline = new Date(Math.floor(statSync(STAMP).mtimeMs / 1000) * 1000).toISOString(); + +const mt = (baseline, path = STAMP) => ({ kind: "file_mtime", path, baseline }); +const jf = (field, baseline, path = MAN) => ({ kind: "json_field", path, field, baseline }); + +const cases = [ + // No predicate => behave exactly as before the feature existed. + ["no metadata", undefined, false], + ["metadata without the key", { foo: "bar" }, false], + ["dormant_unless is a string", { dormant_unless: "release != x" }, false], + ["dormant_unless is empty", { dormant_unless: [] }, false], + ["more than 8 conditions", { dormant_unless: Array(9).fill(mt(atBaseline)) }, false], + + // POSITIVE arms. If these ever read false the feature is inert. + ["file_mtime at baseline", { dormant_unless: [mt(atBaseline)] }, true], + ["json_field at baseline", { dormant_unless: [jf("release_tag", "v1.9.4")] }, true], + ["both at baseline", { dormant_unless: [jf("release_tag", "v1.9.4"), mt(atBaseline)] }, true], + ["dotted field path", { dormant_unless: [jf("nested.deep.leaf", "found")] }, true], + ["number vs string baseline", { dormant_unless: [jf("count", "3")] }, true], + + // The trigger firing: ANY condition differing wakes the ask. + ["file_mtime moved", { dormant_unless: [mt("2020-01-01T00:00:00Z")] }, false], + ["json_field moved", { dormant_unless: [jf("release_tag", "v1.9.5")] }, false], + ["one same, one moved", { dormant_unless: [mt(atBaseline), jf("release_tag", "v1.9.5")] }, false], + + // FAIL-VISIBLE arms: dormancy unproven => announce. + ["missing file", { dormant_unless: [mt(atBaseline, "/nonexistent/path")] }, false], + ["unparseable baseline", { dormant_unless: [mt("not-a-date")] }, false], + ["relative path", { dormant_unless: [{ kind: "file_mtime", path: "etc/hostname", baseline: atBaseline }] }, false], + ["unknown kind", { dormant_unless: [{ kind: "uptime_lt", path: "/etc/hostname", baseline: "1d" }] }, false], + ["missing json field", { dormant_unless: [jf("no_such_field", "x")] }, false], + ["file is not JSON", { dormant_unless: [jf("a", "b", `${W}/notjson.txt`)] }, false], + ["field is not scalar", { dormant_unless: [jf("nested", "x")] }, false], + ["condition is not an object", { dormant_unless: ["release_tag"] }, false], + ["baseline is an object", { dormant_unless: [{ kind: "file_mtime", path: STAMP, baseline: {} }] }, false], +]; + +let pass = 0; +const failed = []; +for (const [name, meta, want] of cases) { + const v = evaluateDormancy(meta); + const ok = v.dormant === want; + if (ok) pass++; + else failed.push(name); + console.log( + ` ${ok ? "PASS" : "FAIL"} dormant=${String(v.dormant).padEnd(5)} want=${String(want).padEnd(5)} ` + + `${name.padEnd(28)} :: ${v.reason}`, + ); +} +console.log(`\n ${pass}/${cases.length} passed`); +if (failed.length) console.log(` FAILED: ${failed.join(", ")}`); +// Recorded because it is the reason file_mtime floors to seconds: a real mtime +// carries sub-second residue that an ISO baseline does not. +console.log(` (stamp mtimeMs=${statSync(STAMP).mtimeMs}, baseline=${atBaseline})`); +process.exit(failed.length === 0 ? 0 : 1); +EOF + +cd "$WORK" || exit 2 +export WORK +node "$WORK/run.mjs" +exit $?