joakimp
25c1265681
The rule was correct and written down twice (global AGENTS.md, this skill)
and was still violated by agents that had just read it: on 2026-09-17 all
five fork briefs in one session carried "do not", one returned confident
verbatim quotes that did not exist, and four had disjoint write boundaries
fork cannot enforce (re-run as pi-task, they passed). Three mechanisms,
none of them wording:
1. fork is a TOOL — its self-recommending description ("implementation,
testing, review…") is in the model's face every turn; pi-task was a CLI
to be remembered and reached through bash with a hand-written JSON.
2. The skill is gone after the first compaction; the tool list never is.
The asymmetry widens in exactly the long sessions where fork is worst.
3. Friction: one string vs a spec file + bash + reading result.json.
extensions/task.ts registers pi-task as the `task` tool. Flat parameters
build the spec; the decision rule sits in the description and in
promptGuidelines (appended to the system prompt, so compaction cannot remove
it). Before spending a model run it rejects the two spec errors that make a
boundary violation certain — write_allowed not an exact subset of roots, and
a writable root nested in a watched-only root (the parent's porcelain would
change every time; pi-task keys deltas by root string) — and it serialises
sibling tasks whose roots overlap (parallel siblings saw each other's writes
as violations, 2026-09-17). Returns the CLI's own parent-facing report; a
FAIL verdict is a result, only a CLI refusal is an error.
extensions/fork-gate.ts is a tool_call hook that BLOCKS a fork whose brief
contains a prohibition, a write boundary, or a clause-initial file-changing
imperative, and returns as the reason the exact task(...) to make instead.
Wording, not intent — the message says so and how to rephrase a genuinely
read-only brief. PI_FORK_GATE=off logs instead; /ext disables.
Evidence: test/fork-gate.test.mjs is two-sided (15 must-block incl. the real
shapes, 10 must-pass incl. "Write a summary…", "Report which files were
modified…", "Give me an update…"); the classifier redirects 5/5 of the real
briefs from the motivating session. test/task.test.mjs pins root overlap and
the pre-launch validation. Live in `pi -p`: the fork was intercepted before
any child spawned (no /tmp/pi-fork-* dir) and the model received the
redirect; task returned PASS with an evidence pointer and audit dir, a
budget-overrun returned a FAIL result (isError=false), and a nested-root spec
was rejected with no audit dir created.
skill/SKILL.md: Part 1 now opens with "decide the rung before the brief"
(the table, the three-question pre-flight, the roots contract, overlap,
what isolation does not fix); the ladder section and quick reference no
longer say pi-task "will never appear in your tool list". package.json gains
"type": "module" and a test script.
74 lines
3.7 KiB
JavaScript
74 lines
3.7 KiB
JavaScript
// Tests for the pure parts of task.ts: root overlap and the pre-launch spec
|
|
// validation that turns a guaranteed false FAIL into an immediate error.
|
|
// task.ts imports typebox / @earendil-works/pi-ai at runtime (only resolvable
|
|
// inside pi), so the functions are cut out of the shipped source by brace
|
|
// matching and type-stripped — the same approach as mempalace-toolkit's tests.
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { readFileSync, writeFileSync, mkdtempSync, mkdirSync } from "node:fs";
|
|
import { stripTypeScriptTypes } from "node:module";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
|
|
const src = readFileSync(new URL("../extensions/task.ts", import.meta.url), "utf8");
|
|
function fn(name) {
|
|
const start = src.indexOf(`export function ${name}(`);
|
|
if (start < 0) throw new Error(`not found: ${name}`);
|
|
let depth = 0, seen = false;
|
|
for (let i = start; i < src.length; i++) {
|
|
if (src[i] === "{") { depth++; seen = true; }
|
|
else if (src[i] === "}") { depth--; if (seen && depth === 0) return src.slice(start, i + 1); }
|
|
}
|
|
throw new Error(`unterminated: ${name}`);
|
|
}
|
|
const work = mkdtempSync(path.join(tmpdir(), "task-test-"));
|
|
const modPath = path.join(work, "pure.mjs");
|
|
writeFileSync(modPath, stripTypeScriptTypes(
|
|
`import { existsSync } from "node:fs";\nimport path from "node:path";\n` +
|
|
[fn("rootsOverlap"), fn("anyOverlap"), fn("validateRoots")].join("\n\n"), { mode: "strip" }));
|
|
const { rootsOverlap, anyOverlap, validateRoots } = await import(modPath);
|
|
|
|
// Real directories, since validateRoots checks existence.
|
|
const repo = path.join(work, "repo"); const docs = path.join(repo, "docs"); const srcd = path.join(repo, "src"); const other = path.join(work, "other");
|
|
for (const d of [docs, srcd, other]) mkdirSync(d, { recursive: true });
|
|
|
|
test("rootsOverlap: identity, containment both ways, siblings, prefix-but-not-parent", () => {
|
|
assert.equal(rootsOverlap(repo, repo), true);
|
|
assert.equal(rootsOverlap(repo, docs), true);
|
|
assert.equal(rootsOverlap(docs, repo), true);
|
|
assert.equal(rootsOverlap(docs, srcd), false);
|
|
assert.equal(rootsOverlap(path.join(work, "repo"), path.join(work, "repo2")), false, "'/x/repo' is not a parent of '/x/repo2'");
|
|
assert.equal(anyOverlap([docs], [srcd, other]), false);
|
|
assert.equal(anyOverlap([docs, other], [srcd, other]), true);
|
|
});
|
|
|
|
test("validateRoots: read-only spec with existing absolute roots is accepted", () => {
|
|
assert.deepEqual(validateRoots([docs, srcd], undefined, true), []);
|
|
});
|
|
|
|
test("validateRoots: the migrate-0N shape (sibling roots, write_allowed subset) is accepted", () => {
|
|
assert.deepEqual(validateRoots([docs, srcd, other], [docs], false), []);
|
|
});
|
|
|
|
test("validateRoots: write task without write_allowed is rejected (violations undetectable)", () => {
|
|
const p = validateRoots([docs], undefined, false);
|
|
assert.equal(p.length, 1); assert.match(p[0], /requires write_allowed/);
|
|
});
|
|
|
|
test("validateRoots: write_allowed must be an exact root string", () => {
|
|
const p = validateRoots([repo], [docs], false);
|
|
assert.ok(p.some((x) => /not one of roots/.test(x)), p.join("\n"));
|
|
});
|
|
|
|
test("validateRoots: writable root nested in a watched-only root is rejected (certain false FAIL)", () => {
|
|
const p = validateRoots([repo, docs], [docs], false);
|
|
assert.ok(p.some((x) => /nested in/.test(x)), p.join("\n"));
|
|
});
|
|
|
|
test("validateRoots: read_only with write_allowed, relative root, missing root are each named", () => {
|
|
const p = validateRoots(["relative/path", path.join(work, "nope"), docs], [docs], true);
|
|
assert.ok(p.some((x) => /not absolute/.test(x)));
|
|
assert.ok(p.some((x) => /does not exist/.test(x)));
|
|
assert.ok(p.some((x) => /pick one/.test(x)));
|
|
});
|