Joakim Persson
3760d90998
A tier's model can be unusable for reasons that have nothing to do with the
task, typically an expired AWS SSO session behind an amazon-bedrock profile.
pi reports that as a normal run: rc 0, agent_end, willRetry=false. The failure
shows only as stopReason="error" + errorMessage on the assistant message
(measured 2026-10-12). pi-task turned it into "no fenced json envelope",
which points at the wrong layer.
- Always surface it, first in the problem list:
"child model error (<provider>/<id>): ...".
- New, optional: pi-task.fallbackProfiles.<effort> = {provider, id[, thinking]}
in ~/.pi/agent/settings.json. It is tried ONCE, and only when the primary
ended in an auth-class error, made zero tool calls, left the boundary
diff empty, and at least 30 s of wall_s remain. A write task that failed
after acting is never re-run on a half-edited tree.
- A fallback without `thinking` inherits the primary's rather than "off".
A fallback identical to the primary is ignored, with a warning.
- The key lives in pi-task's own namespace because the fork tool does not
fall back. pi-fork's parser silently drops unknown profile keys.
- Reporting: "[FALLBACK]" on the verdict line plus a warning line.
result.json gains fallback{} and attempts[]; metrics.model/provider name
the model that produced the verdict. Audit keeps raw.primary.ndjson,
argv.fallback.json and boundary-after.primary.json. The second attempt
gets only the remaining wall budget, so the task tool's (wall_s+90) s kill
still holds.
- Unchanged: exit 0 = PASS / 1 = FAIL, and the closing "audit:" line.
selftest: 14 -> 29 checks, two-sided for every gate (fixtures mirror the
measured event shape; the SSO string is verbatim from a real expired
session). Each gate was mutation-checked: removing it turns its line BAD.
End-to-end, emb-7kj4vr4g, with fake AWS keys in the environment:
- the old pi-task gives FAIL "no fenced json envelope";
- the new one, per tier, gives PASS [FALLBACK] with the correct answer and
real tool calls;
- with real credentials: PASS on the primary, fallback configured but unused;
- with no fallback configured: FAIL, model error listed first.
660 lines
32 KiB
Python
Executable File
660 lines
32 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""pi-task — run a headless pi subtask from an immutable SPEC, then verify the result.
|
|
|
|
Why this exists: `fork` hands the child getHeader()+getBranch(), i.e. the whole
|
|
untrimmed parent branch, so in a long session the child continues the PARENT's
|
|
narrative instead of doing the task (measured 2026-09-06, mbp-m1-2020: 4/4
|
|
dispatches ignored their brief; one filed a diary entry as the parent). This
|
|
runner inverts that default: context is EXPLICIT and default-EMPTY, the child is
|
|
a fresh isolated session, and its answer must parse as a declared envelope or the
|
|
run FAILS.
|
|
|
|
Honest limits, stated so nobody mistakes this for a sandbox:
|
|
* pi has no tool allow/deny list, so --no-extensions removes EXTENSIONS
|
|
(hence the palace write path) but NOT core read/write/edit/bash.
|
|
"read_only" is therefore VERIFIED POST-HOC by a boundary diff, not enforced.
|
|
* cost is checked after the fact; pi has no spend ceiling to pass in.
|
|
|
|
Model fallback (optional): settings.json `pi-task.fallbackProfiles.<effort>` =
|
|
{provider, id[, thinking]}. Tried once, and ONLY when the primary failed with an
|
|
auth-class error before running any tool or changing any watched root, so a
|
|
write task is never re-run on a half-edited tree. The report says so loudly and
|
|
result.json records both attempts. The fork tool does NOT honour this key.
|
|
|
|
Usage:
|
|
pi-task run SPEC.json [--dry-run] [--audit-dir DIR]
|
|
pi-task schema
|
|
pi-task selftest
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse, hashlib, json, os, re, shutil, subprocess, sys, time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
SETTINGS = Path.home() / ".pi/agent/settings.json"
|
|
AUDIT_ROOT = Path(os.environ.get("PI_TASK_AUDIT_DIR", Path.home() / ".pi/agent/pi-task"))
|
|
REQUIRED_SPEC = ("id", "goal", "deliverable", "effort")
|
|
ENVELOPE_KEYS = ("status", "deliverable", "evidence", "unsure", "did_not_do")
|
|
FENCE = re.compile(r"```json\s*(\{.*?\})\s*```", re.DOTALL)
|
|
|
|
|
|
def die(msg: str, code: int = 2):
|
|
print(f"pi-task: FAIL: {msg}", file=sys.stderr)
|
|
sys.exit(code)
|
|
|
|
|
|
def sh(args, cwd=None) -> str:
|
|
p = subprocess.run(args, cwd=cwd, capture_output=True, text=True)
|
|
return p.stdout.strip()
|
|
|
|
|
|
# ---------------------------------------------------------------- boundary diff
|
|
def _git_porcelain(r: str) -> dict:
|
|
"""--ignored is NOT optional: plain `git status --porcelain` omits ignored
|
|
files, so a child writing .env, credentials or build artifacts into a root
|
|
reads back as CLEAN. Measured 2026-09-07: an ignored secret.txt produced zero
|
|
porcelain lines, and `!! secret.txt` with --ignored.
|
|
Huge repos (node_modules) can emit megabytes, so always keep a sha256 and the
|
|
text only when it is small enough to show a human."""
|
|
txt = sh(["git", "-C", r, "status", "--porcelain", "--ignored"])
|
|
return {"sha": hashlib.sha256(txt.encode()).hexdigest()[:16],
|
|
"lines": len(txt.splitlines()),
|
|
"text": txt if len(txt) <= 8000 else None}
|
|
|
|
|
|
def boundary(roots) -> dict:
|
|
"""Cheap, checkable state of each root: git HEAD + porcelain, else file manifest."""
|
|
snap = {}
|
|
for r in roots:
|
|
root = Path(r)
|
|
if not root.exists():
|
|
snap[r] = {"kind": "missing"}
|
|
elif (root / ".git").exists():
|
|
snap[r] = {"kind": "git",
|
|
"head": sh(["git", "-C", r, "rev-parse", "HEAD"]),
|
|
"porcelain": _git_porcelain(r)}
|
|
else:
|
|
man = {}
|
|
for f in sorted(root.rglob("*")):
|
|
if f.is_file():
|
|
man[str(f)] = hashlib.sha256(f.read_bytes()).hexdigest()[:16]
|
|
snap[r] = {"kind": "manifest", "files": man}
|
|
return snap
|
|
|
|
|
|
def boundary_delta(before: dict, after: dict) -> list:
|
|
out = []
|
|
for r in before:
|
|
b, a = before[r], after.get(r, {})
|
|
if b == a:
|
|
continue
|
|
if b.get("kind") != a.get("kind"):
|
|
out.append(f"{r}: root kind changed {b.get('kind')} -> {a.get('kind')}")
|
|
continue
|
|
if b.get("kind") == "git":
|
|
if b.get("head") != a.get("head"):
|
|
out.append(f"{r}: HEAD {b.get('head','?')[:8]} -> {a.get('head','?')[:8]}")
|
|
pb, pa = b.get("porcelain") or {}, a.get("porcelain") or {}
|
|
if pb.get("sha") != pa.get("sha"):
|
|
if pa.get("text") is not None:
|
|
out.append(f"{r}: working tree changed (incl. ignored):\n{pa['text']}")
|
|
else:
|
|
out.append(f"{r}: working tree changed (incl. ignored): "
|
|
f"{pb.get('lines')} -> {pa.get('lines')} entries, "
|
|
f"sha {pb.get('sha')} -> {pa.get('sha')} (text too large to show)")
|
|
else:
|
|
bf, af = b.get("files", {}), a.get("files", {})
|
|
for p in sorted(set(bf) | set(af)):
|
|
if bf.get(p) != af.get(p):
|
|
out.append(f"{r}: {'added' if p not in bf else 'removed' if p not in af else 'modified'} {p}")
|
|
return out
|
|
|
|
|
|
# -------------------------------------------------------------------- the brief
|
|
def writable_roots(spec: dict) -> list:
|
|
"""Which roots the child is allowed to change.
|
|
|
|
`roots` is the WATCHED set; `write_allowed` is the CHANGEABLE subset. Keeping
|
|
them separate is what makes a violation detectable at all: if the two are the
|
|
same list, a write task can never trip the diff, and the only way to provoke
|
|
one is to order the child to defy its own authority line — which a
|
|
well-behaved child simply refuses (measured 2026-09-07, test T2). The real
|
|
hazard is not defiance anyway; it is an INCIDENTAL write, e.g. a verification
|
|
command that drops __pycache__ into a repo the child was only meant to read.
|
|
"""
|
|
if spec.get("read_only", True):
|
|
return []
|
|
aw = spec.get("write_allowed")
|
|
return list(aw) if aw else list(spec.get("roots") or [])
|
|
def build_prompt(spec: dict) -> str:
|
|
ctx = spec.get("context") or {}
|
|
roots = spec.get("roots") or []
|
|
allowed = writable_roots(spec)
|
|
lines = [
|
|
"You are a TASK WORKER invoked by another agent. You are NOT continuing a",
|
|
"conversation and you have NO shared history: there is no 'earlier', no",
|
|
"'as we discussed', and no session to wind down. Do the task below and",
|
|
"return the envelope. Nothing else.",
|
|
"",
|
|
f"## GOAL\n{spec['goal']}",
|
|
f"\n## DELIVERABLE\n{spec['deliverable']}",
|
|
]
|
|
if spec.get("read_only", True):
|
|
lines += ["\n## AUTHORITY\nREAD-ONLY. Do not create, edit or delete any file. Do not run any",
|
|
"command that mutates state (no git commit/push, no installs). A boundary",
|
|
"diff runs after you exit and a violation fails the whole task."]
|
|
else:
|
|
lines += ["\n## AUTHORITY\nYou MAY create and modify files under:"]
|
|
lines += [f" - {a}" for a in allowed]
|
|
watched_only = [r for r in roots if r not in allowed]
|
|
if watched_only:
|
|
lines += ["You may READ these, but must NOT change anything under them:"]
|
|
lines += [f" - {r}" for r in watched_only]
|
|
lines += ["A boundary diff runs after you exit over every path above. ANY change",
|
|
"outside the writable set fails the task — including one made incidentally",
|
|
"by a command you ran rather than by an edit you intended."]
|
|
if ctx.get("facts"):
|
|
lines += ["\n## VERIFIED CONTEXT (asserted by the caller; you need not re-derive it)"]
|
|
lines += [f"- {f}" for f in ctx["facts"]]
|
|
if ctx.get("files"):
|
|
lines += ["\n## RELEVANT PATHS (read them yourself; they are not pasted here)"]
|
|
lines += [f"- {f}" for f in ctx["files"]]
|
|
if ctx.get("commands"):
|
|
lines += ["\n## SUGGESTED COMMANDS"]
|
|
lines += [f"- {c}" for c in ctx["commands"]]
|
|
lines += [
|
|
"",
|
|
"## REQUIRED OUTPUT — a single fenced json block, LAST thing you emit",
|
|
"If it does not parse against this shape the task is recorded as FAILED,",
|
|
"however good the prose was:",
|
|
"```json",
|
|
json.dumps({
|
|
"status": "ok | partial | failed",
|
|
"deliverable": "<the answer asked for, as text>",
|
|
"evidence": [{"claim": "<one claim>", "pointer": "<file:line | exact command | url>"}],
|
|
"unsure": ["<anything you could not establish — empty list only if truly none>"],
|
|
"did_not_do": ["<anything in scope you skipped>"],
|
|
}, indent=2),
|
|
"```",
|
|
"Rules for evidence: every load-bearing claim needs a pointer a third party",
|
|
"can re-run or open. Do not invent quantities ('all N files') you did not count.",
|
|
"If you could not do the task, status=failed with the reason is a CORRECT answer.",
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
# ------------------------------------------------------- envelope validation
|
|
def validate_envelope(text: str):
|
|
"""(envelope|None, problems[]). Deterministic and model-free so `selftest`
|
|
can exercise it two-sidedly — a validator that has only ever returned PASS
|
|
has not been shown to discriminate."""
|
|
problems, env = [], None
|
|
blocks = FENCE.findall(text or "")
|
|
if not blocks:
|
|
problems.append("no fenced json envelope in the final assistant message")
|
|
return None, problems
|
|
try:
|
|
env = json.loads(blocks[-1])
|
|
except json.JSONDecodeError as e:
|
|
problems.append(f"envelope is not valid json: {e}")
|
|
return None, problems
|
|
if not isinstance(env, dict):
|
|
problems.append("envelope is not a json object")
|
|
return None, problems
|
|
for k in ENVELOPE_KEYS:
|
|
if k not in env:
|
|
problems.append(f"envelope missing key '{k}'")
|
|
if env.get("status") not in ("ok", "partial", "failed"):
|
|
problems.append(f"envelope status not ok|partial|failed: {env.get('status')!r}")
|
|
ev_list = env.get("evidence")
|
|
if not isinstance(ev_list, list) or not ev_list:
|
|
problems.append("envelope evidence is empty — no claim is checkable")
|
|
else:
|
|
for i, e in enumerate(ev_list):
|
|
if not isinstance(e, dict) or not e.get("pointer"):
|
|
problems.append(f"evidence[{i}] has no pointer")
|
|
return env, problems
|
|
|
|
|
|
# ------------------------------------------------------------ model fallback
|
|
# A tier's primary model can be unusable for reasons that have nothing to do
|
|
# with the task: an expired SSO session, a revoked key. pi reports that as a
|
|
# perfectly normal run — rc=0, agent_end, willRetry=false — with the failure
|
|
# visible ONLY as stopReason="error" on the assistant message (measured
|
|
# 2026-10-12, Bedrock with an invalid key). Before this, pi-task turned it into
|
|
# the misleading "no fenced json envelope".
|
|
#
|
|
# The retry is deliberately narrow. It happens only when the child provably did
|
|
# NOTHING: an auth-class error, zero tool results, and an unchanged boundary. A
|
|
# child that failed after acting may have half-edited a writable root, and
|
|
# re-running a write task on a dirty tree turns one clean failure into an
|
|
# unauditable mess. So any doubt means no retry, and the original failure
|
|
# stands.
|
|
AUTH_ERROR = re.compile(
|
|
r"token is expired|aws sso login|expiredtoken|unrecognizedclientexception"
|
|
r"|security token included in the request is (?:invalid|expired)"
|
|
r"|could not load credentials|unable to locate credentials|credentialsprovidererror"
|
|
r"|credentials? (?:have|has) expired|\b401\b|unauthori[sz]ed"
|
|
r"|invalid[ _-]?(?:x-)?api[ _-]?key|no api key|authentication (?:failed|error)",
|
|
re.IGNORECASE)
|
|
MIN_FALLBACK_WALL_S = 30
|
|
|
|
|
|
def child_error(events: list):
|
|
"""errorMessage of the LAST assistant message if it ended in error, else None."""
|
|
last = None
|
|
for ev in events:
|
|
m = ev.get("message") or {}
|
|
if ev.get("type") == "message_end" and m.get("role") == "assistant":
|
|
last = m
|
|
if last and last.get("stopReason") == "error":
|
|
return last.get("errorMessage") or "stopReason=error with no errorMessage"
|
|
return None
|
|
|
|
|
|
def fallback_decision(events: list, delta: list, remaining_s: float, has_fallback: bool):
|
|
"""(retry?, reason). Pure, so selftest can exercise both sides of every gate."""
|
|
if not has_fallback:
|
|
return False, "no fallback profile for this effort"
|
|
err = child_error(events)
|
|
if not err:
|
|
return False, "primary did not end in a model error"
|
|
if not AUTH_ERROR.search(err):
|
|
return False, f"primary error is not auth-class: {err[:200]}"
|
|
tools = sum(len(ev.get("toolResults") or []) for ev in events if ev.get("type") == "turn_end")
|
|
if tools:
|
|
return False, f"primary had already run {tools} tool call(s); not safe to re-run"
|
|
if delta:
|
|
return False, "primary changed a watched root; not safe to re-run"
|
|
if remaining_s < MIN_FALLBACK_WALL_S:
|
|
return False, f"only {remaining_s:.0f}s of wall budget left"
|
|
return True, f"primary auth failure before any action: {err[:200]}"
|
|
|
|
|
|
def _profile(raw, inherit_thinking=None):
|
|
if not isinstance(raw, dict) or not raw.get("provider") or not raw.get("id"):
|
|
return None
|
|
out = {"provider": raw["provider"], "id": raw["id"]}
|
|
thinking = raw.get("thinking", inherit_thinking)
|
|
if thinking is not None:
|
|
out["thinking"] = thinking
|
|
return out
|
|
|
|
|
|
def resolve_profiles(settings: dict, effort: str):
|
|
"""(primary|None, fallback|None, warnings[]).
|
|
|
|
The primary comes from pi-fork.effortProfiles, shared with the fork tool on
|
|
purpose. The fallback lives in pi-task's OWN namespace, pi-task.fallbackProfiles,
|
|
because only pi-task honours it. Putting it inside the fork profile would
|
|
suggest fork falls back too — it does not; pi-fork's parser drops unknown keys
|
|
without a word.
|
|
|
|
A fallback that omits `thinking` inherits the primary's. Defaulting to "off",
|
|
as the primary path does, would let a tier lose its reasoning silently."""
|
|
primary = ((settings.get("pi-fork") or {}).get("effortProfiles") or {}).get(effort)
|
|
raw = ((settings.get("pi-task") or {}).get("fallbackProfiles") or {}).get(effort)
|
|
warnings = []
|
|
if raw is None or not primary:
|
|
return primary, None, warnings
|
|
fb = _profile(raw, inherit_thinking=primary.get("thinking"))
|
|
if fb is None:
|
|
warnings.append(f"pi-task.fallbackProfiles.{effort} needs provider and id; ignored")
|
|
elif (fb["provider"], fb["id"]) == (primary.get("provider"), primary.get("id")):
|
|
warnings.append(f"pi-task.fallbackProfiles.{effort} is the primary model itself; ignored")
|
|
fb = None
|
|
return primary, fb, warnings
|
|
|
|
|
|
def _selftest_fallback() -> list:
|
|
"""Fallback decision + profile resolution, two-sided.
|
|
|
|
The event shapes are not invented: they mirror `pi -p --mode json` measured
|
|
2026-10-12 against Bedrock with an invalid key — rc=0, a normal agent_end
|
|
with willRetry=false, and the failure present ONLY as stopReason="error" +
|
|
errorMessage on the assistant message. The SSO string is the verbatim text
|
|
a user saw from an expired `aws sso` session the same day."""
|
|
bad_key = "UnrecognizedClientException: The security token included in the request is invalid."
|
|
sso = ("Token is expired. To refresh this SSO session run 'aws sso login' "
|
|
"with the corresponding profile.")
|
|
|
|
def run(err=None, tools=0, stop=None):
|
|
stop = stop or ("error" if err else "stop")
|
|
msg = {"role": "assistant", "stopReason": stop, "content": [],
|
|
**({"errorMessage": err} if err else {})}
|
|
evs = [{"type": "agent_start"}, {"type": "turn_start"},
|
|
{"type": "message_end", "message": msg}]
|
|
evs += [{"type": "turn_end", "toolResults": [{}] * tools}]
|
|
evs += [{"type": "agent_end", "willRetry": False, "messages": [msg]}]
|
|
return evs
|
|
|
|
def fb(events, delta=(), remaining=600, has_fb=True):
|
|
return fallback_decision(events, list(delta), remaining, has_fb)[0]
|
|
|
|
prim = {"provider": "p", "id": "big", "thinking": "high"}
|
|
alt = {"provider": "q", "id": "alt", "thinking": "medium"}
|
|
s = lambda fbp: {"pi-fork": {"effortProfiles": {"deep": prim}},
|
|
"pi-task": {"fallbackProfiles": {"deep": fbp}}}
|
|
p0, f_ok, _ = resolve_profiles(s(alt), "deep")
|
|
_, f_inh, _ = resolve_profiles(s({"provider": "q", "id": "alt"}), "deep")
|
|
_, f_same, w_same = resolve_profiles(s(dict(prim)), "deep")
|
|
_, f_bad, w_bad = resolve_profiles(s({"provider": "q"}), "deep")
|
|
_, f_none, w_none = resolve_profiles({"pi-fork": {"effortProfiles": {"deep": prim}}}, "deep")
|
|
return [
|
|
("fallback: POSITIVE bad-key auth error, 0 tools, clean tree => retry",
|
|
fb(run(bad_key))),
|
|
("fallback: POSITIVE expired-SSO error (verbatim) => retry", fb(run(sso))),
|
|
("fallback: auth error AFTER a tool call => NO retry", not fb(run(bad_key, tools=1))),
|
|
("fallback: auth error but a root CHANGED => NO retry",
|
|
not fb(run(bad_key), delta=["/r: working tree changed"])),
|
|
("fallback: non-auth error (throttling) => NO retry",
|
|
not fb(run("ThrottlingException: Too many requests, please wait"))),
|
|
("fallback: successful run => NO retry", not fb(run())),
|
|
("fallback: no fallback configured => NO retry", not fb(run(bad_key), has_fb=False)),
|
|
("fallback: under 30 s of wall budget left => NO retry",
|
|
not fb(run(bad_key), remaining=10)),
|
|
("child_error: surfaces the hidden stopReason=error message",
|
|
child_error(run(bad_key)) == bad_key and child_error(run()) is None),
|
|
("profiles: primary resolved from pi-fork.effortProfiles", p0 == prim),
|
|
("profiles: fallback resolved from pi-task.fallbackProfiles", f_ok == alt),
|
|
("profiles: fallback WITHOUT thinking inherits the primary's, not 'off'",
|
|
(f_inh or {}).get("thinking") == "high"),
|
|
("profiles: fallback identical to primary is dropped, with a warning",
|
|
f_same is None and bool(w_same)),
|
|
("profiles: malformed fallback (no id) is dropped, with a warning",
|
|
f_bad is None and bool(w_bad)),
|
|
("profiles: no fallback configured => None, silently", f_none is None and not w_none),
|
|
]
|
|
|
|
|
|
def cmd_selftest(_args):
|
|
"""Two-sided: known-good must pass, each known-bad must fail, and the
|
|
boundary detector must both see a change and see no change. Aborts loudly
|
|
if any pair fails to discriminate."""
|
|
good = json.dumps({"status": "ok", "deliverable": "d",
|
|
"evidence": [{"claim": "c", "pointer": "f.py:1"}],
|
|
"unsure": [], "did_not_do": []})
|
|
fixtures = [
|
|
("POSITIVE valid envelope", f"prose\n```json\n{good}\n```", True),
|
|
("no fence at all", "just fluent prose, no envelope", False),
|
|
("fence but broken json", "```json\n{not json,}\n```", False),
|
|
("missing key did_not_do", "```json\n" + json.dumps(
|
|
{"status": "ok", "deliverable": "d",
|
|
"evidence": [{"claim": "c", "pointer": "p"}], "unsure": []}) + "\n```", False),
|
|
("bad status value", "```json\n" + json.dumps(
|
|
{"status": "done", "deliverable": "d",
|
|
"evidence": [{"claim": "c", "pointer": "p"}],
|
|
"unsure": [], "did_not_do": []}) + "\n```", False),
|
|
("empty evidence", "```json\n" + json.dumps(
|
|
{"status": "ok", "deliverable": "d", "evidence": [],
|
|
"unsure": [], "did_not_do": []}) + "\n```", False),
|
|
("evidence without pointer", "```json\n" + json.dumps(
|
|
{"status": "ok", "deliverable": "d", "evidence": [{"claim": "c"}],
|
|
"unsure": [], "did_not_do": []}) + "\n```", False),
|
|
("json object not last block wins", f"```json\n{{\"junk\":1}}\n```\n```json\n{good}\n```", True),
|
|
]
|
|
fails = 0
|
|
for name, text, want_pass in fixtures:
|
|
_, probs = validate_envelope(text)
|
|
got_pass = not probs
|
|
ok = got_pass == want_pass
|
|
fails += 0 if ok else 1
|
|
print(f" {'ok ' if ok else 'BAD'} {'expect PASS' if want_pass else 'expect FAIL'} {name}"
|
|
+ ("" if ok else f" <-- got {'PASS' if got_pass else 'FAIL'}: {probs}"))
|
|
|
|
import tempfile
|
|
with tempfile.TemporaryDirectory() as td:
|
|
d = Path(td) / "root"; d.mkdir(); (d / "a.txt").write_text("1")
|
|
b1 = boundary([str(d)])
|
|
same = boundary_delta(b1, boundary([str(d)]))
|
|
(d / "b.txt").write_text("2")
|
|
diff = boundary_delta(b1, boundary([str(d)]))
|
|
checks = [("boundary/manifest: unchanged root reports no delta", same == []),
|
|
("boundary/manifest: added file IS detected", len(diff) == 1)]
|
|
|
|
# The GIT path is what every real run uses, and it was previously
|
|
# untested. The ignored-file case below was genuinely broken until
|
|
# --ignored was added, so this is a regression test, not decoration.
|
|
g = Path(td) / "repo"; g.mkdir()
|
|
for cmd in (["git", "init", "-q", "."], ["git", "config", "user.email", "t@t"],
|
|
["git", "config", "user.name", "t"]):
|
|
subprocess.run(cmd, cwd=g, capture_output=True)
|
|
(g / ".gitignore").write_text("secret.txt\n__pycache__/\n")
|
|
(g / "tracked.txt").write_text("v1")
|
|
subprocess.run(["git", "add", "-A"], cwd=g, capture_output=True)
|
|
subprocess.run(["git", "commit", "-qm", "init"], cwd=g, capture_output=True)
|
|
gb = boundary([str(g)])
|
|
checks.append(("boundary/git: clean repo reports no delta",
|
|
boundary_delta(gb, boundary([str(g)])) == []))
|
|
(g / "secret.txt").write_text("exfiltrated")
|
|
checks.append(("boundary/git: IGNORED file IS detected (regression: --ignored)",
|
|
len(boundary_delta(gb, boundary([str(g)]))) == 1))
|
|
(g / "secret.txt").unlink()
|
|
(g / "tracked.txt").write_text("v2")
|
|
checks.append(("boundary/git: modified tracked file IS detected",
|
|
len(boundary_delta(gb, boundary([str(g)]))) == 1))
|
|
subprocess.run(["git", "commit", "-aqm", "v2"], cwd=g, capture_output=True)
|
|
checks.append(("boundary/git: a COMMIT (HEAD move) IS detected",
|
|
any("HEAD" in x for x in boundary_delta(gb, boundary([str(g)])))))
|
|
|
|
checks += _selftest_fallback()
|
|
for name, cond in checks:
|
|
fails += 0 if cond else 1
|
|
print(f" {'ok ' if cond else 'BAD'} {name}")
|
|
|
|
if fails:
|
|
die(f"selftest: {fails} check(s) failed — the validator does not discriminate, "
|
|
"so any PASS it reports is meaningless", 3)
|
|
print(f"selftest: all {len(fixtures) + len(checks)} checks discriminate correctly")
|
|
return 0
|
|
|
|
|
|
# ------------------------------------------------------------------------- main
|
|
def _argv(prof: dict, session_id: str, session_dir: Path, prompt: str) -> list:
|
|
return ["pi", "-p", "--mode", "json",
|
|
"--provider", prof["provider"], "--model", prof["id"],
|
|
"--thinking", prof.get("thinking", "off"),
|
|
"--no-extensions",
|
|
"--session-id", session_id,
|
|
"--session-dir", str(session_dir),
|
|
prompt]
|
|
|
|
|
|
def run_attempt(argv: list, timeout: float) -> dict:
|
|
"""Run one child and parse its NDJSON: final assistant text, cost, tool calls."""
|
|
t0 = time.time()
|
|
try:
|
|
p = subprocess.run(argv, capture_output=True, text=True, timeout=timeout)
|
|
raw, rc = p.stdout, p.returncode
|
|
except subprocess.TimeoutExpired as e:
|
|
raw, rc = (e.stdout or ""), 124
|
|
raw = raw if isinstance(raw, str) else raw.decode()
|
|
events, text, cost, tools = [], "", 0.0, 0
|
|
for line in raw.splitlines():
|
|
try:
|
|
events.append(json.loads(line))
|
|
except json.JSONDecodeError:
|
|
continue
|
|
for ev in events:
|
|
if ev.get("type") == "agent_end":
|
|
for m in ev.get("messages", []):
|
|
cost += (((m.get("usage") or {}).get("cost") or {}).get("total") or 0.0)
|
|
if ev.get("type") == "turn_end":
|
|
tools += len(ev.get("toolResults") or [])
|
|
if ev.get("type") == "message_end" and (ev.get("message") or {}).get("role") == "assistant":
|
|
text = "".join(c.get("text", "") for c in ev["message"].get("content", [])
|
|
if c.get("type") == "text") or text
|
|
return {"raw": raw, "rc": rc, "events": events, "text": text, "cost": cost,
|
|
"tools": tools, "elapsed": time.time() - t0, "error": child_error(events)}
|
|
|
|
|
|
def cmd_run(args):
|
|
spec_path = Path(args.spec)
|
|
spec = json.loads(spec_path.read_text())
|
|
missing = [k for k in REQUIRED_SPEC if k not in spec]
|
|
if missing:
|
|
die(f"spec missing required keys: {missing}")
|
|
|
|
settings = json.loads(SETTINGS.read_text())
|
|
prof, fb_prof, fb_warnings = resolve_profiles(settings, spec["effort"])
|
|
if not prof:
|
|
profiles = (settings.get("pi-fork") or {}).get("effortProfiles") or {}
|
|
die(f"effort '{spec['effort']}' not in {SETTINGS}: pi-fork.effortProfiles ({list(profiles)})")
|
|
for w in fb_warnings:
|
|
print(f"pi-task: warning: {w}", file=sys.stderr)
|
|
|
|
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
audit = Path(args.audit_dir) if args.audit_dir else AUDIT_ROOT / f"{stamp}-{spec['id']}"
|
|
audit.mkdir(parents=True, exist_ok=True)
|
|
prompt = build_prompt(spec)
|
|
roots = spec.get("roots") or []
|
|
budget = spec.get("budget") or {}
|
|
wall = int(budget.get("wall_s", 600))
|
|
|
|
session_id = f"pitask-{spec['id']}-{stamp}"
|
|
argv = _argv(prof, session_id, audit / "sess", prompt)
|
|
fb_argv = _argv(fb_prof, f"{session_id}-fallback", audit / "sess-fallback", prompt) if fb_prof else None
|
|
|
|
(audit / "spec.json").write_text(json.dumps(spec, indent=2))
|
|
(audit / "prompt.txt").write_text(prompt)
|
|
(audit / "argv.json").write_text(json.dumps(argv[:-1] + ["<prompt in prompt.txt>"], indent=2))
|
|
if fb_argv:
|
|
(audit / "argv.fallback.json").write_text(
|
|
json.dumps(fb_argv[:-1] + ["<prompt in prompt.txt>"], indent=2))
|
|
if args.dry_run:
|
|
print(prompt)
|
|
if fb_prof:
|
|
print(f"\n--- fallback if the primary fails auth before acting: "
|
|
f"{fb_prof['provider']}/{fb_prof['id']}")
|
|
print(f"\n--- dry run, nothing launched. audit: {audit}"); return 0
|
|
|
|
before = boundary(roots)
|
|
(audit / "boundary-before.json").write_text(json.dumps(before, indent=2))
|
|
|
|
used, att = prof, run_attempt(argv, wall)
|
|
attempts = [{"provider": prof["provider"], "id": prof["id"], "thinking": prof.get("thinking", "off"),
|
|
"rc": att["rc"], "elapsed_s": round(att["elapsed"], 1), "tool_results": att["tools"],
|
|
"cost_usd": round(att["cost"], 6), "error": att["error"]}]
|
|
after = boundary(roots)
|
|
fallback = {"configured": bool(fb_prof), "used": False, "reason": None}
|
|
if fb_prof:
|
|
retry, why = fallback_decision(att["events"], boundary_delta(before, after),
|
|
wall - att["elapsed"], True)
|
|
fallback["reason"] = why
|
|
if retry:
|
|
# Keep the failed primary's evidence; raw.ndjson is always the run that produced the verdict.
|
|
(audit / "raw.primary.ndjson").write_text(att["raw"])
|
|
(audit / "boundary-after.primary.json").write_text(json.dumps(after, indent=2))
|
|
used, att = fb_prof, run_attempt(fb_argv, wall - att["elapsed"])
|
|
attempts.append({"provider": fb_prof["provider"], "id": fb_prof["id"],
|
|
"thinking": fb_prof.get("thinking", "off"), "rc": att["rc"],
|
|
"elapsed_s": round(att["elapsed"], 1), "tool_results": att["tools"],
|
|
"cost_usd": round(att["cost"], 6), "error": att["error"]})
|
|
fallback.update({"used": True, "from": f"{prof['provider']}/{prof['id']}",
|
|
"to": f"{fb_prof['provider']}/{fb_prof['id']}"})
|
|
after = boundary(roots)
|
|
raw, rc, events, text = att["raw"], att["rc"], att["events"], att["text"]
|
|
cost = sum(a["cost_usd"] for a in attempts) # a failed primary is billed too, if at all
|
|
tools = att["tools"]
|
|
elapsed = sum(a["elapsed_s"] for a in attempts)
|
|
(audit / "raw.ndjson").write_text(raw)
|
|
(audit / "boundary-after.json").write_text(json.dumps(after, indent=2))
|
|
|
|
# ---- validate the envelope: unparseable == FAILED, no matter how fluent
|
|
env, problems = validate_envelope(text)
|
|
if att["error"]:
|
|
# pi exits 0 on a provider error; without this line the only symptom
|
|
# would be "no fenced json envelope", which points at the wrong layer.
|
|
# First: it is the cause, and the missing envelope merely its symptom.
|
|
problems.insert(0, f"child model error ({used['provider']}/{used['id']}): {att['error'][:300]}")
|
|
if rc != 0:
|
|
problems.append(f"child exited rc={rc}" + (" (wall-clock budget)" if rc == 124 else ""))
|
|
if not any(e.get("type") == "agent_end" for e in events):
|
|
problems.append("no agent_end event — child did not complete a turn")
|
|
delta = boundary_delta(before, after)
|
|
allowed = writable_roots(spec)
|
|
violations = [d for d in delta
|
|
if not any(d.startswith(f"{a}:") for a in allowed)]
|
|
if violations:
|
|
if spec.get("read_only", True):
|
|
problems.append("BOUNDARY VIOLATION: read_only task mutated its roots")
|
|
else:
|
|
problems.append("BOUNDARY VIOLATION: task changed a watched root it was "
|
|
f"not authorised to write (authorised: {allowed or 'none'})")
|
|
over = budget.get("usd")
|
|
if over and cost > float(over):
|
|
problems.append(f"cost ${cost:.4f} over budget ${float(over):.4f}")
|
|
|
|
verdict = "PASS" if not problems else "FAIL"
|
|
result = {"verdict": verdict, "problems": problems, "envelope": env,
|
|
"metrics": {"cost_usd": round(cost, 6), "elapsed_s": round(elapsed, 1),
|
|
"tool_results": tools, "rc": rc, "model": used["id"],
|
|
"provider": used["provider"], "effort": spec["effort"]},
|
|
"fallback": fallback, "attempts": attempts,
|
|
"boundary_delta": delta, "boundary_violations": violations,
|
|
"audit_dir": str(audit),
|
|
"raw_text_if_envelope_missing": None if env else text[-2000:]}
|
|
(audit / "result.json").write_text(json.dumps(result, indent=2))
|
|
|
|
# ---- report to the parent: verdict first, then what to spot-check
|
|
print(f"pi-task {verdict} id={spec['id']} {used['id']}"
|
|
+ (" [FALLBACK]" if fallback["used"] else "")
|
|
+ f" ${cost:.4f} {elapsed:.0f}s tools={tools} rc={rc}")
|
|
if fallback["used"]:
|
|
print(f" ⚠ fallback: {fallback['from']} -> {fallback['to']} — {fallback['reason']}")
|
|
print(" This result is from the FALLBACK model; weigh it accordingly.")
|
|
for p_ in problems:
|
|
print(f" ! {p_}")
|
|
if env:
|
|
print(f" status={env.get('status')}")
|
|
print(" deliverable:"); print(" " + str(env.get("deliverable", "")).replace("\n", "\n "))
|
|
print(" evidence (SPOT-CHECK THESE, do not trust the prose):")
|
|
for e in env.get("evidence") or []:
|
|
print(f" - {e.get('claim','')} <= {e.get('pointer','')}")
|
|
for k in ("unsure", "did_not_do"):
|
|
for u in env.get(k) or []:
|
|
print(f" {k}: {u}")
|
|
print(f" audit: {audit}")
|
|
return 0 if verdict == "PASS" else 1
|
|
|
|
|
|
def cmd_schema(_args):
|
|
print(json.dumps({
|
|
"id": "short-slug-used-in-session-id-and-audit-path",
|
|
"goal": "verbatim task statement",
|
|
"deliverable": "the exact shape of answer wanted",
|
|
"effort": "fast | balanced | deep (resolved via ~/.pi/agent/settings.json pi-fork.effortProfiles; "
|
|
"optional per-effort fallback in pi-task.fallbackProfiles, used ONLY when the primary "
|
|
"fails auth before any tool call or file change)",
|
|
"read_only": True,
|
|
"roots": ["/workspace/repo-the-task-touches"],
|
|
"write_allowed": ["subset of roots the child may CHANGE; only meaningful when "
|
|
"read_only is false. Defaults to all of roots, which makes "
|
|
"violations undetectable — set it explicitly for write tasks."],
|
|
"context": {"facts": ["verified fact the caller asserts"],
|
|
"files": ["/abs/path the child should read itself"],
|
|
"commands": ["exact command the child may run"]},
|
|
"budget": {"wall_s": 600, "usd": 0.5},
|
|
}, indent=2))
|
|
return 0
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(prog="pi-task")
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
r = sub.add_parser("run"); r.add_argument("spec")
|
|
r.add_argument("--dry-run", action="store_true"); r.add_argument("--audit-dir")
|
|
r.set_defaults(fn=cmd_run)
|
|
s = sub.add_parser("schema"); s.set_defaults(fn=cmd_schema)
|
|
t = sub.add_parser("selftest"); t.set_defaults(fn=cmd_selftest)
|
|
a = ap.parse_args()
|
|
if not shutil.which("pi"):
|
|
die("pi not on PATH")
|
|
sys.exit(a.fn(a))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|