secret-guard: keep secret env values out of model context, session files and the palace feed
Measured 2026-10-12 on emb-7kj4vr4g. A new session ran `env | grep …` and tried to redact the output itself with a sed. The sed only matched when TOKEN came after the `=`, so MEMPALACE_REMOTE_TOKEN=<value> went through. That raw value was found in 12 local session files back to 2026-08-16, and each of those sessions had sent it to its model provider. The palace stayed clean only because mempalace-toolkit's feeder scrubs transcripts before staging. Nothing guarded the step from tool output to model context. The new extension ports tier 1 of mempalace_redact.py exactly: the exact values of env vars whose name says secret are replaced with <redacted:NAME>. Matching a known value cannot hit anything that is not that secret. - tool_result: content text, structuredContent (always returned when content changes, so it is not dropped) and details. - context: the same pass over every request, for user `!` output, pasted text and resumed sessions that already hold a raw value. - tool_call: block a write/edit whose new text contains a live secret's placeholder, i.e. writing a redacted .env back. The reason says to use bash and let the shell expand "$NAME". PI_SECRET_GUARD=off makes it inert. Tests (16, two-sided): selection, ordering, redaction, deep walk with identity, every handler, the off switch. Two parity tests run mempalace_redact.py on the same synthetic env and compare the selected names and the output text. Thirteen mutations, one per rule, are each caught. Two mutations initially survived and exposed fixture gaps, which are now fixed. Full suite: 45/45. End-to-end in real pi (erdc/deepseek-v4.1-flash). The secret was synthetic, and a test-only probe on before_provider_request counted it in each payload. - control without the guard: 1 raw copy sent to the provider, 1 in the session file; - with the guard: 0 sent and 0 stored, placeholder in both; - resumed control session: 1 sent without the guard, 0 with it; - interactive `!echo $SECRET`: 0 sent, plus a TUI warning; - a write of the placeholder: blocked, file sha unchanged. Not covered, as documented: live progress (tool_execution_update), the stored entry of a user `!` command, secrets not in env (no shape rules), and pi-task children, which run --no-extensions.
This commit is contained in:
Joakim Persson
@@ -0,0 +1,241 @@
|
||||
// Two-sided tests for secret-guard: what MUST be treated as a secret and what
|
||||
// MUST NOT, what MUST be rewritten and what MUST come back untouched (by
|
||||
// reference, so the handlers can return "no change"). A guard that only ever
|
||||
// redacts has not been shown to discriminate. Runs the shipped .ts directly
|
||||
// (node >= 23 strips types natively; the only import is type-only).
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import {
|
||||
envSecrets,
|
||||
placeholder,
|
||||
redactText,
|
||||
redactDeep,
|
||||
toolResultChange,
|
||||
contextChange,
|
||||
writeGuard,
|
||||
guardMode,
|
||||
} from "../extensions/secret-guard.ts";
|
||||
|
||||
// Values are shaped like nothing in particular (no ghp_/sk-/AKIA prefix), so the
|
||||
// Python parity check exercises tier 1 only and tier-2 shape rules stay silent.
|
||||
const V = {
|
||||
a: "Zq7mWc2Lr9Kp4Tn8Vx3Bd6Hs1Fj5Gy0Qe7Ua2Ic9Op4", // 43, like MEMPALACE_REMOTE_TOKEN
|
||||
b: "r2D9xK7pQm4LwZ8vT1cB6nH3sF5jG0yE", // 32
|
||||
c: "Mx8Rt3Lq9Wz2Kc7Vb4Nd1Hp6Js5Fg0Ya1Ue9Io3Pl8Ak2Sd7", // 49
|
||||
d: "uV6tR1eW8qY3iO7pA2sD9fG4", // 24
|
||||
shortish: "abcdefghijkl", // exactly 12: still a secret by length
|
||||
};
|
||||
|
||||
// name -> value. MUST_SELECT: tier-1 secrets. MUST_SKIP: secret-sounding names
|
||||
// that hold a location, a knob or a placeholder, and plain names.
|
||||
const MUST_SELECT = {
|
||||
MEMPALACE_REMOTE_TOKEN: V.a,
|
||||
ERDC_INFERENCE_API_KEY: V.c,
|
||||
GITEA_ACCESS_TOKEN: V.b,
|
||||
CLIENT_SECRET: V.d,
|
||||
DB_PASSWORD: V.shortish,
|
||||
GITHUB_PAT: V.b + "x",
|
||||
PAT: V.d + "y",
|
||||
OAUTH_BEARER: V.a + "z",
|
||||
SESSION_KEY: V.c + "w",
|
||||
api_key: V.d + "q", // lower case: the name rule is case-insensitive
|
||||
AWS_SECRET_ACCESS_KEY: V.b + "AB",
|
||||
SOME_CREDENTIALS: V.d + "cr",
|
||||
};
|
||||
const MUST_SKIP = {
|
||||
SSH_KEY_PATH: "/home/developer/.ssh/id_ed25519", // _PATH suffix
|
||||
GITEA_TOKEN_FILE: "/run/secrets/gitea_token_file_x", // _FILE suffix
|
||||
GITHUB_TOKEN_URL: "https://github.example/token/endpoint", // _URL suffix
|
||||
PI_OM_MAX_TOKENS: "100000", // too short to be a credential
|
||||
SHORT_TOKEN: "abcdefghijk", // 11 chars
|
||||
SPACED_SECRET: "this has spaces in it ok", // credentials have no whitespace
|
||||
PLACEHOLDER_TOKEN: "<your-token-here-please>",
|
||||
REF_TOKEN: "${MEMPALACE_REMOTE_TOKEN}",
|
||||
DOLLAR_TOKEN: "$MEMPALACE_REMOTE_TOKEN",
|
||||
HOME_TOKEN: "~/secrets/token-file-here",
|
||||
TRIVIAL_SECRET: "changeme",
|
||||
STARS_PASSWORD: "****************",
|
||||
HOME: "/home/developer",
|
||||
PATH: "/usr/local/bin:/usr/bin:/bin",
|
||||
MEMPALACE_REMOTE_URL: "https://mempalace.example/mcp",
|
||||
RANDOM_STRING: V.a + "notsecret", // a random value under a plain name is NOT tier 1
|
||||
DEPLOY_PRIVATE_KEY: "~deploy/.ssh/id_ed25519", // ~user/ path: only the path rule catches it
|
||||
};
|
||||
const ENV = { ...MUST_SELECT, ...MUST_SKIP };
|
||||
|
||||
test("envSecrets selects every secret-named credential (positive side)", () => {
|
||||
const names = new Set(envSecrets(ENV).map(([n]) => n));
|
||||
for (const n of Object.keys(MUST_SELECT)) assert.ok(names.has(n), `should select ${n}`);
|
||||
});
|
||||
|
||||
test("envSecrets skips locations, knobs, placeholders and plain names (negative side)", () => {
|
||||
const names = new Set(envSecrets(ENV).map(([n]) => n));
|
||||
for (const n of Object.keys(MUST_SKIP)) assert.ok(!names.has(n), `should skip ${n}`);
|
||||
});
|
||||
|
||||
test("envSecrets orders longest value first so a contained secret cannot fragment a longer one", () => {
|
||||
const lens = envSecrets(ENV).map(([, v]) => v.length);
|
||||
assert.deepEqual(lens, [...lens].sort((x, y) => y - x));
|
||||
});
|
||||
|
||||
// The Python redactor (mempalace-toolkit) is the reference for tier 1. If the two
|
||||
// ever disagree, the palace and the model see different things, so pin them.
|
||||
const REDACT_PY =
|
||||
process.env.MEMPALACE_REDACT_PY ??
|
||||
["/opt/mempalace-toolkit/bin/mempalace_redact.py", path.join(process.env.HOME ?? "", "src/mempalace-toolkit/bin/mempalace_redact.py")].find(
|
||||
(p) => existsSync(p),
|
||||
);
|
||||
|
||||
function runPython(code, input) {
|
||||
return execFileSync("python3", ["-c", code], {
|
||||
input: JSON.stringify(input),
|
||||
env: { PATH: process.env.PATH, PYTHONPATH: path.dirname(REDACT_PY) },
|
||||
encoding: "utf8",
|
||||
});
|
||||
}
|
||||
|
||||
test("envSecrets selects exactly what mempalace_redact.env_secrets() selects", { skip: !REDACT_PY && "mempalace_redact.py not found" }, () => {
|
||||
const py = JSON.parse(
|
||||
runPython(
|
||||
"import json,sys,mempalace_redact as r; print(json.dumps([n for n,_ in r.env_secrets(json.load(sys.stdin))]))",
|
||||
ENV,
|
||||
),
|
||||
);
|
||||
assert.deepEqual(new Set(envSecrets(ENV).map(([n]) => n)), new Set(py));
|
||||
});
|
||||
|
||||
test("redactText output equals the Python redactor's on tier-1 material", { skip: !REDACT_PY && "mempalace_redact.py not found" }, () => {
|
||||
const known = envSecrets(ENV);
|
||||
const text = [
|
||||
`MEMPALACE_REMOTE_TOKEN=${V.a}`,
|
||||
`{"env": {"ERDC_INFERENCE_API_KEY": "${V.c}"}}`,
|
||||
`curl -H "Authorization: token ${V.b}" https://gitea.example/api`,
|
||||
`twice: ${V.d} and ${V.d}`,
|
||||
"nothing secret on this line",
|
||||
].join("\n");
|
||||
const py = runPython(
|
||||
"import json,sys,mempalace_redact as r; d=json.load(sys.stdin); print(r.scrub(d['t'], [tuple(k) for k in d['k']], strict=False)[0], end='')",
|
||||
{ t: text, k: known },
|
||||
);
|
||||
assert.equal(redactText(text, known).text, py);
|
||||
});
|
||||
|
||||
test("redactText: env-dump line, JSON value, header, repeats; counts per name", () => {
|
||||
const known = envSecrets(ENV);
|
||||
const r = redactText(`MEMPALACE_REMOTE_TOKEN=${V.a}\n{"k":"${V.a}"}\nBearer ${V.b} ${V.b}`, known);
|
||||
assert.ok(!r.text.includes(V.a) && !r.text.includes(V.b));
|
||||
assert.match(r.text, /^MEMPALACE_REMOTE_TOKEN=<redacted:MEMPALACE_REMOTE_TOKEN>$/m);
|
||||
assert.equal(r.hits.get("MEMPALACE_REMOTE_TOKEN"), 2);
|
||||
assert.equal(r.hits.get("GITEA_ACCESS_TOKEN"), 2);
|
||||
});
|
||||
|
||||
test("redactText: a longer secret containing a shorter one is replaced whole", () => {
|
||||
const known = envSecrets({ LONG_TOKEN: V.a + V.b, SHORT_TOKEN: V.b });
|
||||
const r = redactText(`x ${V.a + V.b} y ${V.b}`, known);
|
||||
assert.equal(r.text, "x <redacted:LONG_TOKEN> y <redacted:SHORT_TOKEN>");
|
||||
});
|
||||
|
||||
test("redactText: idempotent, and text without secrets is returned as the same string", () => {
|
||||
const known = envSecrets(ENV);
|
||||
const once = redactText(`t=${V.a}`, known).text;
|
||||
assert.equal(redactText(once, known).text, once);
|
||||
const clean = "drawer_pi-devbox_security_85ae3387c82c8211ff30512c and a sha 3760d90998bf269625c97b15061ab1c966bd61b7";
|
||||
const r = redactText(clean, known);
|
||||
assert.equal(r.text, clean);
|
||||
assert.equal(r.hits.size, 0);
|
||||
});
|
||||
|
||||
test("redactDeep rewrites nested strings and returns the SAME reference when nothing matched", () => {
|
||||
const known = envSecrets(ENV);
|
||||
const clean = { a: [1, "x", { b: null, c: true }], d: "y" };
|
||||
assert.equal(redactDeep(clean, known).value, clean);
|
||||
const dirty = { a: [1, `k=${V.a}`, { b: null, c: V.c }], d: "y" };
|
||||
const r = redactDeep(dirty, known);
|
||||
assert.notEqual(r.value, dirty);
|
||||
assert.deepEqual(r.value, { a: [1, "k=<redacted:MEMPALACE_REMOTE_TOKEN>", { b: null, c: "<redacted:ERDC_INFERENCE_API_KEY>" }], d: "y" });
|
||||
assert.equal(dirty.a[1], `k=${V.a}`, "input must not be mutated");
|
||||
});
|
||||
|
||||
const bashEvent = (text, extra = {}) => ({
|
||||
type: "tool_result",
|
||||
toolName: "bash",
|
||||
toolCallId: "c1",
|
||||
input: { command: "env" },
|
||||
content: [{ type: "text", text }, { type: "image", data: "aGVsbG8=", mimeType: "image/png" }],
|
||||
isError: false,
|
||||
details: { fullOutputPath: "/tmp/pi-bash-1.log" },
|
||||
...extra,
|
||||
});
|
||||
|
||||
test("toolResultChange: redacts content, keeps images and structuredContent, reports names", () => {
|
||||
const known = envSecrets(ENV);
|
||||
const sc = { rows: 3 };
|
||||
const ch = toolResultChange(bashEvent(`MEMPALACE_REMOTE_TOKEN=${V.a}\nok`, { structuredContent: sc }), known);
|
||||
assert.ok(ch);
|
||||
assert.equal(ch.result.content[0].text, "MEMPALACE_REMOTE_TOKEN=<redacted:MEMPALACE_REMOTE_TOKEN>\nok");
|
||||
assert.deepEqual(ch.result.content[1], { type: "image", data: "aGVsbG8=", mimeType: "image/png" });
|
||||
// replacing content without returning structuredContent would DROP it (types.d.ts)
|
||||
assert.equal(ch.result.structuredContent, sc);
|
||||
assert.deepEqual([...ch.hits.keys()], ["MEMPALACE_REMOTE_TOKEN"]);
|
||||
assert.ok(!("details" in ch.result), "unchanged details are not returned");
|
||||
});
|
||||
|
||||
test("toolResultChange: secrets in details or structuredContent alone are redacted too", () => {
|
||||
const known = envSecrets(ENV);
|
||||
const ch = toolResultChange(bashEvent("clean", { details: { patch: `+TOKEN=${V.b}` }, structuredContent: { v: V.c } }), known);
|
||||
assert.equal(ch.result.details.patch, "+TOKEN=<redacted:GITEA_ACCESS_TOKEN>");
|
||||
assert.deepEqual(ch.result.structuredContent, { v: "<redacted:ERDC_INFERENCE_API_KEY>" });
|
||||
assert.ok(!("content" in ch.result), "unchanged content is not returned");
|
||||
});
|
||||
|
||||
test("toolResultChange: no secret anywhere -> undefined (handler returns nothing)", () => {
|
||||
assert.equal(toolResultChange(bashEvent("total 12\ndrwx------ 2 developer"), envSecrets(ENV)), undefined);
|
||||
assert.equal(toolResultChange(bashEvent(`x ${V.a}`), []), undefined, "no known secrets -> nothing to do");
|
||||
});
|
||||
|
||||
test("contextChange: redacts any message shape, e.g. a user ! command's output", () => {
|
||||
const known = envSecrets(ENV);
|
||||
const msgs = [
|
||||
{ role: "user", content: [{ type: "text", text: "hi" }] },
|
||||
{ role: "bashExecution", command: "env", output: `ERDC_INFERENCE_API_KEY=${V.c}`, exitCode: 0 },
|
||||
];
|
||||
const ch = contextChange(msgs, known);
|
||||
assert.ok(ch);
|
||||
assert.equal(ch.messages[1].output, "ERDC_INFERENCE_API_KEY=<redacted:ERDC_INFERENCE_API_KEY>");
|
||||
assert.equal(ch.messages[0], msgs[0], "untouched messages keep their identity");
|
||||
assert.equal(contextChange([msgs[0]], known), undefined);
|
||||
});
|
||||
|
||||
test("writeGuard blocks a write/edit that would put a placeholder where a live secret was", () => {
|
||||
const known = envSecrets(ENV);
|
||||
const ph = placeholder("ERDC_INFERENCE_API_KEY");
|
||||
const w = writeGuard({ toolName: "write", input: { path: ".env", content: `ERDC_INFERENCE_API_KEY=${ph}\n` } }, known);
|
||||
assert.ok(w?.block);
|
||||
assert.match(w.reason, /ERDC_INFERENCE_API_KEY/);
|
||||
assert.match(w.reason, /\$ERDC_INFERENCE_API_KEY/, "reason says how to write the real value");
|
||||
const e = writeGuard({ toolName: "edit", input: { path: "a", edits: [{ oldText: "x", newText: `k=${ph}` }] } }, known);
|
||||
assert.ok(e?.block);
|
||||
});
|
||||
|
||||
test("writeGuard lets through what is not that mistake", () => {
|
||||
const known = envSecrets(ENV);
|
||||
const ph = placeholder("ERDC_INFERENCE_API_KEY");
|
||||
// a placeholder for a name that is NOT a live secret here (e.g. docs about the format)
|
||||
assert.equal(writeGuard({ toolName: "write", input: { path: "d.md", content: "<redacted:SOME_OTHER_TOKEN>" } }, known), undefined);
|
||||
// the placeholder only in oldText: the edit cannot write it
|
||||
assert.equal(writeGuard({ toolName: "edit", input: { path: "a", edits: [{ oldText: ph, newText: "y" }] } }, known), undefined);
|
||||
// bash is not guarded: grepping for markers is legitimate
|
||||
assert.equal(writeGuard({ toolName: "bash", input: { command: `grep -c '${ph}' f` } }, known), undefined);
|
||||
// plain writes
|
||||
assert.equal(writeGuard({ toolName: "write", input: { path: "a", content: "hello" } }, known), undefined);
|
||||
});
|
||||
|
||||
test("guardMode: on by default, PI_SECRET_GUARD=off disables", () => {
|
||||
assert.equal(guardMode({}), "on");
|
||||
assert.equal(guardMode({ PI_SECRET_GUARD: "OFF" }), "off");
|
||||
assert.equal(guardMode({ PI_SECRET_GUARD: "on" }), "on");
|
||||
assert.equal(guardMode({ PI_SECRET_GUARD: "bogus" }), "on", "an unknown value must not disable a guard");
|
||||
});
|
||||
Reference in New Issue
Block a user