Joakim Persson
a88fa99d39
Measured in a full-extension session: one `echo "$SECRET"` was reported as "redacted ... x2". pi's bash tool returns its output both as content text and as structuredContent.output, and both are (correctly) redacted. The count now takes, per name, the highest count from any one field instead of the sum. A test pins it: it failed before the change and passes after. Re-run in the full-extension TUI: the warning now reads "redacted SG_TEST_TOKEN from bash output"; 0 raw copies sent, 0 in the session file. Suite: 46/46.
251 lines
12 KiB
JavaScript
251 lines
12 KiB
JavaScript
// Two-sided tests for secret-guard: what MUST be treated as a secret and what
|
|
// MUST NOT, what MUST be rewritten and what MUST come back untouched (by
|
|
// reference, so the handlers can return "no change"). A guard that only ever
|
|
// redacts has not been shown to discriminate. Runs the shipped .ts directly
|
|
// (node >= 23 strips types natively; the only import is type-only).
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { execFileSync } from "node:child_process";
|
|
import { existsSync } from "node:fs";
|
|
import path from "node:path";
|
|
import {
|
|
envSecrets,
|
|
placeholder,
|
|
redactText,
|
|
redactDeep,
|
|
toolResultChange,
|
|
contextChange,
|
|
writeGuard,
|
|
guardMode,
|
|
} from "../extensions/secret-guard.ts";
|
|
|
|
// Values are shaped like nothing in particular (no ghp_/sk-/AKIA prefix), so the
|
|
// Python parity check exercises tier 1 only and tier-2 shape rules stay silent.
|
|
const V = {
|
|
a: "Zq7mWc2Lr9Kp4Tn8Vx3Bd6Hs1Fj5Gy0Qe7Ua2Ic9Op4", // 43, like MEMPALACE_REMOTE_TOKEN
|
|
b: "r2D9xK7pQm4LwZ8vT1cB6nH3sF5jG0yE", // 32
|
|
c: "Mx8Rt3Lq9Wz2Kc7Vb4Nd1Hp6Js5Fg0Ya1Ue9Io3Pl8Ak2Sd7", // 49
|
|
d: "uV6tR1eW8qY3iO7pA2sD9fG4", // 24
|
|
shortish: "abcdefghijkl", // exactly 12: still a secret by length
|
|
};
|
|
|
|
// name -> value. MUST_SELECT: tier-1 secrets. MUST_SKIP: secret-sounding names
|
|
// that hold a location, a knob or a placeholder, and plain names.
|
|
const MUST_SELECT = {
|
|
MEMPALACE_REMOTE_TOKEN: V.a,
|
|
ERDC_INFERENCE_API_KEY: V.c,
|
|
GITEA_ACCESS_TOKEN: V.b,
|
|
CLIENT_SECRET: V.d,
|
|
DB_PASSWORD: V.shortish,
|
|
GITHUB_PAT: V.b + "x",
|
|
PAT: V.d + "y",
|
|
OAUTH_BEARER: V.a + "z",
|
|
SESSION_KEY: V.c + "w",
|
|
api_key: V.d + "q", // lower case: the name rule is case-insensitive
|
|
AWS_SECRET_ACCESS_KEY: V.b + "AB",
|
|
SOME_CREDENTIALS: V.d + "cr",
|
|
};
|
|
const MUST_SKIP = {
|
|
SSH_KEY_PATH: "/home/developer/.ssh/id_ed25519", // _PATH suffix
|
|
GITEA_TOKEN_FILE: "/run/secrets/gitea_token_file_x", // _FILE suffix
|
|
GITHUB_TOKEN_URL: "https://github.example/token/endpoint", // _URL suffix
|
|
PI_OM_MAX_TOKENS: "100000", // too short to be a credential
|
|
SHORT_TOKEN: "abcdefghijk", // 11 chars
|
|
SPACED_SECRET: "this has spaces in it ok", // credentials have no whitespace
|
|
PLACEHOLDER_TOKEN: "<your-token-here-please>",
|
|
REF_TOKEN: "${MEMPALACE_REMOTE_TOKEN}",
|
|
DOLLAR_TOKEN: "$MEMPALACE_REMOTE_TOKEN",
|
|
HOME_TOKEN: "~/secrets/token-file-here",
|
|
TRIVIAL_SECRET: "changeme",
|
|
STARS_PASSWORD: "****************",
|
|
HOME: "/home/developer",
|
|
PATH: "/usr/local/bin:/usr/bin:/bin",
|
|
MEMPALACE_REMOTE_URL: "https://mempalace.example/mcp",
|
|
RANDOM_STRING: V.a + "notsecret", // a random value under a plain name is NOT tier 1
|
|
DEPLOY_PRIVATE_KEY: "~deploy/.ssh/id_ed25519", // ~user/ path: only the path rule catches it
|
|
};
|
|
const ENV = { ...MUST_SELECT, ...MUST_SKIP };
|
|
|
|
test("envSecrets selects every secret-named credential (positive side)", () => {
|
|
const names = new Set(envSecrets(ENV).map(([n]) => n));
|
|
for (const n of Object.keys(MUST_SELECT)) assert.ok(names.has(n), `should select ${n}`);
|
|
});
|
|
|
|
test("envSecrets skips locations, knobs, placeholders and plain names (negative side)", () => {
|
|
const names = new Set(envSecrets(ENV).map(([n]) => n));
|
|
for (const n of Object.keys(MUST_SKIP)) assert.ok(!names.has(n), `should skip ${n}`);
|
|
});
|
|
|
|
test("envSecrets orders longest value first so a contained secret cannot fragment a longer one", () => {
|
|
const lens = envSecrets(ENV).map(([, v]) => v.length);
|
|
assert.deepEqual(lens, [...lens].sort((x, y) => y - x));
|
|
});
|
|
|
|
// The Python redactor (mempalace-toolkit) is the reference for tier 1. If the two
|
|
// ever disagree, the palace and the model see different things, so pin them.
|
|
const REDACT_PY =
|
|
process.env.MEMPALACE_REDACT_PY ??
|
|
["/opt/mempalace-toolkit/bin/mempalace_redact.py", path.join(process.env.HOME ?? "", "src/mempalace-toolkit/bin/mempalace_redact.py")].find(
|
|
(p) => existsSync(p),
|
|
);
|
|
|
|
function runPython(code, input) {
|
|
return execFileSync("python3", ["-c", code], {
|
|
input: JSON.stringify(input),
|
|
env: { PATH: process.env.PATH, PYTHONPATH: path.dirname(REDACT_PY) },
|
|
encoding: "utf8",
|
|
});
|
|
}
|
|
|
|
test("envSecrets selects exactly what mempalace_redact.env_secrets() selects", { skip: !REDACT_PY && "mempalace_redact.py not found" }, () => {
|
|
const py = JSON.parse(
|
|
runPython(
|
|
"import json,sys,mempalace_redact as r; print(json.dumps([n for n,_ in r.env_secrets(json.load(sys.stdin))]))",
|
|
ENV,
|
|
),
|
|
);
|
|
assert.deepEqual(new Set(envSecrets(ENV).map(([n]) => n)), new Set(py));
|
|
});
|
|
|
|
test("redactText output equals the Python redactor's on tier-1 material", { skip: !REDACT_PY && "mempalace_redact.py not found" }, () => {
|
|
const known = envSecrets(ENV);
|
|
const text = [
|
|
`MEMPALACE_REMOTE_TOKEN=${V.a}`,
|
|
`{"env": {"ERDC_INFERENCE_API_KEY": "${V.c}"}}`,
|
|
`curl -H "Authorization: token ${V.b}" https://gitea.example/api`,
|
|
`twice: ${V.d} and ${V.d}`,
|
|
"nothing secret on this line",
|
|
].join("\n");
|
|
const py = runPython(
|
|
"import json,sys,mempalace_redact as r; d=json.load(sys.stdin); print(r.scrub(d['t'], [tuple(k) for k in d['k']], strict=False)[0], end='')",
|
|
{ t: text, k: known },
|
|
);
|
|
assert.equal(redactText(text, known).text, py);
|
|
});
|
|
|
|
test("redactText: env-dump line, JSON value, header, repeats; counts per name", () => {
|
|
const known = envSecrets(ENV);
|
|
const r = redactText(`MEMPALACE_REMOTE_TOKEN=${V.a}\n{"k":"${V.a}"}\nBearer ${V.b} ${V.b}`, known);
|
|
assert.ok(!r.text.includes(V.a) && !r.text.includes(V.b));
|
|
assert.match(r.text, /^MEMPALACE_REMOTE_TOKEN=<redacted:MEMPALACE_REMOTE_TOKEN>$/m);
|
|
assert.equal(r.hits.get("MEMPALACE_REMOTE_TOKEN"), 2);
|
|
assert.equal(r.hits.get("GITEA_ACCESS_TOKEN"), 2);
|
|
});
|
|
|
|
test("redactText: a longer secret containing a shorter one is replaced whole", () => {
|
|
const known = envSecrets({ LONG_TOKEN: V.a + V.b, SHORT_TOKEN: V.b });
|
|
const r = redactText(`x ${V.a + V.b} y ${V.b}`, known);
|
|
assert.equal(r.text, "x <redacted:LONG_TOKEN> y <redacted:SHORT_TOKEN>");
|
|
});
|
|
|
|
test("redactText: idempotent, and text without secrets is returned as the same string", () => {
|
|
const known = envSecrets(ENV);
|
|
const once = redactText(`t=${V.a}`, known).text;
|
|
assert.equal(redactText(once, known).text, once);
|
|
const clean = "drawer_pi-devbox_security_85ae3387c82c8211ff30512c and a sha 3760d90998bf269625c97b15061ab1c966bd61b7";
|
|
const r = redactText(clean, known);
|
|
assert.equal(r.text, clean);
|
|
assert.equal(r.hits.size, 0);
|
|
});
|
|
|
|
test("redactDeep rewrites nested strings and returns the SAME reference when nothing matched", () => {
|
|
const known = envSecrets(ENV);
|
|
const clean = { a: [1, "x", { b: null, c: true }], d: "y" };
|
|
assert.equal(redactDeep(clean, known).value, clean);
|
|
const dirty = { a: [1, `k=${V.a}`, { b: null, c: V.c }], d: "y" };
|
|
const r = redactDeep(dirty, known);
|
|
assert.notEqual(r.value, dirty);
|
|
assert.deepEqual(r.value, { a: [1, "k=<redacted:MEMPALACE_REMOTE_TOKEN>", { b: null, c: "<redacted:ERDC_INFERENCE_API_KEY>" }], d: "y" });
|
|
assert.equal(dirty.a[1], `k=${V.a}`, "input must not be mutated");
|
|
});
|
|
|
|
const bashEvent = (text, extra = {}) => ({
|
|
type: "tool_result",
|
|
toolName: "bash",
|
|
toolCallId: "c1",
|
|
input: { command: "env" },
|
|
content: [{ type: "text", text }, { type: "image", data: "aGVsbG8=", mimeType: "image/png" }],
|
|
isError: false,
|
|
details: { fullOutputPath: "/tmp/pi-bash-1.log" },
|
|
...extra,
|
|
});
|
|
|
|
test("toolResultChange: redacts content, keeps images and structuredContent, reports names", () => {
|
|
const known = envSecrets(ENV);
|
|
const sc = { rows: 3 };
|
|
const ch = toolResultChange(bashEvent(`MEMPALACE_REMOTE_TOKEN=${V.a}\nok`, { structuredContent: sc }), known);
|
|
assert.ok(ch);
|
|
assert.equal(ch.result.content[0].text, "MEMPALACE_REMOTE_TOKEN=<redacted:MEMPALACE_REMOTE_TOKEN>\nok");
|
|
assert.deepEqual(ch.result.content[1], { type: "image", data: "aGVsbG8=", mimeType: "image/png" });
|
|
// replacing content without returning structuredContent would DROP it (types.d.ts)
|
|
assert.equal(ch.result.structuredContent, sc);
|
|
assert.deepEqual([...ch.hits.keys()], ["MEMPALACE_REMOTE_TOKEN"]);
|
|
assert.ok(!("details" in ch.result), "unchanged details are not returned");
|
|
});
|
|
|
|
test("toolResultChange: secrets in details or structuredContent alone are redacted too", () => {
|
|
const known = envSecrets(ENV);
|
|
const ch = toolResultChange(bashEvent("clean", { details: { patch: `+TOKEN=${V.b}` }, structuredContent: { v: V.c } }), known);
|
|
assert.equal(ch.result.details.patch, "+TOKEN=<redacted:GITEA_ACCESS_TOKEN>");
|
|
assert.deepEqual(ch.result.structuredContent, { v: "<redacted:ERDC_INFERENCE_API_KEY>" });
|
|
assert.ok(!("content" in ch.result), "unchanged content is not returned");
|
|
});
|
|
|
|
test("toolResultChange: the same output in content AND structuredContent (bash) counts once", () => {
|
|
// pi's bash returns its output twice: content text and structuredContent.output (measured 2026-10-12).
|
|
const known = envSecrets(ENV);
|
|
const out = `val=${V.a}`;
|
|
const ch = toolResultChange(bashEvent(out, { structuredContent: { output: out, exit_code: 0 } }), known);
|
|
assert.equal(ch.result.structuredContent.output, "val=<redacted:MEMPALACE_REMOTE_TOKEN>");
|
|
assert.equal(ch.hits.get("MEMPALACE_REMOTE_TOKEN"), 1);
|
|
});
|
|
|
|
test("toolResultChange: no secret anywhere -> undefined (handler returns nothing)", () => {
|
|
assert.equal(toolResultChange(bashEvent("total 12\ndrwx------ 2 developer"), envSecrets(ENV)), undefined);
|
|
assert.equal(toolResultChange(bashEvent(`x ${V.a}`), []), undefined, "no known secrets -> nothing to do");
|
|
});
|
|
|
|
test("contextChange: redacts any message shape, e.g. a user ! command's output", () => {
|
|
const known = envSecrets(ENV);
|
|
const msgs = [
|
|
{ role: "user", content: [{ type: "text", text: "hi" }] },
|
|
{ role: "bashExecution", command: "env", output: `ERDC_INFERENCE_API_KEY=${V.c}`, exitCode: 0 },
|
|
];
|
|
const ch = contextChange(msgs, known);
|
|
assert.ok(ch);
|
|
assert.equal(ch.messages[1].output, "ERDC_INFERENCE_API_KEY=<redacted:ERDC_INFERENCE_API_KEY>");
|
|
assert.equal(ch.messages[0], msgs[0], "untouched messages keep their identity");
|
|
assert.equal(contextChange([msgs[0]], known), undefined);
|
|
});
|
|
|
|
test("writeGuard blocks a write/edit that would put a placeholder where a live secret was", () => {
|
|
const known = envSecrets(ENV);
|
|
const ph = placeholder("ERDC_INFERENCE_API_KEY");
|
|
const w = writeGuard({ toolName: "write", input: { path: ".env", content: `ERDC_INFERENCE_API_KEY=${ph}\n` } }, known);
|
|
assert.ok(w?.block);
|
|
assert.match(w.reason, /ERDC_INFERENCE_API_KEY/);
|
|
assert.match(w.reason, /\$ERDC_INFERENCE_API_KEY/, "reason says how to write the real value");
|
|
const e = writeGuard({ toolName: "edit", input: { path: "a", edits: [{ oldText: "x", newText: `k=${ph}` }] } }, known);
|
|
assert.ok(e?.block);
|
|
});
|
|
|
|
test("writeGuard lets through what is not that mistake", () => {
|
|
const known = envSecrets(ENV);
|
|
const ph = placeholder("ERDC_INFERENCE_API_KEY");
|
|
// a placeholder for a name that is NOT a live secret here (e.g. docs about the format)
|
|
assert.equal(writeGuard({ toolName: "write", input: { path: "d.md", content: "<redacted:SOME_OTHER_TOKEN>" } }, known), undefined);
|
|
// the placeholder only in oldText: the edit cannot write it
|
|
assert.equal(writeGuard({ toolName: "edit", input: { path: "a", edits: [{ oldText: ph, newText: "y" }] } }, known), undefined);
|
|
// bash is not guarded: grepping for markers is legitimate
|
|
assert.equal(writeGuard({ toolName: "bash", input: { command: `grep -c '${ph}' f` } }, known), undefined);
|
|
// plain writes
|
|
assert.equal(writeGuard({ toolName: "write", input: { path: "a", content: "hello" } }, known), undefined);
|
|
});
|
|
|
|
test("guardMode: on by default, PI_SECRET_GUARD=off disables", () => {
|
|
assert.equal(guardMode({}), "on");
|
|
assert.equal(guardMode({ PI_SECRET_GUARD: "OFF" }), "off");
|
|
assert.equal(guardMode({ PI_SECRET_GUARD: "on" }), "on");
|
|
assert.equal(guardMode({ PI_SECRET_GUARD: "bogus" }), "on", "an unknown value must not disable a guard");
|
|
});
|