mempalace-toolkit/bin/mempalace-census

#!/usr/bin/env bash
# mempalace-census — RFC 002 Phase A: classify a palace by what a join could move.
#
# Answers "what would joining THIS palace into the primary actually move, and
# what dedupes it?" before any writer exists. Read-only: opens every sqlite file
# with `mode=ro` and never writes, so it is safe to run against a live palace
# while `mempalace-serve` is up.
#
# Classification (RFC 002 §2), by descending signal strength:
#   DIARY           metadata type='diary_entry'  → replay + §7.6 suffix skip
#   MINED           source_file set / id_recipe  → RE-MINE on the target, never
#                                                  replay (ids are path-derived,
#                                                  so replay duplicates)
#   AGENT-AUTHORED  neither                      → replay, idempotent by content id
#
# It also self-verifies rather than trusting the docs. For every classified
# drawer it recomputes the upstream ID from reassembled content and compares to
# the stored ID. That checks three things at once: the ID recipe, the chunk
# reassembly order, and the classification. A mismatch rate above ~0 means one
# of those assumptions is wrong for this palace — investigate before joining.
#
# ⚠ Recipe note: upstream's ids.py DOCSTRINGS claim the hash input is
#   f"{wing}|{room}|{content}", and ids.py:31 defines _DELIM = "|". Both are
#   misleading — _DELIM is dead code and _delimited_sha256() actually
#   length-prefixes each part: "".join(f"{len(p)}:{p}"). Verified empirically:
#   length-prefixed reproduces real IDs 5/5, pipe-joined 0/5. Diary IDs are
#   different again — a PLAIN sha256(entry)[:12], not length-prefixed.
#
# This reads a palace on LOCAL DISK. It is not a client of a remote palace and
# deliberately ignores MEMPALACE_REMOTE_URL — pass --palace to point at a copy
# rsynced from another machine.
#
# Usage:
#   mempalace-census                          # default palace, human report
#   mempalace-census --palace /mnt/tor-ms22/palace
#   mempalace-census --json > manifest.json   # machine-readable, feeds Phase B/C
#   mempalace-census --no-verify-ids          # skip recompute (faster on huge palaces)

set -euo pipefail

# ── Defaults ──────────────────────────────────────────────────────────────────
PALACE="${MEMPALACE_PALACE:-$HOME/.mempalace/palace}"
KG=""
FORMAT="text"
VERIFY="1"

usage() {
  sed -n '2,36p' "$0" | sed 's/^# \{0,1\}//'
}

# ── Argument parsing ──────────────────────────────────────────────────────────
while [[ $# -gt 0 ]]; do
  case "$1" in
    -h|--help)        usage; exit 0 ;;
    --palace)         PALACE="${2:?--palace needs a path}"; shift 2 ;;
    --kg)             KG="${2:?--kg needs a path}"; shift 2 ;;
    --json)           FORMAT="json"; shift ;;
    --no-verify-ids)  VERIFY="0"; shift ;;
    *) echo "mempalace-census: unknown argument '$1' (try --help)" >&2; exit 2 ;;
  esac
done

# ── Path resolution ───────────────────────────────────────────────────────────
DB="$PALACE/chroma.sqlite3"
if [[ ! -f "$DB" ]]; then
  echo "mempalace-census: no chroma.sqlite3 under '$PALACE'" >&2
  echo "  pass --palace /path/to/palace (the dir CONTAINING chroma.sqlite3)" >&2
  exit 2
fi
# KG lives beside the palace dir, not inside it.
[[ -n "$KG" ]] || KG="$(cd "$(dirname "$PALACE")" && pwd)/knowledge_graph.sqlite3"

if [[ "$FORMAT" == "text" && -n "${MEMPALACE_REMOTE_URL:-}" ]]; then
  echo "note: MEMPALACE_REMOTE_URL is set but ignored — this tool reads local disk." >&2
  echo "      censusing: $DB" >&2
fi

# ── Census ────────────────────────────────────────────────────────────────────
PALACE_DB="$DB" KG_DB="$KG" FMT="$FORMAT" VERIFY="$VERIFY" python3 - <<'PY'
import hashlib, json, os, re, sqlite3, sys
from collections import Counter, defaultdict

DB, KG = os.environ["PALACE_DB"], os.environ["KG_DB"]
FMT, VERIFY = os.environ["FMT"], os.environ["VERIFY"] == "1"
CHUNK_RE = re.compile(r"_chunk_(\d+)$")

def ro(path):
    return sqlite3.connect(f"file:{path}?mode=ro", uri=True)

# Upstream ids.py::_delimited_sha256 — length-prefixed, NOT delimiter-joined.
def drawer_hash(parts, trunc=24):
    key = "".join(f"{len(str(p))}:{p}" for p in parts).encode()
    return hashlib.sha256(key).hexdigest()[:trunc]

con = ro(DB)

# Both collections share one sqlite file. Filtering by collection is mandatory:
# an embeddings-wide query over-counts by the closet population (~10%).
counts_by_collection = dict(
    con.execute(
        "SELECT c.name, COUNT(*) FROM embeddings e "
        "JOIN segments s ON s.id = e.segment_id "
        "JOIN collections c ON c.id = s.collection GROUP BY c.name"
    ).fetchall()
)

rows = defaultdict(dict)
for eid, key, sval, ival in con.execute(
    "SELECT e.embedding_id, m.key, m.string_value, m.int_value "
    "FROM embeddings e "
    "JOIN segments s ON s.id = e.segment_id "
    "JOIN collections c ON c.id = s.collection "
    "JOIN embedding_metadata m ON m.id = e.id "
    "WHERE c.name = 'mempalace_drawers'"
):
    # Chroma splits metadata by type across columns — numeric values (chunk_index,
    # source_mtime, line_start, normalize_version) land in int_value and leave
    # string_value NULL. Reading only string_value silently nulls every numeric
    # key, which made the miner-marker cross-check below report 100% conflict.
    rows[eid][key] = sval if sval is not None else ival

# Collapse chunk rows into parent drawers. A chunked drawer has NO parent row
# (verified), so the parent is the id with the _chunk_NNNNNN suffix stripped.
parents = defaultdict(lambda: {"chunks": {}, "meta": None})
for eid, meta in rows.items():
    m = CHUNK_RE.search(eid)
    base = eid[: m.start()] if m else eid
    idx = int(m.group(1)) if m else (meta.get("chunk_index") or 0)
    p = parents[base]
    p["chunks"][idx] = meta.get("chroma:document") or ""
    # Keep the lowest-index row's metadata as canonical for the parent.
    if p["meta"] is None or idx == 0:
        p["meta"] = meta

def classify(base, meta):
    if meta.get("type") == "diary_entry" or base.startswith("diary_"):
        return "diary"
    # id_recipe is NOT a mined-only marker — the server stamps 'v3' on every
    # v3 id, content-hashed ones included. Using it here misclassified all 60
    # agent-authored drawers in the reference palace as mined, which is the
    # dangerous direction: Phase C would try to re-mine drawers that have no
    # source file and silently drop them. A non-empty source_file is the real
    # discriminator (note the miner writes '' rather than omitting the key, so
    # presence-of-key is not enough — it must be truthy after strip()).
    if (meta.get("source_file") or "").strip():
        return "mined"
    return "agent_authored"

cls = Counter()
wings = Counter()
months = Counter()
machines = Counter()
agents = Counter()
verify = {"checked": 0, "match": 0, "mismatch": 0, "drift": 0,
          "samples": [], "drift_samples": []}
manifest = {"agent_authored": [], "diary": []}
signal_conflicts = []

for base, p in sorted(parents.items()):
    meta = p["meta"] or {}
    kind = classify(base, meta)
    cls[kind] += 1
    wings[meta.get("wing") or "?"] += 1
    if meta.get("filed_at"):
        months[str(meta["filed_at"])[:7]] += 1
    if meta.get("source_machine"):
        machines[meta["source_machine"]] += 1
    if meta.get("added_by"):
        agents[meta["added_by"]] += 1

    # Cross-check the classification against the miner's OWN markers
    # (source_mtime / normalize_version are written by the miner and by nothing
    # else). A split here means this palace has a shape the classifier hasn't
    # been taught, and the counts above are soft.
    miner_marked = bool(meta.get("source_mtime") or meta.get("normalize_version"))
    if miner_marked != (kind == "mined"):
        signal_conflicts.append(base)

    if kind == "mined":
        continue

    content = "".join(p["chunks"][i] for i in sorted(p["chunks"]))
    wing, room = meta.get("wing") or "", meta.get("room") or ""

    if kind == "agent_authored":
        expect = f"drawer_{wing}_{room}_{drawer_hash((wing, room, content))}"
        drifted = expect != base
        if VERIFY:
            verify["checked"] += 1
            # A mismatch here is NOT a broken recipe — update_drawer preserves the
            # original id while rewriting (and re-chunking) content, so an edited
            # drawer's content hash legitimately stops reproducing its id. Named
            # separately because it breaks one obvious Phase C strategy: you
            # cannot "recompute the content id and check whether the target has
            # it" — for drifted drawers that lookup misses and you duplicate.
            # Replay by STORED id.
            if drifted:
                verify["drift"] += 1
                if len(verify["drift_samples"]) < 5:
                    verify["drift_samples"].append({"stored": base, "recomputed": expect})
            else:
                verify["match"] += 1
        manifest["agent_authored"].append(
            {"id": base, "wing": wing, "room": room, "chars": len(content),
             "chunks": len(p["chunks"]), "filed_at": meta.get("filed_at"),
             "added_by": meta.get("added_by"),
             "content_id": expect, "edited_since_filing": drifted}
        )
    else:  # diary — id suffix is a PLAIN sha256(entry)[:12]
        suffix = base.rsplit("_", 1)[-1]
        recomputed = hashlib.sha256(content.encode()).hexdigest()[:12]
        if VERIFY:
            verify["checked"] += 1
            if suffix == recomputed:
                verify["match"] += 1
            else:
                verify["mismatch"] += 1
                if len(verify["samples"]) < 5:
                    verify["samples"].append({"stored": base, "recomputed_suffix": recomputed})
        manifest["diary"].append(
            {"id": base, "wing": wing, "chars": len(content), "chunks": len(p["chunks"]),
             "agent": meta.get("agent"), "topic": meta.get("topic"),
             "date": meta.get("date"), "dedup_suffix": suffix,
             "suffix_verified": suffix == recomputed}
        )

# ── Knowledge graph ───────────────────────────────────────────────────────────
kg = {"present": os.path.isfile(KG)}
if kg["present"]:
    k = ro(KG)
    try:
        kg["open_facts"] = k.execute("SELECT COUNT(*) FROM triples WHERE valid_to IS NULL").fetchone()[0]
        kg["closed_facts"] = k.execute("SELECT COUNT(*) FROM triples WHERE valid_to IS NOT NULL").fetchone()[0]
        kg["entities"] = k.execute("SELECT COUNT(*) FROM entities").fetchone()[0]
        kg["predicates"] = dict(
            k.execute("SELECT predicate, COUNT(*) FROM triples GROUP BY 1 ORDER BY 2 DESC LIMIT 10").fetchall()
        )
    except sqlite3.Error as e:
        kg["error"] = str(e)

report = {
    "palace": DB,
    "kg": KG,
    "rows_by_collection": counts_by_collection,
    "parent_drawers": sum(cls.values()),
    "classes": dict(cls),
    "replay_surface": cls["diary"] + cls["agent_authored"],
    "by_wing": dict(wings.most_common()),
    "filed_at_by_month": dict(sorted(months.items())),
    "source_machine": dict(machines),
    "added_by": dict(agents.most_common()),
    "id_verification": verify if VERIFY else "skipped",
    "signal_conflicts": len(signal_conflicts),
    "knowledge_graph": kg,
    "manifest": manifest,
}

if FMT == "json":
    print(json.dumps(report, indent=2, sort_keys=False))
    sys.exit(0)

# ── Human report ──────────────────────────────────────────────────────────────
def bar(n, total, width=28):
    return "█" * max(1, round(width * n / total)) if n and total else ""

print(f"\n  palace : {DB}")
print(f"  kg     : {KG}{'' if kg['present'] else '   (absent)'}")
print("\n  ── rows per collection ─────────────────────────────────────")
for name, n in sorted(counts_by_collection.items()):
    note = "  ← derived at mine time, NOT joinable" if "closet" in name else ""
    print(f"    {name:<20} {n:>7}{note}")

total = sum(cls.values())
print(f"\n  ── parent drawers: {total} ───────────────────────────────────")
labels = {
    "mined": "MINED          re-mine on target, never replay",
    "diary": "DIARY          replay + §7.6 suffix skip",
    "agent_authored": "AGENT-AUTHORED replay, idempotent by content id",
}
for k in ("mined", "diary", "agent_authored"):
    n = cls.get(k, 0)
    pct = 100.0 * n / total if total else 0
    print(f"    {n:>7}  {pct:>5.1f}%  {labels[k]}")
print(f"\n    → REPLAY SURFACE: {report['replay_surface']} records "
      f"({100.0 * report['replay_surface'] / total if total else 0:.1f}% of the palace)")

if VERIFY:
    v = verify
    state = "OK" if v["mismatch"] == 0 else "⚠ MISMATCH"
    print(f"\n  ── id recipe / reassembly self-check: {state} ─────────────")
    print(f"    recomputed {v['checked']} ids — {v['match']} reproduce their stored id, "
          f"{v['mismatch']} unexplained")
    for s in v["samples"]:
        print(f"      stored:     {s.get('stored')}")
        print(f"      recomputed: {s.get('recomputed') or s.get('recomputed_suffix')}")
    if v["drift"]:
        print(f"\n    {v['drift']} agent-authored drawers EDITED SINCE FILING "
              f"(content hash no longer reproduces the id).")
        print("      update_drawer keeps the id and re-chunks, so this is expected — but it")
        print("      means Phase C must replay by STORED id. Recomputing the content id and")
        print("      probing the target for it would miss these and duplicate them.")
        for s in v["drift_samples"][:3]:
            print(f"        {s['stored']}")
if signal_conflicts:
    print(f"\n  ⚠ {len(signal_conflicts)} drawers where the class disagrees with the miner's")
    print("    own markers (source_mtime / normalize_version) — classifier needs teaching")

print("\n  ── by wing ─────────────────────────────────────────────────")
for w, n in wings.most_common(10):
    print(f"    {n:>7}  {w}")

if months:
    print("\n  ── filed_at spread (an MCP replay would flatten all of this) ")
    mx = max(months.values())
    for m, n in sorted(months.items()):
        print(f"    {m}  {n:>6}  {bar(n, mx)}")

if machines:
    print("\n  ── source_machine ──────────────────────────────────────────")
    for m, n in machines.most_common():
        print(f"    {n:>7}  {m}")

if kg["present"] and "error" not in kg:
    print("\n  ── knowledge graph ─────────────────────────────────────────")
    print(f"    {kg['entities']:>7}  entities")
    print(f"    {kg['open_facts']:>7}  open facts    (server guard dedupes → replay as-is)")
    print(f"    {kg['closed_facts']:>7}  closed facts  (NO server guard → client pre-query)")
elif kg.get("error"):
    print(f"\n  ⚠ knowledge graph unreadable: {kg['error']}")

print("\n  next: --json > manifest.json feeds RFC 002 Phase B/C.\n")
PY