From dde575673d6c6a3d919e52c967096ddc53275a26 Mon Sep 17 00:00:00 2001 From: "andrey.epifantsev" Date: Sat, 24 Oct 2026 01:21:53 +0400 Subject: [PATCH] =?UTF-8?q?feat(retro):=20retro=5Fmetrics.py=20=E2=80=94?= =?UTF-8?q?=20=D1=81=D1=80=D0=B5=D0=B7=20=D0=BC=D0=B5=D1=82=D1=80=D0=B8?= =?UTF-8?q?=D0=BA,=20=D0=BA=D0=B0=D0=BD=D0=B4=D0=B8=D0=B4=D0=B0=D1=82?= =?UTF-8?q?=D1=8B=20=D0=B2=20=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA?= =?UTF-8?q?=D0=B8=20=D0=B2=D0=BB=D0=B0=D0=B4=D0=B5=D0=BB=D1=8C=D1=86=D0=B0?= =?UTF-8?q?,=20=D1=81=D1=80=D0=B0=D0=B2=D0=BD=D0=B5=D0=BD=D0=B8=D0=B5=20?= =?UTF-8?q?=D1=81=20=D0=BF=D1=80=D0=BE=D1=88=D0=BB=D1=8B=D0=BC=20=D1=80?= =?UTF-8?q?=D0=B5=D1=82=D1=80=D0=BE=20(#29)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Синтетические сообщения об ошибке API (model ) не считаются ходом модели (O-18, живой прогон). Co-Authored-By: Claude Opus 5.5 --- docs/retro/scripts/retro_metrics.py | 482 +++++++++++++++++++++++ openspec/changes/harness-retro/design.md | 1 + tests/test-retro-metrics.sh | 132 +++++++ 3 files changed, 615 insertions(+) create mode 100644 docs/retro/scripts/retro_metrics.py create mode 100644 tests/test-retro-metrics.sh diff --git a/docs/retro/scripts/retro_metrics.py b/docs/retro/scripts/retro_metrics.py new file mode 100644 index 0000000..84cceea --- /dev/null +++ b/docs/retro/scripts/retro_metrics.py @@ -0,0 +1,482 @@ +#!/usr/bin/env python3 +"""Срез метрик обвязки по транскриптам и памяти одного проекта или workspace (B12, #29). + +Запуск: python3 retro_metrics.py <папка проекта> [--since ГГГГ-ММ-ДД] [--harness <репозиторий обвязки>] + [--projects-root <каталог>] [--labels ] [--out ] + [--markdown [--prev <прошлый.metrics.json>]] [--corrections] +Без флагов вывода печатает JSON метрик (схема 1); --out пишет его в файл; --markdown печатает таблицу «Цифры» +со сравнением с --prev; --corrections — кандидаты в поправки владельца, по JSON-строке на кандидата. +Только чтение транскриптов и памяти. Спека: openspec/specs/harness-retro/spec.md. +""" +import argparse +import collections +import glob +import json +import os +import re +import shlex +import statistics +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +from hook_blocks_export import HOOK_ERROR, hook_of, redact # noqa: E402 + +INTERRUPT = "[Request interrupted by user" +OWNER_DENY = "The user doesn't want to proceed with this tool use" +CLASSIFIER = re.compile(r"denied by the Claude Code auto mode classifier\. Reason: \[([^\]]+)\]") +RECALL = re.compile(r"памят|перечитай|сверься|контекст|вспомни|прочитай .*md", re.I) +MARKERS = re.compile(r"(? os.path.getsize(best[sid]): + best[sid] = f + files = [] + for f in sorted(best.values()): + cwd, ts = first_meta(f) + if not cwd or not (inside(cwd, project) or inside(cwd, project + ".wt")): + continue + if since and (not ts or ts[:10] < since): + continue + files.append(f) + return sorted(set(os.path.basename(os.path.dirname(f)) for f in files)), files + + +def blocks(rec): + c = (rec.get("message") or {}).get("content") + if isinstance(c, str): + return [{"type": "text", "text": c}] + return [b for b in c if isinstance(b, dict)] if isinstance(c, list) else [] + + +def result_text(b): + c = b.get("content", "") + if isinstance(c, list): + return "\n".join(x.get("text", "") for x in c if isinstance(x, dict)) + return c if isinstance(c, str) else "" + + +def prompt_text(rec): + """Текст промпта владельца или None: служебное, tool_result, прерывание, сабагент.""" + if rec.get("type") != "user" or rec.get("isMeta") or rec.get("isSidechain"): + return None + if rec.get("userType", "external") != "external": + return None + bs = blocks(rec) + if any(b.get("type") == "tool_result" for b in bs): + return None + txt = "\n".join(b.get("text") or "" for b in bs if b.get("type") == "text").strip() + if not txt or txt.startswith("<") or txt.startswith(INTERRUPT): + return None + return txt + + +def is_interrupt(rec): + return rec.get("type") == "user" and any( + b.get("type") == "text" and (b.get("text") or "").startswith(INTERRUPT) for b in blocks(rec)) + + +def split_repo(path): + """(основной репозиторий, путь от корня worktree или репозитория) или (None, None).""" + p = os.path.normpath(path) if path else "" + m = re.match(r"^(.+?)/\.claude/worktrees/[^/]+/(.*)$", p) + if m: + return m.group(1), m.group(2) + m = re.match(r"^(.+)/\.wt/[^/]+/([^/]+)/(.*)$", p) + if m: + return m.group(1) + "/" + m.group(2), m.group(3) + m = re.match(r"^(.*[^/])\.wt/[^/]+/(.*)$", p) + if m: + return m.group(1), m.group(2) + d = os.path.dirname(p) + while d and d != os.path.dirname(d): + if os.path.exists(os.path.join(d, ".git")): + return d, os.path.relpath(p, d) + d = os.path.dirname(d) + return None, None + + +def code_prefixes(repo, cache): + if repo not in cache: + cp = os.path.join(repo, ".claude", "code-paths") + if os.path.isfile(cp): + pre = [l.split("#", 1)[0].strip().rstrip("/") for l in open(cp, encoding="utf-8", errors="replace")] + cache[repo] = [x for x in pre if x] + else: + cache[repo] = None + return cache[repo] + + +def is_code(path, cache): + repo, rel = split_repo(path) + if not repo: + return False + pre = code_prefixes(repo, cache) + if pre is None: + return "/src/" in "/" + rel + return any(rel == x or rel.startswith(x + "/") for x in pre) + + +def bash_reads_code(cmd, cwd, cache): + for part in SEPARATORS.split(cmd or ""): + try: + words = shlex.split(part) + except ValueError: + words = part.split() + if not words or not (words[0] in READERS or (words[0] == "sed" and "-n" in words[1:2])): + continue + for w in words[1:]: + if w.startswith("-") or not (os.path.isabs(w) or cwd): + continue + if is_code(w if os.path.isabs(w) else os.path.join(cwd, w), cache): + return True + return False + + +def action_of(b): + name, inp = b.get("name") or "?", b.get("input") or {} + if name == "Bash": + return "Bash: " + " ".join((inp.get("command") or "").split()) + if name in FILE_TOOLS: + return "%s: %s" % (name, inp.get("file_path") or inp.get("notebook_path") or "") + return name + + +def count_tool(s, b, cwd, cache): + name, inp = b.get("name") or "?", b.get("input") or {} + s["tools"][name] += 1 + if name in ("Agent", "Task"): + s["subagents"][inp.get("model") or "не указана"] += 1 + elif name == "Skill": + sk = inp.get("skill") or "?" + s["skills"][sk] += 1 + if s["role"] is None and (sk == "role" or sk.endswith(":role")): + s["role"] = (inp.get("args") or "").strip() or "?" + if "codegraph_explore" in name: + s["cg_explore"] += 1 + elif name == "Read": + s["code_reads"] += is_code(inp.get("file_path") or "", cache) + elif name in ("Grep", "Glob"): + s["code_reads"] += 1 + elif name == "Bash": + cmd = inp.get("command") or "" + s["cg_init"] += "codegraph init" in cmd + s["code_reads"] += bash_reads_code(cmd, cwd, cache) + + +def session_stats(path, cache, bad): + s = {"prompts": 0, "interrupts": 0, "turns": 0, "tools": collections.Counter(), + "subagents": collections.Counter(), "skills": collections.Counter(), "role": None, + "tokens": collections.Counter(), "compactions": 0, "start": None, "blocks": collections.Counter(), + "classifier": collections.Counter(), "owner_denials": 0, "cg_init": 0, "cg_explore": 0, + "code_reads": 0, "recall": 0, "last_ts": ""} + seen, pending = set(), {} + for rec in read_session(path, bad): + ts = rec.get("timestamp") + if isinstance(ts, str) and ts > s["last_ts"]: + s["last_ts"] = ts + t = rec.get("type") + if t == "system" and rec.get("subtype") == "compact_boundary": + s["compactions"] += 1 + elif t == "assistant": + msg = rec.get("message") or {} + if rec.get("isApiErrorMessage") or msg.get("model") == "": + continue # сообщение Claude Code об ошибке API — не ход модели + mid = msg.get("id") + if mid is None or mid not in seen: + if mid is not None: + seen.add(mid) + s["turns"] += 1 + u = msg.get("usage") or {} + if u: + n = lambda k: u.get(k, 0) or 0 + if s["start"] is None: + s["start"] = n("input_tokens") + n("cache_creation_input_tokens") + n("cache_read_input_tokens") + s["tokens"]["output"] += n("output_tokens") + s["tokens"]["cache_read"] += n("cache_read_input_tokens") + s["tokens"]["cache_write"] += n("cache_creation_input_tokens") + for b in blocks(rec): + if b.get("type") == "tool_use": + pending[b.get("id")] = b.get("name") or "?" + count_tool(s, b, rec.get("cwd"), cache) + elif t == "user": + txt = prompt_text(rec) + if txt is not None: + s["prompts"] += 1 + s["recall"] += bool(RECALL.search(txt)) + elif is_interrupt(rec): + s["interrupts"] += 1 + for b in blocks(rec): + if b.get("type") != "tool_result": + continue + rt = result_text(b) + if HOOK_ERROR.match(rt) and "BLOCKED" in rt: + s["blocks"][hook_of(rt, pending.get(b.get("tool_use_id"), "?"))] += 1 + m = CLASSIFIER.search(rt) + if m: + s["classifier"][m.group(1)] += 1 + s["owner_denials"] += OWNER_DENY in rt + return s + + +def aggregate(stats): + total = lambda k: sum(s[k] for s in stats) + merged = lambda k: sum((s[k] for s in stats), collections.Counter()) + tools, blk, cls, sub = merged("tools"), merged("blocks"), merged("classifier"), merged("subagents") + tok = merged("tokens") + starts = [s["start"] for s in stats if s["start"] is not None] + explore, reads = total("cg_explore"), total("code_reads") + return { + "sessions": len(stats), "prompts": total("prompts"), "interrupts": total("interrupts"), + "turns": total("turns"), "tool_calls": sum(tools.values()), "tools": dict(tools.most_common()), + "roles": dict(collections.Counter(s["role"] or "не выбрана" for s in stats).most_common()), + "subagents": sum(sub.values()), "subagent_models": dict(sub.most_common()), + "skills": dict(merged("skills").most_common()), + "tokens": {k: tok.get(k, 0) for k in ("output", "cache_read", "cache_write")}, + "compactions": total("compactions"), + "start_context": {"median": int(statistics.median(starts)), "min": min(starts), "max": max(starts)} + if starts else None, + "hook_blocks": sum(blk.values()), "hook_blocks_by_hook": dict(blk.most_common()), + "classifier_denials": sum(cls.values()), "classifier_by_reason": dict(cls.most_common()), + "owner_denials": total("owner_denials"), + "codegraph": {"init": total("cg_init"), "explore": explore, "code_reads": reads, + "explore_share": round(explore / (explore + reads), 2) if explore + reads else None}, + "recall": total("recall"), + } + + +def memory_stats(root, project): + d = os.path.join(root, slug(project), "memory") + files = [f for f in glob.glob(os.path.join(d, "**", "*"), recursive=True) if os.path.isfile(f)] + idx = os.path.join(d, "MEMORY.md") + return {"index_bytes": os.path.getsize(idx) if os.path.isfile(idx) else 0, "files": len(files), + "bytes": sum(os.path.getsize(f) for f in files)} + + +def false_blocks(labels): + cases = labels.get("cases", {}) + lab = [c for c in cases.values() if "skip" not in c] + allow = [c for c in lab if c.get("expected") == "allow"] + known = [c for c in allow if c.get("known")] + return {"labelled": len(lab), "skipped": len(cases) - len(lab), "false": len(allow), + "share": round(len(allow) / len(lab), 2) if lab else None, "fixed": len(allow) - len(known), + "known": len(known)} + + +def corrections(path): + """Кандидаты в поправки владельца с последним действием и хвостом ответа агента перед ними.""" + out, act, reply, sid = [], "", "", os.path.basename(path)[:8] + for rec in read_session(path, {}): + t = rec.get("type") + if t == "assistant": + for b in blocks(rec): + if b.get("type") == "tool_use": + act = action_of(b) + elif b.get("type") == "text" and (b.get("text") or "").strip(): + reply = b["text"] + continue + if t != "user": + continue + txt, kind = prompt_text(rec), None + if txt is not None: + if len(txt) <= 2000 and MARKERS.search(txt[:200]): + kind = "prompt" + elif is_interrupt(rec): + kind, txt = "interrupt", "[Request interrupted by user]" + elif any(b.get("type") == "tool_result" and OWNER_DENY in result_text(b) for b in blocks(rec)): + kind, txt = "denial", "отказ владельца в разрешении" + if kind: + out.append({"session": sid, "ts": rec.get("timestamp") or "", "kind": kind, "text": redact(txt[:300]), + "action": redact(act[:300]), "reply_tail": redact(" ".join(reply.split())[-200:])}) + return out + + +ROWS = [ + ("sessions", "Сессий"), ("prompts", "Промптов владельца"), ("interrupts", "Прерываний"), + ("turns", "Ходов модели"), ("tool_calls", "Вызовов инструментов"), ("subagents", "Запусков сабагентов"), + ("tokens.output", "Токенов вывода"), ("tokens.cache_read", "Чтение из кэша, токенов"), + ("tokens.cache_write", "Запись в кэш, токенов"), ("compactions", "Компакций контекста"), + ("start_context.median", "Стартовый контекст, медиана"), ("start_context.min", "Стартовый контекст, мин"), + ("start_context.max", "Стартовый контекст, макс"), ("hook_blocks", "Блокировок хуков"), + ("false_blocks.false", "Ложных блокировок"), ("false_blocks.share", "Доля ложных блокировок"), + ("false_blocks.fixed", "Ложных, уже исправлено"), ("false_blocks.known", "Ложных, ещё не исправлено"), + ("classifier_denials", "Отказов классификатора"), ("owner_denials", "Отказов владельца"), + ("codegraph.init", "codegraph init"), ("codegraph.explore", "Вызовов codegraph_explore"), + ("codegraph.code_reads", "Чтений кода (Read/Grep/Glob/Bash)"), + ("codegraph.explore_share", "Доля codegraph_explore"), + ("corrections", "Кандидатов в поправки владельца"), ("recall", "Промптов «перечитай память/контекст»"), + ("memory.index_bytes", "MEMORY.md, байт"), ("memory.files", "Файлов памяти"), ("memory.bytes", "Память, байт"), + ("cd_main", "cd в основной checkout на main"), +] +BREAKDOWNS = [("tools", "Инструменты"), ("roles", "Роли"), ("subagent_models", "Сабагенты по моделям"), + ("skills", "Skills"), ("hook_blocks_by_hook", "Блокировки по хукам"), + ("classifier_by_reason", "Отказы классификатора")] + + +def get(m, key): + for k in key.split("."): + if not isinstance(m, dict) or k not in m: + return None + m = m[k] + return m + + +def _num(v): + return isinstance(v, (int, float)) and not isinstance(v, bool) + + +def fmt(v): + if v is None: + return "—" + if isinstance(v, float): + return "%.2f" % v + if _num(v): + return "{:,}".format(v).replace(",", " ") + return str(v) + + +def cell(label, cur, old): + if isinstance(cur, dict): + cur = None + if isinstance(old, dict) and "retired" in old: + return "| %s | %s | — | не считается: %s |" % (label, fmt(cur), old["retired"]) + if isinstance(old, dict) and "value" in old: + return "| %s | %s | %s (%s) | — |" % (label, fmt(cur), fmt(old["value"]), old.get("note", "иначе считалось")) + d = "—" + if _num(cur) and _num(old): + diff = cur - old + d = ("%+.2f" % diff) if isinstance(diff, float) else "{:+,}".format(diff).replace(",", " ") + return "| %s | %s | %s | %s |" % (label, fmt(cur), fmt(old), d) + + +def to_markdown(doc, prev): + m, pm = doc["metrics"], (prev or {}).get("metrics") or {} + out = ["| Показатель | Сейчас | Прошлое ретро | Δ |", "|---|---|---|---|"] + for key, label in ROWS: + cur, old = get(m, key), get(pm, key) + if cur is None and old is None: + continue + out.append(cell(label, cur, old)) + out.append("") + for key, label in BREAKDOWNS: + d = m.get(key) or {} + if d: + out.append("- %s: %s" % (label, ", ".join("%s %s" % (k, fmt(v)) for k, v in list(d.items())[:10]))) + src, per = doc.get("sources") or {}, doc.get("period") or {} + out += ["", "Период %s — %s; сессий %s; каталоги транскриптов: %s; битых строк %s." % ( + per.get("since") or "начало", per.get("until") or "—", src.get("sessions"), + ", ".join(src.get("dirs") or []) or "—", src.get("bad_lines"))] + return "\n".join(out) + + +def default_since(harness): + names = [os.path.basename(f)[:10] for f in glob.glob(os.path.join(harness, "docs", "retro", "*.md"))] + names = [n for n in names if re.match(r"^\d{4}-\d{2}-\d{2}$", n)] + return max(names) if names else None + + +def parse_args(argv): + cc = os.environ.get("CLAUDE_CONFIG_DIR") or os.path.expanduser("~/.claude") + p = argparse.ArgumentParser(prog="retro_metrics.py", description="Срез метрик обвязки (B12)") + p.add_argument("project") + p.add_argument("--since") + p.add_argument("--harness", default=os.path.normpath(os.path.join(HERE, "..", "..", ".."))) + p.add_argument("--projects-root", default=os.path.join(cc, "projects")) + p.add_argument("--labels") + p.add_argument("--out") + p.add_argument("--markdown", action="store_true") + p.add_argument("--prev") + p.add_argument("--corrections", action="store_true") + return p.parse_args(argv[1:]) + + +def main(argv): + a = parse_args(argv) + project = os.path.abspath(a.project) + since = a.since or default_since(a.harness) + dirs, files = project_sessions(project, a.projects_root, since) + bad, cache = {"n": 0}, {} + stats = [s for s in (session_stats(f, cache, bad) for f in files) if s["prompts"] or s["turns"]] + if not stats: + sys.stderr.write("retro_metrics: нет сессий проекта %s (slug %s) в %s с %s\n" + % (project, slug(project), a.projects_root, since or "начала")) + return 1 + cands = [c for f in files for c in corrections(f)] + if a.corrections: + for c in cands: + print(json.dumps(c, ensure_ascii=False)) + return 0 + m = aggregate(stats) + m["corrections"] = len(cands) + m["memory"] = memory_stats(a.projects_root, project) + if a.labels: + m["false_blocks"] = false_blocks(json.load(open(a.labels, encoding="utf-8"))) + doc = {"schema": 1, "period": {"since": since, "until": max(s["last_ts"] for s in stats)[:10]}, + "project": project, + "sources": {"dirs": dirs, "sessions": len(stats), "bytes": sum(os.path.getsize(f) for f in files), + "bad_lines": bad["n"]}, + "metrics": m} + if a.out: + with open(a.out, "w", encoding="utf-8") as fh: + json.dump(doc, fh, ensure_ascii=False, indent=2) + fh.write("\n") + if a.markdown: + print(to_markdown(doc, json.load(open(a.prev, encoding="utf-8")) if a.prev else None)) + elif not a.out: + print(json.dumps(doc, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/openspec/changes/harness-retro/design.md b/openspec/changes/harness-retro/design.md index 913aa58..f444c5a 100644 --- a/openspec/changes/harness-retro/design.md +++ b/openspec/changes/harness-retro/design.md @@ -174,5 +174,6 @@ Bash-команда считается чтением кода, если хот | 2026-10-24 | Блокировки 11.10 в `metrics.json` — 76 ошибок хука, 72 ложных (по корпусу), а не 103 из таблицы | 103 завышено выводами со словом `BLOCKED` (O-1) | Сравнение с таблицей 11.10 по этой строке расходится с документом | | 2026-10-24 | `retro_stats.py` и `retro_blocks.py` удаляются | Их заменяет `retro_metrics.py`; в ретро 11.10 они упомянуты как история | Чтобы воспроизвести 11.10 буква в букву, нужен git history | | 2026-10-24 | Ходы и токены — по уникальному `message.id`, а не по строкам транскрипта | Строки одного сообщения повторяют `usage`; подсчёт по строкам завышает токены в 2–3 раза | Ходы и токены 11.10 (по строкам) несравнимы; в `2026-10-11.metrics.json` они помечены | +| 2026-10-24 | Записи ассистента с `isApiErrorMessage` или `model: ` — не ход модели: не считаются в ходах, токенах и стартовом контексте | Это сообщения Claude Code об ошибке API с нулевым `usage`; на живых транскриптах давали стартовый контекст 0 (O-18) | Если Claude Code пометит так настоящий ход, он выпадет из подсчёта | | 2026-10-24 | Свежая ветка, совпадающая с `origin/main`, — не хвост | `--merged` считает слитой и ветку без коммитов, например только что созданный worktree | Брошенная пустая ветка не попадёт в хвосты | | 2026-10-24 | Сторона Gitea в хвостах не проверяется | Хрупко (формат описаний PR и issue); агент видит это при чтении | Незакрытая issue после merge не попадёт в метрику | diff --git a/tests/test-retro-metrics.sh b/tests/test-retro-metrics.sh new file mode 100644 index 0000000..3a16525 --- /dev/null +++ b/tests/test-retro-metrics.sh @@ -0,0 +1,132 @@ +# retro_metrics.py: срез метрик по синтетическим транскриптам (спека harness-retro, #29). +M="$ROOT/docs/retro/scripts/retro_metrics.py" +P="$TMP/proj"; CR="$TMP/cc/projects"; mkdir -p "$P/.git" "$P/.claude" "$P/hooks" "$CR" +echo 'hooks/' > "$P/.claude/code-paths" +cat > "$TMP/gen.py" <<'PY' +import json, os, re, sys +P, CR = sys.argv[1], sys.argv[2] +slug = lambda p: re.sub(r"[^A-Za-z0-9]", "-", p) +def w(d, name, recs): + os.makedirs(os.path.join(CR, d), exist_ok=True) + with open(os.path.join(CR, d, name), "w", encoding="utf-8") as fh: + for r in recs: + fh.write((r if isinstance(r, str) else json.dumps(r, ensure_ascii=False)) + "\n") +def u(text, ts, cwd=P, **kw): + return dict({"type": "user", "timestamp": ts, "cwd": cwd, "message": {"role": "user", "content": text}}, **kw) +def tr(tid, text, ts): + return {"type": "user", "timestamp": ts, "cwd": P, + "message": {"content": [{"type": "tool_result", "tool_use_id": tid, "content": text}]}} +def a(mid, content, ts, cwd=P, usage=None): + m = {"id": mid, "role": "assistant", "content": content} + if usage: + m["usage"] = usage + return {"type": "assistant", "timestamp": ts, "cwd": cwd, "message": m} +def tu(tid, tool, **inp): + return {"type": "tool_use", "id": tid, "name": tool, "input": inp} +A = slug(P) +T = "2026-11-02T10:%02d:00Z" +U1 = {"input_tokens": 100, "cache_creation_input_tokens": 1000, "cache_read_input_tokens": 50000, "output_tokens": 10} +w(A, "s1.jsonl", [ + u("сделай X", T % 0), + a("m1", [{"type": "text", "text": "Начинаю."}], T % 1, usage=U1), + a("m1", [tu("t0", "Skill", skill="role", args="dev")], T % 1, usage=U1), + a("m2", [tu("t1", "Read", file_path=P + "/hooks/a.py"), tu("t2", "Read", file_path=P + "/docs/x.md")], T % 2), + a("m3", [tu("t3", "mcp__codegraph__codegraph_explore", query="x"), tu("t4", "mcp__codegraph__codegraph_explore", query="y"), + tu("t5", "Bash", command="grep -rn x hooks/"), tu("t6", "Bash", command="codegraph init"), + tu("t7", "Agent", model="sonnet", prompt="p"), tu("t8", "Agent", prompt="q"), + tu("t9", "Bash", command="echo x > f"), tu("t10", "Bash", command="cat hooks/require-plan.sh")], T % 3), + tr("t9", 'PreToolUse:Bash hook error: ["$HOME/.claude/hooks/guard-main.sh"]: BLOCKED [guard-main]: запись в f', T % 3), + tr("t10", 'echo "BLOCKED: editing without a plan" >&2', T % 3), + tr("t6", "Permission for this action was denied by the Claude Code auto mode classifier. Reason: [Self-Modification]. If", T % 3), + tr("t5", "The user doesn't want to proceed with this tool use. The tool use was rejected", T % 3), + u("/clear", T % 4), + u("служебное", T % 4, isMeta=True), + u("[Request interrupted by user]", T % 4), + a("m4", [{"type": "text", "text": "Вхожу в worktree."}, tu("t11", "EnterWorktree", name="x")], T % 5), + u("почему мне нужно это каждый раз подтверждать?", T % 6), + u("перечитай память", T % 7), + a("m5", [tu("t12", "Bash", command="git branch -m feat/x")], T % 8), + u("нет, tea login add --token 0123456789abcdef0123456789abcdef01234567 не так", T % 9), + {"type": "system", "subtype": "compact_boundary", "timestamp": T % 9}, + "not json", + "[]", +]) +W = P + "/.claude/worktrees/x" +w(A + "--claude-worktrees-x", "s2.jsonl", [ + u("ок", "2026-11-03T09:00:00Z", cwd=W), + {"type": "assistant", "timestamp": "2026-11-03T09:00:30Z", "cwd": W, "isApiErrorMessage": True, # не ход модели + "message": {"id": "syn-1", "model": "", "role": "assistant", "content": [{"type": "text", "text": "API Error"}], + "usage": {"input_tokens": 0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "output_tokens": 0}}}, + a("n1", [tu("x1", "Bash", command='grep "unbalanced hooks/a.py')], "2026-11-03T09:01:00Z", cwd=W, + usage={"input_tokens": 0, "cache_creation_input_tokens": 6000, "cache_read_input_tokens": 50000, "output_tokens": 0}), +]) +s5 = [u("да", "2026-11-04T09:00:00Z", cwd=P + ".wt/y"), + a("k1", [tu("y1", "Grep", pattern="x")], "2026-11-04T09:01:00Z", cwd=P + ".wt/y", + usage={"input_tokens": 0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 61000, "output_tokens": 0})] +w(slug(P + ".wt/y"), "s5.jsonl", s5) +w(A, "s5.jsonl", s5[:1]) # усечённая копия той же сессии — берётся больший файл +w(A, "s0.jsonl", [u("старое", "2026-10-01T09:00:00Z")]) +w(A + "-tools", "s3.jsonl", [u("чужое", "2026-11-02T09:00:00Z", cwd=P + "-tools")]) +os.makedirs(os.path.join(CR, A, "memory"), exist_ok=True) +open(os.path.join(CR, A, "memory", "MEMORY.md"), "w").write("x" * 100) +open(os.path.join(CR, A, "memory", "note.md"), "w").write("y" * 50) +PY +python3 "$TMP/gen.py" "$P" "$CR" +J="$TMP/m.json" +python3 "$M" "$P/" --since 2026-11-01 --projects-root "$CR" --out "$J"; rc=$? +assert_eq "срез: exit 0" "$rc" "0" +q(){ python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); m=d["metrics"]; print(eval(sys.argv[2]))' "$J" "$1"; } +assert_eq "схема 1" "$(q 'd["schema"]')" "1" +assert_eq "сессии: s1, s2 (worktree), s5 (.wt) — без старой и чужой" "$(q 'm["sessions"]')" "3" +assert_eq "каталоги транскриптов среза" "$(q 'len(d["sources"]["dirs"])')" "3" +assert_eq "битые строки" "$(q 'd["sources"]["bad_lines"]')" "1" +assert_eq "промпты владельца без служебных и прерываний" "$(q 'm["prompts"]')" "6" +assert_eq "прерывания" "$(q 'm["interrupts"]')" "1" +assert_eq "ходы — по message.id" "$(q 'm["turns"]')" "7" +assert_eq "вызовы инструментов" "$(q 'm["tool_calls"]')" "15" +assert_eq "токены вывода один раз на сообщение" "$(q 'm["tokens"]["output"]')" "10" +assert_eq "стартовый контекст" "$(q 'sorted(m["start_context"].items())')" "[('max', 61000), ('median', 56000), ('min', 51100)]" +assert_eq "блокировки только по ошибке хука" "$(q 'm["hook_blocks"], m["hook_blocks_by_hook"]')" "(1, {'guard-main(Bash)': 1})" +assert_eq "отказы классификатора по причине" "$(q 'm["classifier_by_reason"]')" "{'Self-Modification': 1}" +assert_eq "отказы владельца" "$(q 'm["owner_denials"]')" "1" +assert_eq "codegraph против чтения кода" "$(q 'sorted(m["codegraph"].items())')" "[('code_reads', 5), ('explore', 2), ('explore_share', 0.29), ('init', 1)]" +assert_eq "роли" "$(q 'sorted(m["roles"].items())')" "[('dev', 1), ('не выбрана', 2)]" +assert_eq "сабагенты по моделям" "$(q 'sorted(m["subagent_models"].items())')" "[('sonnet', 1), ('не указана', 1)]" +assert_eq "skills" "$(q 'm["skills"]')" "{'role': 1}" +assert_eq "компакции, перечитай" "$(q 'm["compactions"], m["recall"]')" "(1, 1)" +assert_eq "кандидаты в поправки" "$(q 'm["corrections"]')" "4" +assert_eq "память" "$(q 'sorted(m["memory"].items())')" "[('bytes', 150), ('files', 2), ('index_bytes', 100)]" +assert_eq "памяти нет — нули" "$(python3 -c "import sys; sys.path.insert(0, '$ROOT/docs/retro/scripts'); import retro_metrics; print(sorted(retro_metrics.memory_stats('$TMP/none', '$P').items()))")" "[('bytes', 0), ('files', 0), ('index_bytes', 0)]" +# кандидаты в поправки: контекст и redact +C="$TMP/corr.jsonl" +python3 "$M" "$P" --since 2026-11-01 --projects-root "$CR" --corrections > "$C" +assert_eq "кандидатов 4" "$(grep -c . "$C")" "4" +assert_eq "«почему» после EnterWorktree" "$(python3 -c 'import json,sys; print([c["action"] for c in map(json.loads, open(sys.argv[1])) if "почему" in c["text"]])' "$C")" "['EnterWorktree']" +assert_eq "последнее действие Bash" "$(python3 -c 'import json,sys; print([c["action"] for c in map(json.loads, open(sys.argv[1])) if c["text"].startswith("нет")])' "$C")" "['Bash: git branch -m feat/x']" +assert_eq "токен замазан" "$(grep -c '0123456789abcdef' "$C")" "0" +assert_grep "токен заменён на " '' "$C" +assert_eq "виды кандидатов" "$(python3 -c 'import json,sys; print(sorted(set(c["kind"] for c in map(json.loads, open(sys.argv[1])))))' "$C")" "['denial', 'interrupt', 'prompt']" +# сравнение с прошлым ретро +cat > "$TMP/prev.json" <<'EOF' +{"schema": 1, "metrics": {"sessions": 44, "turns": {"value": 12121, "note": "по строкам"}, "cd_main": {"retired": "хуки 0.4.0"}}} +EOF +md=$(python3 "$M" "$P" --since 2026-11-01 --projects-root "$CR" --markdown --prev "$TMP/prev.json") +assert_grep "строка с разницей" '^\| Сессий \| 3 \| 44 \| -41 \|$' <(printf '%s' "$md") +assert_grep "новая метрика — прочерки" '^\| Кандидатов в поправки владельца \| 4 \| — \| — \|$' <(printf '%s' "$md") +assert_grep "снятая метрика" '^\| cd в основной checkout на main \| — \| — \| не считается: хуки 0.4.0 \|$' <(printf '%s' "$md") +assert_grep "метрика с другим способом подсчёта" '^\| Ходов модели \| 7 \| 12 121 \(по строкам\) \| — \|$' <(printf '%s' "$md") +# ложные блокировки по разметке +cat > "$TMP/labels.json" <<'EOF' +{"home": "/h", "layout": [], "cases": {"x-001": {"expected": "allow", "why": "a"}, "x-002": {"expected": "allow", "why": "b"}, + "x-003": {"expected": "allow", "why": "c", "known": "RT-21"}, "x-004": {"expected": "block", "why": "d"}, "x-005": {"skip": "s"}}} +EOF +python3 "$M" "$P" --since 2026-11-01 --projects-root "$CR" --labels "$TMP/labels.json" --out "$J" +assert_eq "ложные блокировки" "$(q 'sorted(m["false_blocks"].items())')" "[('false', 3), ('fixed', 2), ('known', 1), ('labelled', 4), ('share', 0.75), ('skipped', 1)]" +# период по умолчанию — от даты последнего ретро +H="$TMP/harness"; mkdir -p "$H/docs/retro"; touch "$H/docs/retro/2026-11-03.md" "$H/docs/retro/2026-10-11.md" +python3 "$M" "$P" --harness "$H" --projects-root "$CR" --out "$J" +assert_eq "период с последнего ретро" "$(q 'd["period"]["since"], m["sessions"]')" "('2026-11-03', 2)" +# нет сессий +err=$(python3 "$M" "$P" --since 2030-01-01 --projects-root "$CR" 2>&1 >/dev/null); rc=$? +assert_eq "нет сессий: exit 1" "$rc" "1" +assert_grep "сообщение со slug" 'proj' <(printf '%s' "$err")