test(hooks): регрессионный корпус хуков из транскриптов, экстрактор только по ошибкам хука (#6)
B2, RT-6, O-1. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
1620d1e806
commit
3d54932709
@ -4,10 +4,14 @@
|
||||
Запуск: python3 hook_blocks_export.py <каталог-транскриптов> [<каталог> ...] > blocks.jsonl
|
||||
Одна строка — одна блокировка: сессия, время, cwd и ветка на момент вызова, инструмент, команда или путь файла,
|
||||
первая строка ответа хука. Поле expected не заполняется: вердикт (block/allow) проставляется при разметке.
|
||||
Выгружаются только ошибки хуков (результат начинается с `PreToolUse:<инструмент> hook error`); вывод команд,
|
||||
в котором встретилось слово BLOCKED (файлы памяти, исходники хуков), не попадает.
|
||||
Похожие на секреты подстроки заменяются на <redacted>.
|
||||
"""
|
||||
import json, sys, glob, os, re
|
||||
|
||||
HOOK_ERROR = re.compile(r"\s*PreToolUse:\S+ hook error")
|
||||
|
||||
RE_SECRET = re.compile(
|
||||
r'(?i)gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9_-]{20,}|\b[0-9a-f]{40}\b'
|
||||
r'|(?P<kv>(?:token|password|passwd|secret|api[_-]?key)\s*[=:]\s*)\S+'
|
||||
@ -22,8 +26,12 @@ def redact(s):
|
||||
return RE_SECRET.sub(sub, s)
|
||||
|
||||
|
||||
def hook_of(txt):
|
||||
if "shell write" in txt:
|
||||
def hook_of(txt, tool):
|
||||
if "[guard-main]" in txt:
|
||||
return "guard-main(Bash)" if tool == "Bash" else "guard-main(Edit/Write)"
|
||||
if "[require-plan]" in txt:
|
||||
return "require-plan"
|
||||
if "shell write" in txt: # формулировки до 0.4.0
|
||||
return "guard-main(Bash)"
|
||||
if "you are on" in txt:
|
||||
return "guard-main(Edit/Write)"
|
||||
@ -57,8 +65,8 @@ for d in sys.argv[1:]:
|
||||
txt = b.get("content", "")
|
||||
if isinstance(txt, list):
|
||||
txt = "\n".join(x.get("text", "") for x in txt if isinstance(x, dict))
|
||||
if not ("BLOCKED" in txt and ("hook" in txt.lower() or "BLOCKED:" in txt[:400])):
|
||||
continue
|
||||
if not HOOK_ERROR.match(txt):
|
||||
continue # обычный вывод команды, даже если в нём есть слово BLOCKED
|
||||
name, inp, cwd, branch, ts = pending.get(b.get("tool_use_id"), ("?", {}, "", "", ""))
|
||||
if name not in ("Bash", "Edit", "Write", "MultiEdit"):
|
||||
continue
|
||||
@ -67,5 +75,5 @@ for d in sys.argv[1:]:
|
||||
"session": sid, "ts": ts, "cwd": cwd, "branch": branch, "tool": name,
|
||||
"command": redact(inp.get("command", "")) if name == "Bash" else None,
|
||||
"file_path": inp.get("file_path") if name != "Bash" else None,
|
||||
"hook": hook_of(txt), "message": redact(msg[:300]), "expected": None,
|
||||
"hook": hook_of(txt, name), "message": redact(msg[:300]), "expected": None,
|
||||
}, ensure_ascii=False))
|
||||
|
||||
94
docs/retro/scripts/hook_cases_build.py
Normal file
94
docs/retro/scripts/hook_cases_build.py
Normal file
@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Сборка регрессионного корпуса хуков из сырой выгрузки блокировок (B2, #6).
|
||||
|
||||
Запуск из корня репозитория:
|
||||
python3 docs/retro/scripts/hook_cases_build.py > skills/harness-doctor/fixtures/hook-cases.jsonl
|
||||
Вход: docs/retro/fixtures/hook-blocks.jsonl (hook_blocks_export.py, 12.10.2026, 126 записей).
|
||||
Выход: первая строка — {"home": …, "layout": […]} (раскладка песочницы), дальше по записи на случай:
|
||||
поля выгрузки + id, expected (allow|block), why (класс). Вердикт размечен вручную по спеке
|
||||
openspec/changes/hooks-write-target/specs/hook-guards/spec.md, а не прогоном хука.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
RAW = os.path.join(HERE, "..", "fixtures", "hook-blocks.jsonl")
|
||||
HOME = "/Users/andrey.epifancev"
|
||||
AW, CH = "projects/awp-ca", "projects/cc-skills/claude-harness"
|
||||
|
||||
LAYOUT = [{"path": AW}, {"path": AW + "/reference-old-arm"}]
|
||||
LAYOUT += [{"path": "%s/%s" % (AW, r), "branch": "main"}
|
||||
for r in ("arm-core", "docs", "tabular-parser", "scan-gateway", "arm-web", "fixtures")]
|
||||
LAYOUT += [{"path": "%s/.wt/%s/%s" % (AW, n, r), "worktree_of": "%s/%s" % (AW, r), "branch": "feat/" + n}
|
||||
for n, r in (("web-proto", "arm-core"), ("web-proto", "arm-web"), ("ocr-q1", "fixtures"),
|
||||
("ocr-q1", "scan-gateway"), ("stmt-template", "arm-web"), ("qa-guide-web", "docs"),
|
||||
("slice-0", "scan-gateway"), ("slice-3b", "arm-core"))]
|
||||
LAYOUT += [{"path": CH, "branch": "main"}]
|
||||
LAYOUT += [{"path": "%s/.wt/%s" % (CH, n), "worktree_of": CH, "branch": b}
|
||||
for n, b in (("retro", "retro/2026-10-11"), ("roles", "feat/2-roles-and-memory"),
|
||||
("guard", "feat/6-guard-main-target"))]
|
||||
LAYOUT += [{"path": ".claude/projects/-Users-andrey-epifancev-projects-awp-ca/memory"},
|
||||
{"path": ".claude/projects/-Users-andrey-epifancev-projects-cc-skills-claude-harness/memory"},
|
||||
{"path": ".claude/hooks"}, {"path": ".claude/jobs"}]
|
||||
|
||||
# Блок по спеке: архивный коммит openspec прямо в main (решение владельца 1 — архив отдельным PR).
|
||||
BLOCK = {"c031", "c069", "c070", "c103"}
|
||||
# Класс для записей, где он не выводится общим правилом ниже.
|
||||
WHY = {
|
||||
"c002": "unparseable", "c008": "var-target", "c023": "outside-repo", "c026": "outside-repo",
|
||||
"c029": "worktree", "c030": "outside-repo", "c040": "worktree-setup", "c044": "worktree", "c045": "worktree",
|
||||
"c055": "outside-repo", "c064": "worktree-setup", "c072": "switch-then-commit", "c073": "worktree-setup",
|
||||
"c078": "outside-repo", "c080": "heredoc", "c081": "heredoc", "c083": "outside-repo", "c091": "heredoc",
|
||||
"c099": "worktree-setup", "c100": "worktree-setup", "c101": "worktree", "c102": "worktree",
|
||||
"c104": "worktree-setup", "c106": "worktree-setup", "c119": "heredoc",
|
||||
}
|
||||
# Ложные блоки этой сессии (15.10.2026), до перехода в worktree: cwd — основной checkout на main.
|
||||
LIVE_RE = ('git show origin/feat/6-guard-main-target:docs/retro/fixtures/hook-blocks.jsonl | python3 -c \'\n'
|
||||
'import sys,json,re,collections\n'
|
||||
'rs=[json.loads(l) for l in sys.stdin if l.strip()]\n'
|
||||
'c=collections.Counter()\n'
|
||||
'for r in rs:\n'
|
||||
' cmd=r.get("command") or ""\n'
|
||||
' if re.search(r"\\d>{1,2}\\s*/dev/null|\\d>&\\d",cmd): c["bash: N>/dev/null или N>&M"]+=1\n'
|
||||
' if re.search(r"mkdir\\s+-p\\s+\\S*\\.wt/",cmd): c["bash: mkdir -p .wt/"]+=1\n'
|
||||
' if re.search(r"\\bsed\\s+-i",cmd): c["bash: sed -i"]+=1\n'
|
||||
'for k,v in c.most_common(): print(f"{v:4} {k}")\'')
|
||||
LIVE = [
|
||||
("l001", "grep-pattern", 'grep -rn "npm i\\|npm install\\|brew install\\|plugin install\\|go install" '
|
||||
'--include="*.md" --include="*.sh" . | grep -v "^./docs/retro/2026" | head -20'),
|
||||
("l002", "devnull", 'grep -rn "plugin add\\|marketplace\\|caveman@\\|superpowers@" --include="*.md" '
|
||||
'--include="*.sh" README.md docs global install.sh 2>/dev/null | head -20'),
|
||||
("l003", "regex-text", LIVE_RE),
|
||||
]
|
||||
DEVNULL = re.compile(r"\d>{1,2}\s*/dev/null|\d>&\d")
|
||||
|
||||
|
||||
def why_of(cid, r):
|
||||
if cid in BLOCK:
|
||||
return "main-git"
|
||||
if cid in WHY:
|
||||
return WHY[cid]
|
||||
if "hook error" not in (r.get("message") or ""):
|
||||
return "not-a-block" # вывод успешной команды, где встретилось слово BLOCKED
|
||||
if r["tool"] != "Bash":
|
||||
return "outside-repo"
|
||||
return "devnull" if DEVNULL.search(r.get("command") or "") else "read-only"
|
||||
|
||||
|
||||
def main():
|
||||
print(json.dumps({"home": HOME, "layout": LAYOUT}, ensure_ascii=False))
|
||||
for n, line in enumerate(l for l in open(RAW, encoding="utf-8") if l.strip()):
|
||||
r = json.loads(line)
|
||||
cid = "c%03d" % (n + 1)
|
||||
r.update({"id": cid, "expected": "block" if cid in BLOCK else "allow", "why": why_of(cid, r)})
|
||||
print(json.dumps(r, ensure_ascii=False))
|
||||
for cid, why, cmd in LIVE:
|
||||
print(json.dumps({"id": cid, "session": "live-1015", "cwd": HOME + "/" + CH, "branch": "main",
|
||||
"tool": "Bash", "command": cmd, "file_path": None, "hook": "guard-main(Bash)",
|
||||
"message": "", "expected": "allow", "why": why}, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -8,6 +8,7 @@ files = sorted(glob.glob(os.path.join(d, "*.jsonl")))
|
||||
RE_REAL = re.compile(r'(^|[^<\d])>{1,2}[^&]|\btee\b|\bsed\s+-i|\bmv\b|\bcp\b|\brm\b|\btouch\b|\bmkdir\b|\bgit\s+(add|commit|push|apply|stash\s+pop)\b|\bnpm\s+(i|install|init)\b|\bpip\s+install\b')
|
||||
RE_STDERR = re.compile(r'\d>{1,2}\s*/dev/null|\d>&\d')
|
||||
RE_HEREDOC_PY = re.compile(r"<<\s*'?(PY|EOF)")
|
||||
RE_HOOK_ERROR = re.compile(r"\s*PreToolUse:\S+ hook error") # только ошибка хука, не вывод команды со словом BLOCKED
|
||||
|
||||
blocks = collections.Counter()
|
||||
by_hook = collections.Counter()
|
||||
@ -63,10 +64,14 @@ for f in files:
|
||||
if isinstance(txt, list):
|
||||
txt = "\n".join(x.get("text", "") for x in txt if isinstance(x, dict))
|
||||
name, inp = pending.get(b.get("tool_use_id"), ("?", {}))
|
||||
if "BLOCKED" in txt and ("hook" in txt.lower() or "BLOCKED:" in txt[:400]):
|
||||
if RE_HOOK_ERROR.match(txt) and "BLOCKED" in txt:
|
||||
if name not in ("Bash", "Edit", "Write", "MultiEdit"):
|
||||
continue
|
||||
if "shell write" in txt:
|
||||
if "[guard-main]" in txt:
|
||||
hook = "guard-main(Bash)" if name == "Bash" else "guard-main(Edit/Write)"
|
||||
elif "[require-plan]" in txt:
|
||||
hook = "require-plan"
|
||||
elif "shell write" in txt:
|
||||
hook = "guard-main(Bash)"
|
||||
elif "you are on" in txt:
|
||||
hook = "guard-main(Edit/Write)"
|
||||
|
||||
130
skills/harness-doctor/fixtures/hook-cases.jsonl
Normal file
130
skills/harness-doctor/fixtures/hook-cases.jsonl
Normal file
File diff suppressed because one or more lines are too long
95
skills/harness-doctor/scripts/hook_corpus.py
Normal file
95
skills/harness-doctor/scripts/hook_corpus.py
Normal file
@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Регрессионный корпус хуков (B2): прогон fixtures/hook-cases.jsonl через guard-main и require-plan в песочнице.
|
||||
|
||||
Запуск: python3 hook_corpus.py <каталог хуков> [--summary] [--fixtures <jsonl>]
|
||||
Песочница строится во временном каталоге по раскладке из первой строки корпуса; префикс домашнего каталога записей
|
||||
заменяется на её home, HOME хуков подменяется туда же. Bash идёт через guard-main, Edit/Write — через guard-main и
|
||||
require-plan (блок, если блокирует любой).
|
||||
Вывод: строка на запись «id<TAB>why<TAB>ожидалось<TAB>получено<TAB>команда» и итог «total N/M»;
|
||||
с --summary — только расхождения и «корпус хуков: N/M». Код выхода 0 — всё совпало, 1 — нет.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
def _git(*args):
|
||||
subprocess.run(["git", "-c", "user.name=corpus", "-c", "user.email=corpus@example.invalid"] + list(args),
|
||||
check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def build(home, layout):
|
||||
for e in layout:
|
||||
p = os.path.join(home, e["path"])
|
||||
if "worktree_of" in e:
|
||||
_git("-C", os.path.join(home, e["worktree_of"]), "worktree", "add", "-q", p, "-b", e["branch"])
|
||||
elif "branch" in e:
|
||||
os.makedirs(p, exist_ok=True)
|
||||
_git("init", "-q", p)
|
||||
_git("-C", p, "symbolic-ref", "HEAD", "refs/heads/" + e["branch"])
|
||||
_git("-C", p, "commit", "-q", "--allow-empty", "-m", "init")
|
||||
else:
|
||||
os.makedirs(p, exist_ok=True)
|
||||
|
||||
|
||||
def verdict(hooks, case, prefix, home, env):
|
||||
swap = lambda s: s.replace(prefix, home) if isinstance(s, str) else s
|
||||
tool = case["tool"]
|
||||
ti = {"command": swap(case.get("command"))} if tool == "Bash" else {"file_path": swap(case.get("file_path"))}
|
||||
payload = json.dumps({"cwd": swap(case.get("cwd")), "tool_name": tool, "tool_input": ti})
|
||||
for name in ["guard-main"] + ([] if tool == "Bash" else ["require-plan"]):
|
||||
r = subprocess.run(["bash", os.path.join(hooks, name + ".sh")], input=payload, universal_newlines=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, timeout=30)
|
||||
if r.returncode == 2:
|
||||
return "block"
|
||||
if r.returncode != 0:
|
||||
return "error"
|
||||
return "allow"
|
||||
|
||||
|
||||
def main(argv):
|
||||
args = argv[1:]
|
||||
summary = "--summary" in args
|
||||
fx = os.path.join(HERE, "..", "fixtures", "hook-cases.jsonl")
|
||||
if "--fixtures" in args:
|
||||
fx = args[args.index("--fixtures") + 1]
|
||||
pos = [a for k, a in enumerate(args) if not a.startswith("--") and (k == 0 or args[k - 1] != "--fixtures")]
|
||||
if not pos:
|
||||
print(__doc__.strip().splitlines()[2])
|
||||
return 1
|
||||
hooks = os.path.abspath(pos[0])
|
||||
lines = [json.loads(l) for l in open(fx, encoding="utf-8") if l.strip()]
|
||||
head, cases = lines[0], lines[1:]
|
||||
tmp = tempfile.mkdtemp(prefix="hook-corpus-")
|
||||
try:
|
||||
home = os.path.join(tmp, "home")
|
||||
try:
|
||||
build(home, head["layout"])
|
||||
except Exception as e:
|
||||
print("корпус хуков: песочница не собрана (%s)" % e)
|
||||
return 1
|
||||
env = {k: v for k, v in os.environ.items() if k not in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE")}
|
||||
env["HOME"] = home
|
||||
ok = 0
|
||||
for c in cases:
|
||||
got = verdict(hooks, c, head["home"], home, env)
|
||||
ok += got == c["expected"]
|
||||
body = (c.get("command") or c.get("file_path") or "").replace(head["home"], "~")
|
||||
row = "\t".join([c["id"], c["why"], c["expected"], got, " ".join(body.split())[:70]])
|
||||
if not summary:
|
||||
print(row)
|
||||
elif got != c["expected"]:
|
||||
print("MISMATCH " + row)
|
||||
print(("корпус хуков: %d/%d" if summary else "total %d/%d") % (ok, len(cases)))
|
||||
return 0 if ok == len(cases) else 1
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
10
tests/test-hook-corpus.sh
Normal file
10
tests/test-hook-corpus.sh
Normal file
@ -0,0 +1,10 @@
|
||||
# hook_corpus.py: регрессионный корпус хуков (B2) через источники hooks/ — assert на запись.
|
||||
R="$ROOT/skills/harness-doctor/scripts/hook_corpus.py"
|
||||
F="$ROOT/skills/harness-doctor/fixtures/hook-cases.jsonl"
|
||||
assert_file "hook_corpus.py существует" "$R"
|
||||
assert_file "корпус hook-cases.jsonl" "$F"
|
||||
assert_eq "корпус совпадает со сборкой из выгрузки" "$(python3 "$ROOT/docs/retro/scripts/hook_cases_build.py" | cmp -s - "$F" && echo same)" "same"
|
||||
while IFS=$'\t' read -r id why exp got cmd; do
|
||||
case "$id" in total*) continue ;; esac
|
||||
assert_eq "корпус $id ($why): $cmd" "$got" "$exp"
|
||||
done < <(python3 "$R" "$ROOT/hooks")
|
||||
15
tests/test-hook-export.sh
Normal file
15
tests/test-hook-export.sh
Normal file
@ -0,0 +1,15 @@
|
||||
# hook_blocks_export.py: выгружаются только ошибки хуков, а не вывод команд со словом BLOCKED.
|
||||
X="$ROOT/docs/retro/scripts/hook_blocks_export.py"
|
||||
T="$TMP/transcripts"; mkdir -p "$T"
|
||||
cat > "$T/s1.jsonl" <<'EOF'
|
||||
{"type":"assistant","cwd":"/r","gitBranch":"main","timestamp":"t1","message":{"content":[{"type":"tool_use","id":"u1","name":"Bash","input":{"command":"echo x > f"}}]}}
|
||||
{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"u1","is_error":true,"content":"PreToolUse:Bash hook error: [\"$HOME/.claude/hooks/guard-main.sh\"]: BLOCKED [guard-main]: запись в f — репозиторий /r на main."}]}}
|
||||
{"type":"assistant","cwd":"/r","gitBranch":"main","timestamp":"t2","message":{"content":[{"type":"tool_use","id":"u2","name":"Bash","input":{"command":"cat hooks/require-plan.sh"}}]}}
|
||||
{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"u2","content":"echo \"BLOCKED: editing $fpath without a plan. Run superpowers:brainstorming\" >&2"}]}}
|
||||
{"type":"assistant","cwd":"/r","gitBranch":"main","timestamp":"t3","message":{"content":[{"type":"tool_use","id":"u3","name":"Edit","input":{"file_path":"/r/a.md"}}]}}
|
||||
{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"u3","is_error":true,"content":[{"type":"text","text":"PreToolUse:Edit hook error: [\"$HOME/.claude/hooks/guard-main.sh\"]: BLOCKED: you are on main. Create an isolated worktree first"}]}]}}
|
||||
EOF
|
||||
out=$(python3 "$X" "$T")
|
||||
assert_eq "две записи: вывод cat с BLOCKED не выгружен" "$(printf '%s\n' "$out" | grep -c .)" "2"
|
||||
assert_eq "новый префикс → guard-main(Bash)" "$(printf '%s\n' "$out" | sed -n 1p | python3 -c 'import json,sys; r=json.load(sys.stdin); print(r["hook"], r["command"])')" "guard-main(Bash) echo x > f"
|
||||
assert_eq "старая фраза → guard-main(Edit/Write)" "$(printf '%s\n' "$out" | sed -n 2p | python3 -c 'import json,sys; r=json.load(sys.stdin); print(r["hook"], r["file_path"])')" "guard-main(Edit/Write) /r/a.md"
|
||||
Loading…
x
Reference in New Issue
Block a user