From bc69d5087a5d0e2518dfcc752d6a4b10f87f739c Mon Sep 17 00:00:00 2001 From: Andrey Epifancev Date: Sat, 24 Oct 2026 02:22:15 +0400 Subject: [PATCH] =?UTF-8?q?wiki:=20=D1=81=D0=B8=D0=BD=D1=85=D1=80=D0=BE?= =?UTF-8?q?=D0=BD=D0=B8=D0=B7=D0=B0=D1=86=D0=B8=D1=8F=20=E2=80=94=20=D0=BF?= =?UTF-8?q?=D0=B0=D1=87=D0=BA=D0=B8=20=D1=81=D1=82=D1=80=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D1=86,=20=D0=BA=D0=BE=D0=BF=D0=B8=D1=8F,=20=D0=BF=D0=BE=D0=B8?= =?UTF-8?q?=D1=81=D0=BA=20(#91)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- src/jira_hub/core/modules.py | 2 + src/jira_hub/core/queue.py | 5 + src/jira_hub/modules/wiki/__init__.py | 5 +- src/jira_hub/modules/wiki/api.py | 11 + src/jira_hub/modules/wiki/commands.py | 11 +- src/jira_hub/modules/wiki/projects.py | 16 +- src/jira_hub/modules/wiki/search.py | 88 ++++ src/jira_hub/modules/wiki/stage.py | 81 ++++ src/jira_hub/modules/wiki/sync.py | 389 +++++++++++++++++ src/jira_hub/modules/wiki/tick.py | 20 +- tests/fixtures/baseline/routes.expected.json | 6 + tests/test_registry.py | 9 +- tests/test_wiki_sync.py | 435 +++++++++++++++++++ 13 files changed, 1062 insertions(+), 16 deletions(-) create mode 100644 src/jira_hub/modules/wiki/api.py create mode 100644 src/jira_hub/modules/wiki/search.py create mode 100644 src/jira_hub/modules/wiki/stage.py create mode 100644 src/jira_hub/modules/wiki/sync.py create mode 100644 tests/test_wiki_sync.py diff --git a/src/jira_hub/core/modules.py b/src/jira_hub/core/modules.py index 696a148..ef4dfd4 100644 --- a/src/jira_hub/core/modules.py +++ b/src/jira_hub/core/modules.py @@ -30,6 +30,8 @@ class CommandHooks: on_result: Callable[[Connection, dict, str, dict | None, int, int, str, Path | None], None] # в транзакции after_result: Callable[[Connection, dict, str], None] # вне транзакции on_late_result: Callable[[Connection, dict, str, dict | None, int, int], bool] + # (conn, cmd, status, result, files_dir): до транзакции отчёта — тяжёлая подготовка (файлы, перевод); None — нет + before_result: Callable[[Connection, dict, str, dict | None, Path | None], None] | None = None @dataclass(frozen=True) diff --git a/src/jira_hub/core/queue.py b/src/jira_hub/core/queue.py index a8adea6..3da999b 100644 --- a/src/jira_hub/core/queue.py +++ b/src/jira_hub/core/queue.py @@ -295,6 +295,11 @@ def report_result(conn, command_id, status, result=None, error=None, jira_calls= if status not in ("done", "failed"): raise ValueError("status — done или failed") error = None if error is None else str(error)[:ERROR_LIMIT] + early = get_command(conn, command_id) + if early is not None and early["status"] == "claimed": + eh = _hooks(early["type"]) + if eh and eh.before_result: # вне транзакции отчёта: модуль готовит тяжёлое, не держа блокировку базы + eh.before_result(conn, early, status, result, files_dir) with db.transaction(conn): cmd = get_command(conn, command_id) if cmd is None: diff --git a/src/jira_hub/modules/wiki/__init__.py b/src/jira_hub/modules/wiki/__init__.py index 4e3657b..1c4418a 100644 --- a/src/jira_hub/modules/wiki/__init__.py +++ b/src/jira_hub/modules/wiki/__init__.py @@ -1,6 +1,6 @@ """Модуль wiki: Markdown-копия вики Eva по проектам, поиск, вложения по запросу.""" from ...core.modules import Module -from . import commands, projects, schema, settings, tick +from . import api, commands, projects, schema, settings, tick MODULE = Module( name="wiki", @@ -10,7 +10,8 @@ MODULE = Module( agent_min_version_key="wiki_min_agent_version", validate_setting=settings.validate_setting, commands=commands.HOOKS, - tick=(tick.auto_sync_tick,), + routes=api.ROUTES, + tick=(tick.housekeeping_tick, tick.auto_sync_tick), project_fields=projects.FIELDS, project_view=projects.view, on_project_created=projects.on_created, diff --git a/src/jira_hub/modules/wiki/api.py b/src/jira_hub/modules/wiki/api.py new file mode 100644 index 0000000..1df7255 --- /dev/null +++ b/src/jira_hub/modules/wiki/api.py @@ -0,0 +1,11 @@ +"""HTTP модуля wiki: маршруты агента (приём пачек страниц).""" +from . import stage + + +def h_agent_pages(hub, conn, m, q, body): + return 200, stage.accept(conn, body.get("command_id"), body.get("pages")) + + +ROUTES = ( + ("POST", r"^/agent/wiki/pages$", h_agent_pages, "agent"), +) diff --git a/src/jira_hub/modules/wiki/commands.py b/src/jira_hub/modules/wiki/commands.py index d54051b..6ba67da 100644 --- a/src/jira_hub/modules/wiki/commands.py +++ b/src/jira_hub/modules/wiki/commands.py @@ -4,6 +4,7 @@ import re from ...core import queue from ...core.errors import CommandError from ...core.modules import CommandHooks +from . import sync TYPES = {"wiki_sync", "wiki_fetch"} SPEC = {"wiki_sync": {"required": ["label", "root"], "optional": []}, @@ -68,14 +69,12 @@ def active_sync(conn, label: str) -> dict | None: return queue.batch_view(conn, row["id"], with_commands=False) if row else None -_noop = lambda *a, **k: None # noqa: E731 - -# Хуки результата пустые: их заполняют приём пачек и отчёт синхронизации +# wiki_fetch: хуки результата заполнит Task 8; wiki_sync — sync.py HOOKS = CommandHooks( types=frozenset(TYPES), read_types=frozenset(TYPES), validate=validate, check_batch=check_batch, tail=lambda cmds: None, pause=lambda conn, ctype: 0, deliver_params=lambda conn, cmd, files_dir: cmd["params"], temp_definer=None, temp_keys_used=lambda cmd: [], substitute=lambda ctype, params, mapping: params, drop_unresolved=lambda cmd, unresolved: unresolved, retry_params=lambda cmd: cmd["params"], - check_confirmed=lambda conn, cmd: (False, None), on_claim=_noop, on_failed=_noop, - check_result=lambda conn, cmd, status, result, files_dir: (status, None), on_result=_noop, after_result=_noop, - on_late_result=lambda *a: False) + check_confirmed=lambda conn, cmd: (False, None), on_claim=sync.on_claim, on_failed=sync.on_failed, + check_result=sync.check_result, on_result=sync.on_result, after_result=sync.after_result, + on_late_result=lambda *a: False, before_result=sync.before_result) diff --git a/src/jira_hub/modules/wiki/projects.py b/src/jira_hub/modules/wiki/projects.py index 8022bbd..8e39f20 100644 --- a/src/jira_hub/modules/wiki/projects.py +++ b/src/jira_hub/modules/wiki/projects.py @@ -1,4 +1,5 @@ """Корень вики проекта: поле wiki_root, значения для GET /projects, хуки создания и правки проекта.""" +import logging import re import shutil @@ -6,6 +7,8 @@ from ...core import db, queue from ...core.modules import ProjectField from . import commands, paths +log = logging.getLogger(__name__) + _CODE = re.compile(r"(? bool: def _drop_copy(conn, label: str) -> None: """Сброс корня: строки проекта, индекс поиска (если он уже создан) и каталог копии.""" if _has_table(conn, "wiki_page_fts"): - conn.execute("DELETE FROM wiki_page_fts WHERE rowid IN (SELECT rowid FROM wiki_page WHERE label = ?)", (label,)) + conn.execute("DELETE FROM wiki_page_fts WHERE label = ?", (label,)) conn.execute("DELETE FROM wiki_page WHERE label = ?", (label,)) conn.execute("DELETE FROM wiki_attachment WHERE label = ?", (label,)) conn.execute("DELETE FROM wiki_project WHERE label = ?", (label,)) try: - shutil.rmtree(paths.project_dir(conn, label), ignore_errors=True) + base = paths.project_dir(conn, label) except RuntimeError: # база без файла (тесты в памяти): копии на диске нет - pass + return + for d in (base, base.with_name(f".{base.name}.new"), base.with_name(f".{base.name}.old")): + try: + shutil.rmtree(d) + except FileNotFoundError: + pass + except OSError as e: + log.warning("вики: каталог %s не удалён: %s", d, e) def on_updated(conn, project, fields, author) -> list[dict]: diff --git a/src/jira_hub/modules/wiki/search.py b/src/jira_hub/modules/wiki/search.py new file mode 100644 index 0000000..aa66b47 --- /dev/null +++ b/src/jira_hub/modules/wiki/search.py @@ -0,0 +1,88 @@ +"""Поиск по копии вики: FTS5 (wiki_page_fts) и запасной LIKE по файлам index.md, если FTS5 в SQLite нет.""" +import logging +import re +import sqlite3 + +from . import paths + +log = logging.getLogger(__name__) + +_WORD = re.compile(r"\w+") +_warned = False + + +def _has_table(conn) -> bool: + return conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'wiki_page_fts'").fetchone() is not None + + +def ensure(conn) -> bool: + """Создать индекс, если его нет. False — FTS5 недоступен (одно предупреждение в лог на процесс).""" + global _warned + if _has_table(conn): + return True + try: + conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS wiki_page_fts USING fts5(label UNINDEXED, code UNINDEXED, " + "title, body, tokenize='unicode61 remove_diacritics 2')") + except sqlite3.OperationalError as e: + if not _warned: + _warned = True + log.warning("FTS5 недоступен, поиск по вики идёт по файлам: %s", e) + return False + return True + + +def replace(conn, label: str, pages: list[tuple[str, str, str]]) -> None: + """Индекс проекта заново: pages — [(код, название, Markdown)]. Без FTS5 — ничего.""" + if not ensure(conn): + return + conn.execute("DELETE FROM wiki_page_fts WHERE label = ?", (label,)) + conn.executemany("INSERT INTO wiki_page_fts(label, code, title, body) VALUES (?, ?, ?, ?)", + [(label, code, title, body) for code, title, body in pages]) + + +def _match(q: str) -> str | None: + """Слова запроса → `"слово"* AND …`: префиксный поиск, кавычки экранируются. Без слов — None.""" + words = [w.replace('"', '""') for w in _WORD.findall(q or "")] + return " AND ".join(f'"{w}"*' for w in words) if words else None + + +def query(conn, label: str, q: str, limit: int) -> list[dict]: + """[{code, title, path, snippet}] по убыванию релевантности.""" + match = _match(q) + if match is None: + return [] + pages = {r["code"]: r for r in conn.execute("SELECT code, title, path FROM wiki_page WHERE label = ?", (label,))} + if _has_table(conn): + try: + rows = conn.execute("SELECT code, title, snippet(wiki_page_fts, 3, '', '', '…', 16) AS snippet " + "FROM wiki_page_fts WHERE wiki_page_fts MATCH ? AND label = ? ORDER BY rank LIMIT ?", + (match, label, limit)).fetchall() + except sqlite3.OperationalError as e: + log.warning("поиск по вики: запрос «%s» отвергнут: %s", q, e) + return [] + return [{"code": r["code"], "title": r["title"], "path": pages[r["code"]]["path"] if r["code"] in pages else None, + "snippet": r["snippet"]} for r in rows] + return _like(conn, label, _WORD.findall(q), pages, limit) + + +def _like(conn, label, words, pages, limit) -> list[dict]: + """Без FTS5: страницы проекта, где есть все слова (без учёта регистра), по числу вхождений.""" + try: + base = paths.project_dir(conn, label) + except RuntimeError: + return [] + found = [] + needles = [w.casefold() for w in words] + for code, r in pages.items(): + try: + text = (base / r["path"] / "index.md").read_text(encoding="utf-8") + except (OSError, TypeError): + continue + low = text.casefold() + if all(n in low for n in needles): + at = low.find(needles[0]) + start = max(0, at - 60) + found.append((-sum(low.count(n) for n in needles), code, r["title"], r["path"], + ("…" if start else "") + text[start:at + 100].replace("\n", " ") + "…")) + found.sort() + return [{"code": c, "title": t, "path": p, "snippet": s} for _, c, t, p, s in found[:limit]] diff --git a/src/jira_hub/modules/wiki/stage.py b/src/jira_hub/modules/wiki/stage.py new file mode 100644 index 0000000..b847777 --- /dev/null +++ b/src/jira_hub/modules/wiki/stage.py @@ -0,0 +1,81 @@ +"""Приём пачек страниц от агента wiki: проверка формы, перевод HTML → Markdown (вне транзакции), стадия команды.""" +import json + +from ...core import db, queue +from ...core.http import ApiError +from . import convert + +MAX_PAGES = 50 + + +def _bad(code, field): + return ApiError(422, "bad_pages", f"{code}: {field}") + + +def _text(page, key, code, required=True, nullable=False): + v = page.get(key) + if v is None and (nullable or not required): + return None + if not isinstance(v, str) or (required and not v.strip()): + raise _bad(code, key) + return v + + +def _check(i, page) -> dict: + """Форма одной страницы → поля для стадии (без html); ApiError 422 `<код>: <поле>`.""" + if not isinstance(page, dict): + raise ApiError(422, "bad_pages", f"страница {i}: не объект") + code = page.get("code") + if not isinstance(code, str) or not code.strip(): + raise ApiError(422, "bad_pages", f"страница {i}: code") + title = _text(page, "title", code) + html = page.get("html") + if not isinstance(html, str): + raise _bad(code, "html") + orderno = page.get("orderno") + if orderno is not None and (not isinstance(orderno, int) or isinstance(orderno, bool)): + raise _bad(code, "orderno") + eva_id = page.get("eva_id") + if eva_id is not None and not isinstance(eva_id, str): + raise _bad(code, "eva_id") + attachments, links = page.get("attachments") or [], page.get("links") or [] + if not isinstance(attachments, list) or not all( + isinstance(a, dict) and isinstance(a.get("ref"), str) and isinstance(a.get("url"), str) + and isinstance(a.get("name") or "", str) for a in attachments): + raise _bad(code, "attachments") + if not isinstance(links, list) or not all( + isinstance(l, dict) and isinstance(l.get("ref"), str) and isinstance(l.get("code"), str) for l in links): + raise _bad(code, "links") + return {"code": code, "eva_id": eva_id, "title": title, "parent_code": _text(page, "parent_code", code, False, True), + "orderno": orderno, "url": _text(page, "url", code, False, True), "html": html, + "attachments": [{"ref": a["ref"], "url": a["url"], "name": a.get("name") or ""} for a in attachments], + "links": [{"ref": l["ref"], "code": l["code"]} for l in links]} + + +def accept(conn, command_id, pages) -> dict: + """Пачка страниц команды wiki_sync → стадия. Перевод HTML идёт до записи, строки пишутся короткой транзакцией.""" + if isinstance(command_id, bool) or not isinstance(command_id, int): + raise ApiError(422, "bad_request", "command_id — целое") + if not isinstance(pages, list) or not pages or len(pages) > MAX_PAGES: + raise ApiError(422, "bad_pages", f"pages — список от 1 до {MAX_PAGES} страниц") + checked = [_check(i, p) for i, p in enumerate(pages, 1)] + cmd = queue.get_command(conn, command_id) + if cmd is None: + raise ApiError(404, "not_found", f"command {command_id}") + if cmd["type"] != "wiki_sync" or cmd["status"] != "claimed": + raise ApiError(409, "not_claimed", f"команда {command_id} не выдана агенту как wiki_sync") + rows = [] + for p in checked: + try: + md, files = convert.to_markdown(p["html"], p["attachments"], p["links"]) + except Exception as e: # noqa: BLE001 недоверенный HTML: сбой разбора — ошибка страницы, не сервера + raise _bad(p["code"], f"html ({type(e).__name__})") from None + payload = {k: p[k] for k in ("code", "eva_id", "title", "parent_code", "orderno", "url")} + payload |= {"markdown": md, "attachments": files} + rows.append((command_id, p["code"], json.dumps(payload, ensure_ascii=False))) + with db.transaction(conn): + cur = conn.execute("SELECT status FROM command WHERE id = ?", (command_id,)).fetchone() + if cur is None or cur["status"] != "claimed": # состояние могло смениться, пока шёл перевод + raise ApiError(409, "not_claimed", f"команда {command_id} уже не в работе") + conn.executemany("INSERT OR REPLACE INTO wiki_sync_stage(command_id, code, payload) VALUES (?, ?, ?)", rows) + return {"accepted": len(rows)} diff --git a/src/jira_hub/modules/wiki/sync.py b/src/jira_hub/modules/wiki/sync.py new file mode 100644 index 0000000..74dd8fd --- /dev/null +++ b/src/jira_hub/modules/wiki/sync.py @@ -0,0 +1,389 @@ +"""Синхронизация вики: сборка Markdown-копии из стадии, подмена каталога, строки wiki_page/вложений/FTS. + +Порядок отчёта `done` (design 10): `prepare` вызывается до транзакции отчёта (CommandHooks.before_result) — готовит +каталог `wiki/.<метка>.new/` и выделяет id вложений; `commit` (в on_result) пишет строки и последним шагом подменяет +каталог. Перевод HTML → Markdown уже сделан при приёме пачек (stage.accept).""" +import json +import logging +import re +import shutil +import threading +from dataclasses import dataclass, field +from pathlib import Path + +from ...core import db +from ...core.paths import secure_dir +from . import convert, paths, search + +log = logging.getLogger(__name__) + +_BAD_CHARS = re.compile(r'[<>:"/\\|?*\x00-\x1f]') +NAME_CHARS = 80 # длина названия, символов (как safe_name в eva_wiki_export.py) +NAME_BYTES = 200 # и байт UTF-8: у файловой системы предел имени 255 байт, «NN_» и «_k» помещаются + + +class SyncFailed(Exception): + """Синхронизация не удалась: текст идёт в error команды и sync_error проекта.""" + + +@dataclass +class Prepared: + cmd_id: int + label: str + root: str + discarded: str | None = None + error: str | None = None + rows: list = field(default_factory=list) + attachments: list = field(default_factory=list) # (page_code, url, name) + kept_ids: set = field(default_factory=set) + new_ids: list = field(default_factory=list) # id вложений, добавленных подготовкой + new_dir: Path | None = None + root_title: str | None = None + + +_PREPARED: dict[int, Prepared] = {} +_lock = threading.Lock() + + +# --- раскладка копии ---------------------------------------------------------- + +def safe_name(title: str) -> str: + """Название каталога: запрещённые символы → «_», до 80 символов и 200 байт, «.» и пробелы по краям убраны.""" + name = _BAD_CHARS.sub("_", title or "")[:NAME_CHARS] + name = name.encode("utf-8")[:NAME_BYTES].decode("utf-8", "ignore").strip(" .") + return name or "untitled" + + +def _sorted(codes, by_code): + return sorted(codes, key=lambda c: (by_code[c]["orderno"] is None, by_code[c]["orderno"] or 0, c)) + + +def _tree(order, by_code) -> dict[str, list[str]]: + """Потомки по parent_code; страница без родителя в наборе (или в цикле) становится дочерней корню.""" + root = order[0] + children = {c: [] for c in order} + parent = {} + for c in order[1:]: + p = by_code[c].get("parent_code") + parent[c] = p if p in children and p != c else root + children[parent[c]].append(c) + + def reach(): + seen, stack = set(), [root] + while stack: + c = stack.pop() + if c not in seen: + seen.add(c) + stack.extend(children[c]) + return seen + + seen = reach() + for c in order[1:]: + if c not in seen: # цикл родителей: оторванное звено цепляем к корню + children[parent[c]].remove(c) + parent[c] = root + children[root].append(c) + seen = reach() + return children + + +def _inside(target: Path, path: Path) -> None: + if not path.resolve().is_relative_to(target.resolve()): + raise ValueError(f"путь {path} вне каталога копии") + + +def build_copy(stage_pages: list[dict], codes: list[str], target: Path, ids: dict[tuple[str, str], int]) -> list[dict]: + """Собрать копию дерева в `target`: index.md страниц и _tree.json. Корень — codes[0]. Возвращает строки wiki_page + (с полем markdown — текст с подставленными id вложений). ValueError — страница или путь негодны.""" + by_code = {p["code"]: p for p in stage_pages} + order = list(dict.fromkeys(codes)) + if not order: + raise ValueError("пустой список страниц") + for c in order: + if c not in by_code: + raise ValueError(f"нет страницы {c}") + children = _tree(order, by_code) + secure_dir(target) + synced_at = db.now() + rows, tree = [], [] + stack = [(order[0], Path(), 0, 0)] + used: dict[str, set[str]] = {} # родитель → занятые имена (без регистра) + while stack: + code, parent_rel, depth, pos = stack.pop() + page = by_code[code] + taken = used.setdefault(str(parent_rel), set()) + name, k = safe_name(page["title"]), 1 + base = name + while name.lower() in taken: + k += 1 + name = f"{base}_{k}" + taken.add(name.lower()) + rel = parent_rel / (name if depth == 0 else f"{pos:02d}_{name}") + full = target / rel + _inside(target, full) + try: + text = convert.fill_ids(page["markdown"], [ids[(code, a["url"])] for a in page["attachments"]]) + except (ValueError, KeyError) as e: + raise ValueError(f"{code}: {e}") from None + secure_dir(full) + title_line = " ".join(str(page["title"]).split()) + head = ["---", f"code: {json.dumps(code, ensure_ascii=False)}", f"title: {json.dumps(page['title'], ensure_ascii=False)}", + f"url: {json.dumps(page.get('url'), ensure_ascii=False)}", f"synced_at: {json.dumps(synced_at)}", "---", "", + f"# {title_line}", ""] + (full / "index.md").write_text("\n".join(head) + "\n" + text, encoding="utf-8") + rows.append({"code": code, "eva_id": page.get("eva_id"), "title": page["title"], "parent_code": page.get("parent_code"), + "orderno": page.get("orderno"), "depth": depth, "path": rel.as_posix(), "url": page.get("url"), + "size": len(text), "synced_at": synced_at, "markdown": text}) + tree.append({"code": code, "title": page["title"], "depth": depth, "path": rel.as_posix()}) + kids = _sorted(children[code], by_code) + for i, kid in reversed(list(enumerate(kids, 1))): # стек: первый ребёнок — следующим + stack.append((kid, rel, depth + 1, i)) + (target / "_tree.json").write_text(json.dumps(tree, ensure_ascii=False, indent=1), encoding="utf-8") + return rows + + +# --- подготовка (до транзакции отчёта) --------------------------------------- + +def _siblings(conn, label: str) -> tuple[Path, Path, Path]: + cur = paths.project_dir(conn, label) + return cur, cur.with_name(f".{cur.name}.new"), cur.with_name(f".{cur.name}.old") + + +def _allocate_ids(conn, label, pages) -> tuple[dict, list, list]: + """id вложений страниц: существующие по (страница, url) сохраняются, новые вставляются короткой транзакцией.""" + ids, new_ids, attach = {}, [], [] + existing = {(r["page_code"], r["url"]): r["id"] for r in conn.execute( + "SELECT id, page_code, url FROM wiki_attachment WHERE label = ?", (label,))} + with db.transaction(conn): + for page in pages: + for a in page["attachments"]: + key = (page["code"], a["url"]) + if key not in existing: + cur = conn.execute("INSERT INTO wiki_attachment(label, page_code, url, name) VALUES (?, ?, ?, ?)", + (label, key[0], key[1], a["name"])) + existing[key] = cur.lastrowid + new_ids.append(cur.lastrowid) + ids[key] = existing[key] + attach.append((page["code"], a["url"], a["name"])) + return ids, new_ids, attach + + +def _abandon(conn, p: Prepared) -> None: + """Подготовка не пригодилась: каталог .new и добавленные ею вложения убираются.""" + if p.new_dir is not None: + shutil.rmtree(p.new_dir, ignore_errors=True) + if p.new_ids: + marks = ",".join("?" * len(p.new_ids)) + conn.execute(f"DELETE FROM wiki_attachment WHERE id IN ({marks}) AND state = 'none'", p.new_ids) + p.new_ids = [] + + +def prepare(conn, cmd: dict, result: dict | None) -> Prepared: + """До транзакции: проверить корень и список кодов, выделить id вложений, собрать `.new`. Ошибка — в p.error.""" + label, root = cmd["params"]["label"], cmd["params"]["root"] + p = Prepared(cmd["id"], label, root) + row = conn.execute("SELECT root_code FROM wiki_project WHERE label = ?", (label,)).fetchone() + if row is None or row["root_code"] != root: + p.discarded = "корень сменился" + return p + try: + codes = (result or {}).get("codes") + if not isinstance(codes, list) or not codes or not all(isinstance(c, str) for c in codes): + raise SyncFailed("в отчёте нет списка codes") + codes = list(dict.fromkeys(codes)) + if codes[0] != root: + raise SyncFailed(f"первой в codes должна быть страница корня {root}") + stage = {r["code"]: json.loads(r["payload"]) for r in conn.execute( + "SELECT code, payload FROM wiki_sync_stage WHERE command_id = ?", (cmd["id"],))} + for c in codes: + if c not in stage: + raise SyncFailed(f"нет страницы {c}") + pages = [stage[c] for c in codes] + _, p.new_dir, _ = _siblings(conn, label) + secure_dir(p.new_dir.parent) + shutil.rmtree(p.new_dir, ignore_errors=True) + ids, p.new_ids, p.attachments = _allocate_ids(conn, label, pages) + p.kept_ids = set(ids.values()) + p.rows = build_copy(pages, codes, p.new_dir, ids) + title = (result or {}).get("root_title") + p.root_title = title if isinstance(title, str) and title.strip() else p.rows[0]["title"] + except SyncFailed as e: + p.error = str(e) + except ValueError as e: # fill_ids, путь вне копии + p.error = str(e) + except OSError as e: + p.error = f"не удалось собрать копию: {e}" + except Exception as e: # noqa: BLE001 сбой не должен оставлять команду без отчёта + log.exception("wiki_sync %s: сборка копии", cmd["id"]) + p.error = f"внутренняя ошибка сборки копии: {type(e).__name__}: {e}" + if p.error: + _abandon(conn, p) + return p + + +def before_result(conn, cmd, status, result, files_dir) -> None: + if cmd["type"] != "wiki_sync" or status != "done": + return + p = prepare(conn, cmd, result) + with _lock: + _PREPARED[cmd["id"]] = p + + +def _prepared(conn, cmd, result) -> Prepared: + with _lock: + p = _PREPARED.get(cmd["id"]) + return p if p is not None else prepare(conn, cmd, result) # в обход before_result: в транзакции, но верно + + +def check_result(conn, cmd, status, result, files_dir) -> tuple[str, str | None]: + if cmd["type"] != "wiki_sync" or status != "done": + return status, None + p = _prepared(conn, cmd, result) + with _lock: + _PREPARED[cmd["id"]] = p + if p.discarded: + if isinstance(result, dict): + result["discarded"] = p.discarded + return "done", None + if p.error: + return "failed", p.error + return "done", None + + +# --- применение (в транзакции отчёта) ---------------------------------------- + +def _swap(cur: Path, new: Path, old: Path) -> None: + """wiki/<метка> → .old, .new → <метка>, .old удаляется. При сбое прежний каталог возвращается на место.""" + shutil.rmtree(old, ignore_errors=True) + had = cur.exists() + try: + if had: + cur.rename(old) + new.rename(cur) + except OSError: + if had and not cur.exists() and old.exists(): + old.rename(cur) + raise + shutil.rmtree(old, ignore_errors=True) + + +def _fail(conn, cmd, error) -> None: + """Команда уже записана как done, но копия не применилась: failed + sync_error.""" + conn.execute("UPDATE command SET status = 'failed', error = ? WHERE id = ?", (error[:1500], cmd["id"])) + on_failed_or_reported_failed(conn, cmd, error) + + +def on_result(conn, cmd, status, result, jira_calls, rate_limited, now, files_dir) -> None: + if cmd["type"] != "wiki_sync": + return + with _lock: + p = _PREPARED.pop(cmd["id"], None) + if status != "done": + error = conn.execute("SELECT error FROM command WHERE id = ?", (cmd["id"],)).fetchone()["error"] + on_failed_or_reported_failed(conn, cmd, error or "синхронизация не удалась") + return + if p is None or p.error: + error = p.error if p else "копия не подготовлена" + on_failed_or_reported_failed(conn, cmd, error) + return + row = conn.execute("SELECT root_code FROM wiki_project WHERE label = ?", (p.label,)).fetchone() + if p.discarded or row is None or row["root_code"] != p.root: # корень сменился, пока шла подготовка + _abandon(conn, p) + if not p.discarded: + out = dict(result or {}, discarded="корень сменился") + conn.execute("UPDATE command SET result = ? WHERE id = ?", (json.dumps(out, ensure_ascii=False), cmd["id"])) + conn.execute("DELETE FROM wiki_sync_stage WHERE command_id = ?", (cmd["id"],)) + conn.execute("UPDATE wiki_project SET failed_at = ? WHERE label = ?", (now, p.label)) + return + try: + with db.transaction(conn): + _commit(conn, p, now) + except OSError as e: + _abandon(conn, p) + _fail(conn, cmd, f"не удалось подменить копию: {e}") + + +def _commit(conn, p: Prepared, now: str) -> None: + search.replace(conn, p.label, [(r["code"], r["title"], r["markdown"]) for r in p.rows]) + conn.execute("DELETE FROM wiki_page WHERE label = ?", (p.label,)) + conn.executemany( + "INSERT INTO wiki_page(label, code, eva_id, title, parent_code, orderno, depth, path, url, size, synced_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [(p.label, r["code"], r["eva_id"], r["title"], r["parent_code"], r["orderno"], r["depth"], r["path"], r["url"], + r["size"], r["synced_at"]) for r in p.rows]) + conn.executemany("UPDATE wiki_attachment SET name = ? WHERE label = ? AND page_code = ? AND url = ?", + [(name, p.label, code, url) for code, url, name in p.attachments]) + for r in conn.execute("SELECT id FROM wiki_attachment WHERE label = ?", (p.label,)).fetchall(): + if r["id"] not in p.kept_ids: + conn.execute("DELETE FROM wiki_attachment WHERE id = ?", (r["id"],)) + conn.execute("UPDATE wiki_project SET synced_at = ?, pages_count = ?, root_title = ?, sync_error = NULL, failed_at = NULL " + "WHERE label = ?", (now, len(p.rows), p.root_title, p.label)) + conn.execute("DELETE FROM wiki_sync_stage WHERE command_id = ?", (p.cmd_id,)) + cur, new, old = _siblings(conn, p.label) + _swap(cur, new, old) # последним: сбой откатывает строки (savepoint) + + +def on_failed_or_reported_failed(conn, cmd: dict, error: str | None) -> None: + """Отчёт failed или предел выдач: sync_error и failed_at проекта (если корень команды ещё текущий), стадия очищена.""" + if cmd["type"] != "wiki_sync": + return + conn.execute("DELETE FROM wiki_sync_stage WHERE command_id = ?", (cmd["id"],)) + conn.execute("UPDATE wiki_project SET sync_error = ?, failed_at = ? WHERE label = ? AND root_code = ?", + ((error or "")[:1500], db.now(), cmd["params"]["label"], cmd["params"]["root"])) + + +def on_failed(conn, cmd, now) -> None: + """Предел выдач: ошибка уже записана в команду.""" + row = conn.execute("SELECT error FROM command WHERE id = ?", (cmd["id"],)).fetchone() + on_failed_or_reported_failed(conn, cmd, row["error"] if row else None) + + +def on_claim(conn, cmd, now, redelivery) -> None: + """Повторная выдача: пачки прошлой выдачи не нужны, агент пришлёт заново.""" + if cmd["type"] == "wiki_sync" and redelivery: + conn.execute("DELETE FROM wiki_sync_stage WHERE command_id = ?", (cmd["id"],)) + + +def after_result(conn, cmd, status) -> None: + with _lock: + _PREPARED.pop(cmd["id"], None) + + +# --- уборка ---------------------------------------------------------------- + +def finish_pending_swaps(root: Path) -> None: + """При старте: `.<имя>.new` без каталога `<имя>` — переименовать (подмена прервана); `.new` рядом с `<имя>` — + остаток неудачной сборки, удаляется; `.<имя>.old` — удаляется. Каталоги, которые собирает идущая подготовка, не трогаются.""" + root = Path(root) + if not root.is_dir(): + return + with _lock: + busy = {p.new_dir for p in _PREPARED.values() if p.new_dir} + for entry in sorted(root.iterdir()): + n = entry.name + if not n.startswith(".") or entry in busy: + continue + try: + if n.endswith(".new"): + target = root / n[1:-4] + if target.exists(): + shutil.rmtree(entry, ignore_errors=True) + else: + entry.rename(target) + elif n.endswith(".old"): + shutil.rmtree(entry) + except OSError as e: + log.warning("вики: не удалось довести подмену %s: %s", entry, e) + + +def housekeeping(conn) -> None: + """Стадия команд, которые уже не в работе (отмена, терминальный статус), очищается; отменённая синхронизация + сдвигает failed_at проекта, чтобы автосинхронизация не ставила пакеты каждый тик.""" + conn.execute("DELETE FROM wiki_sync_stage WHERE command_id NOT IN " + "(SELECT id FROM command WHERE status IN ('queued', 'claimed'))") + for r in conn.execute("SELECT label, synced_at, failed_at FROM wiki_project").fetchall(): + last = max((t for t in (r["synced_at"], r["failed_at"]) if t), default="") + c = conn.execute("SELECT MAX(finished_at) AS t FROM command WHERE type = 'wiki_sync' AND status = 'cancelled' " + "AND json_extract(params, '$.label') = ?", (r["label"],)).fetchone()["t"] + if c and c > last: + conn.execute("UPDATE wiki_project SET failed_at = ? WHERE label = ?", (c, r["label"])) diff --git a/src/jira_hub/modules/wiki/tick.py b/src/jira_hub/modules/wiki/tick.py index 0dafb21..1a5cbeb 100644 --- a/src/jira_hub/modules/wiki/tick.py +++ b/src/jira_hub/modules/wiki/tick.py @@ -2,7 +2,7 @@ from datetime import timedelta from ...core import db, settings -from . import commands +from . import commands, paths, sync def auto_sync_tick(hub, conn) -> None: @@ -20,3 +20,21 @@ def auto_sync_tick(hub, conn) -> None: auto_sync_tick.label = "синхронизация вики" + + +_swept: set[str] = set() + + +def housekeeping_tick(hub, conn) -> None: + """Первый тик на каталог вики: довести прерванную подмену копии; каждый тик: очистка стадий и отмен.""" + try: + root = paths.root(conn) + except RuntimeError: # база без файла: копий на диске нет + root = None + if root is not None and str(root) not in _swept: + _swept.add(str(root)) + sync.finish_pending_swaps(root) + sync.housekeeping(conn) + + +housekeeping_tick.label = "уборка вики" diff --git a/tests/fixtures/baseline/routes.expected.json b/tests/fixtures/baseline/routes.expected.json index d6b2bef..3c6a85c 100644 --- a/tests/fixtures/baseline/routes.expected.json +++ b/tests/fixtures/baseline/routes.expected.json @@ -71,6 +71,12 @@ "agent", "h_agent_state" ], + [ + "POST", + "^/agent/wiki/pages$", + "agent", + "h_agent_pages" + ], [ "GET", "^/agents$", diff --git a/tests/test_registry.py b/tests/test_registry.py index 1467510..5fc7bea 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -51,11 +51,12 @@ def test_route_conflict(): def test_routes_assembled(): from jira_hub.core import api - from jira_hub.modules import jira + from jira_hub.modules import jira, wiki pairs = lambda routes: {(m, p) for m, p, _, _ in routes} # noqa: E731 - assert pairs(api.ROUTES) == pairs(api.CORE_ROUTES) | pairs(jira.MODULE.routes) - assert not pairs(api.CORE_ROUTES) & pairs(jira.MODULE.routes) - assert len(api.ROUTES) == len(api.CORE_ROUTES) + len(jira.MODULE.routes) + mods = jira.MODULE.routes + wiki.MODULE.routes + assert pairs(api.ROUTES) == pairs(api.CORE_ROUTES) | pairs(mods) + assert not pairs(api.CORE_ROUTES) & pairs(mods) + assert len(api.ROUTES) == len(api.CORE_ROUTES) + len(mods) def test_module_of(monkeypatch): diff --git a/tests/test_wiki_sync.py b/tests/test_wiki_sync.py new file mode 100644 index 0000000..c7de22b --- /dev/null +++ b/tests/test_wiki_sync.py @@ -0,0 +1,435 @@ +"""Синхронизация вики: приём пачек, сборка копии, поиск (wiki-module, spec wiki «Синхронизация вики», «Markdown-копия»). +Агента имитирует тест: HTTP на порту 0 токеном агента wiki.""" +import json +import logging +import sqlite3 +from datetime import timedelta +from types import SimpleNamespace + +import pytest + +from jira_hub.core import db, queue, settings +from jira_hub.core.client import HubClient, HubError +from jira_hub.modules.wiki import commands, convert, projects, search, stage, sync, tick + + +def page(code, title, parent=None, orderno=None, html=None, attachments=None, links=None): + return {"code": code, "eva_id": "id-" + code, "title": title, "parent_code": parent, "orderno": orderno, + "url": f"https://eva.example.ru/project/Document/{code}", "html": html if html is not None else f"

текст {title}

", + "attachments": attachments or [], "links": links or []} + + +TREE = [page("DOC-1", "АРМ КА"), page("DOC-2", "Роли/права", "DOC-1", 2), page("DOC-3", "Статусы", "DOC-1", 1, + "

статус «На согласовании» ставит аналитик

")] +CODES = ["DOC-1", "DOC-3", "DOC-2"] + + +@pytest.fixture +def api(hub_server, tmp_path): + hub = hub_server["hub"] + conn = hub.conn() + token = settings.tokens(conn)["agents"]["wiki"] + conn.close() + wiki = HubClient(hub_server["base"], token) + hub_server["client"].post("/projects", {"label": "AWPCA", "name": "АРМ КА", "wiki_root": "DOC-1"}) + + def db_rows(sql, *args): + c = hub.conn() + try: + return [dict(r) for r in c.execute(sql, args)] + finally: + c.close() + + def enqueue(): + c = hub.conn() + try: + with db.transaction(c): + return commands.enqueue_sync(c, "AWPCA", "DOC-1", "test") + finally: + c.close() + + def claim(): + return wiki.post("/agent/claim", {"agent_version": 1})["command"] + + def report(cmd, status="done", result=None, error=None): + return wiki.post(f"/agent/commands/{cmd['id']}/result", {"status": status, "result": result, "error": error}) + + def run(pages, codes=None, root_title="АРМ КА", status="done", error=None): + """Целый обмен: взять команду, прислать страницы, отчитаться. Возвращает команду после отчёта.""" + cmd = claim() + assert cmd and cmd["type"] == "wiki_sync", cmd + if pages: + wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": pages}) + result = {"root_title": root_title, "codes": codes if codes is not None else [p["code"] for p in pages], + "eva_calls": 3} if status == "done" else None + return report(cmd, status, result, error) + + def project(): + return {p["label"]: p for p in hub_server["client"].get("/projects")["projects"]}["AWPCA"] + + return SimpleNamespace(hub=hub, wiki=wiki, client=hub_server["client"], base=hub_server["base"], rows=db_rows, + enqueue=enqueue, claim=claim, report=report, run=run, project=project, + copy=tmp_path / "wiki" / "AWPCA", wiki_dir=tmp_path / "wiki", db_path=tmp_path / "hub.sqlite", + jira=hub_server["agent"]) + + +def first_sync(api): + done = api.run(TREE, CODES) + assert done["status"] == "done" + return done + + +def test_first_sync(api): + first_sync(api) + assert (api.copy / "АРМ КА" / "index.md").is_file() + assert (api.copy / "АРМ КА" / "01_Статусы" / "index.md").is_file() + assert (api.copy / "АРМ КА" / "02_Роли_права" / "index.md").is_file() + assert [t["code"] for t in json.loads((api.copy / "_tree.json").read_text(encoding="utf-8"))] == CODES + text = (api.copy / "АРМ КА" / "01_Статусы" / "index.md").read_text(encoding="utf-8") + assert text.startswith('---\ncode: "DOC-3"\ntitle: "Статусы"\n') and "\n# Статусы\n" in text and "аналитик" in text + p = api.project() + assert p["wiki_pages"] == 3 and p["wiki_root_title"] == "АРМ КА" and p["wiki_synced_at"] and p["wiki_sync_error"] is None + assert api.rows("SELECT COUNT(*) AS n FROM wiki_sync_stage")[0]["n"] == 0 # стадия очищена + assert {r["code"]: r["path"] for r in api.rows("SELECT code, path FROM wiki_page")} == { + "DOC-1": "АРМ КА", "DOC-3": "АРМ КА/01_Статусы", "DOC-2": "АРМ КА/02_Роли_права"} + assert oct(api.copy.stat().st_mode & 0o777) == "0o700" and oct(api.wiki_dir.stat().st_mode & 0o777) == "0o700" + assert oct((api.copy / "АРМ КА" / "01_Статусы").stat().st_mode & 0o777) == "0o700" + + +def test_failed_keeps_copy(api): + first_sync(api) + api.enqueue() + api.run(None, status="failed", error="Eva: HTTP 503") + p = api.project() + assert p["wiki_sync_error"] == "Eva: HTTP 503" and p["wiki_pages"] == 3 + assert (api.copy / "АРМ КА" / "01_Статусы" / "index.md").is_file() + conn = api.hub.conn() + try: + assert [r["code"] for r in search.query(conn, "AWPCA", "согласов", 10)] == ["DOC-3"] + finally: + conn.close() + assert api.rows("SELECT failed_at FROM wiki_project")[0]["failed_at"] + + +def test_missing_page_fails(api): + first_sync(api) + api.enqueue() + done = api.run([TREE[0]], ["DOC-1", "DOC-2"]) + assert done["status"] == "failed" and done["error"] == "нет страницы DOC-2" + assert api.project()["wiki_sync_error"] == "нет страницы DOC-2" + assert (api.copy / "АРМ КА" / "02_Роли_права" / "index.md").is_file() + assert api.rows("SELECT COUNT(*) AS n FROM wiki_sync_stage")[0]["n"] == 0 + + +def test_page_removed(api): + shot = page("DOC-3", "Статусы", "DOC-1", 1, '

см.

', + [{"ref": "/files/x/scheme.png", "url": "https://eva.example.ru/files/x/scheme.png", "name": "scheme.png"}]) + api.run([TREE[0], TREE[1], shot], CODES) + assert len(api.rows("SELECT * FROM wiki_attachment")) == 1 + api.enqueue() + api.run([TREE[0], page("DOC-3", "Статусы", "DOC-1", 1, "

без картинки

")], ["DOC-1", "DOC-3"]) + assert not list(api.copy.rglob("*Роли*")) + assert api.rows("SELECT code FROM wiki_page ORDER BY code") == [{"code": "DOC-1"}, {"code": "DOC-3"}] + assert api.rows("SELECT * FROM wiki_attachment") == [] + conn = api.hub.conn() + try: + assert search.query(conn, "AWPCA", "Роли", 10) == [] + assert api.project()["wiki_pages"] == 2 + finally: + conn.close() + + +def _with_file(text="см. файл"): + att = [{"ref": "/files/x/model.xlsx", "url": "https://eva.example.ru/files/x/model.xlsx", "name": "model.xlsx"}] + return [TREE[0], page("DOC-3", "Статусы", "DOC-1", 1, f'

{text} model

', att)] + + +def test_attachment_id_survives(api): + api.run(_with_file(), ["DOC-1", "DOC-3"]) + (a,) = api.rows("SELECT id, state FROM wiki_attachment") + md = (api.copy / "АРМ КА" / "01_Статусы" / "index.md").read_text(encoding="utf-8") + assert f"[📎 model.xlsx](attachment:{a['id']})" in md and a["state"] == "none" + c = api.hub.conn() + c.execute("UPDATE wiki_attachment SET state = 'ready', file = 'ab', size = 5 WHERE id = ?", (a["id"],)) + c.close() + api.enqueue() + api.run(_with_file("новый текст"), ["DOC-1", "DOC-3"]) + (b,) = api.rows("SELECT id, state, file FROM wiki_attachment") + assert (b["id"], b["state"], b["file"]) == (a["id"], "ready", "ab") + assert f"(attachment:{a['id']})" in (api.copy / "АРМ КА" / "01_Статусы" / "index.md").read_text(encoding="utf-8") + + +def test_stale_root_discarded(api): + cmd = api.claim() + assert cmd["params"]["root"] == "DOC-1" + api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0]]}) + api.client.patch("/projects/AWPCA", {"wiki_root": "DOC-21"}) + done = api.report(cmd, "done", {"root_title": "X", "codes": ["DOC-1"]}) + assert done["status"] == "done" and done["result"]["discarded"] == "корень сменился" + assert not api.copy.exists() and api.rows("SELECT * FROM wiki_page") == [] + p = api.project() + assert p["wiki_root"] == "DOC-21" and p["wiki_pages"] == 0 and p["wiki_synced_at"] is None + assert api.rows("SELECT COUNT(*) AS n FROM wiki_sync_stage")[0]["n"] == 0 + assert api.rows("SELECT failed_at FROM wiki_project")[0]["failed_at"] # часы автосинхронизации сдвинуты + + +def test_redelivery_clears_stage(api): + cmd = api.claim() + api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0]]}) + assert api.rows("SELECT COUNT(*) AS n FROM wiki_sync_stage")[0]["n"] == 1 + again = api.claim() + assert again["id"] == cmd["id"] and api.rows("SELECT COUNT(*) AS n FROM wiki_sync_stage")[0]["n"] == 0 + done = api.report(again, "done", {"root_title": "X", "codes": ["DOC-1"]}) + assert done["status"] == "failed" and done["error"] == "нет страницы DOC-1" + + +def test_pages_wrong_agent(api): + cmd = api.claim() + with pytest.raises(HubError) as e: + api.jira.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0]]}) + assert e.value.status == 401 + + +def test_pages_not_claimed(api): + (row,) = api.rows("SELECT id FROM command WHERE type = 'wiki_sync'") + with pytest.raises(HubError) as e: + api.wiki.post("/agent/wiki/pages", {"command_id": row["id"], "pages": [TREE[0]]}) + assert e.value.status == 409 and e.value.error == "not_claimed" + + +def test_bad_page(api): + cmd = api.claim() + bad = {k: v for k, v in TREE[1].items() if k != "title"} + with pytest.raises(HubError) as e: + api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0], bad]}) + assert e.value.status == 422 and e.value.error == "bad_pages" and e.value.detail == "DOC-2: title" + assert api.rows("SELECT COUNT(*) AS n FROM wiki_sync_stage")[0]["n"] == 0 # пачка не принята целиком + for field, value in (("orderno", "1"), ("parent_code", 5), ("html", None)): + with pytest.raises(HubError) as e: + api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0] | {field: value}]}) + assert e.value.status == 422 and e.value.detail == f"DOC-1: {field}" + + +def test_layout_names_safe(tmp_path): + titles = ["Корень", "../x", "a/b", "😀" * 200, "Статусы", "Статусы", "..", " . ", 'a:"c|?*'] + pages = [{"code": f"D{i}", "title": t, "markdown": "текст\n", "attachments": [], "orderno": i, + "parent_code": None if i == 0 else "D0", "eva_id": None, "url": None} for i, t in enumerate(titles)] + target = tmp_path / "t" + rows = sync.build_copy(pages, [p["code"] for p in pages], target, {}) + paths_ = [r["path"] for r in rows] + for r in rows: + assert (target / r["path"] / "index.md").resolve().is_relative_to(target.resolve()) + assert ".." not in r["path"].split("/") and all(len(part.encode()) <= 255 for part in r["path"].split("/")) + assert "Корень/04_Статусы" in paths_ and "Корень/05_Статусы_2" in paths_ + assert "Корень/06_untitled" in paths_ and "Корень/07_untitled_2" in paths_ + assert "Корень/02_a_b" in paths_ and "Корень/01__x" in paths_ + assert sorted(p.name for p in (target / "Корень").iterdir() if p.is_dir()) == sorted(p.split("/")[1] for p in paths_[1:]) + + +def test_layout_orphans_and_cycles(tmp_path): + def mk(code, parent, order=None): + return {"code": code, "title": code, "markdown": "x\n", "attachments": [], "orderno": order, "parent_code": parent, + "eva_id": None, "url": None} + pages = [mk("R", None), mk("A", "R", 2), mk("B", "R"), mk("C", "GONE"), mk("X", "Y"), mk("Y", "X")] + rows = sync.build_copy(pages, [p["code"] for p in pages], tmp_path / "t", {}) + assert {r["code"] for r in rows} == {"R", "A", "B", "C", "X", "Y"} and len(rows) == 6 # никто не потерян + assert [r["code"] for r in rows if r["depth"] == 1][:2] == ["A", "B"] # orderno, затем None по коду + + +def test_finish_pending_swap(tmp_path): + root = tmp_path / "wiki" + (root / ".AWPCA.new" / "x").mkdir(parents=True) + (root / ".AWPCA.old").mkdir() + (root / ".B.new").mkdir() + (root / "B").mkdir() # остаток неудачной сборки рядом с живой копией + sync.finish_pending_swaps(root) + assert (root / "AWPCA" / "x").is_dir() and not (root / ".AWPCA.old").exists() and not (root / ".AWPCA.new").exists() + assert (root / "B").is_dir() and not (root / ".B.new").exists() + sync.finish_pending_swaps(root / "нет") # нет каталога — не ошибка + + +def test_search_prefix(open_db): + conn = open_db() + conn.execute("INSERT INTO project(label, name, created_at) VALUES ('AWPCA', 'A', '2026-01-01')") + assert search.ensure(conn) is True and search.ensure(conn) is True + search.replace(conn, "AWPCA", [("DOC-3", "Статусы", "статус «На согласовании» ставит аналитик"), + ("DOC-4", "Роли", "роль администратора")]) + search.replace(conn, "OTHER", [("DOC-9", "Чужая", "согласование чужого проекта")]) + conn.execute("INSERT INTO wiki_page(label, code, title, path) VALUES ('AWPCA', 'DOC-3', 'Статусы', 'A/01_Статусы')") + (hit,) = search.query(conn, "AWPCA", "согласов", 10) + assert hit["code"] == "DOC-3" and "На согласовании" in hit["snippet"] and hit["path"] == "A/01_Статусы" + assert search.query(conn, "AWPCA", 'согласов"', 10)[0]["code"] == "DOC-3" # кавычка не роняет + for q in ('"', "", "* AND (", "NEAR(", "статус ОR"): + assert isinstance(search.query(conn, "AWPCA", q, 5), list) + assert [r["code"] for r in search.query(conn, "AWPCA", "роль админ", 10)] == ["DOC-4"] + assert search.query(conn, "AWPCA", "чужого", 10) == [] + + +def test_search_without_fts_uses_files(api, monkeypatch): + first_sync(api) + c = api.hub.conn() + try: + c.execute("DROP TABLE wiki_page_fts") + (hit,) = search.query(c, "AWPCA", "согласов", 10) + assert hit["code"] == "DOC-3" and "На согласовании" in hit["snippet"] + finally: + c.close() + + +# --- пункты контроллера --------------------------------------------------------- + +def test_failed_at_prevents_tick_storm(api): + """Любая неудача пишет failed_at, и автотик не ставит новый пакет сразу.""" + def tick_once(): + c = api.hub.conn() + try: + tick.auto_sync_tick(None, c) + finally: + c.close() + return api.rows("SELECT COUNT(*) AS n FROM batch WHERE project = 'AWPCA'")[0]["n"] + + first_sync(api) + assert api.rows("SELECT synced_at, failed_at FROM wiki_project")[0]["synced_at"] + assert tick_once() == 1 # свежая успешная + c = api.hub.conn() + c.execute("UPDATE wiki_project SET synced_at = ?, failed_at = NULL", ((db.utcnow() - timedelta(hours=30)).isoformat(),)) + c.close() + # failed: пакет выдан, агент отчитался failed + api.enqueue() + api.run(None, status="failed", error="Eva: HTTP 503") + assert tick_once() == 2 + # отброшенная + c = api.hub.conn() + c.execute("UPDATE wiki_project SET synced_at = ?, failed_at = NULL", ((db.utcnow() - timedelta(hours=30)).isoformat(),)) + c.close() + api.enqueue() + claimed = api.claim() + api.client.patch("/projects/AWPCA", {"wiki_root": "DOC-21"}) + api.report(claimed, "done", {"root_title": "X", "codes": ["DOC-1"]}) + assert api.rows("SELECT failed_at FROM wiki_project")[0]["failed_at"] + # отменённая + c = api.hub.conn() + c.execute("UPDATE wiki_project SET synced_at = ?, failed_at = NULL", ((db.utcnow() - timedelta(hours=30)).isoformat(),)) + queued = api.rows("SELECT id FROM command WHERE type = 'wiki_sync' AND status = 'queued'") + for q in queued: + queue.cancel_command(c, q["id"]) + c.close() + c = api.hub.conn() + try: + tick.housekeeping_tick(None, c) + assert api.rows("SELECT failed_at FROM wiki_project")[0]["failed_at"] + before = api.rows("SELECT COUNT(*) AS n FROM batch")[0]["n"] + tick.auto_sync_tick(None, c) + assert api.rows("SELECT COUNT(*) AS n FROM batch")[0]["n"] == before + finally: + c.close() + + +def test_fill_ids_error_becomes_failed(api): + cmd = api.claim() + bad = {"code": "DOC-1", "eva_id": None, "title": "АРМ КА", "parent_code": None, "orderno": None, "url": None, + "markdown": "[📎 f](attachment:@5)\n", "attachments": []} + c = api.hub.conn() + c.execute("INSERT INTO wiki_sync_stage VALUES (?, 'DOC-1', ?)", (cmd["id"], json.dumps(bad))) + c.close() + done = api.report(cmd, "done", {"root_title": "X", "codes": ["DOC-1"]}) + assert done["status"] == "failed" and done["error"].startswith("DOC-1: ") and "вне списка" in done["error"] + assert api.project()["wiki_sync_error"].startswith("DOC-1: ") and not api.copy.exists() + assert not list(api.wiki_dir.glob(".*")) # .new убран + + +def test_long_name_failed_not_crash(hub_server, tmp_path): + label = "/" * 100 # dir_name даёт 300 символов + hub_server["client"].post("/projects", {"label": label, "name": "длинная", "wiki_root": "DOC-1"}) + hub = hub_server["hub"] + conn = hub.conn() + token = settings.tokens(conn)["agents"]["wiki"] + conn.close() + wiki = HubClient(hub_server["base"], token) + cmd = wiki.post("/agent/claim", {})["command"] + wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0]]}) + done = wiki.post(f"/agent/commands/{cmd['id']}/result", {"status": "done", "result": {"root_title": "X", "codes": ["DOC-1"]}}) + assert done["status"] == "failed" and done["error"].startswith("не удалось собрать копию") + conn = hub.conn() + try: + row = conn.execute("SELECT sync_error, failed_at FROM wiki_project WHERE label = ?", (label,)).fetchone() + finally: + conn.close() + assert row["sync_error"].startswith("не удалось собрать копию") and row["failed_at"] + + +def test_reset_root_clears_fts_and_dir(api): + first_sync(api) + c = api.hub.conn() + assert c.execute("SELECT COUNT(*) AS n FROM wiki_page_fts").fetchone()["n"] == 3 + c.close() + api.client.patch("/projects/AWPCA", {"wiki_root": None}) + c = api.hub.conn() + try: + assert c.execute("SELECT COUNT(*) AS n FROM wiki_page_fts").fetchone()["n"] == 0 + assert c.execute("SELECT COUNT(*) AS n FROM wiki_page").fetchone()["n"] == 0 + finally: + c.close() + assert not api.copy.exists() + + +def test_drop_copy_logs_rmtree_failure(api, monkeypatch, caplog): + first_sync(api) + real = projects.shutil.rmtree + + def boom(path, *a, **k): + if "AWPCA" in str(path): + raise PermissionError("нельзя") + return real(path, *a, **k) + + monkeypatch.setattr(projects.shutil, "rmtree", boom) + with caplog.at_level(logging.WARNING, logger=projects.__name__): + api.client.patch("/projects/AWPCA", {"wiki_root": None}) + assert "не удалён" in caplog.text and "нельзя" in caplog.text + + +def test_no_heavy_work_under_write_lock(api, monkeypatch): + """Перевод HTML и запись файлов идут вне BEGIN IMMEDIATE: параллельное соединение может открыть запись.""" + seen = [] + + def probe(tag): + other = sqlite3.connect(api.db_path, timeout=0) + try: + other.execute("BEGIN IMMEDIATE") + other.execute("ROLLBACK") + seen.append((tag, "free")) + except sqlite3.OperationalError: + seen.append((tag, "locked")) + finally: + other.close() + + real_md, real_build = convert.to_markdown, sync.build_copy + monkeypatch.setattr(stage.convert, "to_markdown", lambda *a, **k: (probe("markdown"), real_md(*a, **k))[1]) + monkeypatch.setattr(sync, "build_copy", lambda *a, **k: (probe("files"), real_build(*a, **k))[1]) + first_sync(api) + assert ("markdown", "free") in seen and ("files", "free") in seen and not [s for s in seen if s[1] == "locked"] + + +def test_swap_failure_rolls_back_rows(api, monkeypatch): + first_sync(api) + api.enqueue() + + def boom(cur, new, old): + raise OSError("диск полон") + + monkeypatch.setattr(sync, "_swap", boom) + done = api.run([TREE[0], page("DOC-3", "Новая", "DOC-1", 1)], ["DOC-1", "DOC-3"]) + assert done["status"] == "failed" and "не удалось подменить копию" in done["error"] + assert api.project()["wiki_pages"] == 3 and "диск полон" in api.project()["wiki_sync_error"] + assert [r["code"] for r in api.rows("SELECT code FROM wiki_page ORDER BY code")] == ["DOC-1", "DOC-2", "DOC-3"] + assert (api.copy / "АРМ КА" / "01_Статусы" / "index.md").is_file() and not list(api.wiki_dir.glob(".*")) + + +def test_batch_limit_and_replace(api): + cmd = api.claim() + with pytest.raises(HubError) as e: + api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [page(f"DOC-{i}", "t") for i in range(51)]}) + assert e.value.status == 422 + api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0]]}) + api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0] | {"title": "Новый"}]}) # повтор — замена + assert [json.loads(r["payload"])["title"] for r in api.rows("SELECT payload FROM wiki_sync_stage")] == ["Новый"]