wiki: синхронизация — пачки страниц, копия, поиск (#91)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrey Epifancev
parent
4faf7884ad
commit
bc69d5087a
@ -30,6 +30,8 @@ class CommandHooks:
|
||||
on_result: Callable[[Connection, dict, str, dict | None, int, int, str, Path | None], None] # в транзакции
|
||||
after_result: Callable[[Connection, dict, str], None] # вне транзакции
|
||||
on_late_result: Callable[[Connection, dict, str, dict | None, int, int], bool]
|
||||
# (conn, cmd, status, result, files_dir): до транзакции отчёта — тяжёлая подготовка (файлы, перевод); None — нет
|
||||
before_result: Callable[[Connection, dict, str, dict | None, Path | None], None] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@ -295,6 +295,11 @@ def report_result(conn, command_id, status, result=None, error=None, jira_calls=
|
||||
if status not in ("done", "failed"):
|
||||
raise ValueError("status — done или failed")
|
||||
error = None if error is None else str(error)[:ERROR_LIMIT]
|
||||
early = get_command(conn, command_id)
|
||||
if early is not None and early["status"] == "claimed":
|
||||
eh = _hooks(early["type"])
|
||||
if eh and eh.before_result: # вне транзакции отчёта: модуль готовит тяжёлое, не держа блокировку базы
|
||||
eh.before_result(conn, early, status, result, files_dir)
|
||||
with db.transaction(conn):
|
||||
cmd = get_command(conn, command_id)
|
||||
if cmd is None:
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
"""Модуль wiki: Markdown-копия вики Eva по проектам, поиск, вложения по запросу."""
|
||||
from ...core.modules import Module
|
||||
from . import commands, projects, schema, settings, tick
|
||||
from . import api, commands, projects, schema, settings, tick
|
||||
|
||||
MODULE = Module(
|
||||
name="wiki",
|
||||
@ -10,7 +10,8 @@ MODULE = Module(
|
||||
agent_min_version_key="wiki_min_agent_version",
|
||||
validate_setting=settings.validate_setting,
|
||||
commands=commands.HOOKS,
|
||||
tick=(tick.auto_sync_tick,),
|
||||
routes=api.ROUTES,
|
||||
tick=(tick.housekeeping_tick, tick.auto_sync_tick),
|
||||
project_fields=projects.FIELDS,
|
||||
project_view=projects.view,
|
||||
on_project_created=projects.on_created,
|
||||
|
||||
11
src/jira_hub/modules/wiki/api.py
Normal file
11
src/jira_hub/modules/wiki/api.py
Normal file
@ -0,0 +1,11 @@
|
||||
"""HTTP модуля wiki: маршруты агента (приём пачек страниц)."""
|
||||
from . import stage
|
||||
|
||||
|
||||
def h_agent_pages(hub, conn, m, q, body):
|
||||
return 200, stage.accept(conn, body.get("command_id"), body.get("pages"))
|
||||
|
||||
|
||||
ROUTES = (
|
||||
("POST", r"^/agent/wiki/pages$", h_agent_pages, "agent"),
|
||||
)
|
||||
@ -4,6 +4,7 @@ import re
|
||||
from ...core import queue
|
||||
from ...core.errors import CommandError
|
||||
from ...core.modules import CommandHooks
|
||||
from . import sync
|
||||
|
||||
TYPES = {"wiki_sync", "wiki_fetch"}
|
||||
SPEC = {"wiki_sync": {"required": ["label", "root"], "optional": []},
|
||||
@ -68,14 +69,12 @@ def active_sync(conn, label: str) -> dict | None:
|
||||
return queue.batch_view(conn, row["id"], with_commands=False) if row else None
|
||||
|
||||
|
||||
_noop = lambda *a, **k: None # noqa: E731
|
||||
|
||||
# Хуки результата пустые: их заполняют приём пачек и отчёт синхронизации
|
||||
# wiki_fetch: хуки результата заполнит Task 8; wiki_sync — sync.py
|
||||
HOOKS = CommandHooks(
|
||||
types=frozenset(TYPES), read_types=frozenset(TYPES), validate=validate, check_batch=check_batch,
|
||||
tail=lambda cmds: None, pause=lambda conn, ctype: 0, deliver_params=lambda conn, cmd, files_dir: cmd["params"],
|
||||
temp_definer=None, temp_keys_used=lambda cmd: [], substitute=lambda ctype, params, mapping: params,
|
||||
drop_unresolved=lambda cmd, unresolved: unresolved, retry_params=lambda cmd: cmd["params"],
|
||||
check_confirmed=lambda conn, cmd: (False, None), on_claim=_noop, on_failed=_noop,
|
||||
check_result=lambda conn, cmd, status, result, files_dir: (status, None), on_result=_noop, after_result=_noop,
|
||||
on_late_result=lambda *a: False)
|
||||
check_confirmed=lambda conn, cmd: (False, None), on_claim=sync.on_claim, on_failed=sync.on_failed,
|
||||
check_result=sync.check_result, on_result=sync.on_result, after_result=sync.after_result,
|
||||
on_late_result=lambda *a: False, before_result=sync.before_result)
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
"""Корень вики проекта: поле wiki_root, значения для GET /projects, хуки создания и правки проекта."""
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
|
||||
@ -6,6 +7,8 @@ from ...core import db, queue
|
||||
from ...core.modules import ProjectField
|
||||
from . import commands, paths
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_CODE = re.compile(r"(?<![A-Za-z0-9])DOC-(\d+)", re.IGNORECASE)
|
||||
|
||||
|
||||
@ -63,14 +66,21 @@ def _has_table(conn, name: str) -> bool:
|
||||
def _drop_copy(conn, label: str) -> None:
|
||||
"""Сброс корня: строки проекта, индекс поиска (если он уже создан) и каталог копии."""
|
||||
if _has_table(conn, "wiki_page_fts"):
|
||||
conn.execute("DELETE FROM wiki_page_fts WHERE rowid IN (SELECT rowid FROM wiki_page WHERE label = ?)", (label,))
|
||||
conn.execute("DELETE FROM wiki_page_fts WHERE label = ?", (label,))
|
||||
conn.execute("DELETE FROM wiki_page WHERE label = ?", (label,))
|
||||
conn.execute("DELETE FROM wiki_attachment WHERE label = ?", (label,))
|
||||
conn.execute("DELETE FROM wiki_project WHERE label = ?", (label,))
|
||||
try:
|
||||
shutil.rmtree(paths.project_dir(conn, label), ignore_errors=True)
|
||||
base = paths.project_dir(conn, label)
|
||||
except RuntimeError: # база без файла (тесты в памяти): копии на диске нет
|
||||
pass
|
||||
return
|
||||
for d in (base, base.with_name(f".{base.name}.new"), base.with_name(f".{base.name}.old")):
|
||||
try:
|
||||
shutil.rmtree(d)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError as e:
|
||||
log.warning("вики: каталог %s не удалён: %s", d, e)
|
||||
|
||||
|
||||
def on_updated(conn, project, fields, author) -> list[dict]:
|
||||
|
||||
88
src/jira_hub/modules/wiki/search.py
Normal file
88
src/jira_hub/modules/wiki/search.py
Normal file
@ -0,0 +1,88 @@
|
||||
"""Поиск по копии вики: FTS5 (wiki_page_fts) и запасной LIKE по файлам index.md, если FTS5 в SQLite нет."""
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
from . import paths
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_WORD = re.compile(r"\w+")
|
||||
_warned = False
|
||||
|
||||
|
||||
def _has_table(conn) -> bool:
|
||||
return conn.execute("SELECT 1 FROM sqlite_master WHERE name = 'wiki_page_fts'").fetchone() is not None
|
||||
|
||||
|
||||
def ensure(conn) -> bool:
|
||||
"""Создать индекс, если его нет. False — FTS5 недоступен (одно предупреждение в лог на процесс)."""
|
||||
global _warned
|
||||
if _has_table(conn):
|
||||
return True
|
||||
try:
|
||||
conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS wiki_page_fts USING fts5(label UNINDEXED, code UNINDEXED, "
|
||||
"title, body, tokenize='unicode61 remove_diacritics 2')")
|
||||
except sqlite3.OperationalError as e:
|
||||
if not _warned:
|
||||
_warned = True
|
||||
log.warning("FTS5 недоступен, поиск по вики идёт по файлам: %s", e)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def replace(conn, label: str, pages: list[tuple[str, str, str]]) -> None:
|
||||
"""Индекс проекта заново: pages — [(код, название, Markdown)]. Без FTS5 — ничего."""
|
||||
if not ensure(conn):
|
||||
return
|
||||
conn.execute("DELETE FROM wiki_page_fts WHERE label = ?", (label,))
|
||||
conn.executemany("INSERT INTO wiki_page_fts(label, code, title, body) VALUES (?, ?, ?, ?)",
|
||||
[(label, code, title, body) for code, title, body in pages])
|
||||
|
||||
|
||||
def _match(q: str) -> str | None:
|
||||
"""Слова запроса → `"слово"* AND …`: префиксный поиск, кавычки экранируются. Без слов — None."""
|
||||
words = [w.replace('"', '""') for w in _WORD.findall(q or "")]
|
||||
return " AND ".join(f'"{w}"*' for w in words) if words else None
|
||||
|
||||
|
||||
def query(conn, label: str, q: str, limit: int) -> list[dict]:
|
||||
"""[{code, title, path, snippet}] по убыванию релевантности."""
|
||||
match = _match(q)
|
||||
if match is None:
|
||||
return []
|
||||
pages = {r["code"]: r for r in conn.execute("SELECT code, title, path FROM wiki_page WHERE label = ?", (label,))}
|
||||
if _has_table(conn):
|
||||
try:
|
||||
rows = conn.execute("SELECT code, title, snippet(wiki_page_fts, 3, '', '', '…', 16) AS snippet "
|
||||
"FROM wiki_page_fts WHERE wiki_page_fts MATCH ? AND label = ? ORDER BY rank LIMIT ?",
|
||||
(match, label, limit)).fetchall()
|
||||
except sqlite3.OperationalError as e:
|
||||
log.warning("поиск по вики: запрос «%s» отвергнут: %s", q, e)
|
||||
return []
|
||||
return [{"code": r["code"], "title": r["title"], "path": pages[r["code"]]["path"] if r["code"] in pages else None,
|
||||
"snippet": r["snippet"]} for r in rows]
|
||||
return _like(conn, label, _WORD.findall(q), pages, limit)
|
||||
|
||||
|
||||
def _like(conn, label, words, pages, limit) -> list[dict]:
|
||||
"""Без FTS5: страницы проекта, где есть все слова (без учёта регистра), по числу вхождений."""
|
||||
try:
|
||||
base = paths.project_dir(conn, label)
|
||||
except RuntimeError:
|
||||
return []
|
||||
found = []
|
||||
needles = [w.casefold() for w in words]
|
||||
for code, r in pages.items():
|
||||
try:
|
||||
text = (base / r["path"] / "index.md").read_text(encoding="utf-8")
|
||||
except (OSError, TypeError):
|
||||
continue
|
||||
low = text.casefold()
|
||||
if all(n in low for n in needles):
|
||||
at = low.find(needles[0])
|
||||
start = max(0, at - 60)
|
||||
found.append((-sum(low.count(n) for n in needles), code, r["title"], r["path"],
|
||||
("…" if start else "") + text[start:at + 100].replace("\n", " ") + "…"))
|
||||
found.sort()
|
||||
return [{"code": c, "title": t, "path": p, "snippet": s} for _, c, t, p, s in found[:limit]]
|
||||
81
src/jira_hub/modules/wiki/stage.py
Normal file
81
src/jira_hub/modules/wiki/stage.py
Normal file
@ -0,0 +1,81 @@
|
||||
"""Приём пачек страниц от агента wiki: проверка формы, перевод HTML → Markdown (вне транзакции), стадия команды."""
|
||||
import json
|
||||
|
||||
from ...core import db, queue
|
||||
from ...core.http import ApiError
|
||||
from . import convert
|
||||
|
||||
MAX_PAGES = 50
|
||||
|
||||
|
||||
def _bad(code, field):
|
||||
return ApiError(422, "bad_pages", f"{code}: {field}")
|
||||
|
||||
|
||||
def _text(page, key, code, required=True, nullable=False):
|
||||
v = page.get(key)
|
||||
if v is None and (nullable or not required):
|
||||
return None
|
||||
if not isinstance(v, str) or (required and not v.strip()):
|
||||
raise _bad(code, key)
|
||||
return v
|
||||
|
||||
|
||||
def _check(i, page) -> dict:
|
||||
"""Форма одной страницы → поля для стадии (без html); ApiError 422 `<код>: <поле>`."""
|
||||
if not isinstance(page, dict):
|
||||
raise ApiError(422, "bad_pages", f"страница {i}: не объект")
|
||||
code = page.get("code")
|
||||
if not isinstance(code, str) or not code.strip():
|
||||
raise ApiError(422, "bad_pages", f"страница {i}: code")
|
||||
title = _text(page, "title", code)
|
||||
html = page.get("html")
|
||||
if not isinstance(html, str):
|
||||
raise _bad(code, "html")
|
||||
orderno = page.get("orderno")
|
||||
if orderno is not None and (not isinstance(orderno, int) or isinstance(orderno, bool)):
|
||||
raise _bad(code, "orderno")
|
||||
eva_id = page.get("eva_id")
|
||||
if eva_id is not None and not isinstance(eva_id, str):
|
||||
raise _bad(code, "eva_id")
|
||||
attachments, links = page.get("attachments") or [], page.get("links") or []
|
||||
if not isinstance(attachments, list) or not all(
|
||||
isinstance(a, dict) and isinstance(a.get("ref"), str) and isinstance(a.get("url"), str)
|
||||
and isinstance(a.get("name") or "", str) for a in attachments):
|
||||
raise _bad(code, "attachments")
|
||||
if not isinstance(links, list) or not all(
|
||||
isinstance(l, dict) and isinstance(l.get("ref"), str) and isinstance(l.get("code"), str) for l in links):
|
||||
raise _bad(code, "links")
|
||||
return {"code": code, "eva_id": eva_id, "title": title, "parent_code": _text(page, "parent_code", code, False, True),
|
||||
"orderno": orderno, "url": _text(page, "url", code, False, True), "html": html,
|
||||
"attachments": [{"ref": a["ref"], "url": a["url"], "name": a.get("name") or ""} for a in attachments],
|
||||
"links": [{"ref": l["ref"], "code": l["code"]} for l in links]}
|
||||
|
||||
|
||||
def accept(conn, command_id, pages) -> dict:
|
||||
"""Пачка страниц команды wiki_sync → стадия. Перевод HTML идёт до записи, строки пишутся короткой транзакцией."""
|
||||
if isinstance(command_id, bool) or not isinstance(command_id, int):
|
||||
raise ApiError(422, "bad_request", "command_id — целое")
|
||||
if not isinstance(pages, list) or not pages or len(pages) > MAX_PAGES:
|
||||
raise ApiError(422, "bad_pages", f"pages — список от 1 до {MAX_PAGES} страниц")
|
||||
checked = [_check(i, p) for i, p in enumerate(pages, 1)]
|
||||
cmd = queue.get_command(conn, command_id)
|
||||
if cmd is None:
|
||||
raise ApiError(404, "not_found", f"command {command_id}")
|
||||
if cmd["type"] != "wiki_sync" or cmd["status"] != "claimed":
|
||||
raise ApiError(409, "not_claimed", f"команда {command_id} не выдана агенту как wiki_sync")
|
||||
rows = []
|
||||
for p in checked:
|
||||
try:
|
||||
md, files = convert.to_markdown(p["html"], p["attachments"], p["links"])
|
||||
except Exception as e: # noqa: BLE001 недоверенный HTML: сбой разбора — ошибка страницы, не сервера
|
||||
raise _bad(p["code"], f"html ({type(e).__name__})") from None
|
||||
payload = {k: p[k] for k in ("code", "eva_id", "title", "parent_code", "orderno", "url")}
|
||||
payload |= {"markdown": md, "attachments": files}
|
||||
rows.append((command_id, p["code"], json.dumps(payload, ensure_ascii=False)))
|
||||
with db.transaction(conn):
|
||||
cur = conn.execute("SELECT status FROM command WHERE id = ?", (command_id,)).fetchone()
|
||||
if cur is None or cur["status"] != "claimed": # состояние могло смениться, пока шёл перевод
|
||||
raise ApiError(409, "not_claimed", f"команда {command_id} уже не в работе")
|
||||
conn.executemany("INSERT OR REPLACE INTO wiki_sync_stage(command_id, code, payload) VALUES (?, ?, ?)", rows)
|
||||
return {"accepted": len(rows)}
|
||||
389
src/jira_hub/modules/wiki/sync.py
Normal file
389
src/jira_hub/modules/wiki/sync.py
Normal file
@ -0,0 +1,389 @@
|
||||
"""Синхронизация вики: сборка Markdown-копии из стадии, подмена каталога, строки wiki_page/вложений/FTS.
|
||||
|
||||
Порядок отчёта `done` (design 10): `prepare` вызывается до транзакции отчёта (CommandHooks.before_result) — готовит
|
||||
каталог `wiki/.<метка>.new/` и выделяет id вложений; `commit` (в on_result) пишет строки и последним шагом подменяет
|
||||
каталог. Перевод HTML → Markdown уже сделан при приёме пачек (stage.accept)."""
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from ...core import db
|
||||
from ...core.paths import secure_dir
|
||||
from . import convert, paths, search
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_BAD_CHARS = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
|
||||
NAME_CHARS = 80 # длина названия, символов (как safe_name в eva_wiki_export.py)
|
||||
NAME_BYTES = 200 # и байт UTF-8: у файловой системы предел имени 255 байт, «NN_» и «_k» помещаются
|
||||
|
||||
|
||||
class SyncFailed(Exception):
|
||||
"""Синхронизация не удалась: текст идёт в error команды и sync_error проекта."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Prepared:
|
||||
cmd_id: int
|
||||
label: str
|
||||
root: str
|
||||
discarded: str | None = None
|
||||
error: str | None = None
|
||||
rows: list = field(default_factory=list)
|
||||
attachments: list = field(default_factory=list) # (page_code, url, name)
|
||||
kept_ids: set = field(default_factory=set)
|
||||
new_ids: list = field(default_factory=list) # id вложений, добавленных подготовкой
|
||||
new_dir: Path | None = None
|
||||
root_title: str | None = None
|
||||
|
||||
|
||||
_PREPARED: dict[int, Prepared] = {}
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
# --- раскладка копии ----------------------------------------------------------
|
||||
|
||||
def safe_name(title: str) -> str:
|
||||
"""Название каталога: запрещённые символы → «_», до 80 символов и 200 байт, «.» и пробелы по краям убраны."""
|
||||
name = _BAD_CHARS.sub("_", title or "")[:NAME_CHARS]
|
||||
name = name.encode("utf-8")[:NAME_BYTES].decode("utf-8", "ignore").strip(" .")
|
||||
return name or "untitled"
|
||||
|
||||
|
||||
def _sorted(codes, by_code):
|
||||
return sorted(codes, key=lambda c: (by_code[c]["orderno"] is None, by_code[c]["orderno"] or 0, c))
|
||||
|
||||
|
||||
def _tree(order, by_code) -> dict[str, list[str]]:
|
||||
"""Потомки по parent_code; страница без родителя в наборе (или в цикле) становится дочерней корню."""
|
||||
root = order[0]
|
||||
children = {c: [] for c in order}
|
||||
parent = {}
|
||||
for c in order[1:]:
|
||||
p = by_code[c].get("parent_code")
|
||||
parent[c] = p if p in children and p != c else root
|
||||
children[parent[c]].append(c)
|
||||
|
||||
def reach():
|
||||
seen, stack = set(), [root]
|
||||
while stack:
|
||||
c = stack.pop()
|
||||
if c not in seen:
|
||||
seen.add(c)
|
||||
stack.extend(children[c])
|
||||
return seen
|
||||
|
||||
seen = reach()
|
||||
for c in order[1:]:
|
||||
if c not in seen: # цикл родителей: оторванное звено цепляем к корню
|
||||
children[parent[c]].remove(c)
|
||||
parent[c] = root
|
||||
children[root].append(c)
|
||||
seen = reach()
|
||||
return children
|
||||
|
||||
|
||||
def _inside(target: Path, path: Path) -> None:
|
||||
if not path.resolve().is_relative_to(target.resolve()):
|
||||
raise ValueError(f"путь {path} вне каталога копии")
|
||||
|
||||
|
||||
def build_copy(stage_pages: list[dict], codes: list[str], target: Path, ids: dict[tuple[str, str], int]) -> list[dict]:
|
||||
"""Собрать копию дерева в `target`: index.md страниц и _tree.json. Корень — codes[0]. Возвращает строки wiki_page
|
||||
(с полем markdown — текст с подставленными id вложений). ValueError — страница или путь негодны."""
|
||||
by_code = {p["code"]: p for p in stage_pages}
|
||||
order = list(dict.fromkeys(codes))
|
||||
if not order:
|
||||
raise ValueError("пустой список страниц")
|
||||
for c in order:
|
||||
if c not in by_code:
|
||||
raise ValueError(f"нет страницы {c}")
|
||||
children = _tree(order, by_code)
|
||||
secure_dir(target)
|
||||
synced_at = db.now()
|
||||
rows, tree = [], []
|
||||
stack = [(order[0], Path(), 0, 0)]
|
||||
used: dict[str, set[str]] = {} # родитель → занятые имена (без регистра)
|
||||
while stack:
|
||||
code, parent_rel, depth, pos = stack.pop()
|
||||
page = by_code[code]
|
||||
taken = used.setdefault(str(parent_rel), set())
|
||||
name, k = safe_name(page["title"]), 1
|
||||
base = name
|
||||
while name.lower() in taken:
|
||||
k += 1
|
||||
name = f"{base}_{k}"
|
||||
taken.add(name.lower())
|
||||
rel = parent_rel / (name if depth == 0 else f"{pos:02d}_{name}")
|
||||
full = target / rel
|
||||
_inside(target, full)
|
||||
try:
|
||||
text = convert.fill_ids(page["markdown"], [ids[(code, a["url"])] for a in page["attachments"]])
|
||||
except (ValueError, KeyError) as e:
|
||||
raise ValueError(f"{code}: {e}") from None
|
||||
secure_dir(full)
|
||||
title_line = " ".join(str(page["title"]).split())
|
||||
head = ["---", f"code: {json.dumps(code, ensure_ascii=False)}", f"title: {json.dumps(page['title'], ensure_ascii=False)}",
|
||||
f"url: {json.dumps(page.get('url'), ensure_ascii=False)}", f"synced_at: {json.dumps(synced_at)}", "---", "",
|
||||
f"# {title_line}", ""]
|
||||
(full / "index.md").write_text("\n".join(head) + "\n" + text, encoding="utf-8")
|
||||
rows.append({"code": code, "eva_id": page.get("eva_id"), "title": page["title"], "parent_code": page.get("parent_code"),
|
||||
"orderno": page.get("orderno"), "depth": depth, "path": rel.as_posix(), "url": page.get("url"),
|
||||
"size": len(text), "synced_at": synced_at, "markdown": text})
|
||||
tree.append({"code": code, "title": page["title"], "depth": depth, "path": rel.as_posix()})
|
||||
kids = _sorted(children[code], by_code)
|
||||
for i, kid in reversed(list(enumerate(kids, 1))): # стек: первый ребёнок — следующим
|
||||
stack.append((kid, rel, depth + 1, i))
|
||||
(target / "_tree.json").write_text(json.dumps(tree, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
return rows
|
||||
|
||||
|
||||
# --- подготовка (до транзакции отчёта) ---------------------------------------
|
||||
|
||||
def _siblings(conn, label: str) -> tuple[Path, Path, Path]:
|
||||
cur = paths.project_dir(conn, label)
|
||||
return cur, cur.with_name(f".{cur.name}.new"), cur.with_name(f".{cur.name}.old")
|
||||
|
||||
|
||||
def _allocate_ids(conn, label, pages) -> tuple[dict, list, list]:
|
||||
"""id вложений страниц: существующие по (страница, url) сохраняются, новые вставляются короткой транзакцией."""
|
||||
ids, new_ids, attach = {}, [], []
|
||||
existing = {(r["page_code"], r["url"]): r["id"] for r in conn.execute(
|
||||
"SELECT id, page_code, url FROM wiki_attachment WHERE label = ?", (label,))}
|
||||
with db.transaction(conn):
|
||||
for page in pages:
|
||||
for a in page["attachments"]:
|
||||
key = (page["code"], a["url"])
|
||||
if key not in existing:
|
||||
cur = conn.execute("INSERT INTO wiki_attachment(label, page_code, url, name) VALUES (?, ?, ?, ?)",
|
||||
(label, key[0], key[1], a["name"]))
|
||||
existing[key] = cur.lastrowid
|
||||
new_ids.append(cur.lastrowid)
|
||||
ids[key] = existing[key]
|
||||
attach.append((page["code"], a["url"], a["name"]))
|
||||
return ids, new_ids, attach
|
||||
|
||||
|
||||
def _abandon(conn, p: Prepared) -> None:
|
||||
"""Подготовка не пригодилась: каталог .new и добавленные ею вложения убираются."""
|
||||
if p.new_dir is not None:
|
||||
shutil.rmtree(p.new_dir, ignore_errors=True)
|
||||
if p.new_ids:
|
||||
marks = ",".join("?" * len(p.new_ids))
|
||||
conn.execute(f"DELETE FROM wiki_attachment WHERE id IN ({marks}) AND state = 'none'", p.new_ids)
|
||||
p.new_ids = []
|
||||
|
||||
|
||||
def prepare(conn, cmd: dict, result: dict | None) -> Prepared:
|
||||
"""До транзакции: проверить корень и список кодов, выделить id вложений, собрать `.new`. Ошибка — в p.error."""
|
||||
label, root = cmd["params"]["label"], cmd["params"]["root"]
|
||||
p = Prepared(cmd["id"], label, root)
|
||||
row = conn.execute("SELECT root_code FROM wiki_project WHERE label = ?", (label,)).fetchone()
|
||||
if row is None or row["root_code"] != root:
|
||||
p.discarded = "корень сменился"
|
||||
return p
|
||||
try:
|
||||
codes = (result or {}).get("codes")
|
||||
if not isinstance(codes, list) or not codes or not all(isinstance(c, str) for c in codes):
|
||||
raise SyncFailed("в отчёте нет списка codes")
|
||||
codes = list(dict.fromkeys(codes))
|
||||
if codes[0] != root:
|
||||
raise SyncFailed(f"первой в codes должна быть страница корня {root}")
|
||||
stage = {r["code"]: json.loads(r["payload"]) for r in conn.execute(
|
||||
"SELECT code, payload FROM wiki_sync_stage WHERE command_id = ?", (cmd["id"],))}
|
||||
for c in codes:
|
||||
if c not in stage:
|
||||
raise SyncFailed(f"нет страницы {c}")
|
||||
pages = [stage[c] for c in codes]
|
||||
_, p.new_dir, _ = _siblings(conn, label)
|
||||
secure_dir(p.new_dir.parent)
|
||||
shutil.rmtree(p.new_dir, ignore_errors=True)
|
||||
ids, p.new_ids, p.attachments = _allocate_ids(conn, label, pages)
|
||||
p.kept_ids = set(ids.values())
|
||||
p.rows = build_copy(pages, codes, p.new_dir, ids)
|
||||
title = (result or {}).get("root_title")
|
||||
p.root_title = title if isinstance(title, str) and title.strip() else p.rows[0]["title"]
|
||||
except SyncFailed as e:
|
||||
p.error = str(e)
|
||||
except ValueError as e: # fill_ids, путь вне копии
|
||||
p.error = str(e)
|
||||
except OSError as e:
|
||||
p.error = f"не удалось собрать копию: {e}"
|
||||
except Exception as e: # noqa: BLE001 сбой не должен оставлять команду без отчёта
|
||||
log.exception("wiki_sync %s: сборка копии", cmd["id"])
|
||||
p.error = f"внутренняя ошибка сборки копии: {type(e).__name__}: {e}"
|
||||
if p.error:
|
||||
_abandon(conn, p)
|
||||
return p
|
||||
|
||||
|
||||
def before_result(conn, cmd, status, result, files_dir) -> None:
|
||||
if cmd["type"] != "wiki_sync" or status != "done":
|
||||
return
|
||||
p = prepare(conn, cmd, result)
|
||||
with _lock:
|
||||
_PREPARED[cmd["id"]] = p
|
||||
|
||||
|
||||
def _prepared(conn, cmd, result) -> Prepared:
|
||||
with _lock:
|
||||
p = _PREPARED.get(cmd["id"])
|
||||
return p if p is not None else prepare(conn, cmd, result) # в обход before_result: в транзакции, но верно
|
||||
|
||||
|
||||
def check_result(conn, cmd, status, result, files_dir) -> tuple[str, str | None]:
|
||||
if cmd["type"] != "wiki_sync" or status != "done":
|
||||
return status, None
|
||||
p = _prepared(conn, cmd, result)
|
||||
with _lock:
|
||||
_PREPARED[cmd["id"]] = p
|
||||
if p.discarded:
|
||||
if isinstance(result, dict):
|
||||
result["discarded"] = p.discarded
|
||||
return "done", None
|
||||
if p.error:
|
||||
return "failed", p.error
|
||||
return "done", None
|
||||
|
||||
|
||||
# --- применение (в транзакции отчёта) ----------------------------------------
|
||||
|
||||
def _swap(cur: Path, new: Path, old: Path) -> None:
|
||||
"""wiki/<метка> → .old, .new → <метка>, .old удаляется. При сбое прежний каталог возвращается на место."""
|
||||
shutil.rmtree(old, ignore_errors=True)
|
||||
had = cur.exists()
|
||||
try:
|
||||
if had:
|
||||
cur.rename(old)
|
||||
new.rename(cur)
|
||||
except OSError:
|
||||
if had and not cur.exists() and old.exists():
|
||||
old.rename(cur)
|
||||
raise
|
||||
shutil.rmtree(old, ignore_errors=True)
|
||||
|
||||
|
||||
def _fail(conn, cmd, error) -> None:
|
||||
"""Команда уже записана как done, но копия не применилась: failed + sync_error."""
|
||||
conn.execute("UPDATE command SET status = 'failed', error = ? WHERE id = ?", (error[:1500], cmd["id"]))
|
||||
on_failed_or_reported_failed(conn, cmd, error)
|
||||
|
||||
|
||||
def on_result(conn, cmd, status, result, jira_calls, rate_limited, now, files_dir) -> None:
|
||||
if cmd["type"] != "wiki_sync":
|
||||
return
|
||||
with _lock:
|
||||
p = _PREPARED.pop(cmd["id"], None)
|
||||
if status != "done":
|
||||
error = conn.execute("SELECT error FROM command WHERE id = ?", (cmd["id"],)).fetchone()["error"]
|
||||
on_failed_or_reported_failed(conn, cmd, error or "синхронизация не удалась")
|
||||
return
|
||||
if p is None or p.error:
|
||||
error = p.error if p else "копия не подготовлена"
|
||||
on_failed_or_reported_failed(conn, cmd, error)
|
||||
return
|
||||
row = conn.execute("SELECT root_code FROM wiki_project WHERE label = ?", (p.label,)).fetchone()
|
||||
if p.discarded or row is None or row["root_code"] != p.root: # корень сменился, пока шла подготовка
|
||||
_abandon(conn, p)
|
||||
if not p.discarded:
|
||||
out = dict(result or {}, discarded="корень сменился")
|
||||
conn.execute("UPDATE command SET result = ? WHERE id = ?", (json.dumps(out, ensure_ascii=False), cmd["id"]))
|
||||
conn.execute("DELETE FROM wiki_sync_stage WHERE command_id = ?", (cmd["id"],))
|
||||
conn.execute("UPDATE wiki_project SET failed_at = ? WHERE label = ?", (now, p.label))
|
||||
return
|
||||
try:
|
||||
with db.transaction(conn):
|
||||
_commit(conn, p, now)
|
||||
except OSError as e:
|
||||
_abandon(conn, p)
|
||||
_fail(conn, cmd, f"не удалось подменить копию: {e}")
|
||||
|
||||
|
||||
def _commit(conn, p: Prepared, now: str) -> None:
|
||||
search.replace(conn, p.label, [(r["code"], r["title"], r["markdown"]) for r in p.rows])
|
||||
conn.execute("DELETE FROM wiki_page WHERE label = ?", (p.label,))
|
||||
conn.executemany(
|
||||
"INSERT INTO wiki_page(label, code, eva_id, title, parent_code, orderno, depth, path, url, size, synced_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
[(p.label, r["code"], r["eva_id"], r["title"], r["parent_code"], r["orderno"], r["depth"], r["path"], r["url"],
|
||||
r["size"], r["synced_at"]) for r in p.rows])
|
||||
conn.executemany("UPDATE wiki_attachment SET name = ? WHERE label = ? AND page_code = ? AND url = ?",
|
||||
[(name, p.label, code, url) for code, url, name in p.attachments])
|
||||
for r in conn.execute("SELECT id FROM wiki_attachment WHERE label = ?", (p.label,)).fetchall():
|
||||
if r["id"] not in p.kept_ids:
|
||||
conn.execute("DELETE FROM wiki_attachment WHERE id = ?", (r["id"],))
|
||||
conn.execute("UPDATE wiki_project SET synced_at = ?, pages_count = ?, root_title = ?, sync_error = NULL, failed_at = NULL "
|
||||
"WHERE label = ?", (now, len(p.rows), p.root_title, p.label))
|
||||
conn.execute("DELETE FROM wiki_sync_stage WHERE command_id = ?", (p.cmd_id,))
|
||||
cur, new, old = _siblings(conn, p.label)
|
||||
_swap(cur, new, old) # последним: сбой откатывает строки (savepoint)
|
||||
|
||||
|
||||
def on_failed_or_reported_failed(conn, cmd: dict, error: str | None) -> None:
|
||||
"""Отчёт failed или предел выдач: sync_error и failed_at проекта (если корень команды ещё текущий), стадия очищена."""
|
||||
if cmd["type"] != "wiki_sync":
|
||||
return
|
||||
conn.execute("DELETE FROM wiki_sync_stage WHERE command_id = ?", (cmd["id"],))
|
||||
conn.execute("UPDATE wiki_project SET sync_error = ?, failed_at = ? WHERE label = ? AND root_code = ?",
|
||||
((error or "")[:1500], db.now(), cmd["params"]["label"], cmd["params"]["root"]))
|
||||
|
||||
|
||||
def on_failed(conn, cmd, now) -> None:
|
||||
"""Предел выдач: ошибка уже записана в команду."""
|
||||
row = conn.execute("SELECT error FROM command WHERE id = ?", (cmd["id"],)).fetchone()
|
||||
on_failed_or_reported_failed(conn, cmd, row["error"] if row else None)
|
||||
|
||||
|
||||
def on_claim(conn, cmd, now, redelivery) -> None:
|
||||
"""Повторная выдача: пачки прошлой выдачи не нужны, агент пришлёт заново."""
|
||||
if cmd["type"] == "wiki_sync" and redelivery:
|
||||
conn.execute("DELETE FROM wiki_sync_stage WHERE command_id = ?", (cmd["id"],))
|
||||
|
||||
|
||||
def after_result(conn, cmd, status) -> None:
|
||||
with _lock:
|
||||
_PREPARED.pop(cmd["id"], None)
|
||||
|
||||
|
||||
# --- уборка ----------------------------------------------------------------
|
||||
|
||||
def finish_pending_swaps(root: Path) -> None:
|
||||
"""При старте: `.<имя>.new` без каталога `<имя>` — переименовать (подмена прервана); `.new` рядом с `<имя>` —
|
||||
остаток неудачной сборки, удаляется; `.<имя>.old` — удаляется. Каталоги, которые собирает идущая подготовка, не трогаются."""
|
||||
root = Path(root)
|
||||
if not root.is_dir():
|
||||
return
|
||||
with _lock:
|
||||
busy = {p.new_dir for p in _PREPARED.values() if p.new_dir}
|
||||
for entry in sorted(root.iterdir()):
|
||||
n = entry.name
|
||||
if not n.startswith(".") or entry in busy:
|
||||
continue
|
||||
try:
|
||||
if n.endswith(".new"):
|
||||
target = root / n[1:-4]
|
||||
if target.exists():
|
||||
shutil.rmtree(entry, ignore_errors=True)
|
||||
else:
|
||||
entry.rename(target)
|
||||
elif n.endswith(".old"):
|
||||
shutil.rmtree(entry)
|
||||
except OSError as e:
|
||||
log.warning("вики: не удалось довести подмену %s: %s", entry, e)
|
||||
|
||||
|
||||
def housekeeping(conn) -> None:
|
||||
"""Стадия команд, которые уже не в работе (отмена, терминальный статус), очищается; отменённая синхронизация
|
||||
сдвигает failed_at проекта, чтобы автосинхронизация не ставила пакеты каждый тик."""
|
||||
conn.execute("DELETE FROM wiki_sync_stage WHERE command_id NOT IN "
|
||||
"(SELECT id FROM command WHERE status IN ('queued', 'claimed'))")
|
||||
for r in conn.execute("SELECT label, synced_at, failed_at FROM wiki_project").fetchall():
|
||||
last = max((t for t in (r["synced_at"], r["failed_at"]) if t), default="")
|
||||
c = conn.execute("SELECT MAX(finished_at) AS t FROM command WHERE type = 'wiki_sync' AND status = 'cancelled' "
|
||||
"AND json_extract(params, '$.label') = ?", (r["label"],)).fetchone()["t"]
|
||||
if c and c > last:
|
||||
conn.execute("UPDATE wiki_project SET failed_at = ? WHERE label = ?", (c, r["label"]))
|
||||
@ -2,7 +2,7 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from ...core import db, settings
|
||||
from . import commands
|
||||
from . import commands, paths, sync
|
||||
|
||||
|
||||
def auto_sync_tick(hub, conn) -> None:
|
||||
@ -20,3 +20,21 @@ def auto_sync_tick(hub, conn) -> None:
|
||||
|
||||
|
||||
auto_sync_tick.label = "синхронизация вики"
|
||||
|
||||
|
||||
_swept: set[str] = set()
|
||||
|
||||
|
||||
def housekeeping_tick(hub, conn) -> None:
|
||||
"""Первый тик на каталог вики: довести прерванную подмену копии; каждый тик: очистка стадий и отмен."""
|
||||
try:
|
||||
root = paths.root(conn)
|
||||
except RuntimeError: # база без файла: копий на диске нет
|
||||
root = None
|
||||
if root is not None and str(root) not in _swept:
|
||||
_swept.add(str(root))
|
||||
sync.finish_pending_swaps(root)
|
||||
sync.housekeeping(conn)
|
||||
|
||||
|
||||
housekeeping_tick.label = "уборка вики"
|
||||
|
||||
6
tests/fixtures/baseline/routes.expected.json
vendored
6
tests/fixtures/baseline/routes.expected.json
vendored
@ -71,6 +71,12 @@
|
||||
"agent",
|
||||
"h_agent_state"
|
||||
],
|
||||
[
|
||||
"POST",
|
||||
"^/agent/wiki/pages$",
|
||||
"agent",
|
||||
"h_agent_pages"
|
||||
],
|
||||
[
|
||||
"GET",
|
||||
"^/agents$",
|
||||
|
||||
@ -51,11 +51,12 @@ def test_route_conflict():
|
||||
|
||||
def test_routes_assembled():
|
||||
from jira_hub.core import api
|
||||
from jira_hub.modules import jira
|
||||
from jira_hub.modules import jira, wiki
|
||||
pairs = lambda routes: {(m, p) for m, p, _, _ in routes} # noqa: E731
|
||||
assert pairs(api.ROUTES) == pairs(api.CORE_ROUTES) | pairs(jira.MODULE.routes)
|
||||
assert not pairs(api.CORE_ROUTES) & pairs(jira.MODULE.routes)
|
||||
assert len(api.ROUTES) == len(api.CORE_ROUTES) + len(jira.MODULE.routes)
|
||||
mods = jira.MODULE.routes + wiki.MODULE.routes
|
||||
assert pairs(api.ROUTES) == pairs(api.CORE_ROUTES) | pairs(mods)
|
||||
assert not pairs(api.CORE_ROUTES) & pairs(mods)
|
||||
assert len(api.ROUTES) == len(api.CORE_ROUTES) + len(mods)
|
||||
|
||||
|
||||
def test_module_of(monkeypatch):
|
||||
|
||||
435
tests/test_wiki_sync.py
Normal file
435
tests/test_wiki_sync.py
Normal file
@ -0,0 +1,435 @@
|
||||
"""Синхронизация вики: приём пачек, сборка копии, поиск (wiki-module, spec wiki «Синхронизация вики», «Markdown-копия»).
|
||||
Агента имитирует тест: HTTP на порту 0 токеном агента wiki."""
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import timedelta
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from jira_hub.core import db, queue, settings
|
||||
from jira_hub.core.client import HubClient, HubError
|
||||
from jira_hub.modules.wiki import commands, convert, projects, search, stage, sync, tick
|
||||
|
||||
|
||||
def page(code, title, parent=None, orderno=None, html=None, attachments=None, links=None):
|
||||
return {"code": code, "eva_id": "id-" + code, "title": title, "parent_code": parent, "orderno": orderno,
|
||||
"url": f"https://eva.example.ru/project/Document/{code}", "html": html if html is not None else f"<p>текст {title}</p>",
|
||||
"attachments": attachments or [], "links": links or []}
|
||||
|
||||
|
||||
TREE = [page("DOC-1", "АРМ КА"), page("DOC-2", "Роли/права", "DOC-1", 2), page("DOC-3", "Статусы", "DOC-1", 1,
|
||||
"<p>статус «На согласовании» ставит аналитик</p>")]
|
||||
CODES = ["DOC-1", "DOC-3", "DOC-2"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api(hub_server, tmp_path):
|
||||
hub = hub_server["hub"]
|
||||
conn = hub.conn()
|
||||
token = settings.tokens(conn)["agents"]["wiki"]
|
||||
conn.close()
|
||||
wiki = HubClient(hub_server["base"], token)
|
||||
hub_server["client"].post("/projects", {"label": "AWPCA", "name": "АРМ КА", "wiki_root": "DOC-1"})
|
||||
|
||||
def db_rows(sql, *args):
|
||||
c = hub.conn()
|
||||
try:
|
||||
return [dict(r) for r in c.execute(sql, args)]
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
def enqueue():
|
||||
c = hub.conn()
|
||||
try:
|
||||
with db.transaction(c):
|
||||
return commands.enqueue_sync(c, "AWPCA", "DOC-1", "test")
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
def claim():
|
||||
return wiki.post("/agent/claim", {"agent_version": 1})["command"]
|
||||
|
||||
def report(cmd, status="done", result=None, error=None):
|
||||
return wiki.post(f"/agent/commands/{cmd['id']}/result", {"status": status, "result": result, "error": error})
|
||||
|
||||
def run(pages, codes=None, root_title="АРМ КА", status="done", error=None):
|
||||
"""Целый обмен: взять команду, прислать страницы, отчитаться. Возвращает команду после отчёта."""
|
||||
cmd = claim()
|
||||
assert cmd and cmd["type"] == "wiki_sync", cmd
|
||||
if pages:
|
||||
wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": pages})
|
||||
result = {"root_title": root_title, "codes": codes if codes is not None else [p["code"] for p in pages],
|
||||
"eva_calls": 3} if status == "done" else None
|
||||
return report(cmd, status, result, error)
|
||||
|
||||
def project():
|
||||
return {p["label"]: p for p in hub_server["client"].get("/projects")["projects"]}["AWPCA"]
|
||||
|
||||
return SimpleNamespace(hub=hub, wiki=wiki, client=hub_server["client"], base=hub_server["base"], rows=db_rows,
|
||||
enqueue=enqueue, claim=claim, report=report, run=run, project=project,
|
||||
copy=tmp_path / "wiki" / "AWPCA", wiki_dir=tmp_path / "wiki", db_path=tmp_path / "hub.sqlite",
|
||||
jira=hub_server["agent"])
|
||||
|
||||
|
||||
def first_sync(api):
|
||||
done = api.run(TREE, CODES)
|
||||
assert done["status"] == "done"
|
||||
return done
|
||||
|
||||
|
||||
def test_first_sync(api):
|
||||
first_sync(api)
|
||||
assert (api.copy / "АРМ КА" / "index.md").is_file()
|
||||
assert (api.copy / "АРМ КА" / "01_Статусы" / "index.md").is_file()
|
||||
assert (api.copy / "АРМ КА" / "02_Роли_права" / "index.md").is_file()
|
||||
assert [t["code"] for t in json.loads((api.copy / "_tree.json").read_text(encoding="utf-8"))] == CODES
|
||||
text = (api.copy / "АРМ КА" / "01_Статусы" / "index.md").read_text(encoding="utf-8")
|
||||
assert text.startswith('---\ncode: "DOC-3"\ntitle: "Статусы"\n') and "\n# Статусы\n" in text and "аналитик" in text
|
||||
p = api.project()
|
||||
assert p["wiki_pages"] == 3 and p["wiki_root_title"] == "АРМ КА" and p["wiki_synced_at"] and p["wiki_sync_error"] is None
|
||||
assert api.rows("SELECT COUNT(*) AS n FROM wiki_sync_stage")[0]["n"] == 0 # стадия очищена
|
||||
assert {r["code"]: r["path"] for r in api.rows("SELECT code, path FROM wiki_page")} == {
|
||||
"DOC-1": "АРМ КА", "DOC-3": "АРМ КА/01_Статусы", "DOC-2": "АРМ КА/02_Роли_права"}
|
||||
assert oct(api.copy.stat().st_mode & 0o777) == "0o700" and oct(api.wiki_dir.stat().st_mode & 0o777) == "0o700"
|
||||
assert oct((api.copy / "АРМ КА" / "01_Статусы").stat().st_mode & 0o777) == "0o700"
|
||||
|
||||
|
||||
def test_failed_keeps_copy(api):
|
||||
first_sync(api)
|
||||
api.enqueue()
|
||||
api.run(None, status="failed", error="Eva: HTTP 503")
|
||||
p = api.project()
|
||||
assert p["wiki_sync_error"] == "Eva: HTTP 503" and p["wiki_pages"] == 3
|
||||
assert (api.copy / "АРМ КА" / "01_Статусы" / "index.md").is_file()
|
||||
conn = api.hub.conn()
|
||||
try:
|
||||
assert [r["code"] for r in search.query(conn, "AWPCA", "согласов", 10)] == ["DOC-3"]
|
||||
finally:
|
||||
conn.close()
|
||||
assert api.rows("SELECT failed_at FROM wiki_project")[0]["failed_at"]
|
||||
|
||||
|
||||
def test_missing_page_fails(api):
|
||||
first_sync(api)
|
||||
api.enqueue()
|
||||
done = api.run([TREE[0]], ["DOC-1", "DOC-2"])
|
||||
assert done["status"] == "failed" and done["error"] == "нет страницы DOC-2"
|
||||
assert api.project()["wiki_sync_error"] == "нет страницы DOC-2"
|
||||
assert (api.copy / "АРМ КА" / "02_Роли_права" / "index.md").is_file()
|
||||
assert api.rows("SELECT COUNT(*) AS n FROM wiki_sync_stage")[0]["n"] == 0
|
||||
|
||||
|
||||
def test_page_removed(api):
|
||||
shot = page("DOC-3", "Статусы", "DOC-1", 1, '<p>см. <img src="/files/x/scheme.png"></p>',
|
||||
[{"ref": "/files/x/scheme.png", "url": "https://eva.example.ru/files/x/scheme.png", "name": "scheme.png"}])
|
||||
api.run([TREE[0], TREE[1], shot], CODES)
|
||||
assert len(api.rows("SELECT * FROM wiki_attachment")) == 1
|
||||
api.enqueue()
|
||||
api.run([TREE[0], page("DOC-3", "Статусы", "DOC-1", 1, "<p>без картинки</p>")], ["DOC-1", "DOC-3"])
|
||||
assert not list(api.copy.rglob("*Роли*"))
|
||||
assert api.rows("SELECT code FROM wiki_page ORDER BY code") == [{"code": "DOC-1"}, {"code": "DOC-3"}]
|
||||
assert api.rows("SELECT * FROM wiki_attachment") == []
|
||||
conn = api.hub.conn()
|
||||
try:
|
||||
assert search.query(conn, "AWPCA", "Роли", 10) == []
|
||||
assert api.project()["wiki_pages"] == 2
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _with_file(text="см. файл"):
|
||||
att = [{"ref": "/files/x/model.xlsx", "url": "https://eva.example.ru/files/x/model.xlsx", "name": "model.xlsx"}]
|
||||
return [TREE[0], page("DOC-3", "Статусы", "DOC-1", 1, f'<p>{text} <a href="/files/x/model.xlsx">model</a></p>', att)]
|
||||
|
||||
|
||||
def test_attachment_id_survives(api):
|
||||
api.run(_with_file(), ["DOC-1", "DOC-3"])
|
||||
(a,) = api.rows("SELECT id, state FROM wiki_attachment")
|
||||
md = (api.copy / "АРМ КА" / "01_Статусы" / "index.md").read_text(encoding="utf-8")
|
||||
assert f"[📎 model.xlsx](attachment:{a['id']})" in md and a["state"] == "none"
|
||||
c = api.hub.conn()
|
||||
c.execute("UPDATE wiki_attachment SET state = 'ready', file = 'ab', size = 5 WHERE id = ?", (a["id"],))
|
||||
c.close()
|
||||
api.enqueue()
|
||||
api.run(_with_file("новый текст"), ["DOC-1", "DOC-3"])
|
||||
(b,) = api.rows("SELECT id, state, file FROM wiki_attachment")
|
||||
assert (b["id"], b["state"], b["file"]) == (a["id"], "ready", "ab")
|
||||
assert f"(attachment:{a['id']})" in (api.copy / "АРМ КА" / "01_Статусы" / "index.md").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_stale_root_discarded(api):
|
||||
cmd = api.claim()
|
||||
assert cmd["params"]["root"] == "DOC-1"
|
||||
api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0]]})
|
||||
api.client.patch("/projects/AWPCA", {"wiki_root": "DOC-21"})
|
||||
done = api.report(cmd, "done", {"root_title": "X", "codes": ["DOC-1"]})
|
||||
assert done["status"] == "done" and done["result"]["discarded"] == "корень сменился"
|
||||
assert not api.copy.exists() and api.rows("SELECT * FROM wiki_page") == []
|
||||
p = api.project()
|
||||
assert p["wiki_root"] == "DOC-21" and p["wiki_pages"] == 0 and p["wiki_synced_at"] is None
|
||||
assert api.rows("SELECT COUNT(*) AS n FROM wiki_sync_stage")[0]["n"] == 0
|
||||
assert api.rows("SELECT failed_at FROM wiki_project")[0]["failed_at"] # часы автосинхронизации сдвинуты
|
||||
|
||||
|
||||
def test_redelivery_clears_stage(api):
|
||||
cmd = api.claim()
|
||||
api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0]]})
|
||||
assert api.rows("SELECT COUNT(*) AS n FROM wiki_sync_stage")[0]["n"] == 1
|
||||
again = api.claim()
|
||||
assert again["id"] == cmd["id"] and api.rows("SELECT COUNT(*) AS n FROM wiki_sync_stage")[0]["n"] == 0
|
||||
done = api.report(again, "done", {"root_title": "X", "codes": ["DOC-1"]})
|
||||
assert done["status"] == "failed" and done["error"] == "нет страницы DOC-1"
|
||||
|
||||
|
||||
def test_pages_wrong_agent(api):
|
||||
cmd = api.claim()
|
||||
with pytest.raises(HubError) as e:
|
||||
api.jira.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0]]})
|
||||
assert e.value.status == 401
|
||||
|
||||
|
||||
def test_pages_not_claimed(api):
|
||||
(row,) = api.rows("SELECT id FROM command WHERE type = 'wiki_sync'")
|
||||
with pytest.raises(HubError) as e:
|
||||
api.wiki.post("/agent/wiki/pages", {"command_id": row["id"], "pages": [TREE[0]]})
|
||||
assert e.value.status == 409 and e.value.error == "not_claimed"
|
||||
|
||||
|
||||
def test_bad_page(api):
|
||||
cmd = api.claim()
|
||||
bad = {k: v for k, v in TREE[1].items() if k != "title"}
|
||||
with pytest.raises(HubError) as e:
|
||||
api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0], bad]})
|
||||
assert e.value.status == 422 and e.value.error == "bad_pages" and e.value.detail == "DOC-2: title"
|
||||
assert api.rows("SELECT COUNT(*) AS n FROM wiki_sync_stage")[0]["n"] == 0 # пачка не принята целиком
|
||||
for field, value in (("orderno", "1"), ("parent_code", 5), ("html", None)):
|
||||
with pytest.raises(HubError) as e:
|
||||
api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0] | {field: value}]})
|
||||
assert e.value.status == 422 and e.value.detail == f"DOC-1: {field}"
|
||||
|
||||
|
||||
def test_layout_names_safe(tmp_path):
|
||||
titles = ["Корень", "../x", "a/b", "😀" * 200, "Статусы", "Статусы", "..", " . ", 'a<b>:"c|?*']
|
||||
pages = [{"code": f"D{i}", "title": t, "markdown": "текст\n", "attachments": [], "orderno": i,
|
||||
"parent_code": None if i == 0 else "D0", "eva_id": None, "url": None} for i, t in enumerate(titles)]
|
||||
target = tmp_path / "t"
|
||||
rows = sync.build_copy(pages, [p["code"] for p in pages], target, {})
|
||||
paths_ = [r["path"] for r in rows]
|
||||
for r in rows:
|
||||
assert (target / r["path"] / "index.md").resolve().is_relative_to(target.resolve())
|
||||
assert ".." not in r["path"].split("/") and all(len(part.encode()) <= 255 for part in r["path"].split("/"))
|
||||
assert "Корень/04_Статусы" in paths_ and "Корень/05_Статусы_2" in paths_
|
||||
assert "Корень/06_untitled" in paths_ and "Корень/07_untitled_2" in paths_
|
||||
assert "Корень/02_a_b" in paths_ and "Корень/01__x" in paths_
|
||||
assert sorted(p.name for p in (target / "Корень").iterdir() if p.is_dir()) == sorted(p.split("/")[1] for p in paths_[1:])
|
||||
|
||||
|
||||
def test_layout_orphans_and_cycles(tmp_path):
|
||||
def mk(code, parent, order=None):
|
||||
return {"code": code, "title": code, "markdown": "x\n", "attachments": [], "orderno": order, "parent_code": parent,
|
||||
"eva_id": None, "url": None}
|
||||
pages = [mk("R", None), mk("A", "R", 2), mk("B", "R"), mk("C", "GONE"), mk("X", "Y"), mk("Y", "X")]
|
||||
rows = sync.build_copy(pages, [p["code"] for p in pages], tmp_path / "t", {})
|
||||
assert {r["code"] for r in rows} == {"R", "A", "B", "C", "X", "Y"} and len(rows) == 6 # никто не потерян
|
||||
assert [r["code"] for r in rows if r["depth"] == 1][:2] == ["A", "B"] # orderno, затем None по коду
|
||||
|
||||
|
||||
def test_finish_pending_swap(tmp_path):
|
||||
root = tmp_path / "wiki"
|
||||
(root / ".AWPCA.new" / "x").mkdir(parents=True)
|
||||
(root / ".AWPCA.old").mkdir()
|
||||
(root / ".B.new").mkdir()
|
||||
(root / "B").mkdir() # остаток неудачной сборки рядом с живой копией
|
||||
sync.finish_pending_swaps(root)
|
||||
assert (root / "AWPCA" / "x").is_dir() and not (root / ".AWPCA.old").exists() and not (root / ".AWPCA.new").exists()
|
||||
assert (root / "B").is_dir() and not (root / ".B.new").exists()
|
||||
sync.finish_pending_swaps(root / "нет") # нет каталога — не ошибка
|
||||
|
||||
|
||||
def test_search_prefix(open_db):
|
||||
conn = open_db()
|
||||
conn.execute("INSERT INTO project(label, name, created_at) VALUES ('AWPCA', 'A', '2026-01-01')")
|
||||
assert search.ensure(conn) is True and search.ensure(conn) is True
|
||||
search.replace(conn, "AWPCA", [("DOC-3", "Статусы", "статус «На согласовании» ставит аналитик"),
|
||||
("DOC-4", "Роли", "роль администратора")])
|
||||
search.replace(conn, "OTHER", [("DOC-9", "Чужая", "согласование чужого проекта")])
|
||||
conn.execute("INSERT INTO wiki_page(label, code, title, path) VALUES ('AWPCA', 'DOC-3', 'Статусы', 'A/01_Статусы')")
|
||||
(hit,) = search.query(conn, "AWPCA", "согласов", 10)
|
||||
assert hit["code"] == "DOC-3" and "На согласовании" in hit["snippet"] and hit["path"] == "A/01_Статусы"
|
||||
assert search.query(conn, "AWPCA", 'согласов"', 10)[0]["code"] == "DOC-3" # кавычка не роняет
|
||||
for q in ('"', "", "* AND (", "NEAR(", "статус ОR"):
|
||||
assert isinstance(search.query(conn, "AWPCA", q, 5), list)
|
||||
assert [r["code"] for r in search.query(conn, "AWPCA", "роль админ", 10)] == ["DOC-4"]
|
||||
assert search.query(conn, "AWPCA", "чужого", 10) == []
|
||||
|
||||
|
||||
def test_search_without_fts_uses_files(api, monkeypatch):
|
||||
first_sync(api)
|
||||
c = api.hub.conn()
|
||||
try:
|
||||
c.execute("DROP TABLE wiki_page_fts")
|
||||
(hit,) = search.query(c, "AWPCA", "согласов", 10)
|
||||
assert hit["code"] == "DOC-3" and "На согласовании" in hit["snippet"]
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
|
||||
# --- пункты контроллера ---------------------------------------------------------
|
||||
|
||||
def test_failed_at_prevents_tick_storm(api):
|
||||
"""Любая неудача пишет failed_at, и автотик не ставит новый пакет сразу."""
|
||||
def tick_once():
|
||||
c = api.hub.conn()
|
||||
try:
|
||||
tick.auto_sync_tick(None, c)
|
||||
finally:
|
||||
c.close()
|
||||
return api.rows("SELECT COUNT(*) AS n FROM batch WHERE project = 'AWPCA'")[0]["n"]
|
||||
|
||||
first_sync(api)
|
||||
assert api.rows("SELECT synced_at, failed_at FROM wiki_project")[0]["synced_at"]
|
||||
assert tick_once() == 1 # свежая успешная
|
||||
c = api.hub.conn()
|
||||
c.execute("UPDATE wiki_project SET synced_at = ?, failed_at = NULL", ((db.utcnow() - timedelta(hours=30)).isoformat(),))
|
||||
c.close()
|
||||
# failed: пакет выдан, агент отчитался failed
|
||||
api.enqueue()
|
||||
api.run(None, status="failed", error="Eva: HTTP 503")
|
||||
assert tick_once() == 2
|
||||
# отброшенная
|
||||
c = api.hub.conn()
|
||||
c.execute("UPDATE wiki_project SET synced_at = ?, failed_at = NULL", ((db.utcnow() - timedelta(hours=30)).isoformat(),))
|
||||
c.close()
|
||||
api.enqueue()
|
||||
claimed = api.claim()
|
||||
api.client.patch("/projects/AWPCA", {"wiki_root": "DOC-21"})
|
||||
api.report(claimed, "done", {"root_title": "X", "codes": ["DOC-1"]})
|
||||
assert api.rows("SELECT failed_at FROM wiki_project")[0]["failed_at"]
|
||||
# отменённая
|
||||
c = api.hub.conn()
|
||||
c.execute("UPDATE wiki_project SET synced_at = ?, failed_at = NULL", ((db.utcnow() - timedelta(hours=30)).isoformat(),))
|
||||
queued = api.rows("SELECT id FROM command WHERE type = 'wiki_sync' AND status = 'queued'")
|
||||
for q in queued:
|
||||
queue.cancel_command(c, q["id"])
|
||||
c.close()
|
||||
c = api.hub.conn()
|
||||
try:
|
||||
tick.housekeeping_tick(None, c)
|
||||
assert api.rows("SELECT failed_at FROM wiki_project")[0]["failed_at"]
|
||||
before = api.rows("SELECT COUNT(*) AS n FROM batch")[0]["n"]
|
||||
tick.auto_sync_tick(None, c)
|
||||
assert api.rows("SELECT COUNT(*) AS n FROM batch")[0]["n"] == before
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
|
||||
def test_fill_ids_error_becomes_failed(api):
|
||||
cmd = api.claim()
|
||||
bad = {"code": "DOC-1", "eva_id": None, "title": "АРМ КА", "parent_code": None, "orderno": None, "url": None,
|
||||
"markdown": "[📎 f](attachment:@5)\n", "attachments": []}
|
||||
c = api.hub.conn()
|
||||
c.execute("INSERT INTO wiki_sync_stage VALUES (?, 'DOC-1', ?)", (cmd["id"], json.dumps(bad)))
|
||||
c.close()
|
||||
done = api.report(cmd, "done", {"root_title": "X", "codes": ["DOC-1"]})
|
||||
assert done["status"] == "failed" and done["error"].startswith("DOC-1: ") and "вне списка" in done["error"]
|
||||
assert api.project()["wiki_sync_error"].startswith("DOC-1: ") and not api.copy.exists()
|
||||
assert not list(api.wiki_dir.glob(".*")) # .new убран
|
||||
|
||||
|
||||
def test_long_name_failed_not_crash(hub_server, tmp_path):
|
||||
label = "/" * 100 # dir_name даёт 300 символов
|
||||
hub_server["client"].post("/projects", {"label": label, "name": "длинная", "wiki_root": "DOC-1"})
|
||||
hub = hub_server["hub"]
|
||||
conn = hub.conn()
|
||||
token = settings.tokens(conn)["agents"]["wiki"]
|
||||
conn.close()
|
||||
wiki = HubClient(hub_server["base"], token)
|
||||
cmd = wiki.post("/agent/claim", {})["command"]
|
||||
wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0]]})
|
||||
done = wiki.post(f"/agent/commands/{cmd['id']}/result", {"status": "done", "result": {"root_title": "X", "codes": ["DOC-1"]}})
|
||||
assert done["status"] == "failed" and done["error"].startswith("не удалось собрать копию")
|
||||
conn = hub.conn()
|
||||
try:
|
||||
row = conn.execute("SELECT sync_error, failed_at FROM wiki_project WHERE label = ?", (label,)).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
assert row["sync_error"].startswith("не удалось собрать копию") and row["failed_at"]
|
||||
|
||||
|
||||
def test_reset_root_clears_fts_and_dir(api):
|
||||
first_sync(api)
|
||||
c = api.hub.conn()
|
||||
assert c.execute("SELECT COUNT(*) AS n FROM wiki_page_fts").fetchone()["n"] == 3
|
||||
c.close()
|
||||
api.client.patch("/projects/AWPCA", {"wiki_root": None})
|
||||
c = api.hub.conn()
|
||||
try:
|
||||
assert c.execute("SELECT COUNT(*) AS n FROM wiki_page_fts").fetchone()["n"] == 0
|
||||
assert c.execute("SELECT COUNT(*) AS n FROM wiki_page").fetchone()["n"] == 0
|
||||
finally:
|
||||
c.close()
|
||||
assert not api.copy.exists()
|
||||
|
||||
|
||||
def test_drop_copy_logs_rmtree_failure(api, monkeypatch, caplog):
|
||||
first_sync(api)
|
||||
real = projects.shutil.rmtree
|
||||
|
||||
def boom(path, *a, **k):
|
||||
if "AWPCA" in str(path):
|
||||
raise PermissionError("нельзя")
|
||||
return real(path, *a, **k)
|
||||
|
||||
monkeypatch.setattr(projects.shutil, "rmtree", boom)
|
||||
with caplog.at_level(logging.WARNING, logger=projects.__name__):
|
||||
api.client.patch("/projects/AWPCA", {"wiki_root": None})
|
||||
assert "не удалён" in caplog.text and "нельзя" in caplog.text
|
||||
|
||||
|
||||
def test_no_heavy_work_under_write_lock(api, monkeypatch):
|
||||
"""Перевод HTML и запись файлов идут вне BEGIN IMMEDIATE: параллельное соединение может открыть запись."""
|
||||
seen = []
|
||||
|
||||
def probe(tag):
|
||||
other = sqlite3.connect(api.db_path, timeout=0)
|
||||
try:
|
||||
other.execute("BEGIN IMMEDIATE")
|
||||
other.execute("ROLLBACK")
|
||||
seen.append((tag, "free"))
|
||||
except sqlite3.OperationalError:
|
||||
seen.append((tag, "locked"))
|
||||
finally:
|
||||
other.close()
|
||||
|
||||
real_md, real_build = convert.to_markdown, sync.build_copy
|
||||
monkeypatch.setattr(stage.convert, "to_markdown", lambda *a, **k: (probe("markdown"), real_md(*a, **k))[1])
|
||||
monkeypatch.setattr(sync, "build_copy", lambda *a, **k: (probe("files"), real_build(*a, **k))[1])
|
||||
first_sync(api)
|
||||
assert ("markdown", "free") in seen and ("files", "free") in seen and not [s for s in seen if s[1] == "locked"]
|
||||
|
||||
|
||||
def test_swap_failure_rolls_back_rows(api, monkeypatch):
|
||||
first_sync(api)
|
||||
api.enqueue()
|
||||
|
||||
def boom(cur, new, old):
|
||||
raise OSError("диск полон")
|
||||
|
||||
monkeypatch.setattr(sync, "_swap", boom)
|
||||
done = api.run([TREE[0], page("DOC-3", "Новая", "DOC-1", 1)], ["DOC-1", "DOC-3"])
|
||||
assert done["status"] == "failed" and "не удалось подменить копию" in done["error"]
|
||||
assert api.project()["wiki_pages"] == 3 and "диск полон" in api.project()["wiki_sync_error"]
|
||||
assert [r["code"] for r in api.rows("SELECT code FROM wiki_page ORDER BY code")] == ["DOC-1", "DOC-2", "DOC-3"]
|
||||
assert (api.copy / "АРМ КА" / "01_Статусы" / "index.md").is_file() and not list(api.wiki_dir.glob(".*"))
|
||||
|
||||
|
||||
def test_batch_limit_and_replace(api):
|
||||
cmd = api.claim()
|
||||
with pytest.raises(HubError) as e:
|
||||
api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [page(f"DOC-{i}", "t") for i in range(51)]})
|
||||
assert e.value.status == 422
|
||||
api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0]]})
|
||||
api.wiki.post("/agent/wiki/pages", {"command_id": cmd["id"], "pages": [TREE[0] | {"title": "Новый"}]}) # повтор — замена
|
||||
assert [json.loads(r["payload"])["title"] for r in api.rows("SELECT payload FROM wiki_sync_stage")] == ["Новый"]
|
||||
Loading…
x
Reference in New Issue
Block a user