283 lines
15 KiB
Python
283 lines
15 KiB
Python
import json
|
||
import os
|
||
import tempfile
|
||
import unittest
|
||
|
||
from _common import SETTINGS, agent, eva_cfg, no_sleep
|
||
from fake_eva import FakeEva
|
||
import fake_hub
|
||
from fake_hub import FakeHub
|
||
|
||
GOOD = {"page_code": "DOC-1", "code": "DOCH-1", "created_at": "2026-10-27T17:59:00+03:00",
|
||
"author": {"login": None, "name": None}, "html": "", "attachments": [], "links": []}
|
||
|
||
|
||
class FakeHubVersionsTest(unittest.TestCase):
|
||
def test_check_versions(self):
|
||
claimed = {"type": "wiki_sync", "status": "claimed"}
|
||
self.assertIsNone(fake_hub.check_versions(claimed, 1, [GOOD]))
|
||
self.assertEqual(fake_hub.check_versions(claimed, 1, [GOOD] * 51)[0], 422)
|
||
self.assertEqual(fake_hub.check_versions({"type": "wiki_sync", "status": "done"}, 1, [GOOD])[0], 409)
|
||
self.assertEqual(fake_hub.check_versions(claimed, 1, [dict(GOOD, created_at="2026-10-27T17:59:00")]),
|
||
(422, {"error": "bad_versions", "detail": "DOCH-1: created_at"}))
|
||
self.assertEqual(fake_hub.check_versions(claimed, 1, [dict(GOOD, author={"login": 1, "name": None})]),
|
||
(422, {"error": "bad_versions", "detail": "DOCH-1: author"}))
|
||
|
||
|
||
class SyncBase(unittest.TestCase):
|
||
def setUp(self):
|
||
self.eva = FakeEva()
|
||
self.hub = FakeHub(SETTINGS)
|
||
self.addCleanup(self.eva.stop)
|
||
self.addCleanup(self.hub.stop)
|
||
self.cfg = eva_cfg(self.eva.base_url, hub_url=self.hub.base_url, hub_token=self.hub.token)
|
||
self.client = agent.EvaClient(self.cfg)
|
||
self.client.sleep = no_sleep
|
||
self.ex = agent.Executor(self.client, agent.HubClient(self.cfg))
|
||
|
||
def sync(self, root="DOC-1", label="tz"):
|
||
cid = self.hub.enqueue("wiki_sync", {"label": label, "root": root})
|
||
self.hub.queue.remove(cid)
|
||
self.hub.commands[cid]["status"] = "claimed"
|
||
self.cid = cid
|
||
return self.ex.run({"id": cid, "type": "wiki_sync", "params": {"label": label, "root": root}})
|
||
|
||
def sent(self):
|
||
return [p for body in self.hub.pages for p in body["pages"]]
|
||
|
||
|
||
class WikiSyncTest(SyncBase):
|
||
def test_sync_tree_order(self):
|
||
e = self.eva
|
||
e.add_doc(1, "DOC-1", "Корень", project="P", text="<p>корень</p>")
|
||
e.add_doc(2, "DOC-2", "Б", parent="DOC-1", project="P", orderno=2, text="<p>б</p>")
|
||
e.add_doc(3, "DOC-3", "А", parent="DOC-1", project="P", orderno=1, text="<p>а</p>")
|
||
e.add_doc(4, "DOC-4", "Внук", parent="DOC-3", project="P", text="<p>внук</p>")
|
||
e.add_doc(5, "DOC-5", "Чужой", parent="DOC-1", project="OTHER")
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "done", out)
|
||
self.assertEqual(out["result"]["codes"], ["DOC-1", "DOC-3", "DOC-4", "DOC-2"])
|
||
self.assertEqual(out["result"]["root_title"], "Корень")
|
||
self.assertEqual(out["result"]["eva_calls"], self.client.calls)
|
||
pages = {p["code"]: p for p in self.sent()}
|
||
self.assertEqual([p["code"] for p in self.sent()], out["result"]["codes"])
|
||
self.assertIsNone(pages["DOC-1"]["parent_code"])
|
||
self.assertEqual(pages["DOC-3"]["parent_code"], "DOC-1")
|
||
self.assertEqual(pages["DOC-4"]["parent_code"], "DOC-3")
|
||
self.assertEqual(pages["DOC-3"]["orderno"], 1)
|
||
self.assertIsNone(pages["DOC-4"]["orderno"])
|
||
self.assertEqual(pages["DOC-3"]["url"], self.eva.base_url + "/project/Document/DOC-3")
|
||
self.assertEqual(pages["DOC-3"]["eva_id"], "3")
|
||
self.assertEqual(pages["DOC-3"]["title"], "А")
|
||
self.assertEqual(pages["DOC-3"]["html"], "<p>а</p>")
|
||
self.assertEqual(self.hub.pages[0]["command_id"], self.cid)
|
||
|
||
def test_sync_passes_links_and_attachments(self):
|
||
self.eva.add_doc(1, "DOC-1", "К", project="P", text=(
|
||
'<p><a href="/project/Document/DOC-7">x</a><img src="/files/a/b.png" alt="Схема"></p>'))
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "done", out)
|
||
page = self.sent()[0]
|
||
self.assertEqual(page["links"], [{"ref": "/project/Document/DOC-7", "code": "DOC-7"}])
|
||
self.assertEqual(page["attachments"], [{"ref": "/files/a/b.png", "url": self.eva.base_url + "/files/a/b.png",
|
||
"name": "Схема"}])
|
||
|
||
def test_sync_batches_50(self):
|
||
self.eva.add_doc(1, "DOC-1", "Корень", project="P")
|
||
for i in range(2, 121):
|
||
self.eva.add_doc(i, "DOC-%d" % i, "С%03d" % i, parent="DOC-1", project="P", orderno=i)
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "done", out)
|
||
self.assertEqual([len(b["pages"]) for b in self.hub.pages], [50, 50, 20])
|
||
self.assertEqual(len(out["result"]["codes"]), 120)
|
||
self.assertEqual(out["result"]["codes"][0], "DOC-1")
|
||
|
||
def test_sync_batch_bytes(self):
|
||
big = "<p>" + "x" * (1500 * 1024) + "</p>"
|
||
self.eva.add_doc(1, "DOC-1", "Корень", project="P", text=big)
|
||
for i in (2, 3):
|
||
self.eva.add_doc(i, "DOC-%d" % i, "С", parent="DOC-1", project="P", orderno=i, text=big)
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "done", out)
|
||
sizes = [len(json.dumps(b, ensure_ascii=False).encode("utf-8")) for b in self.hub.pages]
|
||
self.assertTrue(all(s <= 4 * 1024 * 1024 for s in sizes), sizes)
|
||
self.assertEqual([len(b["pages"]) for b in self.hub.pages], [2, 1])
|
||
|
||
def test_tree_cycle_and_two_parents(self):
|
||
e = self.eva
|
||
e.add_doc(1, "DOC-1", "Корень", project="P")
|
||
e.add_doc(3, "DOC-3", "Три", parent="DOC-1", project="P", orderno=1)
|
||
e.add_doc(10, "DOC-A", "А", parent="DOC-B", tree_parent="DOC-1", project="P", orderno=2)
|
||
e.add_doc(11, "DOC-B", "Б", tree_parent="DOC-A", project="P")
|
||
e.add_doc(12, "DOC-C", "Ц", parent="DOC-1", tree_parent="DOC-3", project="P", orderno=3)
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "done", out)
|
||
codes = out["result"]["codes"]
|
||
self.assertEqual(len(codes), len(set(codes)))
|
||
self.assertEqual(sorted(codes), ["DOC-1", "DOC-3", "DOC-A", "DOC-B", "DOC-C"])
|
||
self.assertEqual([p["code"] for p in self.sent()], codes)
|
||
|
||
def fill_forms_tree(self):
|
||
e = self.eva
|
||
e.add_doc(101, "DOC-1", "Корень", project="P")
|
||
e.add_doc(103, "DOC-3", "Три", parent="DOC-1", project="P", orderno=1) # только parent
|
||
e.add_doc(104, "DOC-4", "Внук", tree_parent="DOC-3", project="P") # только tree_parent
|
||
e.add_doc(110, "DOC-A", "А", parent="DOC-B", tree_parent="DOC-1", project="P", orderno=2)
|
||
e.add_doc(111, "DOC-B", "Б", tree_parent="DOC-A", project="P") # цикл A <-> B
|
||
e.add_doc(112, "DOC-C", "Ц", parent="DOC-1", tree_parent="DOC-3", project="P", orderno=3) # два родителя
|
||
|
||
def test_all_ref_forms(self):
|
||
"""Ссылки parent/tree_parent бывают {id, code}, {id} и голой строкой-id; id документа не равен коду."""
|
||
for form in ("full", "id_only", "string", "code_id"):
|
||
with self.subTest(form=form):
|
||
self.setUp()
|
||
self.eva.ref_form = form
|
||
self.fill_forms_tree()
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "done", out)
|
||
codes = out["result"]["codes"]
|
||
self.assertEqual(codes[:4], ["DOC-1", "DOC-3", "DOC-C", "DOC-4"])
|
||
self.assertEqual(len(codes), len(set(codes)))
|
||
self.assertEqual(sorted(codes), ["DOC-1", "DOC-3", "DOC-4", "DOC-A", "DOC-B", "DOC-C"])
|
||
parents = {p["code"]: p["parent_code"] for p in self.sent()}
|
||
self.assertEqual(parents["DOC-4"], "DOC-3")
|
||
self.assertEqual(parents["DOC-A"], "DOC-1")
|
||
self.assertEqual(parents["DOC-B"], "DOC-A")
|
||
self.assertEqual(parents["DOC-C"], "DOC-3") # первый родитель по обходу в глубину
|
||
self.assertEqual(self.sent()[0]["eva_id"], "101")
|
||
self.eva.stop()
|
||
self.hub.stop()
|
||
|
||
def test_root_without_project(self):
|
||
e = self.eva
|
||
e.add_doc(1, "DOC-1", "Корень", text="<p>к</p>")
|
||
e.add_doc(2, "DOC-2", "Б", tree_parent="DOC-1", orderno=2)
|
||
e.add_doc(3, "DOC-3", "А", tree_parent="DOC-1", orderno=1)
|
||
e.add_doc(4, "DOC-4", "Внук", tree_parent="DOC-3")
|
||
e.add_doc(5, "DOC-5", "Посторонний", tree_parent="DOC-9")
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "done", out)
|
||
self.assertEqual(out["result"]["codes"], ["DOC-1", "DOC-3", "DOC-4", "DOC-2"])
|
||
self.assertEqual({p["code"]: p["parent_code"] for p in self.sent()},
|
||
{"DOC-1": None, "DOC-3": "DOC-1", "DOC-4": "DOC-3", "DOC-2": "DOC-1"})
|
||
|
||
def test_root_missing(self):
|
||
out = self.sync("DOC-9")
|
||
self.assertEqual(out["status"], "failed")
|
||
self.assertEqual(out["error"], "страница DOC-9 не найдена или нет прав")
|
||
self.assertEqual(self.hub.pages, [])
|
||
|
||
def test_eva_failure_reported(self):
|
||
self.eva.add_doc(1, "DOC-1", "Корень", project="P")
|
||
self.eva.fail_next(503, 4)
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "failed")
|
||
self.assertIn("503", out["error"])
|
||
self.assertNotIn(FakeEva.token, out["error"])
|
||
self.assertEqual(self.hub.pages, [])
|
||
|
||
def test_hub_rejects_batch(self):
|
||
self.eva.add_doc(1, "DOC-1", "Корень", project="P")
|
||
self.hub.reject_pages = (422, {"error": "bad_pages", "detail": "DOC-1: html"})
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "failed")
|
||
self.assertIn("422", out["error"])
|
||
self.assertIn("DOC-1: html", out["error"])
|
||
|
||
def test_eva_rpc_error_reported(self):
|
||
"""M-1: `error` JSON-RPC Eva — failed «Eva: <текст>», без повтора и без «внутренней ошибки агента»."""
|
||
self.eva.add_doc(1, "DOC-1", "Корень", project="P")
|
||
self.eva.rpc_error_next("нет прав, токен eva-secret")
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "failed")
|
||
self.assertEqual(out["error"], "Eva: CmfDocument.get: нет прав, токен ***")
|
||
self.assertFalse(out.get("retryable"))
|
||
self.assertEqual(self.hub.pages, [])
|
||
|
||
def test_eva_rpc_error_in_runner(self):
|
||
self.eva.add_doc(1, "DOC-1", "Корень", project="P")
|
||
self.eva.rpc_error_next("нет прав")
|
||
runner = agent.Runner(self.cfg, self.ex.hub, self.client, agent.Journal(os.devnull))
|
||
runner.executor = self.ex
|
||
self.assertEqual(runner.run_once({"id": 1, "type": "wiki_sync", "params": {"root": "DOC-1"}})["error"],
|
||
"Eva: CmfDocument.get: нет прав")
|
||
|
||
def test_hub_413_single_page(self):
|
||
"""M-3: хаб не принял тело (413) — в ошибке названа страница."""
|
||
self.eva.add_doc(1, "DOC-1", "Корень", project="P")
|
||
self.hub.reject_pages = (413, {"error": "too_large", "detail": "тело больше 16777216 байт"})
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "failed")
|
||
self.assertIn("страница DOC-1 больше 16 МБ", out["error"])
|
||
self.assertFalse(out.get("retryable"))
|
||
|
||
def test_hub_413_batch(self):
|
||
self.eva.add_doc(1, "DOC-1", "Корень", project="P")
|
||
for i in (2, 3):
|
||
self.eva.add_doc(i, "DOC-%d" % i, "С", parent="DOC-1", project="P", orderno=i)
|
||
self.hub.reject_pages = (413, {"error": "too_large", "detail": "тело больше 16777216 байт"})
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "failed")
|
||
self.assertIn("пачка страниц DOC-1…DOC-3 (3 шт.) больше 16 МБ", out["error"])
|
||
|
||
def test_empty_name_falls_back_to_code(self):
|
||
self.eva.add_doc(1, "DOC-1", "", project="P") # хаб отклоняет пустой title (stage._check)
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "done", out)
|
||
self.assertEqual(self.sent()[0]["title"], "DOC-1")
|
||
self.assertEqual(out["result"]["root_title"], "DOC-1")
|
||
|
||
def test_page_vanished(self):
|
||
e = self.eva
|
||
e.add_doc(1, "DOC-1", "Корень", project="P")
|
||
e.add_doc(2, "DOC-2", "Б", parent="DOC-1", project="P")
|
||
real = self.client.get_doc
|
||
self.client.get_doc = lambda code, fields: None if code == "DOC-2" else real(code, fields)
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "failed")
|
||
self.assertIn("DOC-2", out["error"])
|
||
|
||
def test_counts(self):
|
||
e = self.eva
|
||
e.add_doc(1, "DOC-1", "Корень", project="P")
|
||
e.add_doc(2, "DOC-2", "Б", parent="DOC-1", project="P")
|
||
e.fail_next(429, 2, retry_after=0)
|
||
out = self.sync()
|
||
self.assertEqual(out["status"], "done", out)
|
||
self.assertEqual(out["jira_calls"], len(e.requests))
|
||
self.assertEqual(out["jira_calls"], self.client.calls)
|
||
self.assertEqual(out["rate_limited"], 2)
|
||
|
||
|
||
class WikiSyncRecoverTest(unittest.TestCase):
|
||
def test_recover_resyncs(self):
|
||
eva, hub = FakeEva(), FakeHub(SETTINGS)
|
||
tmp = tempfile.TemporaryDirectory()
|
||
for x in (eva.stop, hub.stop, tmp.cleanup):
|
||
self.addCleanup(x)
|
||
eva.add_doc(1, "DOC-1", "Корень", project="P")
|
||
for i in range(2, 61):
|
||
eva.add_doc(i, "DOC-%d" % i, "С", parent="DOC-1", project="P", orderno=i)
|
||
cid = hub.enqueue("wiki_sync", {"label": "tz", "root": "DOC-1"})
|
||
hub.queue.remove(cid)
|
||
hub.commands[cid]["status"] = "claimed"
|
||
hub.in_flight = cid
|
||
journal = agent.Journal(os.path.join(tmp.name, "journal.jsonl"))
|
||
journal.claimed(cid)
|
||
journal.started(cid) # перезапуск посреди выполнения: finished нет
|
||
cfg = eva_cfg(eva.base_url, hub_url=hub.base_url, hub_token=hub.token)
|
||
runner = agent.Runner(cfg, agent.HubClient(cfg), agent.EvaClient(cfg), journal)
|
||
runner.sleep = lambda s: None
|
||
runner.eva.sleep = no_sleep
|
||
runner.idle_limit = 1
|
||
runner.loop()
|
||
self.assertEqual(hub.results[cid]["status"], "done", hub.results[cid])
|
||
self.assertEqual(len(hub.results[cid]["result"]["codes"]), 60)
|
||
self.assertEqual([len(b["pages"]) for b in hub.pages], [50, 10])
|
||
self.assertEqual(len([r for r in journal.records() if r["kind"] == "finished"]), 1)
|
||
self.assertEqual(journal.unfinished_started(), [])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|