eva-agent/tests/test_wiki_sync.py

import json
import os
import tempfile
import unittest

from _common import SETTINGS, agent, eva_cfg, no_sleep
from fake_eva import FakeEva
from fake_hub import FakeHub


class SyncBase(unittest.TestCase):
    def setUp(self):
        self.eva = FakeEva()
        self.hub = FakeHub(SETTINGS)
        self.addCleanup(self.eva.stop)
        self.addCleanup(self.hub.stop)
        self.cfg = eva_cfg(self.eva.base_url, hub_url=self.hub.base_url, hub_token=self.hub.token)
        self.client = agent.EvaClient(self.cfg)
        self.client.sleep = no_sleep
        self.ex = agent.Executor(self.client, agent.HubClient(self.cfg), dict(SETTINGS))

    def sync(self, root="DOC-1", label="tz"):
        cid = self.hub.enqueue("wiki_sync", {"label": label, "root": root})
        self.hub.queue.remove(cid)
        self.hub.commands[cid]["status"] = "claimed"
        self.cid = cid
        return self.ex.run({"id": cid, "type": "wiki_sync", "params": {"label": label, "root": root}})

    def sent(self):
        return [p for body in self.hub.pages for p in body["pages"]]


class WikiSyncTest(SyncBase):
    def test_sync_tree_order(self):
        e = self.eva
        e.add_doc(1, "DOC-1", "Корень", project="P", text="<p>корень</p>")
        e.add_doc(2, "DOC-2", "Б", parent="DOC-1", project="P", orderno=2, text="<p>б</p>")
        e.add_doc(3, "DOC-3", "А", parent="DOC-1", project="P", orderno=1, text="<p>а</p>")
        e.add_doc(4, "DOC-4", "Внук", parent="DOC-3", project="P", text="<p>внук</p>")
        e.add_doc(5, "DOC-5", "Чужой", parent="DOC-1", project="OTHER")
        out = self.sync()
        self.assertEqual(out["status"], "done", out)
        self.assertEqual(out["result"]["codes"], ["DOC-1", "DOC-3", "DOC-4", "DOC-2"])
        self.assertEqual(out["result"]["root_title"], "Корень")
        self.assertEqual(out["result"]["eva_calls"], self.client.calls)
        pages = {p["code"]: p for p in self.sent()}
        self.assertEqual([p["code"] for p in self.sent()], out["result"]["codes"])
        self.assertIsNone(pages["DOC-1"]["parent_code"])
        self.assertEqual(pages["DOC-3"]["parent_code"], "DOC-1")
        self.assertEqual(pages["DOC-4"]["parent_code"], "DOC-3")
        self.assertEqual(pages["DOC-3"]["orderno"], 1)
        self.assertIsNone(pages["DOC-4"]["orderno"])
        self.assertEqual(pages["DOC-3"]["url"], self.eva.base_url + "/project/Document/DOC-3")
        self.assertEqual(pages["DOC-3"]["eva_id"], "3")
        self.assertEqual(pages["DOC-3"]["title"], "А")
        self.assertEqual(pages["DOC-3"]["html"], "<p>а</p>")
        self.assertEqual(self.hub.pages[0]["command_id"], self.cid)

    def test_sync_passes_links_and_attachments(self):
        self.eva.add_doc(1, "DOC-1", "К", project="P", text=(
            '<p><a href="/project/Document/DOC-7">x</a><img src="/files/a/b.png" alt="Схема"></p>'))
        out = self.sync()
        self.assertEqual(out["status"], "done", out)
        page = self.sent()[0]
        self.assertEqual(page["links"], [{"ref": "/project/Document/DOC-7", "code": "DOC-7"}])
        self.assertEqual(page["attachments"], [{"ref": "/files/a/b.png", "url": self.eva.base_url + "/files/a/b.png",
                                                "name": "Схема"}])

    def test_sync_batches_50(self):
        self.eva.add_doc(1, "DOC-1", "Корень", project="P")
        for i in range(2, 121):
            self.eva.add_doc(i, "DOC-%d" % i, "С%03d" % i, parent="DOC-1", project="P", orderno=i)
        out = self.sync()
        self.assertEqual(out["status"], "done", out)
        self.assertEqual([len(b["pages"]) for b in self.hub.pages], [50, 50, 20])
        self.assertEqual(len(out["result"]["codes"]), 120)
        self.assertEqual(out["result"]["codes"][0], "DOC-1")

    def test_sync_batch_bytes(self):
        big = "<p>" + "x" * (1500 * 1024) + "</p>"
        self.eva.add_doc(1, "DOC-1", "Корень", project="P", text=big)
        for i in (2, 3):
            self.eva.add_doc(i, "DOC-%d" % i, "С", parent="DOC-1", project="P", orderno=i, text=big)
        out = self.sync()
        self.assertEqual(out["status"], "done", out)
        sizes = [len(json.dumps(b, ensure_ascii=False).encode("utf-8")) for b in self.hub.pages]
        self.assertTrue(all(s <= 4 * 1024 * 1024 for s in sizes), sizes)
        self.assertEqual([len(b["pages"]) for b in self.hub.pages], [2, 1])

    def test_tree_cycle_and_two_parents(self):
        e = self.eva
        e.add_doc(1, "DOC-1", "Корень", project="P")
        e.add_doc(3, "DOC-3", "Три", parent="DOC-1", project="P", orderno=1)
        e.add_doc(10, "DOC-A", "А", parent="DOC-B", tree_parent="DOC-1", project="P", orderno=2)
        e.add_doc(11, "DOC-B", "Б", tree_parent="DOC-A", project="P")
        e.add_doc(12, "DOC-C", "Ц", parent="DOC-1", tree_parent="DOC-3", project="P", orderno=3)
        out = self.sync()
        self.assertEqual(out["status"], "done", out)
        codes = out["result"]["codes"]
        self.assertEqual(len(codes), len(set(codes)))
        self.assertEqual(sorted(codes), ["DOC-1", "DOC-3", "DOC-A", "DOC-B", "DOC-C"])
        self.assertEqual([p["code"] for p in self.sent()], codes)

    def fill_forms_tree(self):
        e = self.eva
        e.add_doc(101, "DOC-1", "Корень", project="P")
        e.add_doc(103, "DOC-3", "Три", parent="DOC-1", project="P", orderno=1)       # только parent
        e.add_doc(104, "DOC-4", "Внук", tree_parent="DOC-3", project="P")             # только tree_parent
        e.add_doc(110, "DOC-A", "А", parent="DOC-B", tree_parent="DOC-1", project="P", orderno=2)
        e.add_doc(111, "DOC-B", "Б", tree_parent="DOC-A", project="P")                # цикл A <-> B
        e.add_doc(112, "DOC-C", "Ц", parent="DOC-1", tree_parent="DOC-3", project="P", orderno=3)   # два родителя

    def test_all_ref_forms(self):
        """Ссылки parent/tree_parent бывают {id, code}, {id} и голой строкой-id; id документа не равен коду."""
        for form in ("full", "id_only", "string", "code_id"):
            with self.subTest(form=form):
                self.setUp()
                self.eva.ref_form = form
                self.fill_forms_tree()
                out = self.sync()
                self.assertEqual(out["status"], "done", out)
                codes = out["result"]["codes"]
                self.assertEqual(codes[:4], ["DOC-1", "DOC-3", "DOC-C", "DOC-4"])
                self.assertEqual(len(codes), len(set(codes)))
                self.assertEqual(sorted(codes), ["DOC-1", "DOC-3", "DOC-4", "DOC-A", "DOC-B", "DOC-C"])
                parents = {p["code"]: p["parent_code"] for p in self.sent()}
                self.assertEqual(parents["DOC-4"], "DOC-3")
                self.assertEqual(parents["DOC-A"], "DOC-1")
                self.assertEqual(parents["DOC-B"], "DOC-A")
                self.assertEqual(parents["DOC-C"], "DOC-3")      # первый родитель по обходу в глубину
                self.assertEqual(self.sent()[0]["eva_id"], "101")
                self.eva.stop()
                self.hub.stop()

    def test_root_without_project(self):
        e = self.eva
        e.add_doc(1, "DOC-1", "Корень", text="<p>к</p>")
        e.add_doc(2, "DOC-2", "Б", tree_parent="DOC-1", orderno=2)
        e.add_doc(3, "DOC-3", "А", tree_parent="DOC-1", orderno=1)
        e.add_doc(4, "DOC-4", "Внук", tree_parent="DOC-3")
        e.add_doc(5, "DOC-5", "Посторонний", tree_parent="DOC-9")
        out = self.sync()
        self.assertEqual(out["status"], "done", out)
        self.assertEqual(out["result"]["codes"], ["DOC-1", "DOC-3", "DOC-4", "DOC-2"])
        self.assertEqual({p["code"]: p["parent_code"] for p in self.sent()},
                         {"DOC-1": None, "DOC-3": "DOC-1", "DOC-4": "DOC-3", "DOC-2": "DOC-1"})

    def test_root_missing(self):
        out = self.sync("DOC-9")
        self.assertEqual(out["status"], "failed")
        self.assertEqual(out["error"], "страница DOC-9 не найдена или нет прав")
        self.assertEqual(self.hub.pages, [])

    def test_eva_failure_reported(self):
        self.eva.add_doc(1, "DOC-1", "Корень", project="P")
        self.eva.fail_next(503, 4)
        out = self.sync()
        self.assertEqual(out["status"], "failed")
        self.assertIn("503", out["error"])
        self.assertNotIn(FakeEva.token, out["error"])
        self.assertEqual(self.hub.pages, [])

    def test_hub_rejects_batch(self):
        self.eva.add_doc(1, "DOC-1", "Корень", project="P")
        self.hub.reject_pages = (422, {"error": "bad_pages", "detail": "DOC-1: html"})
        out = self.sync()
        self.assertEqual(out["status"], "failed")
        self.assertIn("422", out["error"])
        self.assertIn("DOC-1: html", out["error"])

    def test_empty_name_falls_back_to_code(self):
        self.eva.add_doc(1, "DOC-1", "", project="P")        # хаб отклоняет пустой title (stage._check)
        out = self.sync()
        self.assertEqual(out["status"], "done", out)
        self.assertEqual(self.sent()[0]["title"], "DOC-1")
        self.assertEqual(out["result"]["root_title"], "DOC-1")

    def test_page_vanished(self):
        e = self.eva
        e.add_doc(1, "DOC-1", "Корень", project="P")
        e.add_doc(2, "DOC-2", "Б", parent="DOC-1", project="P")
        real = self.client.get_doc
        self.client.get_doc = lambda code, fields: None if code == "DOC-2" else real(code, fields)
        out = self.sync()
        self.assertEqual(out["status"], "failed")
        self.assertIn("DOC-2", out["error"])

    def test_counts(self):
        e = self.eva
        e.add_doc(1, "DOC-1", "Корень", project="P")
        e.add_doc(2, "DOC-2", "Б", parent="DOC-1", project="P")
        e.fail_next(429, 2, retry_after=0)
        out = self.sync()
        self.assertEqual(out["status"], "done", out)
        self.assertEqual(out["jira_calls"], len(e.requests))
        self.assertEqual(out["jira_calls"], self.client.calls)
        self.assertEqual(out["rate_limited"], 2)


class WikiSyncRecoverTest(unittest.TestCase):
    def test_recover_resyncs(self):
        eva, hub = FakeEva(), FakeHub(SETTINGS)
        tmp = tempfile.TemporaryDirectory()
        for x in (eva.stop, hub.stop, tmp.cleanup):
            self.addCleanup(x)
        eva.add_doc(1, "DOC-1", "Корень", project="P")
        for i in range(2, 61):
            eva.add_doc(i, "DOC-%d" % i, "С", parent="DOC-1", project="P", orderno=i)
        cid = hub.enqueue("wiki_sync", {"label": "tz", "root": "DOC-1"})
        hub.queue.remove(cid)
        hub.commands[cid]["status"] = "claimed"
        hub.in_flight = cid
        journal = agent.Journal(os.path.join(tmp.name, "journal.jsonl"))
        journal.claimed(cid)
        journal.started(cid)                  # перезапуск посреди выполнения: finished нет
        cfg = eva_cfg(eva.base_url, hub_url=hub.base_url, hub_token=hub.token)
        runner = agent.Runner(cfg, agent.HubClient(cfg), agent.EvaClient(cfg), journal)
        runner.sleep = lambda s: None
        runner.eva.sleep = no_sleep
        runner.idle_limit = 1
        runner.loop()
        self.assertEqual(hub.results[cid]["status"], "done", hub.results[cid])
        self.assertEqual(len(hub.results[cid]["result"]["codes"]), 60)
        self.assertEqual([len(b["pages"]) for b in hub.pages], [50, 10])
        self.assertEqual(len([r for r in journal.records() if r["kind"] == "finished"]), 1)
        self.assertEqual(journal.unfinished_started(), [])


if __name__ == "__main__":
    unittest.main()