import hashlib import json import os import signal import subprocess import sys import tempfile import textwrap import time import unittest from _common import SRC if SRC not in sys.path: sys.path.insert(0, SRC) import run # noqa: E402 # Поддельные агенты: маленькие скрипты, которые ведут себя как agent.py по контракту (коды выхода, update.json). STATE_HELPERS = textwrap.dedent(''' import json, os, sys HERE = os.path.dirname(os.path.abspath(__file__)) STATE = os.path.join(HERE, "update.json") def state(): try: return json.load(open(STATE, encoding="utf-8")) except Exception: return None def write(**kw): json.dump(kw, open(STATE, "w", encoding="utf-8")) def mark(name): open(os.path.join(HERE, "runs.log"), "a").write(name + "\\n") def human_stop(code=0): # Ctrl-C в терминале получает вся группа: запускатель тоже видит SIGINT, затем агент выходит import signal os.kill(os.getppid(), signal.SIGINT) sys.exit(code) ''') def fake(name, body): return STATE_HELPERS + "mark(%r)\n" % name + textwrap.dedent(body) # v5 «нашёл» новую версию: кладёт agent.py.new из next.py, пишет staged и выходит с 75; после отката/отказа — выход 0. V5_UPDATES = fake("v5", ''' import hashlib st = state() if st is None: data = open(os.path.join(HERE, "next.py"), "rb").read() open(os.path.join(HERE, "agent.py.new"), "wb").write(data) write(state="staged", from_version=5, to_version=6, sha256=hashlib.sha256(data).hexdigest()) sys.exit(75) human_stop() ''') # v5 с подменённым после проверки файлом: sha в staged не сходится с agent.py.new. V5_BAD_STAGE = fake("v5", ''' st = state() if st is None: open(os.path.join(HERE, "agent.py.new"), "w").write("мусор") write(state="staged", from_version=5, to_version=6, sha256="чужой") sys.exit(75) human_stop() ''') V6_OK_THEN_STOP = fake("v6", ''' st = state() write(**dict(st, state="ok")) human_stop() ''') V6_CRASH = fake("v6", 'sys.exit(1)\n') V6_HANG = fake("v6", 'import time\ntime.sleep(60)\n') V6_SEES_STATE = fake("v6", ''' mark("seen:" + (state() or {}).get("state", "-")) write(**dict(state(), state="ok")) human_stop() ''') V6_OK_ON_TERM = fake("v6", ''' import signal, time if len(open(os.path.join(HERE, "runs.log")).read().split()) > 2: human_stop() def term(*a): write(**dict(state(), state="ok")) # ответ claim пришёл в последнюю секунду пробного периода sys.exit(0) signal.signal(signal.SIGTERM, term) time.sleep(60) ''') # v6 подтвердился (ok), через delay с записал rejected для v10 с другим sha (как Updater._reject) и упал с кодом 1. V6_OK_THEN_REJECT_NEXT = fake("v6", ''' import time n = len(open(os.path.join(HERE, "runs.log")).read().split()) if n > 2: human_stop() st = state() write(**dict(st, state="ok")) time.sleep(DELAY) write(state="rejected", from_version=6, to_version=10, sha256="чужой", error="отвергнута") sys.exit(1) ''') class LauncherTest(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.dir = self.tmp.name self.l = run.Launcher(self.dir, ["--journal", "j.jsonl"]) self.pauses = [] self.l.pause = lambda s: (self.pauses.append(s), True)[1] self.old_sigint = signal.signal(signal.SIGINT, self.l.on_sigint) # как в run.main: Ctrl-C видит и запускатель def tearDown(self): signal.signal(signal.SIGINT, self.old_sigint) self.tmp.cleanup() def put(self, name, text): with open(os.path.join(self.dir, name), "w", encoding="utf-8") as f: f.write(text) def read(self, name): with open(os.path.join(self.dir, name), encoding="utf-8") as f: return f.read() def state(self): return json.loads(self.read("update.json")) def runs(self): return self.read("runs.log").split() def next_version(self, text): self.put("next.py", text) return hashlib.sha256(text.encode("utf-8")).hexdigest() def stage(self, new_text, to_version=6): self.put("agent.py.new", new_text) sha = hashlib.sha256(new_text.encode("utf-8")).hexdigest() self.put("update.json", json.dumps({"state": "staged", "from_version": 5, "to_version": to_version, "sha256": sha})) return sha def sha(self, text): return hashlib.sha256(text.encode("utf-8")).hexdigest() def path(self, name): return os.path.join(self.dir, name) def test_update_applied_and_confirmed(self): self.put("agent.py", V5_UPDATES) sha = self.next_version(V6_OK_THEN_STOP) self.assertEqual(self.l.run(), 0) self.assertEqual(self.read("agent.py"), V6_OK_THEN_STOP) self.assertEqual(self.read("agent.py.prev"), V5_UPDATES) self.assertFalse(os.path.exists(os.path.join(self.dir, "agent.py.new"))) st = self.state() self.assertEqual((st["state"], st["from_version"], st["to_version"], st["sha256"]), ("ok", 5, 6, sha)) self.assertEqual(self.runs(), ["v5", "v6"]) def test_crash_in_trial_rolls_back(self): self.put("agent.py", V5_UPDATES) sha = self.next_version(V6_CRASH) self.assertEqual(self.l.run(), 0) self.assertEqual(self.read("agent.py"), V5_UPDATES) st = self.state() self.assertEqual((st["state"], st["to_version"], st["sha256"]), ("rolled_back", 6, sha)) self.assertRegex(st["error"], r"^код 1 через \d+ с$") self.assertEqual(self.runs(), ["v5", "v6", "v5"]) self.assertEqual(self.pauses, []) # откат без паузы def test_hang_in_trial_rolls_back(self): self.l.trial_seconds = 1 self.l.kill_grace = 5 self.put("agent.py", V5_UPDATES) self.next_version(V6_HANG) self.assertEqual(self.l.run(), 0) self.assertEqual(self.read("agent.py"), V5_UPDATES) self.assertEqual(self.state()["error"], "не вышла на связь за 10 мин") def test_crash_after_ok_restarts_with_backoff(self): self.put("update.json", json.dumps({"state": "ok", "from_version": 5, "to_version": 6, "sha256": "s"})) self.put("agent.py", fake("v6", ''' n = len(open(os.path.join(HERE, "runs.log")).read().split()) human_stop() if n > 2 else sys.exit(1) ''')) self.assertEqual(self.l.run(), 0) self.assertEqual(self.pauses, [5, 10]) self.assertEqual(self.state()["state"], "ok") # отката нет def test_backoff_caps_and_resets(self): self.assertEqual([self.l.next_backoff(10) for _ in range(8)], [5, 10, 20, 40, 80, 160, 300, 300]) self.assertEqual(self.l.next_backoff(600), 5) # проработал 10 минут — с начала def test_75_without_staged_rejected(self): self.put("agent.py", V5_BAD_STAGE) self.assertEqual(self.l.run(), 0) self.assertEqual(self.read("agent.py"), V5_BAD_STAGE) self.assertEqual(self.runs(), ["v5", "v5"]) self.assertFalse(os.path.exists(os.path.join(self.dir, "agent.py.new"))) st = self.state() self.assertEqual((st["state"], st["error"]), ("rejected", "файл изменился после проверки")) def test_staged_at_start_is_applied(self): self.put("agent.py", fake("v5", "human_stop()\n")) self.stage(V6_OK_THEN_STOP) self.assertEqual(self.l.run(), 0) self.assertEqual(self.runs(), ["v6"]) self.assertEqual(self.state()["state"], "ok") def test_stray_new_removed_at_start(self): self.put("agent.py", fake("v5", "human_stop()\n")) self.put("agent.py.new", "остаток") self.assertEqual(self.l.run(), 0) self.assertFalse(os.path.exists(os.path.join(self.dir, "agent.py.new"))) def test_missing_agent_restored_from_prev(self): self.put("agent.py.prev", fake("v5", "human_stop()\n")) self.assertEqual(self.l.run(), 0) self.assertEqual(self.runs(), ["v5"]) def test_garbage_update_json_is_no_state(self): self.put("update.json", '{"state": "tri') self.put("agent.py", fake("v5", "human_stop()\n")) self.assertEqual(self.l.run(), 0) def test_env_and_args_passed(self): self.put("agent.py", fake("v5", ''' open(os.path.join(HERE, "env.txt"), "w").write(os.environ.get("JIRA_AGENT_LAUNCHER", "") + " " + " ".join(sys.argv[1:])) human_stop(130) ''')) self.assertEqual(self.l.run(), 130) self.assertEqual(self.read("env.txt"), "2 run --journal j.jsonl") def test_sigint_without_child_stops(self): self.l.on_sigint(signal.SIGINT, None) self.assertTrue(self.l.stopping) def test_sigint_with_child_ignored(self): self.l.child = object() self.l.on_sigint(signal.SIGINT, None) self.assertFalse(self.l.stopping) self.assertTrue(self.l.interrupted) # выход агента с 0 — остановка человеком def test_exit_0_without_signal_in_trial_rolls_back(self): self.put("agent.py", V5_UPDATES) self.next_version(fake("v6", "sys.exit(0)\n")) # сама вышла, никто не нажимал Ctrl-C (I-1) self.assertEqual(self.l.run(), 0) self.assertEqual(self.read("agent.py"), V5_UPDATES) st = self.state() self.assertEqual(st["state"], "rolled_back") self.assertRegex(st["error"], r"^код 0 через \d+ с$") self.assertEqual(self.runs(), ["v5", "v6", "v5"]) def test_exit_0_without_signal_after_ok_restarts(self): self.put("update.json", json.dumps({"state": "ok", "from_version": 5, "to_version": 6, "sha256": "s"})) self.put("agent.py", fake("v6", ''' n = len(open(os.path.join(HERE, "runs.log")).read().split()) human_stop() if n > 1 else sys.exit(0) ''')) self.assertEqual(self.l.run(), 0) self.assertEqual(self.pauses, [5]) # выход 0 без сигнала — как падение self.assertEqual(self.runs(), ["v6", "v6"]) def test_ctrl_c_with_75_applies_and_stops(self): self.put("next.py", V6_OK_THEN_STOP) self.put("agent.py", fake("v5", ''' import hashlib, signal data = open(os.path.join(HERE, "next.py"), "rb").read() open(os.path.join(HERE, "agent.py.new"), "wb").write(data) write(state="staged", from_version=5, to_version=6, sha256=hashlib.sha256(data).hexdigest()) os.kill(os.getppid(), signal.SIGINT) # Ctrl-C пришёл, когда версия уже подготовлена sys.exit(75) ''')) self.assertEqual(self.l.run(), 0) self.assertEqual(self.read("agent.py"), V6_OK_THEN_STOP) # подмена сделана, новая не запущена self.assertEqual(self.state()["state"], "trial") # пробный период начнётся при старте self.assertEqual(self.runs(), ["v5"]) def test_stale_interrupt_does_not_hide_exit_0(self): self.put("update.json", json.dumps({"state": "ok", "from_version": 5, "to_version": 6, "sha256": "s"})) self.put("agent.py", fake("v6", ''' import signal n = len(open(os.path.join(HERE, "runs.log")).read().split()) if n == 1: os.kill(os.getppid(), signal.SIGINT) # Ctrl-C до установки обработчиков агента sys.exit(1) human_stop() if n > 2 else sys.exit(0) ''')) self.l.stopping = False orig = self.l.on_sigint def sigint_child_alive(signum, frame): # агент жив — запускатель только запоминает Ctrl-C self.l.child = self.l.child or object() orig(signum, frame) signal.signal(signal.SIGINT, sigint_child_alive) self.assertEqual(self.l.run(), 0) self.assertEqual(self.runs(), ["v6", "v6", "v6"]) # второй выход 0 без сигнала — перезапуск self.assertEqual(len(self.pauses), 2) def test_stop_before_spawn(self): self.put("agent.py", fake("v5", "human_stop()\n")) self.l.stopping = True # SIGTERM пришёл между запусками (M-2) self.assertEqual(self.l.run(), 0) self.assertFalse(os.path.exists(os.path.join(self.dir, "runs.log"))) def test_pause_never_sleeps_negative(self): clock = iter([0.0, 0.5, 1.5, 1.6]) # остаток перешёл через ноль между вызовами (M-1) slept = [] orig_m, orig_s = run.time.monotonic, run.time.sleep run.time.monotonic, run.time.sleep = (lambda: next(clock)), slept.append try: self.assertTrue(run.Launcher.pause(self.l, 1.0)) finally: run.time.monotonic, run.time.sleep = orig_m, orig_s self.assertTrue(all(x >= 0 for x in slept), slept) def test_second_launcher_refused(self): with open(os.path.join(SRC, "run.py"), encoding="utf-8") as f: self.put("run.py", f.read()) self.put("agent.py", fake("v5", "sys.exit(0)\n")) lock = run.acquire_lock(self.dir) try: p = subprocess.run([sys.executable, os.path.join(self.dir, "run.py")], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=30) finally: lock.close() self.assertEqual(p.returncode, 1) self.assertIn("запускатель уже работает", p.stdout.decode("utf-8")) self.assertFalse(os.path.exists(os.path.join(self.dir, "runs.log"))) def test_staged_already_replaced_resumes_trial(self): """M-5: запускатель упал между os.replace(new, agent) и записью trial.""" self.stage(V6_SEES_STATE) os.replace(self.path("agent.py.new"), self.path("agent.py")) self.put("agent.py.prev", fake("v5", "human_stop()\n")) self.assertEqual(self.l.run(), 0) self.assertEqual(self.runs(), ["v6", "seen:trial"]) # пробный период, а не staged навсегда self.assertEqual(self.state()["state"], "ok") def test_staged_without_file_rejected(self): self.put("agent.py", fake("v5", "human_stop()\n")) self.put("update.json", json.dumps({"state": "staged", "from_version": 5, "to_version": 6, "sha256": "a" * 64})) self.assertEqual(self.l.run(), 0) st = self.state() self.assertEqual((st["state"], st["error"]), ("rejected", "подготовленный файл потерян")) self.assertEqual(self.runs(), ["v5"]) def test_stale_trial_marked_rolled_back(self): """M-3: откат прерван — agent.py уже прежняя версия, update.json всё ещё trial.""" self.l.trial_seconds = 1 self.put("agent.py", fake("v8", "import time\ntime.sleep(2)\nhuman_stop()\n")) # дольше пробного периода self.put("update.json", json.dumps({"state": "trial", "from_version": 8, "to_version": 9, "sha256": "a" * 64})) self.assertEqual(self.l.run(), 0) st = self.state() self.assertEqual((st["state"], st["to_version"], st["error"]), ("rolled_back", 9, "пробный период прерван: agent.py не v9")) self.assertEqual(self.runs(), ["v8"]) # не остановлен через пробный период def test_trial_without_sha_is_stale(self): self.put("agent.py", fake("v8", "human_stop()\n")) self.put("update.json", json.dumps({"state": "trial", "from_version": 8, "to_version": 9})) self.assertEqual(self.l.run(), 0) self.assertEqual(self.state()["state"], "rolled_back") def test_trial_survives_launcher_restart(self): """M-10 #22: trial при перезапуске run.py — пробный период продолжается, падение даёт откат.""" self.put("agent.py", V6_CRASH) self.put("agent.py.prev", fake("v5", "human_stop()\n")) self.put("update.json", json.dumps({"state": "trial", "from_version": 5, "to_version": 6, "sha256": self.sha(V6_CRASH)})) self.assertEqual(self.l.run(), 0) self.assertEqual(self.runs(), ["v6", "v5"]) self.assertEqual(self.state()["state"], "rolled_back") def test_ok_at_last_moment_no_rollback(self): """M-4: пробный период истёк, но агент записал ok, получив SIGTERM.""" self.l.trial_seconds = 1 self.l.kill_grace = 5 self.put("agent.py", V5_UPDATES) self.next_version(V6_OK_ON_TERM) self.assertEqual(self.l.run(), 0) self.assertEqual(self.read("agent.py"), V6_OK_ON_TERM) self.assertEqual(self.state()["state"], "ok") self.assertEqual(self.runs(), ["v5", "v6", "v6"]) self.assertEqual(self.pauses, []) # перезапуск без паузы def _confirmed_then_rejected_next(self, delay): self.put("agent.py", V5_UPDATES) self.next_version(V6_OK_THEN_REJECT_NEXT.replace("DELAY", repr(delay))) self.assertEqual(self.l.run(), 0) self.assertEqual(self.read("agent.py"), V6_OK_THEN_REJECT_NEXT.replace("DELAY", repr(delay))) self.assertEqual(self.runs(), ["v5", "v6", "v6"]) st = self.state() self.assertEqual((st["state"], st["to_version"]), ("rejected", 10)) # не rolled_back: v10 не заблокирована def test_confirmed_then_next_rejected_no_rollback(self): """I-1: агент записал ok, потом сам перезаписал update.json (rejected для v10) и упал — v6 здоровая, откат не нужен.""" self._confirmed_then_rejected_next(2.5) def test_confirmed_and_rejected_within_poll_no_rollback(self): """I-1: ok и rejected записаны быстрее, чем запускатель опрашивает файл раз в секунду.""" self._confirmed_then_rejected_next(0) def test_locked_in_trial_no_rollback(self): """M-7: журнал держит осиротевший агент — код 73 не откатывает новую версию.""" v6 = fake("v6", ''' n = len(open(os.path.join(HERE, "runs.log")).read().split()) human_stop() if n > 1 else sys.exit(73) ''') self.put("agent.py", v6) self.put("agent.py.prev", fake("v5", "human_stop()\n")) self.put("update.json", json.dumps({"state": "trial", "from_version": 5, "to_version": 6, "sha256": self.sha(v6)})) self.assertEqual(self.l.run(), 0) self.assertEqual(self.runs(), ["v6", "v6"]) self.assertEqual(self.pauses, [5]) self.assertEqual(self.read("agent.py"), v6) self.assertEqual(self.state()["state"], "trial") def test_locked_outside_trial_backs_off(self): self.put("agent.py", fake("v6", ''' n = len(open(os.path.join(HERE, "runs.log")).read().split()) human_stop() if n > 3 else sys.exit(73) ''')) self.assertEqual(self.l.run(), 0) self.assertEqual(self.pauses, [5, 10, 20]) def test_os_error_before_replace_rejected(self): """M-6: копия в agent.py.prev.tmp не удалась (здесь — на месте файла каталог).""" self.put("agent.py", V5_UPDATES) self.next_version(V6_OK_THEN_STOP) os.mkdir(self.path("agent.py.prev.tmp")) self.assertEqual(self.l.run(), 0) self.assertEqual(self.read("agent.py"), V5_UPDATES) st = self.state() self.assertEqual(st["state"], "rejected") self.assertTrue(st["error"].startswith("подмена не удалась: "), st["error"]) self.assertFalse(os.path.exists(self.path("agent.py.new"))) self.assertEqual(self.runs(), ["v5", "v5"]) def test_trial_not_written_reverts(self): orig = run.write_state def flaky(path, state, base, error=None): return False if state == "trial" else orig(path, state, base, error) run.write_state = flaky self.addCleanup(setattr, run, "write_state", orig) self.put("agent.py", V5_UPDATES) self.next_version(V6_OK_THEN_STOP) self.assertEqual(self.l.run(), 0) self.assertEqual(self.read("agent.py"), V5_UPDATES) # agent.py.prev вернулся на место st = self.state() self.assertEqual((st["state"], st["error"]), ("rejected", "подмена не удалась: update.json не записан")) self.assertEqual(self.runs(), ["v5", "v5"]) def test_write_state_error_does_not_raise(self): self.assertFalse(run.write_state(self.path("нет/update.json"), "ok", {})) def test_sha256_file_unreadable_is_none(self): self.assertIsNone(run.sha256_file(self.path("нет-файла"))) def test_tmp_files_removed_at_start(self): self.put("agent.py", fake("v5", "human_stop()\n")) for name in ("agent.py.new.tmp", "agent.py.prev.tmp", "update.json.tmp"): self.put(name, "остаток") self.assertEqual(self.l.run(), 0) for name in ("agent.py.new.tmp", "agent.py.prev.tmp", "update.json.tmp"): self.assertFalse(os.path.exists(self.path(name)), name) def test_sigterm_forwarded_and_no_restart(self): """M-10 #22: on_sigterm — пересылка агенту, нового запуска нет.""" import threading self.put("agent.py", fake("v5", ''' import signal, time def term(*a): mark("term") sys.exit(0) signal.signal(signal.SIGTERM, term) open(os.path.join(HERE, "ready"), "w").close() time.sleep(30) ''')) def send(): for _ in range(300): if os.path.exists(self.path("ready")): break time.sleep(0.05) self.l.on_sigterm(signal.SIGTERM, None) t = threading.Thread(target=send) t.start() self.assertEqual(self.l.run(), 0) t.join() self.assertEqual(self.runs(), ["v5", "term"]) class RunPyIsCopyTest(unittest.TestCase): """run.py — дословная копия запускателя jira-agent (design 2): правки общих частей делаются в обоих репозиториях.""" def test_run_py_is_copy(self): # от worktree (<корень>/eva-agent/.wt/<срез>) и от основного checkout (<корень>/eva-agent) — разная глубина here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) candidates = [os.path.normpath(os.path.join(here, up, "jira-agent", "src", "run.py")) for up in ("..", os.path.join("..", ".."), os.path.join("..", "..", ".."))] origin = next((c for c in candidates if os.path.exists(c)), None) if origin is None: self.skipTest("рядом нет jira-agent/src/run.py (искал: %s)" % ", ".join(candidates)) with open(origin, "rb") as a, open(os.path.join(SRC, "run.py"), "rb") as b: self.assertEqual(a.read(), b.read(), "src/run.py разошёлся с %s" % origin) if __name__ == "__main__": unittest.main()