fix(agent): OSError при подготовке версии — временный сбой; формат agent_release (M-6, M-10 #22)
Диск полон или нет прав при записи agent.py.new, selftest или update.json больше не роняет агент трейсбеком: это временный сбой, повтор через 5 минут. Ошибка записи update.json при отказе версии только логируется. Неполная публикация (version не int, sha256 не 64 hex в нижнем регистре) отбрасывается с подсказкой не чаще раза в час; таймеры подсказок разделены по видам. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrey Epifancev
parent
47a7424eca
commit
c51651717c
55
src/agent.py
55
src/agent.py
@ -954,6 +954,7 @@ class Updater:
|
||||
HINT_SECONDS = 3600 # подсказка «запускайте через run.py» — не чаще (R-10)
|
||||
SELFTEST_TIMEOUT = 120
|
||||
VERSION_RE = re.compile(r"^AGENT_VERSION = (\d+)[ \t]*\r?$", re.M)
|
||||
SHA_RE = re.compile(r"[0-9a-f]{64}")
|
||||
|
||||
def __init__(self, hub, base_dir: str, launcher_version: Optional[int], enabled: bool = True,
|
||||
config_path: Optional[str] = None):
|
||||
@ -963,7 +964,7 @@ class Updater:
|
||||
self.new_path = os.path.join(base_dir, "agent.py.new")
|
||||
self.agent_path = os.path.join(base_dir, "agent.py")
|
||||
self.retry_after = {} # (version, sha256) -> clock(), раньше которого не повторять
|
||||
self.hint_at = None # type: Optional[float]
|
||||
self.hints = {} # вид подсказки -> clock() последней записи в лог (не чаще HINT_SECONDS)
|
||||
self.clock = time.monotonic
|
||||
self.should_stop = lambda: False
|
||||
self.run_selftest = self._run_selftest
|
||||
@ -978,13 +979,20 @@ class Updater:
|
||||
write_update_state(self.state_path, "ok", st.get("from_version"), AGENT_VERSION, st.get("sha256"))
|
||||
log("обновление до v%d подтверждено" % AGENT_VERSION)
|
||||
|
||||
def _hint(self, kind: str, msg: str) -> None:
|
||||
now = self.clock()
|
||||
at = self.hints.get(kind)
|
||||
if at is None or now - at >= self.HINT_SECONDS:
|
||||
self.hints[kind] = now
|
||||
log(msg)
|
||||
|
||||
def check(self, release) -> bool:
|
||||
"""Вызывается после claim без команды. True — agent.py.new проверен, update.json = staged: run выходит с 75."""
|
||||
if not self.enabled or not isinstance(release, dict):
|
||||
return False
|
||||
try:
|
||||
version, sha = int(release.get("version")), str(release.get("sha256"))
|
||||
except (TypeError, ValueError):
|
||||
version, sha = release.get("version"), release.get("sha256")
|
||||
if type(version) is not int or not isinstance(sha, str) or not self.SHA_RE.fullmatch(sha):
|
||||
self._hint("format", "хаб прислал неполную публикацию: version=%r, sha256=%r" % (version, sha)) # design 10
|
||||
return False
|
||||
if version <= AGENT_VERSION:
|
||||
return False
|
||||
@ -993,10 +1001,7 @@ class Updater:
|
||||
and st.get("sha256") == sha:
|
||||
return False
|
||||
if self.launcher_version is None:
|
||||
now = self.clock()
|
||||
if self.hint_at is None or now - self.hint_at >= self.HINT_SECONDS:
|
||||
self.hint_at = now
|
||||
log("доступна v%d; для самообновления запускайте через run.py" % version)
|
||||
self._hint("launcher", "доступна v%d; для самообновления запускайте через run.py" % version)
|
||||
return False
|
||||
until = self.retry_after.get((version, sha))
|
||||
if until is not None and self.clock() < until:
|
||||
@ -1022,22 +1027,31 @@ class Updater:
|
||||
if self.VERSION_RE.findall(content) != [str(version)]:
|
||||
return self._reject(version, sha, "AGENT_VERSION в файле не %d" % version)
|
||||
tmp = self.new_path + ".tmp"
|
||||
with open(tmp, "wb") as f:
|
||||
f.write(data)
|
||||
try:
|
||||
os.chmod(tmp, os.stat(self.agent_path).st_mode & 0o777)
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(tmp, self.new_path)
|
||||
try: # диск полон, нет прав — временный сбой, не трейсбек (design 9)
|
||||
with open(tmp, "wb") as f:
|
||||
f.write(data)
|
||||
try:
|
||||
os.chmod(tmp, os.stat(self.agent_path).st_mode & 0o777)
|
||||
except OSError:
|
||||
pass
|
||||
os.replace(tmp, self.new_path)
|
||||
except OSError as e:
|
||||
return self._transient(version, sha, "файлы: %s" % e)
|
||||
if self.should_stop():
|
||||
self._drop_new()
|
||||
return False
|
||||
code, out = self.run_selftest(self.new_path)
|
||||
try:
|
||||
code, out = self.run_selftest(self.new_path)
|
||||
except OSError as e:
|
||||
return self._transient(version, sha, "selftest: %s" % e)
|
||||
if self.should_stop(): # Ctrl-C уронил и selftest — это не отказ версии
|
||||
self._drop_new()
|
||||
return False
|
||||
if code == 0:
|
||||
write_update_state(self.state_path, "staged", AGENT_VERSION, version, sha)
|
||||
try:
|
||||
write_update_state(self.state_path, "staged", AGENT_VERSION, version, sha)
|
||||
except OSError as e:
|
||||
return self._transient(version, sha, "update.json: %s" % e)
|
||||
log("v%d готова, перезапуск через run.py" % version)
|
||||
return True
|
||||
if code is None:
|
||||
@ -1058,7 +1072,7 @@ class Updater:
|
||||
for path in (self.new_path, self.new_path + ".tmp"):
|
||||
try:
|
||||
os.remove(path)
|
||||
except FileNotFoundError:
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _transient(self, version, sha, reason) -> bool:
|
||||
@ -1069,7 +1083,10 @@ class Updater:
|
||||
|
||||
def _reject(self, version, sha, reason) -> bool:
|
||||
self._drop_new()
|
||||
write_update_state(self.state_path, "rejected", AGENT_VERSION, version, sha, reason)
|
||||
try:
|
||||
write_update_state(self.state_path, "rejected", AGENT_VERSION, version, sha, reason)
|
||||
except OSError as e:
|
||||
log("не удалось записать update.json: %s" % e)
|
||||
log("v%d отвергнута: %s" % (version, reason))
|
||||
return False
|
||||
|
||||
|
||||
@ -78,6 +78,68 @@ class UpdaterCheckTest(unittest.TestCase):
|
||||
def ref(self, rel):
|
||||
return {"version": rel["version"], "sha256": rel["sha256"]}
|
||||
|
||||
def capture_logs(self):
|
||||
logs = []
|
||||
orig = agent.log
|
||||
agent.log = logs.append
|
||||
self.addCleanup(setattr, agent, "log", orig)
|
||||
return logs
|
||||
|
||||
def test_incomplete_release_ignored_with_hourly_hint(self):
|
||||
logs = self.capture_logs()
|
||||
for ref in ({"version": NEXT, "sha256": None}, {"version": str(NEXT), "sha256": "a" * 64},
|
||||
{"version": True, "sha256": "a" * 64}, {"version": NEXT, "sha256": "A" * 64},
|
||||
{"version": NEXT, "sha256": "a" * 63}, {"version": NEXT, "sha256": "a" * 64 + "\n"}):
|
||||
self.assertFalse(self.u.check(ref))
|
||||
self.now[0] += 3600
|
||||
self.assertFalse(self.u.check({"version": NEXT, "sha256": None}))
|
||||
self.assertEqual(sum(1 for l in logs if l.startswith("хаб прислал неполную публикацию")), 2)
|
||||
self.assertEqual(self.hub.release_requests, 0)
|
||||
self.assertIsNone(self.u.state())
|
||||
|
||||
def test_hints_have_separate_timers(self):
|
||||
logs = self.capture_logs()
|
||||
self.u.launcher_version = None
|
||||
self.hub.release = make_release(NEXT)
|
||||
self.assertFalse(self.u.check({"version": NEXT, "sha256": None}))
|
||||
self.assertFalse(self.u.check(self.ref(self.hub.release)))
|
||||
self.assertEqual(sum(1 for l in logs if l.startswith("хаб прислал неполную публикацию")), 1)
|
||||
self.assertEqual(logs.count("доступна v%d; для самообновления запускайте через run.py" % NEXT), 1)
|
||||
|
||||
def test_write_error_is_transient(self):
|
||||
self.hub.release = make_release(NEXT)
|
||||
ref = self.ref(self.hub.release)
|
||||
self.u.new_path = os.path.join(self.dir.name, "нет-каталога", "agent.py.new") # open() → FileNotFoundError
|
||||
self.assertFalse(self.u.check(ref))
|
||||
self.assertIsNone(self.u.state())
|
||||
self.assertEqual(self.selftests, [])
|
||||
self.now[0] += 299
|
||||
self.assertFalse(self.u.check(ref))
|
||||
self.assertEqual(self.hub.release_requests, 1) # раньше 5 минут не повторяет
|
||||
|
||||
def test_selftest_os_error_is_transient(self):
|
||||
self.hub.release = make_release(NEXT)
|
||||
def boom(path):
|
||||
raise OSError(28, "No space left on device")
|
||||
self.u.run_selftest = boom
|
||||
self.assertFalse(self.u.check(self.ref(self.hub.release)))
|
||||
self.assertIsNone(self.u.state())
|
||||
self.assertFalse(os.path.exists(self.u.new_path))
|
||||
|
||||
def test_staged_write_error_is_transient(self):
|
||||
self.hub.release = make_release(NEXT)
|
||||
self.u.state_path = os.path.join(self.dir.name, "нет-каталога", "update.json")
|
||||
self.assertFalse(self.u.check(self.ref(self.hub.release))) # без staged выходить с 75 нельзя
|
||||
self.assertFalse(os.path.exists(self.u.new_path))
|
||||
|
||||
def test_reject_write_error_only_logged(self):
|
||||
rel = make_release(NEXT)
|
||||
self.hub.release = dict(rel, content=rel["content"] + "# подмена\n")
|
||||
self.u.state_path = os.path.join(self.dir.name, "нет-каталога", "update.json")
|
||||
logs = self.capture_logs()
|
||||
self.assertFalse(self.u.check(self.ref(rel))) # не бросает
|
||||
self.assertTrue(any(l.startswith("не удалось записать update.json") for l in logs), logs)
|
||||
|
||||
def test_good_version_is_staged(self):
|
||||
rel = make_release(NEXT)
|
||||
self.hub.release = rel
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user