arm-scan-gateway/api/tests/unit/test_local_hotfolder.py
Andrey Epifancev cb8c71fc26 fix(hotfolder): повтор задания при существующем назначении не трогает .part (R-17)
LocalHotFolder.put сначала проверяет назначение: если файл во входящем каталоге уже есть, он считается доставленным,
ничего не пишется, а .part снимается только при включённой очистке. Иначе удаление старого .part подавляется (остаток
прерванной записи не роняет задание); если он остался, усечение разрешено только при st_nlink == 1, ссылка на чужой файл
не трогается и исключение поднимается как есть.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012vJkmBGKKqMoEMzvsXG3y2
2026-10-07 03:44:41 +04:00

791 lines
35 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Горячая папка в обычном каталоге (spec image-recognition-gateway, D-1, R-4/R-5, slice-0-skeleton).
Быстрые тесты на `tmp_path`: атомарная запись входящего файла через служебный каталог `Tmp`, опрос набора ответов ИС
с готовностью по стабилизации (settle, часы подменены), проверка здоровья, валидация имён.
"""
import errno
import os
import uuid
from pathlib import Path
from typing import Any
import pytest
from src.gateway.hotfolder import AnswerFile, HotFolder, LocalHotFolder, SettleTracker, inbox_name
def _job_id() -> str:
return str(uuid.uuid4())
class FakeClock:
"""Монотонные часы теста: время двигается только вручную."""
def __init__(self) -> None:
self.now = 100.0
def __call__(self) -> float:
return self.now
def _write_answer(hf: LocalHotFolder, job_id: str, data: bytes, name: str | None = None) -> Path:
path = hf.answer_dir(job_id) / (name or f"{job_id}.json")
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
return path
def _settled(tmp_path: Path, settle: float = 2) -> tuple[LocalHotFolder, FakeClock]:
clock = FakeClock()
return LocalHotFolder(tmp_path, settle_seconds=settle, clock=clock), clock
class TestInboxName:
def test_job_id_and_lowercase_extension(self) -> None:
job_id = _job_id()
assert inbox_name(job_id, "aep_5.PDF") == f"{job_id}.pdf"
assert inbox_name(job_id, "dir/sub/Отчёт за 2024.Jpeg") == f"{job_id}.jpeg"
assert inbox_name(job_id, "C:\\scans\\report.pdf") == f"{job_id}.pdf"
@pytest.mark.parametrize(
"original",
["scan", "", ".", "..", "report.", "a/..", "x.a:b", "x." + "p" * 17, "name.тар"],
)
def test_rejects_names_without_safe_extension(self, original: str) -> None:
with pytest.raises(ValueError, match="расширени"):
inbox_name("job", original)
class TestPutWithoutReplace:
def test_existing_destination_is_kept(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path)
hf.put("job.pdf", b"first")
hf.put("job.pdf", b"second")
assert (hf.input_dir / "job.pdf").read_bytes() == b"first"
# R-17: назначение есть — повтор ничего не пишет; прежнего `.part` после успешной публикации нет
assert not (hf.tmp_dir / ".job.pdf.part").exists()
def test_existing_destination_part_removed_when_cleanup_enabled(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path, cleanup=True)
hf.put("job.pdf", b"first")
hf.put("job.pdf", b"second")
assert (hf.input_dir / "job.pdf").read_bytes() == b"first"
assert not (hf.tmp_dir / ".job.pdf.part").exists()
def test_no_temp_files_in_input(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path)
hf.put("job.pdf", b"x")
assert [p.name for p in hf.input_dir.iterdir()] == ["job.pdf"]
assert not (hf.tmp_dir / ".job.pdf.part").exists() # после успешной публикации .part нет
class TestCleanup:
def _fill(self, hf: LocalHotFolder, job_id: str) -> None:
hf.put(f"{job_id}.pdf", b"x")
(hf.input_dir / f"{job_id}.pdf.done").write_bytes(b"")
(hf.tmp_dir / f".{job_id}.pdf.part").write_bytes(b"orphan")
_write_answer(hf, job_id, b"{}")
_write_answer(hf, job_id, b"{}", name=f"{job_id}_ofr.json")
(hf.answer_dir(job_id) / "readme.txt").write_bytes(b"keep")
def test_disabled_touches_nothing(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path)
job_id = _job_id()
self._fill(hf, job_id)
assert hf.cleanup(job_id, f"{job_id}.pdf") is False
assert (hf.input_dir / f"{job_id}.pdf").exists() and hf.answer_dir(job_id).is_dir()
assert (hf.tmp_dir / f".{job_id}.pdf.part").exists()
def test_enabled_removes_job_files(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path, cleanup=True)
job_id = _job_id()
self._fill(hf, job_id)
assert hf.cleanup(job_id, f"{job_id}.pdf") is True
assert not (hf.input_dir / f"{job_id}.pdf").exists()
assert not (hf.input_dir / f"{job_id}.pdf.done").exists()
assert not (hf.tmp_dir / f".{job_id}.pdf.part").exists()
assert not hf.answer_dir(job_id).exists() # каталог удалён вместе с readme.txt
assert hf.cleanup(job_id, f"{job_id}.pdf") is True # повтор — не ошибка
def test_missing_files_are_fine(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path, cleanup=True)
assert hf.cleanup(_job_id(), "nothing.pdf") is True
def test_permission_error_propagates(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
hf = LocalHotFolder(tmp_path, cleanup=True)
job_id = _job_id()
self._fill(hf, job_id)
def deny(path: Any, *args: Any, **kwargs: Any) -> None:
raise PermissionError(f"нет прав: {path}")
monkeypatch.setattr(os, "unlink", deny)
with pytest.raises(PermissionError):
hf.cleanup(job_id, f"{job_id}.pdf")
def test_rejects_unsafe_names(self, tmp_path: Path) -> None:
"""cleanup: validate_name защищает от выхода за пределы папки (T11.4)."""
hf = LocalHotFolder(tmp_path, cleanup=True)
with pytest.raises(ValueError, match="имя файла"):
hf.cleanup(_job_id(), "../x.pdf")
# ничего не удалено
assert not hf.input_dir.exists()
class TestPaths:
def test_default_layout(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path)
job_id = _job_id()
assert hf.root == tmp_path
assert hf.input_dir == tmp_path / "Input"
assert hf.output_dir == tmp_path / "Output"
assert hf.tmp_dir == tmp_path / "Tmp"
assert hf.answer_dir(job_id) == tmp_path / "Output" / job_id
assert hf.answer_path(job_id) == tmp_path / "Output" / job_id / f"{job_id}.json"
def test_custom_dir_names_and_str_root(self, tmp_path: Path) -> None:
hf = LocalHotFolder(str(tmp_path), input_dir="Входящие", output_dir="Исходящие", tmp_dir="Служебный")
assert hf.root == tmp_path
assert hf.input_dir == tmp_path / "Входящие"
assert hf.output_dir == tmp_path / "Исходящие"
assert hf.tmp_dir == tmp_path / "Служебный"
assert hf.answer_path("abc") == tmp_path / "Исходящие" / "abc" / "abc.json"
@pytest.mark.parametrize("job_id", ["", ".", "..", "../x", "a/b", "a\\b", "a:b", "a?b", "a*b", "a\x00b", "a\nb"])
def test_answer_path_rejects_unsafe_job_id(self, tmp_path: Path, job_id: str) -> None:
hf = LocalHotFolder(tmp_path)
with pytest.raises(ValueError, match="job_id"):
hf.answer_path(job_id)
with pytest.raises(ValueError, match="job_id"):
hf.answer_dir(job_id)
with pytest.raises(ValueError, match="job_id"):
hf.poll(job_id)
def test_negative_settle_is_rejected(self, tmp_path: Path) -> None:
with pytest.raises(ValueError, match="settle"):
LocalHotFolder(tmp_path, settle_seconds=-1)
def test_implements_protocol(self, tmp_path: Path) -> None:
hf: HotFolder = LocalHotFolder(tmp_path)
assert hf.healthy() is True
assert hasattr(hf, "cleanup")
class TestPut:
def test_writes_bytes_under_inbox_name(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path)
job_id = _job_id()
name = inbox_name(job_id, "aep_5.pdf")
data = bytes(range(256)) * 10
hf.put(name, data)
assert (tmp_path / "Input" / name).read_bytes() == data
def test_leaves_no_part_files(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path)
job_id = _job_id()
name = inbox_name(job_id, "aep_5.pdf")
hf.put(name, b"data")
assert [p.name for p in hf.input_dir.iterdir()] == [name]
assert list(hf.tmp_dir.iterdir()) == []
def test_part_is_written_in_tmp_not_in_input(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""OQ-79: во время записи во входящем каталоге нет временного файла — он в `Tmp/`, затем link в `Input`."""
hf = LocalHotFolder(tmp_path)
job_id = _job_id()
name = inbox_name(job_id, "aep_5.pdf")
seen: dict[str, Any] = {}
real_link = os.link
def spy(src: Any, dst: Any) -> None:
seen["src"], seen["dst"] = Path(src), Path(dst)
seen["input"] = sorted(p.name for p in hf.input_dir.iterdir())
seen["tmp"] = sorted(p.name for p in hf.tmp_dir.iterdir())
seen["part"] = Path(src).read_bytes()
real_link(src, dst)
monkeypatch.setattr("src.gateway.hotfolder.os.link", spy)
hf.put(name, b"data")
assert seen == {
"src": hf.tmp_dir / f".{name}.part",
"dst": hf.input_dir / name,
"input": [],
"tmp": [f".{name}.part"],
"part": b"data",
}
assert [p.name for p in hf.input_dir.iterdir()] == [name]
assert list(hf.tmp_dir.iterdir()) == []
def test_creates_input_and_tmp_dirs_only(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path)
assert not hf.input_dir.exists()
assert not hf.tmp_dir.exists()
hf.put("a.pdf", b"1")
assert hf.input_dir.is_dir()
assert hf.tmp_dir.is_dir()
assert not hf.output_dir.exists()
def test_existing_tmp_dir_is_fine(self, tmp_path: Path) -> None:
(tmp_path / "Tmp").mkdir()
LocalHotFolder(tmp_path).put("a.pdf", b"1")
assert (tmp_path / "Input" / "a.pdf").read_bytes() == b"1"
def test_custom_tmp_dir_name(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path, tmp_dir="Служебный")
hf.put("a.pdf", b"1")
assert (tmp_path / "Служебный").is_dir()
assert not (tmp_path / "Tmp").exists()
def test_existing_input_dir_is_fine(self, tmp_path: Path) -> None:
(tmp_path / "Input").mkdir()
LocalHotFolder(tmp_path).put("a.pdf", b"1")
assert (tmp_path / "Input" / "a.pdf").read_bytes() == b"1"
def test_missing_root_raises_and_creates_nothing(self, tmp_path: Path) -> None:
root = tmp_path / "absent"
with pytest.raises(FileNotFoundError):
LocalHotFolder(root).put("a.pdf", b"1")
assert not root.exists()
assert list(tmp_path.iterdir()) == []
def test_empty_data_is_written(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path)
hf.put("empty.pdf", b"")
assert (hf.input_dir / "empty.pdf").read_bytes() == b""
@pytest.mark.parametrize(
"name",
["", "../escape.pdf", "sub/a.pdf", "sub\\a.pdf", "/abs.pdf", ".hidden.pdf", ".a.pdf.part", "..", ".", "a\x00b"],
)
def test_rejects_invalid_names(self, tmp_path: Path, name: str) -> None:
hf = LocalHotFolder(tmp_path)
with pytest.raises(ValueError, match="имя файла"):
hf.put(name, b"x")
assert not hf.input_dir.exists()
assert not (tmp_path.parent / "escape.pdf").exists()
@pytest.mark.parametrize("name", ["a:b.pdf", "a<b.pdf", "a>b.pdf", 'a"b.pdf', "a|b.pdf", "a?b.pdf", "a*b.pdf"])
def test_rejects_names_illegal_on_windows(self, tmp_path: Path, name: str) -> None:
hf = LocalHotFolder(tmp_path)
with pytest.raises(ValueError, match="имя файла"):
hf.put(name, b"x")
assert not hf.input_dir.exists()
@pytest.mark.parametrize("name", ["a\x01b.pdf", "a\tb.pdf", "a\nb.pdf", "a\x1fb.pdf"])
def test_rejects_control_chars(self, tmp_path: Path, name: str) -> None:
with pytest.raises(ValueError, match="имя файла"):
LocalHotFolder(tmp_path).put(name, b"x")
def test_accepts_inbox_names(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path)
job_id_1 = _job_id()
job_id_2 = _job_id()
names = [inbox_name(job_id_1, "Отчёт за 2024 год (копия).PDF"), inbox_name(job_id_2, "aep_8.pdf")]
for name in names:
hf.put(name, b"x")
assert sorted(p.name for p in hf.input_dir.iterdir()) == sorted(names)
def test_cleans_part_file_on_failure(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""При cleanup=True и ошибке link: .part удаляется."""
hf = LocalHotFolder(tmp_path, cleanup=True)
def boom(src: Any, dst: Any) -> None:
raise OSError("диск недоступен")
monkeypatch.setattr("src.gateway.hotfolder.os.link", boom)
with pytest.raises(OSError, match="диск недоступен"):
hf.put("a.pdf", b"data")
assert list(hf.input_dir.iterdir()) == []
assert list(hf.tmp_dir.iterdir()) == []
def test_keeps_part_file_on_failure_when_cleanup_disabled(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""При cleanup=False и ошибке link: .part остаётся в Tmp (T11.1)."""
hf = LocalHotFolder(tmp_path, cleanup=False)
def boom(src: Any, dst: Any) -> None:
raise OSError("диск недоступен")
monkeypatch.setattr("src.gateway.hotfolder.os.link", boom)
with pytest.raises(OSError, match="диск недоступен"):
hf.put("a.pdf", b"data")
assert list(hf.input_dir.iterdir()) == []
assert [p.name for p in hf.tmp_dir.iterdir()] == [".a.pdf.part"]
def test_successful_put_despite_final_unlink_failure(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""Успешный link, но ошибка в завершающем unlink .part (R-8): put НЕ падает, файл доставлен (T11.2)."""
hf = LocalHotFolder(tmp_path)
unlink_calls = []
real_unlink = os.unlink
def unlink_with_failure(path: Any) -> None:
unlink_calls.append(path)
if str(path).endswith(".part"):
raise PermissionError("удаление запрещено")
real_unlink(path)
monkeypatch.setattr("src.gateway.hotfolder.os.unlink", unlink_with_failure)
hf.put("a.pdf", b"data") # НЕ падает несмотря на ошибку unlink
assert (hf.input_dir / "a.pdf").read_bytes() == b"data"
assert (hf.tmp_dir / ".a.pdf.part").exists() # .part не удалён из-за ошибки
assert unlink_calls # попытка удаления была
def test_cleanup_failure_does_not_mask_original_error(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Ошибка удаления `.part` на пути ошибки link подавляется (T11.1); исходное исключение проходит."""
hf = LocalHotFolder(tmp_path, cleanup=True)
def boom(src: Any, dst: Any) -> None:
raise OSError("диск недоступен")
def unlink_fails(self: Path, missing_ok: bool = False) -> None:
raise PermissionError("удаление .part запрещено")
monkeypatch.setattr("src.gateway.hotfolder.os.link", boom)
monkeypatch.setattr(Path, "unlink", unlink_fails)
with pytest.raises(OSError, match="диск недоступен"):
hf.put("a.pdf", b"data")
class TestPartIsCreatedFresh:
"""Решение №32: `.part` создаётся заново (удаление старого + `O_EXCL`), fallback — только вокруг `os.link`."""
@pytest.mark.parametrize("cleanup", [False, True])
def test_existing_destination_is_not_touched_by_repeat(self, tmp_path: Path, cleanup: bool) -> None:
"""R-17: назначение есть, `.part` — жёсткая ссылка на него: байты не меняются, put не падает."""
hf = LocalHotFolder(tmp_path, cleanup=cleanup)
hf.put("job.pdf", b"first")
published = hf.input_dir / "job.pdf"
stale = hf.tmp_dir / ".job.pdf.part"
os.link(published, stale)
hf.put("job.pdf", b"different data")
assert published.read_bytes() == b"first"
assert stale.exists() is (not cleanup) # очистка снимает `.part`, иначе он остаётся как был
if not cleanup:
assert stale.read_bytes() == b"first" and published.stat().st_nlink == 2
def test_existing_destination_with_undeletable_part_does_not_fail(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
hf = LocalHotFolder(tmp_path) # очистка выключена: удаление `.part` вообще не пытается
hf.put("job.pdf", b"first")
os.link(hf.input_dir / "job.pdf", hf.tmp_dir / ".job.pdf.part")
def deny(self: Path, missing_ok: bool = False) -> None:
raise PermissionError("удаление запрещено")
monkeypatch.setattr(Path, "unlink", deny)
hf.put("job.pdf", b"second")
assert (hf.input_dir / "job.pdf").read_bytes() == b"first"
def test_undeletable_stale_part_with_single_link_is_truncated(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Назначения нет, старый `.part` не удаляется, но это обычный файл (nlink == 1): put успешен."""
hf = LocalHotFolder(tmp_path)
hf.tmp_dir.mkdir()
stale = hf.tmp_dir / ".job.pdf.part"
stale.write_bytes(b"old and longer content")
def deny(self: Path, missing_ok: bool = False) -> None:
raise PermissionError("удаление запрещено")
monkeypatch.setattr(Path, "unlink", deny)
hf.put("job.pdf", b"new")
assert (hf.input_dir / "job.pdf").read_bytes() == b"new"
def test_undeletable_stale_part_with_other_links_is_not_written(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Назначения нет, старый `.part` не удаляется и имеет nlink == 2 (синтетически): запись не идёт, ошибка не
маскируется, второй файл не изменён."""
hf = LocalHotFolder(tmp_path)
hf.tmp_dir.mkdir()
other = tmp_path / "other.bin"
other.write_bytes(b"precious")
os.link(other, hf.tmp_dir / ".job.pdf.part")
def deny(self: Path, missing_ok: bool = False) -> None:
raise PermissionError("удаление запрещено")
monkeypatch.setattr(Path, "unlink", deny)
with pytest.raises(FileExistsError):
hf.put("job.pdf", b"new")
assert other.read_bytes() == b"precious"
assert not (hf.input_dir / "job.pdf").exists()
def test_stale_part_is_replaced_before_publication(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path)
hf.tmp_dir.mkdir()
(hf.tmp_dir / ".job.pdf.part").write_bytes(b"old and longer content")
hf.put("job.pdf", b"new")
assert (hf.input_dir / "job.pdf").read_bytes() == b"new"
assert not (hf.tmp_dir / ".job.pdf.part").exists()
def test_part_is_opened_exclusively(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
hf = LocalHotFolder(tmp_path)
flags: list[int] = []
real_open = os.open
def spy(path: Any, flag: int, mode: int = 0o777, **kwargs: Any) -> int:
flags.append(flag)
return real_open(path, flag, mode, **kwargs)
monkeypatch.setattr("src.gateway.hotfolder.os.open", spy)
hf.put("job.pdf", b"x")
assert flags and flags[0] & os.O_EXCL and flags[0] & os.O_CREAT
def test_write_error_propagates_as_itself(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""EPERM записи не превращается в `FileNotFoundError` от `os.replace` несуществующего `.part`."""
hf = LocalHotFolder(tmp_path, cleanup=True)
class FailingFile:
def __enter__(self) -> "FailingFile":
return self
def __exit__(self, *exc: object) -> None:
return None
def write(self, data: bytes) -> int:
raise OSError(errno.EPERM, "запись запрещена")
monkeypatch.setattr("src.gateway.hotfolder.os.fdopen", lambda fd, mode: (os.close(fd), FailingFile())[1])
with pytest.raises(OSError, match="запись запрещена") as caught:
hf.put("job.pdf", b"data")
assert caught.value.errno == errno.EPERM and not isinstance(caught.value, FileNotFoundError)
assert list(hf.input_dir.iterdir()) == [] and list(hf.tmp_dir.iterdir()) == []
@pytest.mark.parametrize("code", [errno.EPERM, errno.EXDEV, errno.ENOTSUP, errno.EOPNOTSUPP])
def test_fallback_to_replace_when_link_unsupported(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, code: int
) -> None:
hf = LocalHotFolder(tmp_path)
def no_links(src: Any, dst: Any) -> None:
raise OSError(code, "жёсткие ссылки не поддерживаются")
monkeypatch.setattr("src.gateway.hotfolder.os.link", no_links)
hf.put("job.pdf", b"data")
assert (hf.input_dir / "job.pdf").read_bytes() == b"data"
assert list(hf.tmp_dir.iterdir()) == [] # `.part` стал назначением
def test_no_fallback_when_destination_appeared_during_link(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Назначение появилось между проверкой и линковкой, а `os.link` дал EPERM: `os.replace` не затирает."""
hf = LocalHotFolder(tmp_path, cleanup=True)
hf.input_dir.mkdir()
def no_links(src: Any, dst: Any) -> None:
Path(dst).write_bytes(b"first") # файл успел появиться (параллельный повтор того же задания)
raise OSError(errno.EPERM, "жёсткие ссылки не поддерживаются")
monkeypatch.setattr("src.gateway.hotfolder.os.link", no_links)
with pytest.raises(OSError, match="жёсткие ссылки"):
hf.put("job.pdf", b"second")
assert (hf.input_dir / "job.pdf").read_bytes() == b"first"
def test_replace_failure_after_link_fallback_propagates(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
hf = LocalHotFolder(tmp_path, cleanup=True)
def no_links(src: Any, dst: Any) -> None:
raise OSError(errno.EPERM, "жёсткие ссылки не поддерживаются")
def broken_replace(src: Any, dst: Any) -> None:
raise OSError(errno.EIO, "замена не удалась")
monkeypatch.setattr("src.gateway.hotfolder.os.link", no_links)
monkeypatch.setattr("src.gateway.hotfolder.os.replace", broken_replace)
with pytest.raises(OSError, match="замена не удалась"):
hf.put("job.pdf", b"data")
assert list(hf.tmp_dir.iterdir()) == [] # при очистке `.part` снят
class TestPoll:
"""`poll` — все `*.json` каталога `<output>/<job_id>/` по имени, когда набор настоялся `settle_seconds`."""
def test_none_until_answer_dir_has_json(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path, settle_seconds=0)
job_id = _job_id()
assert hf.poll(job_id) is None
(tmp_path / "Output").mkdir()
assert hf.poll(job_id) is None
(tmp_path / "Output" / job_id).mkdir()
assert hf.poll(job_id) is None
def test_missing_root_is_none(self, tmp_path: Path) -> None:
assert LocalHotFolder(tmp_path / "absent", settle_seconds=0).poll(_job_id()) is None
def test_answer_dir_is_a_file_is_none(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path, settle_seconds=0)
job_id = _job_id()
(tmp_path / "Output").mkdir()
(tmp_path / "Output" / job_id).write_bytes(b"{}")
assert hf.poll(job_id) is None
def test_settle_zero_returns_at_once(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path, settle_seconds=0)
job_id = _job_id()
payload = b'{"file_uid": "x", "data_ocr": {}}'
_write_answer(hf, job_id, payload)
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", payload)]
def test_one_json_after_settle(self, tmp_path: Path) -> None:
hf, clock = _settled(tmp_path)
job_id = _job_id()
_write_answer(hf, job_id, b"{}")
assert hf.poll(job_id) is None # набор впервые увиден
clock.now += 1.9
assert hf.poll(job_id) is None
clock.now += 0.1
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b"{}")]
def test_two_jsons_ascending_by_name(self, tmp_path: Path) -> None:
hf, clock = _settled(tmp_path)
job_id = _job_id()
_write_answer(hf, job_id, b'{"n": 2}', f"{job_id}_ofr.json")
_write_answer(hf, job_id, b'{"n": 1}', f"{job_id}.json")
assert hf.poll(job_id) is None
clock.now += 2
assert hf.poll(job_id) == [
AnswerFile(f"{job_id}.json", b'{"n": 1}'),
AnswerFile(f"{job_id}_ofr.json", b'{"n": 2}'),
]
def test_non_json_files_and_subdirs_are_ignored(self, tmp_path: Path) -> None:
hf, clock = _settled(tmp_path)
job_id = _job_id()
_write_answer(hf, job_id, b"text", "readme.txt")
_write_answer(hf, job_id, b"{}", f".{job_id}.json.part")
(hf.answer_dir(job_id) / "nested.json").mkdir()
assert hf.poll(job_id) is None
clock.now += 10
assert hf.poll(job_id) is None # json-файлов нет — ответа нет, сколько ни жди
_write_answer(hf, job_id, b"{}")
assert hf.poll(job_id) is None
clock.now += 2
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b"{}")]
def test_json_extension_is_case_insensitive(self, tmp_path: Path) -> None:
hf = LocalHotFolder(tmp_path, settle_seconds=0)
job_id = _job_id()
_write_answer(hf, job_id, b"{}", "B.JSON")
_write_answer(hf, job_id, b"[]", "a.Json")
assert hf.poll(job_id) == [AnswerFile("B.JSON", b"{}"), AnswerFile("a.Json", b"[]")]
def test_growing_file_is_not_ready_until_stable(self, tmp_path: Path) -> None:
hf, clock = _settled(tmp_path)
job_id = _job_id()
path = _write_answer(hf, job_id, b'{"part": ')
assert hf.poll(job_id) is None
clock.now += 1.5
path.write_bytes(b'{"part": 1, "more": ') # размер вырос — таймер заново
assert hf.poll(job_id) is None
clock.now += 1.5
path.write_bytes(b'{"part": 1, "more": 2}')
assert hf.poll(job_id) is None
clock.now += 1.9
assert hf.poll(job_id) is None
clock.now += 0.1
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b'{"part": 1, "more": 2}')]
def test_new_file_resets_timer(self, tmp_path: Path) -> None:
hf, clock = _settled(tmp_path)
job_id = _job_id()
_write_answer(hf, job_id, b"{}")
assert hf.poll(job_id) is None
clock.now += 1.5
_write_answer(hf, job_id, b"[]", f"{job_id}_ofr.json")
assert hf.poll(job_id) is None
clock.now += 1.5
assert hf.poll(job_id) is None # с момента появления второго файла прошло 1,5 с
clock.now += 0.5
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b"{}"), AnswerFile(f"{job_id}_ofr.json", b"[]")]
def test_removed_file_resets_timer(self, tmp_path: Path) -> None:
hf, clock = _settled(tmp_path)
job_id = _job_id()
_write_answer(hf, job_id, b"{}")
extra = _write_answer(hf, job_id, b"[]", f"{job_id}_ofr.json")
assert hf.poll(job_id) is None
clock.now += 1.5
extra.unlink()
assert hf.poll(job_id) is None
clock.now += 1.5
assert hf.poll(job_id) is None
clock.now += 0.5
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b"{}")]
def test_removal_of_all_json_is_none_and_restarts(self, tmp_path: Path) -> None:
hf, clock = _settled(tmp_path)
job_id = _job_id()
path = _write_answer(hf, job_id, b"{}")
assert hf.poll(job_id) is None
clock.now += 1
path.unlink()
assert hf.poll(job_id) is None
clock.now += 5
path.write_bytes(b"{}") # тот же набор снова: настаивается заново, а не «с прошлого раза»
assert hf.poll(job_id) is None
clock.now += 2
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b"{}")]
def test_state_is_dropped_after_answer(self, tmp_path: Path) -> None:
hf, clock = _settled(tmp_path)
job_id = _job_id()
_write_answer(hf, job_id, b"{}")
assert hf.poll(job_id) is None
clock.now += 2
assert hf.poll(job_id) is not None
assert hf.pending_jobs() == 0
assert hf.poll(job_id) is None # состояние сброшено: повторный опрос настаивает набор заново
def test_other_job_is_unaffected(self, tmp_path: Path) -> None:
hf, clock = _settled(tmp_path)
job_id, other = _job_id(), _job_id()
_write_answer(hf, other, b"{}")
assert hf.poll(other) is None
clock.now += 1
_write_answer(hf, job_id, b"[]")
assert hf.poll(job_id) is None
clock.now += 1
assert hf.poll(job_id) is None
assert hf.poll(other) == [AnswerFile(f"{other}.json", b"{}")]
clock.now += 1
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b"[]")]
def test_file_changed_between_listing_and_read_is_none(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
hf = LocalHotFolder(tmp_path, settle_seconds=0)
job_id = _job_id()
_write_answer(hf, job_id, b"{}")
monkeypatch.setattr(Path, "read_bytes", lambda self: b'{"longer": true}')
assert hf.poll(job_id) is None
def test_file_removed_between_listing_and_read_is_none(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
hf = LocalHotFolder(tmp_path, settle_seconds=0)
job_id = _job_id()
path = _write_answer(hf, job_id, b"{}")
def vanished(self: Path) -> bytes:
path.unlink()
raise FileNotFoundError(str(self))
monkeypatch.setattr(Path, "read_bytes", vanished)
assert hf.poll(job_id) is None
def test_new_file_appeared_while_reading_is_none_and_set_resettles(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""T2.1: после чтения каталог листается заново; набор изменился (появился файл) — `None`, набор настаивается
заново и отдаётся целиком."""
hf, clock = _settled(tmp_path)
job_id = _job_id()
_write_answer(hf, job_id, b"{}")
assert hf.poll(job_id) is None
clock.now += 2
real_read = Path.read_bytes
def read_then_add(self: Path) -> bytes:
data = real_read(self)
_write_answer(hf, job_id, b'{"n": 2}', f"{job_id}_ofr.json")
return data
monkeypatch.setattr(Path, "read_bytes", read_then_add)
assert hf.poll(job_id) is None
monkeypatch.setattr(Path, "read_bytes", real_read)
assert hf.poll(job_id) is None # состояние сброшено: новый набор настаивается с нуля
clock.now += 2
assert hf.poll(job_id) == [
AnswerFile(f"{job_id}.json", b"{}"),
AnswerFile(f"{job_id}_ofr.json", b'{"n": 2}'),
]
def test_file_removed_after_read_is_none(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""T2.1: файл прочитан, но пропал до повторного листинга — набор изменился, `None`."""
hf = LocalHotFolder(tmp_path, settle_seconds=0)
job_id = _job_id()
_write_answer(hf, job_id, b"{}")
second = _write_answer(hf, job_id, b"[]", f"{job_id}_ofr.json")
real_read = Path.read_bytes
def read_then_remove(self: Path) -> bytes:
data = real_read(self)
if self == second:
second.unlink()
return data
monkeypatch.setattr(Path, "read_bytes", read_then_remove)
assert hf.poll(job_id) is None
class TestSettleTracker:
def test_snapshot_must_be_stable_for_settle_seconds(self) -> None:
clock = FakeClock()
tracker = SettleTracker(2, clock)
snap = (("a.json", 2),)
assert tracker.ready("job", snap) is False
clock.now += 2
assert tracker.ready("job", snap) is True
assert len(tracker) == 0
def test_empty_snapshot_is_never_ready(self) -> None:
assert SettleTracker(0).ready("job", ()) is False
def test_zero_settle_keeps_no_state(self) -> None:
tracker = SettleTracker(0)
assert tracker.ready("job", (("a.json", 2),)) is True
assert len(tracker) == 0
def test_state_is_bounded(self) -> None:
"""Задания, ушедшие по таймауту с ненастоявшимся набором, не копятся: старые вытесняются."""
clock = FakeClock()
tracker = SettleTracker(2, clock, max_jobs=3)
for job in ("j1", "j2", "j3", "j4"):
assert tracker.ready(job, (("a.json", 1),)) is False
assert len(tracker) == 3
clock.now += 2
assert tracker.ready("j1", (("a.json", 1),)) is False # вытеснено — настаивается заново
assert tracker.ready("j4", (("a.json", 1),)) is True
def test_forget(self) -> None:
clock = FakeClock()
tracker = SettleTracker(2, clock)
tracker.ready("job", (("a.json", 1),))
tracker.forget("job")
tracker.forget("unknown")
assert len(tracker) == 0
def test_negative_settle_is_rejected(self) -> None:
with pytest.raises(ValueError, match="settle"):
SettleTracker(-0.1)
class TestHealthy:
def test_true_for_existing_dir(self, tmp_path: Path) -> None:
assert LocalHotFolder(tmp_path).healthy() is True
def test_false_for_missing_root(self, tmp_path: Path) -> None:
assert LocalHotFolder(tmp_path / "absent").healthy() is False
def test_false_for_file_root(self, tmp_path: Path) -> None:
file_root = tmp_path / "file"
file_root.write_bytes(b"x")
assert LocalHotFolder(file_root).healthy() is False
def test_never_raises(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
def boom(self: Path) -> bool:
raise PermissionError("нет доступа")
monkeypatch.setattr(Path, "is_dir", boom)
assert LocalHotFolder(tmp_path).healthy() is False