LocalHotFolder.put сначала проверяет назначение: если файл во входящем каталоге уже есть, он считается доставленным, ничего не пишется, а .part снимается только при включённой очистке. Иначе удаление старого .part подавляется (остаток прерванной записи не роняет задание); если он остался, усечение разрешено только при st_nlink == 1, ссылка на чужой файл не трогается и исключение поднимается как есть. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vJkmBGKKqMoEMzvsXG3y2
791 lines
35 KiB
Python
791 lines
35 KiB
Python
"""Горячая папка в обычном каталоге (spec image-recognition-gateway, D-1, R-4/R-5, slice-0-skeleton).
|
||
|
||
Быстрые тесты на `tmp_path`: атомарная запись входящего файла через служебный каталог `Tmp`, опрос набора ответов ИС
|
||
с готовностью по стабилизации (settle, часы подменены), проверка здоровья, валидация имён.
|
||
"""
|
||
|
||
import errno
|
||
import os
|
||
import uuid
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
|
||
from src.gateway.hotfolder import AnswerFile, HotFolder, LocalHotFolder, SettleTracker, inbox_name
|
||
|
||
|
||
def _job_id() -> str:
|
||
return str(uuid.uuid4())
|
||
|
||
|
||
class FakeClock:
|
||
"""Монотонные часы теста: время двигается только вручную."""
|
||
|
||
def __init__(self) -> None:
|
||
self.now = 100.0
|
||
|
||
def __call__(self) -> float:
|
||
return self.now
|
||
|
||
|
||
def _write_answer(hf: LocalHotFolder, job_id: str, data: bytes, name: str | None = None) -> Path:
|
||
path = hf.answer_dir(job_id) / (name or f"{job_id}.json")
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_bytes(data)
|
||
return path
|
||
|
||
|
||
def _settled(tmp_path: Path, settle: float = 2) -> tuple[LocalHotFolder, FakeClock]:
|
||
clock = FakeClock()
|
||
return LocalHotFolder(tmp_path, settle_seconds=settle, clock=clock), clock
|
||
|
||
|
||
class TestInboxName:
|
||
def test_job_id_and_lowercase_extension(self) -> None:
|
||
job_id = _job_id()
|
||
assert inbox_name(job_id, "aep_5.PDF") == f"{job_id}.pdf"
|
||
assert inbox_name(job_id, "dir/sub/Отчёт за 2024.Jpeg") == f"{job_id}.jpeg"
|
||
assert inbox_name(job_id, "C:\\scans\\report.pdf") == f"{job_id}.pdf"
|
||
|
||
@pytest.mark.parametrize(
|
||
"original",
|
||
["scan", "", ".", "..", "report.", "a/..", "x.a:b", "x." + "p" * 17, "name.тар"],
|
||
)
|
||
def test_rejects_names_without_safe_extension(self, original: str) -> None:
|
||
with pytest.raises(ValueError, match="расширени"):
|
||
inbox_name("job", original)
|
||
|
||
|
||
class TestPutWithoutReplace:
|
||
def test_existing_destination_is_kept(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
hf.put("job.pdf", b"first")
|
||
hf.put("job.pdf", b"second")
|
||
assert (hf.input_dir / "job.pdf").read_bytes() == b"first"
|
||
# R-17: назначение есть — повтор ничего не пишет; прежнего `.part` после успешной публикации нет
|
||
assert not (hf.tmp_dir / ".job.pdf.part").exists()
|
||
|
||
def test_existing_destination_part_removed_when_cleanup_enabled(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path, cleanup=True)
|
||
hf.put("job.pdf", b"first")
|
||
hf.put("job.pdf", b"second")
|
||
assert (hf.input_dir / "job.pdf").read_bytes() == b"first"
|
||
assert not (hf.tmp_dir / ".job.pdf.part").exists()
|
||
|
||
def test_no_temp_files_in_input(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
hf.put("job.pdf", b"x")
|
||
assert [p.name for p in hf.input_dir.iterdir()] == ["job.pdf"]
|
||
assert not (hf.tmp_dir / ".job.pdf.part").exists() # после успешной публикации .part нет
|
||
|
||
|
||
class TestCleanup:
|
||
def _fill(self, hf: LocalHotFolder, job_id: str) -> None:
|
||
hf.put(f"{job_id}.pdf", b"x")
|
||
(hf.input_dir / f"{job_id}.pdf.done").write_bytes(b"")
|
||
(hf.tmp_dir / f".{job_id}.pdf.part").write_bytes(b"orphan")
|
||
_write_answer(hf, job_id, b"{}")
|
||
_write_answer(hf, job_id, b"{}", name=f"{job_id}_ofr.json")
|
||
(hf.answer_dir(job_id) / "readme.txt").write_bytes(b"keep")
|
||
|
||
def test_disabled_touches_nothing(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
job_id = _job_id()
|
||
self._fill(hf, job_id)
|
||
assert hf.cleanup(job_id, f"{job_id}.pdf") is False
|
||
assert (hf.input_dir / f"{job_id}.pdf").exists() and hf.answer_dir(job_id).is_dir()
|
||
assert (hf.tmp_dir / f".{job_id}.pdf.part").exists()
|
||
|
||
def test_enabled_removes_job_files(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path, cleanup=True)
|
||
job_id = _job_id()
|
||
self._fill(hf, job_id)
|
||
assert hf.cleanup(job_id, f"{job_id}.pdf") is True
|
||
assert not (hf.input_dir / f"{job_id}.pdf").exists()
|
||
assert not (hf.input_dir / f"{job_id}.pdf.done").exists()
|
||
assert not (hf.tmp_dir / f".{job_id}.pdf.part").exists()
|
||
assert not hf.answer_dir(job_id).exists() # каталог удалён вместе с readme.txt
|
||
assert hf.cleanup(job_id, f"{job_id}.pdf") is True # повтор — не ошибка
|
||
|
||
def test_missing_files_are_fine(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path, cleanup=True)
|
||
assert hf.cleanup(_job_id(), "nothing.pdf") is True
|
||
|
||
def test_permission_error_propagates(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
hf = LocalHotFolder(tmp_path, cleanup=True)
|
||
job_id = _job_id()
|
||
self._fill(hf, job_id)
|
||
|
||
def deny(path: Any, *args: Any, **kwargs: Any) -> None:
|
||
raise PermissionError(f"нет прав: {path}")
|
||
|
||
monkeypatch.setattr(os, "unlink", deny)
|
||
with pytest.raises(PermissionError):
|
||
hf.cleanup(job_id, f"{job_id}.pdf")
|
||
|
||
def test_rejects_unsafe_names(self, tmp_path: Path) -> None:
|
||
"""cleanup: validate_name защищает от выхода за пределы папки (T11.4)."""
|
||
hf = LocalHotFolder(tmp_path, cleanup=True)
|
||
with pytest.raises(ValueError, match="имя файла"):
|
||
hf.cleanup(_job_id(), "../x.pdf")
|
||
# ничего не удалено
|
||
assert not hf.input_dir.exists()
|
||
|
||
|
||
class TestPaths:
|
||
def test_default_layout(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
job_id = _job_id()
|
||
assert hf.root == tmp_path
|
||
assert hf.input_dir == tmp_path / "Input"
|
||
assert hf.output_dir == tmp_path / "Output"
|
||
assert hf.tmp_dir == tmp_path / "Tmp"
|
||
assert hf.answer_dir(job_id) == tmp_path / "Output" / job_id
|
||
assert hf.answer_path(job_id) == tmp_path / "Output" / job_id / f"{job_id}.json"
|
||
|
||
def test_custom_dir_names_and_str_root(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(str(tmp_path), input_dir="Входящие", output_dir="Исходящие", tmp_dir="Служебный")
|
||
assert hf.root == tmp_path
|
||
assert hf.input_dir == tmp_path / "Входящие"
|
||
assert hf.output_dir == tmp_path / "Исходящие"
|
||
assert hf.tmp_dir == tmp_path / "Служебный"
|
||
assert hf.answer_path("abc") == tmp_path / "Исходящие" / "abc" / "abc.json"
|
||
|
||
@pytest.mark.parametrize("job_id", ["", ".", "..", "../x", "a/b", "a\\b", "a:b", "a?b", "a*b", "a\x00b", "a\nb"])
|
||
def test_answer_path_rejects_unsafe_job_id(self, tmp_path: Path, job_id: str) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
with pytest.raises(ValueError, match="job_id"):
|
||
hf.answer_path(job_id)
|
||
with pytest.raises(ValueError, match="job_id"):
|
||
hf.answer_dir(job_id)
|
||
with pytest.raises(ValueError, match="job_id"):
|
||
hf.poll(job_id)
|
||
|
||
def test_negative_settle_is_rejected(self, tmp_path: Path) -> None:
|
||
with pytest.raises(ValueError, match="settle"):
|
||
LocalHotFolder(tmp_path, settle_seconds=-1)
|
||
|
||
def test_implements_protocol(self, tmp_path: Path) -> None:
|
||
hf: HotFolder = LocalHotFolder(tmp_path)
|
||
assert hf.healthy() is True
|
||
assert hasattr(hf, "cleanup")
|
||
|
||
|
||
class TestPut:
|
||
def test_writes_bytes_under_inbox_name(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
job_id = _job_id()
|
||
name = inbox_name(job_id, "aep_5.pdf")
|
||
data = bytes(range(256)) * 10
|
||
hf.put(name, data)
|
||
assert (tmp_path / "Input" / name).read_bytes() == data
|
||
|
||
def test_leaves_no_part_files(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
job_id = _job_id()
|
||
name = inbox_name(job_id, "aep_5.pdf")
|
||
hf.put(name, b"data")
|
||
assert [p.name for p in hf.input_dir.iterdir()] == [name]
|
||
assert list(hf.tmp_dir.iterdir()) == []
|
||
|
||
def test_part_is_written_in_tmp_not_in_input(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""OQ-79: во время записи во входящем каталоге нет временного файла — он в `Tmp/`, затем link в `Input`."""
|
||
hf = LocalHotFolder(tmp_path)
|
||
job_id = _job_id()
|
||
name = inbox_name(job_id, "aep_5.pdf")
|
||
seen: dict[str, Any] = {}
|
||
real_link = os.link
|
||
|
||
def spy(src: Any, dst: Any) -> None:
|
||
seen["src"], seen["dst"] = Path(src), Path(dst)
|
||
seen["input"] = sorted(p.name for p in hf.input_dir.iterdir())
|
||
seen["tmp"] = sorted(p.name for p in hf.tmp_dir.iterdir())
|
||
seen["part"] = Path(src).read_bytes()
|
||
real_link(src, dst)
|
||
|
||
monkeypatch.setattr("src.gateway.hotfolder.os.link", spy)
|
||
hf.put(name, b"data")
|
||
assert seen == {
|
||
"src": hf.tmp_dir / f".{name}.part",
|
||
"dst": hf.input_dir / name,
|
||
"input": [],
|
||
"tmp": [f".{name}.part"],
|
||
"part": b"data",
|
||
}
|
||
assert [p.name for p in hf.input_dir.iterdir()] == [name]
|
||
assert list(hf.tmp_dir.iterdir()) == []
|
||
|
||
def test_creates_input_and_tmp_dirs_only(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
assert not hf.input_dir.exists()
|
||
assert not hf.tmp_dir.exists()
|
||
hf.put("a.pdf", b"1")
|
||
assert hf.input_dir.is_dir()
|
||
assert hf.tmp_dir.is_dir()
|
||
assert not hf.output_dir.exists()
|
||
|
||
def test_existing_tmp_dir_is_fine(self, tmp_path: Path) -> None:
|
||
(tmp_path / "Tmp").mkdir()
|
||
LocalHotFolder(tmp_path).put("a.pdf", b"1")
|
||
assert (tmp_path / "Input" / "a.pdf").read_bytes() == b"1"
|
||
|
||
def test_custom_tmp_dir_name(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path, tmp_dir="Служебный")
|
||
hf.put("a.pdf", b"1")
|
||
assert (tmp_path / "Служебный").is_dir()
|
||
assert not (tmp_path / "Tmp").exists()
|
||
|
||
def test_existing_input_dir_is_fine(self, tmp_path: Path) -> None:
|
||
(tmp_path / "Input").mkdir()
|
||
LocalHotFolder(tmp_path).put("a.pdf", b"1")
|
||
assert (tmp_path / "Input" / "a.pdf").read_bytes() == b"1"
|
||
|
||
def test_missing_root_raises_and_creates_nothing(self, tmp_path: Path) -> None:
|
||
root = tmp_path / "absent"
|
||
with pytest.raises(FileNotFoundError):
|
||
LocalHotFolder(root).put("a.pdf", b"1")
|
||
assert not root.exists()
|
||
assert list(tmp_path.iterdir()) == []
|
||
|
||
def test_empty_data_is_written(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
hf.put("empty.pdf", b"")
|
||
assert (hf.input_dir / "empty.pdf").read_bytes() == b""
|
||
|
||
@pytest.mark.parametrize(
|
||
"name",
|
||
["", "../escape.pdf", "sub/a.pdf", "sub\\a.pdf", "/abs.pdf", ".hidden.pdf", ".a.pdf.part", "..", ".", "a\x00b"],
|
||
)
|
||
def test_rejects_invalid_names(self, tmp_path: Path, name: str) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
with pytest.raises(ValueError, match="имя файла"):
|
||
hf.put(name, b"x")
|
||
assert not hf.input_dir.exists()
|
||
assert not (tmp_path.parent / "escape.pdf").exists()
|
||
|
||
@pytest.mark.parametrize("name", ["a:b.pdf", "a<b.pdf", "a>b.pdf", 'a"b.pdf', "a|b.pdf", "a?b.pdf", "a*b.pdf"])
|
||
def test_rejects_names_illegal_on_windows(self, tmp_path: Path, name: str) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
with pytest.raises(ValueError, match="имя файла"):
|
||
hf.put(name, b"x")
|
||
assert not hf.input_dir.exists()
|
||
|
||
@pytest.mark.parametrize("name", ["a\x01b.pdf", "a\tb.pdf", "a\nb.pdf", "a\x1fb.pdf"])
|
||
def test_rejects_control_chars(self, tmp_path: Path, name: str) -> None:
|
||
with pytest.raises(ValueError, match="имя файла"):
|
||
LocalHotFolder(tmp_path).put(name, b"x")
|
||
|
||
def test_accepts_inbox_names(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
job_id_1 = _job_id()
|
||
job_id_2 = _job_id()
|
||
names = [inbox_name(job_id_1, "Отчёт за 2024 год (копия).PDF"), inbox_name(job_id_2, "aep_8.pdf")]
|
||
for name in names:
|
||
hf.put(name, b"x")
|
||
assert sorted(p.name for p in hf.input_dir.iterdir()) == sorted(names)
|
||
|
||
def test_cleans_part_file_on_failure(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""При cleanup=True и ошибке link: .part удаляется."""
|
||
hf = LocalHotFolder(tmp_path, cleanup=True)
|
||
|
||
def boom(src: Any, dst: Any) -> None:
|
||
raise OSError("диск недоступен")
|
||
|
||
monkeypatch.setattr("src.gateway.hotfolder.os.link", boom)
|
||
with pytest.raises(OSError, match="диск недоступен"):
|
||
hf.put("a.pdf", b"data")
|
||
assert list(hf.input_dir.iterdir()) == []
|
||
assert list(hf.tmp_dir.iterdir()) == []
|
||
|
||
def test_keeps_part_file_on_failure_when_cleanup_disabled(
|
||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
"""При cleanup=False и ошибке link: .part остаётся в Tmp (T11.1)."""
|
||
hf = LocalHotFolder(tmp_path, cleanup=False)
|
||
|
||
def boom(src: Any, dst: Any) -> None:
|
||
raise OSError("диск недоступен")
|
||
|
||
monkeypatch.setattr("src.gateway.hotfolder.os.link", boom)
|
||
with pytest.raises(OSError, match="диск недоступен"):
|
||
hf.put("a.pdf", b"data")
|
||
assert list(hf.input_dir.iterdir()) == []
|
||
assert [p.name for p in hf.tmp_dir.iterdir()] == [".a.pdf.part"]
|
||
|
||
def test_successful_put_despite_final_unlink_failure(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Успешный link, но ошибка в завершающем unlink .part (R-8): put НЕ падает, файл доставлен (T11.2)."""
|
||
hf = LocalHotFolder(tmp_path)
|
||
|
||
unlink_calls = []
|
||
|
||
real_unlink = os.unlink
|
||
|
||
def unlink_with_failure(path: Any) -> None:
|
||
unlink_calls.append(path)
|
||
if str(path).endswith(".part"):
|
||
raise PermissionError("удаление запрещено")
|
||
real_unlink(path)
|
||
|
||
monkeypatch.setattr("src.gateway.hotfolder.os.unlink", unlink_with_failure)
|
||
hf.put("a.pdf", b"data") # НЕ падает несмотря на ошибку unlink
|
||
assert (hf.input_dir / "a.pdf").read_bytes() == b"data"
|
||
assert (hf.tmp_dir / ".a.pdf.part").exists() # .part не удалён из-за ошибки
|
||
assert unlink_calls # попытка удаления была
|
||
|
||
def test_cleanup_failure_does_not_mask_original_error(
|
||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
"""Ошибка удаления `.part` на пути ошибки link подавляется (T11.1); исходное исключение проходит."""
|
||
hf = LocalHotFolder(tmp_path, cleanup=True)
|
||
|
||
def boom(src: Any, dst: Any) -> None:
|
||
raise OSError("диск недоступен")
|
||
|
||
def unlink_fails(self: Path, missing_ok: bool = False) -> None:
|
||
raise PermissionError("удаление .part запрещено")
|
||
|
||
monkeypatch.setattr("src.gateway.hotfolder.os.link", boom)
|
||
monkeypatch.setattr(Path, "unlink", unlink_fails)
|
||
with pytest.raises(OSError, match="диск недоступен"):
|
||
hf.put("a.pdf", b"data")
|
||
|
||
|
||
class TestPartIsCreatedFresh:
|
||
"""Решение №32: `.part` создаётся заново (удаление старого + `O_EXCL`), fallback — только вокруг `os.link`."""
|
||
|
||
@pytest.mark.parametrize("cleanup", [False, True])
|
||
def test_existing_destination_is_not_touched_by_repeat(self, tmp_path: Path, cleanup: bool) -> None:
|
||
"""R-17: назначение есть, `.part` — жёсткая ссылка на него: байты не меняются, put не падает."""
|
||
hf = LocalHotFolder(tmp_path, cleanup=cleanup)
|
||
hf.put("job.pdf", b"first")
|
||
published = hf.input_dir / "job.pdf"
|
||
stale = hf.tmp_dir / ".job.pdf.part"
|
||
os.link(published, stale)
|
||
hf.put("job.pdf", b"different data")
|
||
assert published.read_bytes() == b"first"
|
||
assert stale.exists() is (not cleanup) # очистка снимает `.part`, иначе он остаётся как был
|
||
if not cleanup:
|
||
assert stale.read_bytes() == b"first" and published.stat().st_nlink == 2
|
||
|
||
def test_existing_destination_with_undeletable_part_does_not_fail(
|
||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
hf = LocalHotFolder(tmp_path) # очистка выключена: удаление `.part` вообще не пытается
|
||
hf.put("job.pdf", b"first")
|
||
os.link(hf.input_dir / "job.pdf", hf.tmp_dir / ".job.pdf.part")
|
||
|
||
def deny(self: Path, missing_ok: bool = False) -> None:
|
||
raise PermissionError("удаление запрещено")
|
||
|
||
monkeypatch.setattr(Path, "unlink", deny)
|
||
hf.put("job.pdf", b"second")
|
||
assert (hf.input_dir / "job.pdf").read_bytes() == b"first"
|
||
|
||
def test_undeletable_stale_part_with_single_link_is_truncated(
|
||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
"""Назначения нет, старый `.part` не удаляется, но это обычный файл (nlink == 1): put успешен."""
|
||
hf = LocalHotFolder(tmp_path)
|
||
hf.tmp_dir.mkdir()
|
||
stale = hf.tmp_dir / ".job.pdf.part"
|
||
stale.write_bytes(b"old and longer content")
|
||
|
||
def deny(self: Path, missing_ok: bool = False) -> None:
|
||
raise PermissionError("удаление запрещено")
|
||
|
||
monkeypatch.setattr(Path, "unlink", deny)
|
||
hf.put("job.pdf", b"new")
|
||
assert (hf.input_dir / "job.pdf").read_bytes() == b"new"
|
||
|
||
def test_undeletable_stale_part_with_other_links_is_not_written(
|
||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
"""Назначения нет, старый `.part` не удаляется и имеет nlink == 2 (синтетически): запись не идёт, ошибка не
|
||
маскируется, второй файл не изменён."""
|
||
hf = LocalHotFolder(tmp_path)
|
||
hf.tmp_dir.mkdir()
|
||
other = tmp_path / "other.bin"
|
||
other.write_bytes(b"precious")
|
||
os.link(other, hf.tmp_dir / ".job.pdf.part")
|
||
|
||
def deny(self: Path, missing_ok: bool = False) -> None:
|
||
raise PermissionError("удаление запрещено")
|
||
|
||
monkeypatch.setattr(Path, "unlink", deny)
|
||
with pytest.raises(FileExistsError):
|
||
hf.put("job.pdf", b"new")
|
||
assert other.read_bytes() == b"precious"
|
||
assert not (hf.input_dir / "job.pdf").exists()
|
||
|
||
def test_stale_part_is_replaced_before_publication(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
hf.tmp_dir.mkdir()
|
||
(hf.tmp_dir / ".job.pdf.part").write_bytes(b"old and longer content")
|
||
hf.put("job.pdf", b"new")
|
||
assert (hf.input_dir / "job.pdf").read_bytes() == b"new"
|
||
assert not (hf.tmp_dir / ".job.pdf.part").exists()
|
||
|
||
def test_part_is_opened_exclusively(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
flags: list[int] = []
|
||
real_open = os.open
|
||
|
||
def spy(path: Any, flag: int, mode: int = 0o777, **kwargs: Any) -> int:
|
||
flags.append(flag)
|
||
return real_open(path, flag, mode, **kwargs)
|
||
|
||
monkeypatch.setattr("src.gateway.hotfolder.os.open", spy)
|
||
hf.put("job.pdf", b"x")
|
||
assert flags and flags[0] & os.O_EXCL and flags[0] & os.O_CREAT
|
||
|
||
def test_write_error_propagates_as_itself(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""EPERM записи не превращается в `FileNotFoundError` от `os.replace` несуществующего `.part`."""
|
||
hf = LocalHotFolder(tmp_path, cleanup=True)
|
||
|
||
class FailingFile:
|
||
def __enter__(self) -> "FailingFile":
|
||
return self
|
||
|
||
def __exit__(self, *exc: object) -> None:
|
||
return None
|
||
|
||
def write(self, data: bytes) -> int:
|
||
raise OSError(errno.EPERM, "запись запрещена")
|
||
|
||
monkeypatch.setattr("src.gateway.hotfolder.os.fdopen", lambda fd, mode: (os.close(fd), FailingFile())[1])
|
||
with pytest.raises(OSError, match="запись запрещена") as caught:
|
||
hf.put("job.pdf", b"data")
|
||
assert caught.value.errno == errno.EPERM and not isinstance(caught.value, FileNotFoundError)
|
||
assert list(hf.input_dir.iterdir()) == [] and list(hf.tmp_dir.iterdir()) == []
|
||
|
||
@pytest.mark.parametrize("code", [errno.EPERM, errno.EXDEV, errno.ENOTSUP, errno.EOPNOTSUPP])
|
||
def test_fallback_to_replace_when_link_unsupported(
|
||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, code: int
|
||
) -> None:
|
||
hf = LocalHotFolder(tmp_path)
|
||
|
||
def no_links(src: Any, dst: Any) -> None:
|
||
raise OSError(code, "жёсткие ссылки не поддерживаются")
|
||
|
||
monkeypatch.setattr("src.gateway.hotfolder.os.link", no_links)
|
||
hf.put("job.pdf", b"data")
|
||
assert (hf.input_dir / "job.pdf").read_bytes() == b"data"
|
||
assert list(hf.tmp_dir.iterdir()) == [] # `.part` стал назначением
|
||
|
||
def test_no_fallback_when_destination_appeared_during_link(
|
||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
"""Назначение появилось между проверкой и линковкой, а `os.link` дал EPERM: `os.replace` не затирает."""
|
||
hf = LocalHotFolder(tmp_path, cleanup=True)
|
||
hf.input_dir.mkdir()
|
||
|
||
def no_links(src: Any, dst: Any) -> None:
|
||
Path(dst).write_bytes(b"first") # файл успел появиться (параллельный повтор того же задания)
|
||
raise OSError(errno.EPERM, "жёсткие ссылки не поддерживаются")
|
||
|
||
monkeypatch.setattr("src.gateway.hotfolder.os.link", no_links)
|
||
with pytest.raises(OSError, match="жёсткие ссылки"):
|
||
hf.put("job.pdf", b"second")
|
||
assert (hf.input_dir / "job.pdf").read_bytes() == b"first"
|
||
|
||
def test_replace_failure_after_link_fallback_propagates(
|
||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
hf = LocalHotFolder(tmp_path, cleanup=True)
|
||
|
||
def no_links(src: Any, dst: Any) -> None:
|
||
raise OSError(errno.EPERM, "жёсткие ссылки не поддерживаются")
|
||
|
||
def broken_replace(src: Any, dst: Any) -> None:
|
||
raise OSError(errno.EIO, "замена не удалась")
|
||
|
||
monkeypatch.setattr("src.gateway.hotfolder.os.link", no_links)
|
||
monkeypatch.setattr("src.gateway.hotfolder.os.replace", broken_replace)
|
||
with pytest.raises(OSError, match="замена не удалась"):
|
||
hf.put("job.pdf", b"data")
|
||
assert list(hf.tmp_dir.iterdir()) == [] # при очистке `.part` снят
|
||
|
||
|
||
class TestPoll:
|
||
"""`poll` — все `*.json` каталога `<output>/<job_id>/` по имени, когда набор настоялся `settle_seconds`."""
|
||
|
||
def test_none_until_answer_dir_has_json(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path, settle_seconds=0)
|
||
job_id = _job_id()
|
||
assert hf.poll(job_id) is None
|
||
(tmp_path / "Output").mkdir()
|
||
assert hf.poll(job_id) is None
|
||
(tmp_path / "Output" / job_id).mkdir()
|
||
assert hf.poll(job_id) is None
|
||
|
||
def test_missing_root_is_none(self, tmp_path: Path) -> None:
|
||
assert LocalHotFolder(tmp_path / "absent", settle_seconds=0).poll(_job_id()) is None
|
||
|
||
def test_answer_dir_is_a_file_is_none(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path, settle_seconds=0)
|
||
job_id = _job_id()
|
||
(tmp_path / "Output").mkdir()
|
||
(tmp_path / "Output" / job_id).write_bytes(b"{}")
|
||
assert hf.poll(job_id) is None
|
||
|
||
def test_settle_zero_returns_at_once(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path, settle_seconds=0)
|
||
job_id = _job_id()
|
||
payload = b'{"file_uid": "x", "data_ocr": {}}'
|
||
_write_answer(hf, job_id, payload)
|
||
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", payload)]
|
||
|
||
def test_one_json_after_settle(self, tmp_path: Path) -> None:
|
||
hf, clock = _settled(tmp_path)
|
||
job_id = _job_id()
|
||
_write_answer(hf, job_id, b"{}")
|
||
assert hf.poll(job_id) is None # набор впервые увиден
|
||
clock.now += 1.9
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 0.1
|
||
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b"{}")]
|
||
|
||
def test_two_jsons_ascending_by_name(self, tmp_path: Path) -> None:
|
||
hf, clock = _settled(tmp_path)
|
||
job_id = _job_id()
|
||
_write_answer(hf, job_id, b'{"n": 2}', f"{job_id}_ofr.json")
|
||
_write_answer(hf, job_id, b'{"n": 1}', f"{job_id}.json")
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 2
|
||
assert hf.poll(job_id) == [
|
||
AnswerFile(f"{job_id}.json", b'{"n": 1}'),
|
||
AnswerFile(f"{job_id}_ofr.json", b'{"n": 2}'),
|
||
]
|
||
|
||
def test_non_json_files_and_subdirs_are_ignored(self, tmp_path: Path) -> None:
|
||
hf, clock = _settled(tmp_path)
|
||
job_id = _job_id()
|
||
_write_answer(hf, job_id, b"text", "readme.txt")
|
||
_write_answer(hf, job_id, b"{}", f".{job_id}.json.part")
|
||
(hf.answer_dir(job_id) / "nested.json").mkdir()
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 10
|
||
assert hf.poll(job_id) is None # json-файлов нет — ответа нет, сколько ни жди
|
||
_write_answer(hf, job_id, b"{}")
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 2
|
||
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b"{}")]
|
||
|
||
def test_json_extension_is_case_insensitive(self, tmp_path: Path) -> None:
|
||
hf = LocalHotFolder(tmp_path, settle_seconds=0)
|
||
job_id = _job_id()
|
||
_write_answer(hf, job_id, b"{}", "B.JSON")
|
||
_write_answer(hf, job_id, b"[]", "a.Json")
|
||
assert hf.poll(job_id) == [AnswerFile("B.JSON", b"{}"), AnswerFile("a.Json", b"[]")]
|
||
|
||
def test_growing_file_is_not_ready_until_stable(self, tmp_path: Path) -> None:
|
||
hf, clock = _settled(tmp_path)
|
||
job_id = _job_id()
|
||
path = _write_answer(hf, job_id, b'{"part": ')
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 1.5
|
||
path.write_bytes(b'{"part": 1, "more": ') # размер вырос — таймер заново
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 1.5
|
||
path.write_bytes(b'{"part": 1, "more": 2}')
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 1.9
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 0.1
|
||
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b'{"part": 1, "more": 2}')]
|
||
|
||
def test_new_file_resets_timer(self, tmp_path: Path) -> None:
|
||
hf, clock = _settled(tmp_path)
|
||
job_id = _job_id()
|
||
_write_answer(hf, job_id, b"{}")
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 1.5
|
||
_write_answer(hf, job_id, b"[]", f"{job_id}_ofr.json")
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 1.5
|
||
assert hf.poll(job_id) is None # с момента появления второго файла прошло 1,5 с
|
||
clock.now += 0.5
|
||
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b"{}"), AnswerFile(f"{job_id}_ofr.json", b"[]")]
|
||
|
||
def test_removed_file_resets_timer(self, tmp_path: Path) -> None:
|
||
hf, clock = _settled(tmp_path)
|
||
job_id = _job_id()
|
||
_write_answer(hf, job_id, b"{}")
|
||
extra = _write_answer(hf, job_id, b"[]", f"{job_id}_ofr.json")
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 1.5
|
||
extra.unlink()
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 1.5
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 0.5
|
||
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b"{}")]
|
||
|
||
def test_removal_of_all_json_is_none_and_restarts(self, tmp_path: Path) -> None:
|
||
hf, clock = _settled(tmp_path)
|
||
job_id = _job_id()
|
||
path = _write_answer(hf, job_id, b"{}")
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 1
|
||
path.unlink()
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 5
|
||
path.write_bytes(b"{}") # тот же набор снова: настаивается заново, а не «с прошлого раза»
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 2
|
||
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b"{}")]
|
||
|
||
def test_state_is_dropped_after_answer(self, tmp_path: Path) -> None:
|
||
hf, clock = _settled(tmp_path)
|
||
job_id = _job_id()
|
||
_write_answer(hf, job_id, b"{}")
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 2
|
||
assert hf.poll(job_id) is not None
|
||
assert hf.pending_jobs() == 0
|
||
assert hf.poll(job_id) is None # состояние сброшено: повторный опрос настаивает набор заново
|
||
|
||
def test_other_job_is_unaffected(self, tmp_path: Path) -> None:
|
||
hf, clock = _settled(tmp_path)
|
||
job_id, other = _job_id(), _job_id()
|
||
_write_answer(hf, other, b"{}")
|
||
assert hf.poll(other) is None
|
||
clock.now += 1
|
||
_write_answer(hf, job_id, b"[]")
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 1
|
||
assert hf.poll(job_id) is None
|
||
assert hf.poll(other) == [AnswerFile(f"{other}.json", b"{}")]
|
||
clock.now += 1
|
||
assert hf.poll(job_id) == [AnswerFile(f"{job_id}.json", b"[]")]
|
||
|
||
def test_file_changed_between_listing_and_read_is_none(
|
||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
hf = LocalHotFolder(tmp_path, settle_seconds=0)
|
||
job_id = _job_id()
|
||
_write_answer(hf, job_id, b"{}")
|
||
monkeypatch.setattr(Path, "read_bytes", lambda self: b'{"longer": true}')
|
||
assert hf.poll(job_id) is None
|
||
|
||
def test_file_removed_between_listing_and_read_is_none(
|
||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
hf = LocalHotFolder(tmp_path, settle_seconds=0)
|
||
job_id = _job_id()
|
||
path = _write_answer(hf, job_id, b"{}")
|
||
|
||
def vanished(self: Path) -> bytes:
|
||
path.unlink()
|
||
raise FileNotFoundError(str(self))
|
||
|
||
monkeypatch.setattr(Path, "read_bytes", vanished)
|
||
assert hf.poll(job_id) is None
|
||
|
||
def test_new_file_appeared_while_reading_is_none_and_set_resettles(
|
||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||
) -> None:
|
||
"""T2.1: после чтения каталог листается заново; набор изменился (появился файл) — `None`, набор настаивается
|
||
заново и отдаётся целиком."""
|
||
hf, clock = _settled(tmp_path)
|
||
job_id = _job_id()
|
||
_write_answer(hf, job_id, b"{}")
|
||
assert hf.poll(job_id) is None
|
||
clock.now += 2
|
||
real_read = Path.read_bytes
|
||
|
||
def read_then_add(self: Path) -> bytes:
|
||
data = real_read(self)
|
||
_write_answer(hf, job_id, b'{"n": 2}', f"{job_id}_ofr.json")
|
||
return data
|
||
|
||
monkeypatch.setattr(Path, "read_bytes", read_then_add)
|
||
assert hf.poll(job_id) is None
|
||
monkeypatch.setattr(Path, "read_bytes", real_read)
|
||
assert hf.poll(job_id) is None # состояние сброшено: новый набор настаивается с нуля
|
||
clock.now += 2
|
||
assert hf.poll(job_id) == [
|
||
AnswerFile(f"{job_id}.json", b"{}"),
|
||
AnswerFile(f"{job_id}_ofr.json", b'{"n": 2}'),
|
||
]
|
||
|
||
def test_file_removed_after_read_is_none(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""T2.1: файл прочитан, но пропал до повторного листинга — набор изменился, `None`."""
|
||
hf = LocalHotFolder(tmp_path, settle_seconds=0)
|
||
job_id = _job_id()
|
||
_write_answer(hf, job_id, b"{}")
|
||
second = _write_answer(hf, job_id, b"[]", f"{job_id}_ofr.json")
|
||
real_read = Path.read_bytes
|
||
|
||
def read_then_remove(self: Path) -> bytes:
|
||
data = real_read(self)
|
||
if self == second:
|
||
second.unlink()
|
||
return data
|
||
|
||
monkeypatch.setattr(Path, "read_bytes", read_then_remove)
|
||
assert hf.poll(job_id) is None
|
||
|
||
|
||
class TestSettleTracker:
|
||
def test_snapshot_must_be_stable_for_settle_seconds(self) -> None:
|
||
clock = FakeClock()
|
||
tracker = SettleTracker(2, clock)
|
||
snap = (("a.json", 2),)
|
||
assert tracker.ready("job", snap) is False
|
||
clock.now += 2
|
||
assert tracker.ready("job", snap) is True
|
||
assert len(tracker) == 0
|
||
|
||
def test_empty_snapshot_is_never_ready(self) -> None:
|
||
assert SettleTracker(0).ready("job", ()) is False
|
||
|
||
def test_zero_settle_keeps_no_state(self) -> None:
|
||
tracker = SettleTracker(0)
|
||
assert tracker.ready("job", (("a.json", 2),)) is True
|
||
assert len(tracker) == 0
|
||
|
||
def test_state_is_bounded(self) -> None:
|
||
"""Задания, ушедшие по таймауту с ненастоявшимся набором, не копятся: старые вытесняются."""
|
||
clock = FakeClock()
|
||
tracker = SettleTracker(2, clock, max_jobs=3)
|
||
for job in ("j1", "j2", "j3", "j4"):
|
||
assert tracker.ready(job, (("a.json", 1),)) is False
|
||
assert len(tracker) == 3
|
||
clock.now += 2
|
||
assert tracker.ready("j1", (("a.json", 1),)) is False # вытеснено — настаивается заново
|
||
assert tracker.ready("j4", (("a.json", 1),)) is True
|
||
|
||
def test_forget(self) -> None:
|
||
clock = FakeClock()
|
||
tracker = SettleTracker(2, clock)
|
||
tracker.ready("job", (("a.json", 1),))
|
||
tracker.forget("job")
|
||
tracker.forget("unknown")
|
||
assert len(tracker) == 0
|
||
|
||
def test_negative_settle_is_rejected(self) -> None:
|
||
with pytest.raises(ValueError, match="settle"):
|
||
SettleTracker(-0.1)
|
||
|
||
|
||
class TestHealthy:
|
||
def test_true_for_existing_dir(self, tmp_path: Path) -> None:
|
||
assert LocalHotFolder(tmp_path).healthy() is True
|
||
|
||
def test_false_for_missing_root(self, tmp_path: Path) -> None:
|
||
assert LocalHotFolder(tmp_path / "absent").healthy() is False
|
||
|
||
def test_false_for_file_root(self, tmp_path: Path) -> None:
|
||
file_root = tmp_path / "file"
|
||
file_root.write_bytes(b"x")
|
||
assert LocalHotFolder(file_root).healthy() is False
|
||
|
||
def test_never_raises(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||
def boom(self: Path) -> bool:
|
||
raise PermissionError("нет доступа")
|
||
|
||
monkeypatch.setattr(Path, "is_dir", boom)
|
||
assert LocalHotFolder(tmp_path).healthy() is False
|